From f42bca8e35bda433485092d46421784b6e226eb5 Mon Sep 17 00:00:00 2001 From: Robert Findley Date: Mon, 7 Aug 2023 15:10:26 -0400 Subject: [PATCH 001/178] refactor/satisfy: (re)allow composite lits of pointer type CL 513775 fixed a bug related to composite lits of type parameter type, but inadvertently removed handling for lits of pointer type (the 'deref' call was replaced, rather than wrapped). This results in a panic for unnamed literals of pointer type (such as in []*X{{}}). Replace the defer. Fixes golang/go#61813 Change-Id: I2a66f6bfebd6d6d11a5fb7bf4c1cf453bcf02d4b Reviewed-on: https://go-review.googlesource.com/c/tools/+/516775 Reviewed-by: Hyang-Ah Hana Kim Run-TryBot: Robert Findley TryBot-Result: Gopher Robot gopls-CI: kokoro --- .../marker/testdata/rename/issue61813.txt | 18 ++++++++++++++++++ refactor/satisfy/find.go | 4 ++-- 2 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 gopls/internal/regtest/marker/testdata/rename/issue61813.txt diff --git a/gopls/internal/regtest/marker/testdata/rename/issue61813.txt b/gopls/internal/regtest/marker/testdata/rename/issue61813.txt new file mode 100644 index 00000000000..ae5162b84a4 --- /dev/null +++ b/gopls/internal/regtest/marker/testdata/rename/issue61813.txt @@ -0,0 +1,18 @@ +This test exercises the panic reported in golang/go#61813. + +-- p.go -- +package p + +type P struct{} + +func (P) M() {} //@rename("M", N, MToN) + +var x = []*P{{}} +-- @MToN/p.go -- +package p + +type P struct{} + +func (P) N() {} //@rename("M", N, MToN) + +var x = []*P{{}} diff --git a/refactor/satisfy/find.go b/refactor/satisfy/find.go index 47dc97e471c..9e60af3b618 100644 --- a/refactor/satisfy/find.go +++ b/refactor/satisfy/find.go @@ -355,7 +355,7 @@ func (f *Finder) expr(e ast.Expr) types.Type { f.sig = saved case *ast.CompositeLit: - switch T := coreType(tv.Type).(type) { + switch T := coreType(deref(tv.Type)).(type) { case *types.Struct: for i, elem := range e.Elts { if kv, ok := elem.(*ast.KeyValueExpr); ok { @@ -386,7 +386,7 @@ func (f *Finder) expr(e ast.Expr) types.Type { } default: - panic("unexpected composite literal type: " + tv.Type.String()) + panic(fmt.Sprintf("unexpected composite literal type %T: %v", tv.Type, tv.Type.String())) } case *ast.ParenExpr: From 732ad6f9f8bf74d273ff8a0dd7ab9b6c5bd425c6 Mon Sep 17 00:00:00 2001 From: Gopher Robot Date: Tue, 8 Aug 2023 15:10:32 +0000 Subject: [PATCH 002/178] internal/imports: update stdlib index for Go 1.21.0 For golang/go#38706. Change-Id: I278b5edd7caf6ddb0585a3ea8f47777753457ddf Reviewed-on: https://go-review.googlesource.com/c/tools/+/517075 Run-TryBot: Michael Knyszek Reviewed-by: Michael Knyszek Reviewed-by: David Chase TryBot-Result: Gopher Robot Auto-Submit: Gopher Robot Run-TryBot: Gopher Robot Auto-Submit: Michael Knyszek --- internal/imports/zstdlib.go | 230 ++++++++++++++++++++++++++++++++++++ 1 file changed, 230 insertions(+) diff --git a/internal/imports/zstdlib.go b/internal/imports/zstdlib.go index 31a75949cdc..9f992c2bec8 100644 --- a/internal/imports/zstdlib.go +++ b/internal/imports/zstdlib.go @@ -93,6 +93,7 @@ var stdlib = map[string][]string{ "Compare", "Contains", "ContainsAny", + "ContainsFunc", "ContainsRune", "Count", "Cut", @@ -147,6 +148,11 @@ var stdlib = map[string][]string{ "TrimSpace", "TrimSuffix", }, + "cmp": { + "Compare", + "Less", + "Ordered", + }, "compress/bzip2": { "NewReader", "StructuralError", @@ -228,6 +234,7 @@ var stdlib = map[string][]string{ "Ring", }, "context": { + "AfterFunc", "Background", "CancelCauseFunc", "CancelFunc", @@ -239,8 +246,11 @@ var stdlib = map[string][]string{ "WithCancel", "WithCancelCause", "WithDeadline", + "WithDeadlineCause", "WithTimeout", + "WithTimeoutCause", "WithValue", + "WithoutCancel", }, "crypto": { "BLAKE2b_256", @@ -445,6 +455,7 @@ var stdlib = map[string][]string{ "XORBytes", }, "crypto/tls": { + "AlertError", "Certificate", "CertificateRequestInfo", "CertificateVerificationError", @@ -476,6 +487,7 @@ var stdlib = map[string][]string{ "LoadX509KeyPair", "NewLRUClientSessionCache", "NewListener", + "NewResumptionState", "NoClientCert", "PKCS1WithSHA1", "PKCS1WithSHA256", @@ -484,6 +496,27 @@ var stdlib = map[string][]string{ "PSSWithSHA256", "PSSWithSHA384", "PSSWithSHA512", + "ParseSessionState", + "QUICClient", + "QUICConfig", + "QUICConn", + "QUICEncryptionLevel", + "QUICEncryptionLevelApplication", + "QUICEncryptionLevelEarly", + "QUICEncryptionLevelHandshake", + "QUICEncryptionLevelInitial", + "QUICEvent", + "QUICEventKind", + "QUICHandshakeDone", + "QUICNoEvent", + "QUICRejectedEarlyData", + "QUICServer", + "QUICSessionTicketOptions", + "QUICSetReadSecret", + "QUICSetWriteSecret", + "QUICTransportParameters", + "QUICTransportParametersRequired", + "QUICWriteData", "RecordHeaderError", "RenegotiateFreelyAsClient", "RenegotiateNever", @@ -493,6 +526,7 @@ var stdlib = map[string][]string{ "RequireAndVerifyClientCert", "RequireAnyClientCert", "Server", + "SessionState", "SignatureScheme", "TLS_AES_128_GCM_SHA256", "TLS_AES_256_GCM_SHA384", @@ -523,6 +557,7 @@ var stdlib = map[string][]string{ "TLS_RSA_WITH_AES_256_GCM_SHA384", "TLS_RSA_WITH_RC4_128_SHA", "VerifyClientCertIfGiven", + "VersionName", "VersionSSL30", "VersionTLS10", "VersionTLS11", @@ -618,6 +653,7 @@ var stdlib = map[string][]string{ "PureEd25519", "RSA", "RevocationList", + "RevocationListEntry", "SHA1WithRSA", "SHA256WithRSA", "SHA256WithRSAPSS", @@ -1002,10 +1038,42 @@ var stdlib = map[string][]string{ "COMPRESS_LOOS", "COMPRESS_LOPROC", "COMPRESS_ZLIB", + "COMPRESS_ZSTD", "Chdr32", "Chdr64", "Class", "CompressionType", + "DF_1_CONFALT", + "DF_1_DIRECT", + "DF_1_DISPRELDNE", + "DF_1_DISPRELPND", + "DF_1_EDITED", + "DF_1_ENDFILTEE", + "DF_1_GLOBAL", + "DF_1_GLOBAUDIT", + "DF_1_GROUP", + "DF_1_IGNMULDEF", + "DF_1_INITFIRST", + "DF_1_INTERPOSE", + "DF_1_KMOD", + "DF_1_LOADFLTR", + "DF_1_NOCOMMON", + "DF_1_NODEFLIB", + "DF_1_NODELETE", + "DF_1_NODIRECT", + "DF_1_NODUMP", + "DF_1_NOHDR", + "DF_1_NOKSYMS", + "DF_1_NOOPEN", + "DF_1_NORELOC", + "DF_1_NOW", + "DF_1_ORIGIN", + "DF_1_PIE", + "DF_1_SINGLETON", + "DF_1_STUB", + "DF_1_SYMINTPOSE", + "DF_1_TRANS", + "DF_1_WEAKFILTER", "DF_BIND_NOW", "DF_ORIGIN", "DF_STATIC_TLS", @@ -1144,6 +1212,7 @@ var stdlib = map[string][]string{ "Dyn32", "Dyn64", "DynFlag", + "DynFlag1", "DynTag", "EI_ABIVERSION", "EI_CLASS", @@ -2111,6 +2180,7 @@ var stdlib = map[string][]string{ "R_PPC64_REL16_LO", "R_PPC64_REL24", "R_PPC64_REL24_NOTOC", + "R_PPC64_REL24_P9NOTOC", "R_PPC64_REL30", "R_PPC64_REL32", "R_PPC64_REL64", @@ -2848,6 +2918,7 @@ var stdlib = map[string][]string{ "MaxVarintLen16", "MaxVarintLen32", "MaxVarintLen64", + "NativeEndian", "PutUvarint", "PutVarint", "Read", @@ -2963,6 +3034,7 @@ var stdlib = map[string][]string{ }, "errors": { "As", + "ErrUnsupported", "Is", "Join", "New", @@ -2989,6 +3061,7 @@ var stdlib = map[string][]string{ "Arg", "Args", "Bool", + "BoolFunc", "BoolVar", "CommandLine", "ContinueOnError", @@ -3119,6 +3192,7 @@ var stdlib = map[string][]string{ "Inspect", "InterfaceType", "IsExported", + "IsGenerated", "KeyValueExpr", "LabeledStmt", "Lbl", @@ -3169,6 +3243,7 @@ var stdlib = map[string][]string{ "ArchChar", "Context", "Default", + "Directive", "FindOnly", "IgnoreVendor", "Import", @@ -3184,6 +3259,7 @@ var stdlib = map[string][]string{ "go/build/constraint": { "AndExpr", "Expr", + "GoVersion", "IsGoBuild", "IsPlusBuild", "NotExpr", @@ -3626,6 +3702,7 @@ var stdlib = map[string][]string{ "ErrBadHTML", "ErrBranchEnd", "ErrEndContext", + "ErrJSTemplate", "ErrNoSuchTemplate", "ErrOutputContext", "ErrPartialCharset", @@ -3870,6 +3947,8 @@ var stdlib = map[string][]string{ "FileInfo", "FileInfoToDirEntry", "FileMode", + "FormatDirEntry", + "FormatFileInfo", "Glob", "GlobFS", "ModeAppend", @@ -3942,6 +4021,78 @@ var stdlib = map[string][]string{ "SetPrefix", "Writer", }, + "log/slog": { + "Any", + "AnyValue", + "Attr", + "Bool", + "BoolValue", + "Debug", + "DebugContext", + "Default", + "Duration", + "DurationValue", + "Error", + "ErrorContext", + "Float64", + "Float64Value", + "Group", + "GroupValue", + "Handler", + "HandlerOptions", + "Info", + "InfoContext", + "Int", + "Int64", + "Int64Value", + "IntValue", + "JSONHandler", + "Kind", + "KindAny", + "KindBool", + "KindDuration", + "KindFloat64", + "KindGroup", + "KindInt64", + "KindLogValuer", + "KindString", + "KindTime", + "KindUint64", + "Level", + "LevelDebug", + "LevelError", + "LevelInfo", + "LevelKey", + "LevelVar", + "LevelWarn", + "Leveler", + "Log", + "LogAttrs", + "LogValuer", + "Logger", + "MessageKey", + "New", + "NewJSONHandler", + "NewLogLogger", + "NewRecord", + "NewTextHandler", + "Record", + "SetDefault", + "Source", + "SourceKey", + "String", + "StringValue", + "TextHandler", + "Time", + "TimeKey", + "TimeValue", + "Uint64", + "Uint64Value", + "Value", + "Warn", + "WarnContext", + "With", + }, "log/syslog": { "Dial", "LOG_ALERT", @@ -3977,6 +4128,13 @@ var stdlib = map[string][]string{ "Priority", "Writer", }, + "maps": { + "Clone", + "Copy", + "DeleteFunc", + "Equal", + "EqualFunc", + }, "math": { "Abs", "Acos", @@ -4371,6 +4529,7 @@ var stdlib = map[string][]string{ "ErrNoLocation", "ErrNotMultipart", "ErrNotSupported", + "ErrSchemeMismatch", "ErrServerClosed", "ErrShortBody", "ErrSkipAltProtocol", @@ -5084,6 +5243,8 @@ var stdlib = map[string][]string{ "NumCPU", "NumCgoCall", "NumGoroutine", + "PanicNilError", + "Pinner", "ReadMemStats", "ReadTrace", "SetBlockProfileRate", @@ -5172,6 +5333,37 @@ var stdlib = map[string][]string{ "Task", "WithRegion", }, + "slices": { + "BinarySearch", + "BinarySearchFunc", + "Clip", + "Clone", + "Compact", + "CompactFunc", + "Compare", + "CompareFunc", + "Contains", + "ContainsFunc", + "Delete", + "DeleteFunc", + "Equal", + "EqualFunc", + "Grow", + "Index", + "IndexFunc", + "Insert", + "IsSorted", + "IsSortedFunc", + "Max", + "MaxFunc", + "Min", + "MinFunc", + "Replace", + "Reverse", + "Sort", + "SortFunc", + "SortStableFunc", + }, "sort": { "Find", "Float64Slice", @@ -5242,6 +5434,7 @@ var stdlib = map[string][]string{ "Compare", "Contains", "ContainsAny", + "ContainsFunc", "ContainsRune", "Count", "Cut", @@ -5299,6 +5492,9 @@ var stdlib = map[string][]string{ "Mutex", "NewCond", "Once", + "OnceFunc", + "OnceValue", + "OnceValues", "Pool", "RWMutex", "WaitGroup", @@ -9135,10 +9331,12 @@ var stdlib = map[string][]string{ "SYS_AIO_CANCEL", "SYS_AIO_ERROR", "SYS_AIO_FSYNC", + "SYS_AIO_MLOCK", "SYS_AIO_READ", "SYS_AIO_RETURN", "SYS_AIO_SUSPEND", "SYS_AIO_SUSPEND_NOCANCEL", + "SYS_AIO_WAITCOMPLETE", "SYS_AIO_WRITE", "SYS_ALARM", "SYS_ARCH_PRCTL", @@ -9368,6 +9566,7 @@ var stdlib = map[string][]string{ "SYS_GET_MEMPOLICY", "SYS_GET_ROBUST_LIST", "SYS_GET_THREAD_AREA", + "SYS_GSSD_SYSCALL", "SYS_GTTY", "SYS_IDENTITYSVC", "SYS_IDLE", @@ -9411,8 +9610,24 @@ var stdlib = map[string][]string{ "SYS_KLDSYM", "SYS_KLDUNLOAD", "SYS_KLDUNLOADF", + "SYS_KMQ_NOTIFY", + "SYS_KMQ_OPEN", + "SYS_KMQ_SETATTR", + "SYS_KMQ_TIMEDRECEIVE", + "SYS_KMQ_TIMEDSEND", + "SYS_KMQ_UNLINK", "SYS_KQUEUE", "SYS_KQUEUE1", + "SYS_KSEM_CLOSE", + "SYS_KSEM_DESTROY", + "SYS_KSEM_GETVALUE", + "SYS_KSEM_INIT", + "SYS_KSEM_OPEN", + "SYS_KSEM_POST", + "SYS_KSEM_TIMEDWAIT", + "SYS_KSEM_TRYWAIT", + "SYS_KSEM_UNLINK", + "SYS_KSEM_WAIT", "SYS_KTIMER_CREATE", "SYS_KTIMER_DELETE", "SYS_KTIMER_GETOVERRUN", @@ -9504,11 +9719,14 @@ var stdlib = map[string][]string{ "SYS_NFSSVC", "SYS_NFSTAT", "SYS_NICE", + "SYS_NLM_SYSCALL", "SYS_NLSTAT", "SYS_NMOUNT", "SYS_NSTAT", "SYS_NTP_ADJTIME", "SYS_NTP_GETTIME", + "SYS_NUMA_GETAFFINITY", + "SYS_NUMA_SETAFFINITY", "SYS_OABI_SYSCALL_BASE", "SYS_OBREAK", "SYS_OLDFSTAT", @@ -9891,6 +10109,7 @@ var stdlib = map[string][]string{ "SYS___ACL_SET_FD", "SYS___ACL_SET_FILE", "SYS___ACL_SET_LINK", + "SYS___CAP_RIGHTS_GET", "SYS___CLONE", "SYS___DISABLE_THREADSIGNAL", "SYS___GETCWD", @@ -10574,6 +10793,7 @@ var stdlib = map[string][]string{ "Short", "T", "TB", + "Testing", "Verbose", }, "testing/fstest": { @@ -10603,6 +10823,9 @@ var stdlib = map[string][]string{ "SetupError", "Value", }, + "testing/slogtest": { + "TestHandler", + }, "text/scanner": { "Char", "Comment", @@ -10826,6 +11049,7 @@ var stdlib = map[string][]string{ "Cs", "Cuneiform", "Cypriot", + "Cypro_Minoan", "Cyrillic", "Dash", "Deprecated", @@ -10889,6 +11113,7 @@ var stdlib = map[string][]string{ "Kaithi", "Kannada", "Katakana", + "Kawi", "Kayah_Li", "Kharoshthi", "Khitan_Small_Script", @@ -10943,6 +11168,7 @@ var stdlib = map[string][]string{ "Myanmar", "N", "Nabataean", + "Nag_Mundari", "Nandinagari", "Nd", "New_Tai_Lue", @@ -10964,6 +11190,7 @@ var stdlib = map[string][]string{ "Old_Sogdian", "Old_South_Arabian", "Old_Turkic", + "Old_Uyghur", "Oriya", "Osage", "Osmanya", @@ -11038,6 +11265,7 @@ var stdlib = map[string][]string{ "Tai_Viet", "Takri", "Tamil", + "Tangsa", "Tangut", "Telugu", "Terminal_Punctuation", @@ -11052,6 +11280,7 @@ var stdlib = map[string][]string{ "ToLower", "ToTitle", "ToUpper", + "Toto", "TurkishCase", "Ugaritic", "Unified_Ideograph", @@ -11061,6 +11290,7 @@ var stdlib = map[string][]string{ "Vai", "Variation_Selector", "Version", + "Vithkuqi", "Wancho", "Warang_Citi", "White_Space", From 33da5c0a6ab7e78d68dc892becda564f6d36d68d Mon Sep 17 00:00:00 2001 From: "Hana (Hyang-Ah) Kim" Date: Thu, 3 Aug 2023 18:26:07 -0400 Subject: [PATCH 003/178] gopls/internal/telemetry: record Go version used in each view Change-Id: I59233ca205b59726a75791db04dfc1fe8dd4fff4 Reviewed-on: https://go-review.googlesource.com/c/tools/+/515558 Run-TryBot: Hyang-Ah Hana Kim Reviewed-by: Jamal Carvalho Reviewed-by: Robert Findley TryBot-Result: Gopher Robot gopls-CI: kokoro --- gopls/internal/lsp/general.go | 1 + gopls/internal/telemetry/telemetry.go | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/gopls/internal/lsp/general.go b/gopls/internal/lsp/general.go index b57992fed5e..9c10d9377a1 100644 --- a/gopls/internal/lsp/general.go +++ b/gopls/internal/lsp/general.go @@ -317,6 +317,7 @@ func (s *Server) checkViewGoVersions() { if oldestVersion == -1 || viewVersion < oldestVersion { oldestVersion, fromBuild = viewVersion, false } + telemetry.RecordViewGoVersion(viewVersion) } if msg, mType := versionMessage(oldestVersion, fromBuild); msg != "" { diff --git a/gopls/internal/telemetry/telemetry.go b/gopls/internal/telemetry/telemetry.go index 67ab45adb41..62c89b7610d 100644 --- a/gopls/internal/telemetry/telemetry.go +++ b/gopls/internal/telemetry/telemetry.go @@ -5,6 +5,7 @@ package telemetry import ( + "fmt" "os" "golang.org/x/telemetry/counter" @@ -50,3 +51,12 @@ func RecordClientInfo(params *protocol.ParamInitialize) { } counter.Inc(client) } + +// RecordViewGoVersion records the Go minor version number (1.x) used for a view. +func RecordViewGoVersion(x int) { + if x < 0 { + return + } + name := fmt.Sprintf("gopls/goversion:1.%d", x) + counter.Inc(name) +} From 9abb02c1b5524bf655a0985bc101496a6b53ff71 Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Mon, 7 Aug 2023 17:17:52 -0400 Subject: [PATCH 004/178] gopls/internal/regtest/bench: add an oracle completion benchmark The enormous dataintegration package in oracle demonstrates a clear regression from gopls@v0.11.0 in completion CPU utilization during autocompletion. On my machine latency appears about the same, but on more resource-constrained machines this may not be the case. For golang/go#61207 Change-Id: I59631f34fe0d8d5d3329c9444d4e485840ad85ed Reviewed-on: https://go-review.googlesource.com/c/tools/+/516678 TryBot-Result: Gopher Robot gopls-CI: kokoro Run-TryBot: Robert Findley Reviewed-by: Hyang-Ah Hana Kim --- gopls/internal/regtest/bench/completion_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/gopls/internal/regtest/bench/completion_test.go b/gopls/internal/regtest/bench/completion_test.go index a0cf5a043f9..0400e70b0bd 100644 --- a/gopls/internal/regtest/bench/completion_test.go +++ b/gopls/internal/regtest/bench/completion_test.go @@ -180,6 +180,18 @@ func (kl *Kubelet) _() { `, `kl\.()`, }, + { + "oracle", + "dataintegration/pivot2.go", + ` +package dataintegration + +func (p *Pivot) _() { + p. +} +`, + `p\.()`, + }, } for _, test := range tests { From 582ef29c0d0ca1a82d225643517fce7323cad29f Mon Sep 17 00:00:00 2001 From: Peter Weinberger Date: Tue, 8 Aug 2023 14:41:44 -0400 Subject: [PATCH 005/178] gopls/internal/protocol: remove // line comments from tsprotocol.go The change just removes comment lines that change as the version of the LSP changes. While they are useful while debugging, they are just noise while trying to understand changes. Change-Id: If4c85191760baef916911130ca315773d2adda1f Reviewed-on: https://go-review.googlesource.com/c/tools/+/517275 Reviewed-by: Robert Findley TryBot-Result: Gopher Robot Run-TryBot: Peter Weinberger --- gopls/internal/lsp/protocol/tsprotocol.go | 1356 +++++++++++---------- 1 file changed, 692 insertions(+), 664 deletions(-) diff --git a/gopls/internal/lsp/protocol/tsprotocol.go b/gopls/internal/lsp/protocol/tsprotocol.go index 19cdd817773..f571be379a8 100644 --- a/gopls/internal/lsp/protocol/tsprotocol.go +++ b/gopls/internal/lsp/protocol/tsprotocol.go @@ -15,14 +15,14 @@ import "encoding/json" // A special text edit with an additional change annotation. // // @since 3.16.0. -type AnnotatedTextEdit struct { // line 9702 +type AnnotatedTextEdit struct { // The actual identifier of the change annotation AnnotationID ChangeAnnotationIdentifier `json:"annotationId"` TextEdit } // The parameters passed via an apply workspace edit request. -type ApplyWorkspaceEditParams struct { // line 6220 +type ApplyWorkspaceEditParams struct { // An optional label of the workspace edit. This label is // presented in the user interface for example on an undo // stack to undo the workspace edit. @@ -34,7 +34,7 @@ type ApplyWorkspaceEditParams struct { // line 6220 // The result returned from the apply workspace edit request. // // @since 3.17 renamed from ApplyWorkspaceEditResponse -type ApplyWorkspaceEditResult struct { // line 6243 +type ApplyWorkspaceEditResult struct { // Indicates whether the edit was applied or not. Applied bool `json:"applied"` // An optional textual description for why the edit was not applied. @@ -48,7 +48,7 @@ type ApplyWorkspaceEditResult struct { // line 6243 } // A base for all symbol information. -type BaseSymbolInformation struct { // line 9284 +type BaseSymbolInformation struct { // The name of this symbol. Name string `json:"name"` // The kind of this symbol. @@ -65,7 +65,7 @@ type BaseSymbolInformation struct { // line 9284 } // @since 3.16.0 -type CallHierarchyClientCapabilities struct { // line 12517 +type CallHierarchyClientCapabilities struct { // Whether implementation supports dynamic registration. If this is set to `true` // the client supports the new `(TextDocumentRegistrationOptions & StaticRegistrationOptions)` // return value for the corresponding server capability as well. @@ -75,7 +75,7 @@ type CallHierarchyClientCapabilities struct { // line 12517 // Represents an incoming call, e.g. a caller of a method or constructor. // // @since 3.16.0 -type CallHierarchyIncomingCall struct { // line 2852 +type CallHierarchyIncomingCall struct { // The item that makes the call. From CallHierarchyItem `json:"from"` // The ranges at which the calls appear. This is relative to the caller @@ -86,7 +86,7 @@ type CallHierarchyIncomingCall struct { // line 2852 // The parameter of a `callHierarchy/incomingCalls` request. // // @since 3.16.0 -type CallHierarchyIncomingCallsParams struct { // line 2828 +type CallHierarchyIncomingCallsParams struct { Item CallHierarchyItem `json:"item"` WorkDoneProgressParams PartialResultParams @@ -96,7 +96,7 @@ type CallHierarchyIncomingCallsParams struct { // line 2828 // of call hierarchy. // // @since 3.16.0 -type CallHierarchyItem struct { // line 2729 +type CallHierarchyItem struct { // The name of this item. Name string `json:"name"` // The kind of this item. @@ -120,14 +120,14 @@ type CallHierarchyItem struct { // line 2729 // Call hierarchy options used during static registration. // // @since 3.16.0 -type CallHierarchyOptions struct { // line 6770 +type CallHierarchyOptions struct { WorkDoneProgressOptions } // Represents an outgoing call, e.g. calling a getter from a method or a method from a constructor etc. // // @since 3.16.0 -type CallHierarchyOutgoingCall struct { // line 2902 +type CallHierarchyOutgoingCall struct { // The item that is called. To CallHierarchyItem `json:"to"` // The range at which this item is called. This is the range relative to the caller, e.g the item @@ -139,7 +139,7 @@ type CallHierarchyOutgoingCall struct { // line 2902 // The parameter of a `callHierarchy/outgoingCalls` request. // // @since 3.16.0 -type CallHierarchyOutgoingCallsParams struct { // line 2878 +type CallHierarchyOutgoingCallsParams struct { Item CallHierarchyItem `json:"item"` WorkDoneProgressParams PartialResultParams @@ -148,7 +148,7 @@ type CallHierarchyOutgoingCallsParams struct { // line 2878 // The parameter of a `textDocument/prepareCallHierarchy` request. // // @since 3.16.0 -type CallHierarchyPrepareParams struct { // line 2711 +type CallHierarchyPrepareParams struct { TextDocumentPositionParams WorkDoneProgressParams } @@ -156,12 +156,12 @@ type CallHierarchyPrepareParams struct { // line 2711 // Call hierarchy options used during static or dynamic registration. // // @since 3.16.0 -type CallHierarchyRegistrationOptions struct { // line 2806 +type CallHierarchyRegistrationOptions struct { TextDocumentRegistrationOptions CallHierarchyOptions StaticRegistrationOptions } -type CancelParams struct { // line 6415 +type CancelParams struct { // The request id to cancel. ID interface{} `json:"id"` } @@ -169,7 +169,7 @@ type CancelParams struct { // line 6415 // Additional information that describes document changes. // // @since 3.16.0 -type ChangeAnnotation struct { // line 7067 +type ChangeAnnotation struct { // A human-readable string describing the actual change. The string // is rendered prominent in the user interface. Label string `json:"label"` @@ -184,7 +184,7 @@ type ChangeAnnotation struct { // line 7067 // An identifier to refer to a change annotation stored with a workspace edit. type ChangeAnnotationIdentifier = string // (alias) line 14391 // Defines the capabilities provided by the client. -type ClientCapabilities struct { // line 10028 +type ClientCapabilities struct { // Workspace specific client capabilities. Workspace WorkspaceClientCapabilities `json:"workspace,omitempty"` // Text document specific client capabilities. @@ -207,7 +207,7 @@ type ClientCapabilities struct { // line 10028 // to refactor code. // // A CodeAction must set either `edit` and/or a `command`. If both are supplied, the `edit` is applied first, then the `command` is executed. -type CodeAction struct { // line 5577 +type CodeAction struct { // A short, human-readable, title for this code action. Title string `json:"title"` // The kind of the code action. @@ -254,7 +254,7 @@ type CodeAction struct { // line 5577 } // The Client Capabilities of a {@link CodeActionRequest}. -type CodeActionClientCapabilities struct { // line 12086 +type CodeActionClientCapabilities struct { // Whether code action supports dynamic registration. DynamicRegistration bool `json:"dynamicRegistration,omitempty"` // The client support code action literals of type `CodeAction` as a valid @@ -294,7 +294,7 @@ type CodeActionClientCapabilities struct { // line 12086 // Contains additional diagnostic information about the context in which // a {@link CodeActionProvider.provideCodeActions code action} is run. -type CodeActionContext struct { // line 9350 +type CodeActionContext struct { // An array of diagnostics known on the client side overlapping the range provided to the // `textDocument/codeAction` request. They are provided so that the server knows which // errors are currently presented to the user for the given range. There is no guarantee @@ -313,9 +313,10 @@ type CodeActionContext struct { // line 9350 } // A set of predefined code action kinds -type CodeActionKind string // line 13719 +type CodeActionKind string + // Provider options for a {@link CodeActionRequest}. -type CodeActionOptions struct { // line 9389 +type CodeActionOptions struct { // CodeActionKinds that this server may return. // // The list of kinds may be generic, such as `CodeActionKind.Refactor`, or the server @@ -330,7 +331,7 @@ type CodeActionOptions struct { // line 9389 } // The parameters of a {@link CodeActionRequest}. -type CodeActionParams struct { // line 5503 +type CodeActionParams struct { // The document in which the command was invoked. TextDocument TextDocumentIdentifier `json:"textDocument"` // The range for which the command was invoked. @@ -342,7 +343,7 @@ type CodeActionParams struct { // line 5503 } // Registration options for a {@link CodeActionRequest}. -type CodeActionRegistrationOptions struct { // line 5671 +type CodeActionRegistrationOptions struct { TextDocumentRegistrationOptions CodeActionOptions } @@ -350,11 +351,12 @@ type CodeActionRegistrationOptions struct { // line 5671 // The reason why code actions were requested. // // @since 3.17.0 -type CodeActionTriggerKind uint32 // line 14021 +type CodeActionTriggerKind uint32 + // Structure to capture a description for an error code. // // @since 3.16.0 -type CodeDescription struct { // line 10380 +type CodeDescription struct { // An URI to open with more information about the diagnostic error. Href URI `json:"href"` } @@ -364,7 +366,7 @@ type CodeDescription struct { // line 10380 // // A code lens is _unresolved_ when no command is associated to it. For performance // reasons the creation of a code lens and resolving should be done in two stages. -type CodeLens struct { // line 5794 +type CodeLens struct { // The range in which this code lens is valid. Should only span a single line. Range Range `json:"range"` // The command this code lens represents. @@ -376,20 +378,20 @@ type CodeLens struct { // line 5794 } // The client capabilities of a {@link CodeLensRequest}. -type CodeLensClientCapabilities struct { // line 12200 +type CodeLensClientCapabilities struct { // Whether code lens supports dynamic registration. DynamicRegistration bool `json:"dynamicRegistration,omitempty"` } // Code Lens provider options of a {@link CodeLensRequest}. -type CodeLensOptions struct { // line 9445 +type CodeLensOptions struct { // Code lens has a resolve provider as well. ResolveProvider bool `json:"resolveProvider,omitempty"` WorkDoneProgressOptions } // The parameters of a {@link CodeLensRequest}. -type CodeLensParams struct { // line 5770 +type CodeLensParams struct { // The document to request code lens for. TextDocument TextDocumentIdentifier `json:"textDocument"` WorkDoneProgressParams @@ -397,13 +399,13 @@ type CodeLensParams struct { // line 5770 } // Registration options for a {@link CodeLensRequest}. -type CodeLensRegistrationOptions struct { // line 5826 +type CodeLensRegistrationOptions struct { TextDocumentRegistrationOptions CodeLensOptions } // @since 3.16.0 -type CodeLensWorkspaceClientCapabilities struct { // line 11358 +type CodeLensWorkspaceClientCapabilities struct { // Whether the client implementation supports a refresh request sent from the // server to the client. // @@ -415,7 +417,7 @@ type CodeLensWorkspaceClientCapabilities struct { // line 11358 } // Represents a color in RGBA space. -type Color struct { // line 6669 +type Color struct { // The red component of this color in the range [0-1]. Red float64 `json:"red"` // The green component of this color in the range [0-1]. @@ -427,13 +429,13 @@ type Color struct { // line 6669 } // Represents a color range from a document. -type ColorInformation struct { // line 2312 +type ColorInformation struct { // The range in the document where this color appears. Range Range `json:"range"` // The actual color value for this color range. Color Color `json:"color"` } -type ColorPresentation struct { // line 2394 +type ColorPresentation struct { // The label of this color presentation. It will be shown on the color // picker header. By default this is also the text that is inserted when selecting // this color presentation. @@ -448,7 +450,7 @@ type ColorPresentation struct { // line 2394 } // Parameters for a {@link ColorPresentationRequest}. -type ColorPresentationParams struct { // line 2354 +type ColorPresentationParams struct { // The text document. TextDocument TextDocumentIdentifier `json:"textDocument"` // The color to request presentations for. @@ -463,7 +465,7 @@ type ColorPresentationParams struct { // line 2354 // will be used to represent a command in the UI and, optionally, // an array of arguments which will be passed to the command handler // function when invoked. -type Command struct { // line 5543 +type Command struct { // Title of the command, like `save`. Title string `json:"title"` // The identifier of the actual command handler. @@ -474,7 +476,7 @@ type Command struct { // line 5543 } // Completion client capabilities -type CompletionClientCapabilities struct { // line 11533 +type CompletionClientCapabilities struct { // Whether completion supports dynamic registration. DynamicRegistration bool `json:"dynamicRegistration,omitempty"` // The client supports the following `CompletionItem` specific @@ -498,7 +500,7 @@ type CompletionClientCapabilities struct { // line 11533 } // Contains additional information about the context in which a completion request is triggered. -type CompletionContext struct { // line 8946 +type CompletionContext struct { // How the completion was triggered. TriggerKind CompletionTriggerKind `json:"triggerKind"` // The trigger character (a single character) that has trigger code complete. @@ -508,7 +510,7 @@ type CompletionContext struct { // line 8946 // A completion item represents a text snippet that is // proposed to complete text that is being typed. -type CompletionItem struct { // line 4723 +type CompletionItem struct { // The label of this completion item. // // The label property is also by default the text that @@ -629,11 +631,12 @@ type CompletionItem struct { // line 4723 } // The kind of a completion entry. -type CompletionItemKind uint32 // line 13527 +type CompletionItemKind uint32 + // Additional details for a completion item label. // // @since 3.17.0 -type CompletionItemLabelDetails struct { // line 8969 +type CompletionItemLabelDetails struct { // An optional string which is rendered less prominently directly after {@link CompletionItem.label label}, // without any spacing. Should be used for function signatures and type annotations. Detail string `json:"detail,omitempty"` @@ -646,10 +649,11 @@ type CompletionItemLabelDetails struct { // line 8969 // item. // // @since 3.15.0 -type CompletionItemTag uint32 // line 13637 +type CompletionItemTag uint32 + // Represents a collection of {@link CompletionItem completion items} to be presented // in the editor. -type CompletionList struct { // line 4932 +type CompletionList struct { // This list it not complete. Further typing results in recomputing this list. // // Recomputed lists have all their items replaced (not appended) in the @@ -674,7 +678,7 @@ type CompletionList struct { // line 4932 } // Completion options. -type CompletionOptions struct { // line 9025 +type CompletionOptions struct { // Most tools trigger completion request automatically without explicitly requesting // it using a keyboard shortcut (e.g. Ctrl+Space). Typically they do so when the user // starts to type an identifier. For example if the user types `c` in a JavaScript file @@ -705,7 +709,7 @@ type CompletionOptions struct { // line 9025 } // Completion parameters -type CompletionParams struct { // line 4692 +type CompletionParams struct { // The completion context. This is only available it the client specifies // to send this using the client capability `textDocument.completion.contextSupport === true` Context CompletionContext `json:"context,omitempty"` @@ -715,14 +719,14 @@ type CompletionParams struct { // line 4692 } // Registration options for a {@link CompletionRequest}. -type CompletionRegistrationOptions struct { // line 5049 +type CompletionRegistrationOptions struct { TextDocumentRegistrationOptions CompletionOptions } // How a completion was triggered -type CompletionTriggerKind uint32 // line 13970 -type ConfigurationItem struct { // line 6632 +type CompletionTriggerKind uint32 +type ConfigurationItem struct { // The scope to get the configuration section for. ScopeURI string `json:"scopeUri,omitempty"` // The configuration section asked for. @@ -730,12 +734,12 @@ type ConfigurationItem struct { // line 6632 } // The parameters of a configuration request. -type ConfigurationParams struct { // line 2272 +type ConfigurationParams struct { Items []ConfigurationItem `json:"items"` } // Create file operation. -type CreateFile struct { // line 6948 +type CreateFile struct { // A create Kind string `json:"kind"` // The resource to create. @@ -746,7 +750,7 @@ type CreateFile struct { // line 6948 } // Options to create a file. -type CreateFileOptions struct { // line 9747 +type CreateFileOptions struct { // Overwrite existing file. Overwrite wins over `ignoreIfExists` Overwrite bool `json:"overwrite,omitempty"` // Ignore if exists. @@ -757,7 +761,7 @@ type CreateFileOptions struct { // line 9747 // files. // // @since 3.16.0 -type CreateFilesParams struct { // line 3248 +type CreateFilesParams struct { // An array of all files/folders created in this operation. Files []FileCreate `json:"files"` } @@ -765,7 +769,7 @@ type CreateFilesParams struct { // line 3248 // The declaration of a symbol representation as one or many {@link Location locations}. type Declaration = []Location // (alias) line 14248 // @since 3.14.0 -type DeclarationClientCapabilities struct { // line 11874 +type DeclarationClientCapabilities struct { // Whether declaration supports dynamic registration. If this is set to `true` // the client supports the new `DeclarationRegistrationOptions` return value // for the corresponding server capability as well. @@ -782,15 +786,15 @@ type DeclarationClientCapabilities struct { // line 11874 // Servers should prefer returning `DeclarationLink` over `Declaration` if supported // by the client. type DeclarationLink = LocationLink // (alias) line 14268 -type DeclarationOptions struct { // line 6727 +type DeclarationOptions struct { WorkDoneProgressOptions } -type DeclarationParams struct { // line 2567 +type DeclarationParams struct { TextDocumentPositionParams WorkDoneProgressParams PartialResultParams } -type DeclarationRegistrationOptions struct { // line 2587 +type DeclarationRegistrationOptions struct { DeclarationOptions TextDocumentRegistrationOptions StaticRegistrationOptions @@ -804,7 +808,7 @@ type DeclarationRegistrationOptions struct { // line 2587 // by the client. type Definition = Or_Definition // (alias) line 14166 // Client Capabilities for a {@link DefinitionRequest}. -type DefinitionClientCapabilities struct { // line 11899 +type DefinitionClientCapabilities struct { // Whether definition supports dynamic registration. DynamicRegistration bool `json:"dynamicRegistration,omitempty"` // The client supports additional metadata in the form of definition links. @@ -819,25 +823,25 @@ type DefinitionClientCapabilities struct { // line 11899 // the defining symbol type DefinitionLink = LocationLink // (alias) line 14186 // Server Capabilities for a {@link DefinitionRequest}. -type DefinitionOptions struct { // line 9237 +type DefinitionOptions struct { WorkDoneProgressOptions } // Parameters for a {@link DefinitionRequest}. -type DefinitionParams struct { // line 5213 +type DefinitionParams struct { TextDocumentPositionParams WorkDoneProgressParams PartialResultParams } // Registration options for a {@link DefinitionRequest}. -type DefinitionRegistrationOptions struct { // line 5234 +type DefinitionRegistrationOptions struct { TextDocumentRegistrationOptions DefinitionOptions } // Delete file operation -type DeleteFile struct { // line 7030 +type DeleteFile struct { // A delete Kind string `json:"kind"` // The file to delete. @@ -848,7 +852,7 @@ type DeleteFile struct { // line 7030 } // Delete file options -type DeleteFileOptions struct { // line 9795 +type DeleteFileOptions struct { // Delete the content recursively if a folder is denoted. Recursive bool `json:"recursive,omitempty"` // Ignore the operation if the file doesn't exist. @@ -859,14 +863,14 @@ type DeleteFileOptions struct { // line 9795 // files. // // @since 3.16.0 -type DeleteFilesParams struct { // line 3373 +type DeleteFilesParams struct { // An array of all files/folders deleted in this operation. Files []FileDelete `json:"files"` } // Represents a diagnostic, such as a compiler error or warning. Diagnostic objects // are only valid in the scope of a resource. -type Diagnostic struct { // line 8843 +type Diagnostic struct { // The range at which the message applies Range Range `json:"range"` // The diagnostic's severity. Can be omitted. If omitted it is up to the @@ -902,7 +906,7 @@ type Diagnostic struct { // line 8843 // Client capabilities specific to diagnostic pull requests. // // @since 3.17.0 -type DiagnosticClientCapabilities struct { // line 12784 +type DiagnosticClientCapabilities struct { // Whether implementation supports dynamic registration. If this is set to `true` // the client supports the new `(TextDocumentRegistrationOptions & StaticRegistrationOptions)` // return value for the corresponding server capability as well. @@ -914,7 +918,7 @@ type DiagnosticClientCapabilities struct { // line 12784 // Diagnostic options. // // @since 3.17.0 -type DiagnosticOptions struct { // line 7529 +type DiagnosticOptions struct { // An optional identifier under which the diagnostics are // managed by the client. Identifier string `json:"identifier,omitempty"` @@ -931,7 +935,7 @@ type DiagnosticOptions struct { // line 7529 // Diagnostic registration options. // // @since 3.17.0 -type DiagnosticRegistrationOptions struct { // line 3928 +type DiagnosticRegistrationOptions struct { TextDocumentRegistrationOptions DiagnosticOptions StaticRegistrationOptions @@ -940,7 +944,7 @@ type DiagnosticRegistrationOptions struct { // line 3928 // Represents a related message and source code location for a diagnostic. This should be // used to point to code locations that cause or related to a diagnostics, e.g when duplicating // a symbol in a scope. -type DiagnosticRelatedInformation struct { // line 10395 +type DiagnosticRelatedInformation struct { // The location of this related diagnostic information. Location Location `json:"location"` // The message of this related diagnostic information. @@ -950,20 +954,22 @@ type DiagnosticRelatedInformation struct { // line 10395 // Cancellation data returned from a diagnostic request. // // @since 3.17.0 -type DiagnosticServerCancellationData struct { // line 3914 +type DiagnosticServerCancellationData struct { RetriggerRequest bool `json:"retriggerRequest"` } // The diagnostic's severity. -type DiagnosticSeverity uint32 // line 13919 +type DiagnosticSeverity uint32 + // The diagnostic tags. // // @since 3.15.0 -type DiagnosticTag uint32 // line 13949 +type DiagnosticTag uint32 + // Workspace client capabilities specific to diagnostic pull requests. // // @since 3.17.0 -type DiagnosticWorkspaceClientCapabilities struct { // line 11476 +type DiagnosticWorkspaceClientCapabilities struct { // Whether the client implementation supports a refresh request sent from // the server to the client. // @@ -973,24 +979,24 @@ type DiagnosticWorkspaceClientCapabilities struct { // line 11476 // change that requires such a calculation. RefreshSupport bool `json:"refreshSupport,omitempty"` } -type DidChangeConfigurationClientCapabilities struct { // line 11202 +type DidChangeConfigurationClientCapabilities struct { // Did change configuration notification supports dynamic registration. DynamicRegistration bool `json:"dynamicRegistration,omitempty"` } // The parameters of a change configuration notification. -type DidChangeConfigurationParams struct { // line 4339 +type DidChangeConfigurationParams struct { // The actual changed settings Settings interface{} `json:"settings"` } -type DidChangeConfigurationRegistrationOptions struct { // line 4353 +type DidChangeConfigurationRegistrationOptions struct { Section *OrPSection_workspace_didChangeConfiguration `json:"section,omitempty"` } // The params sent in a change notebook document notification. // // @since 3.17.0 -type DidChangeNotebookDocumentParams struct { // line 4047 +type DidChangeNotebookDocumentParams struct { // The notebook document that did change. The version number points // to the version after all provided changes have been applied. If // only the text document content of a cell changes the notebook version @@ -1014,7 +1020,7 @@ type DidChangeNotebookDocumentParams struct { // line 4047 } // The change text document notification's parameters. -type DidChangeTextDocumentParams struct { // line 4482 +type DidChangeTextDocumentParams struct { // The document that did change. The version number points // to the version after all provided content changes have // been applied. @@ -1033,7 +1039,7 @@ type DidChangeTextDocumentParams struct { // line 4482 // you receive them. ContentChanges []TextDocumentContentChangeEvent `json:"contentChanges"` } -type DidChangeWatchedFilesClientCapabilities struct { // line 11216 +type DidChangeWatchedFilesClientCapabilities struct { // Did change watched files notification supports dynamic registration. Please note // that the current protocol doesn't support static configuration for file changes // from the server side. @@ -1046,19 +1052,19 @@ type DidChangeWatchedFilesClientCapabilities struct { // line 11216 } // The watched files change notification's parameters. -type DidChangeWatchedFilesParams struct { // line 4623 +type DidChangeWatchedFilesParams struct { // The actual file events. Changes []FileEvent `json:"changes"` } // Describe options to be used when registered for text document change events. -type DidChangeWatchedFilesRegistrationOptions struct { // line 4640 +type DidChangeWatchedFilesRegistrationOptions struct { // The watchers to register. Watchers []FileSystemWatcher `json:"watchers"` } // The parameters of a `workspace/didChangeWorkspaceFolders` notification. -type DidChangeWorkspaceFoldersParams struct { // line 2258 +type DidChangeWorkspaceFoldersParams struct { // The actual workspace folder change event. Event WorkspaceFoldersChangeEvent `json:"event"` } @@ -1066,7 +1072,7 @@ type DidChangeWorkspaceFoldersParams struct { // line 2258 // The params sent in a close notebook document notification. // // @since 3.17.0 -type DidCloseNotebookDocumentParams struct { // line 4085 +type DidCloseNotebookDocumentParams struct { // The notebook document that got closed. NotebookDocument NotebookDocumentIdentifier `json:"notebookDocument"` // The text documents that represent the content @@ -1075,7 +1081,7 @@ type DidCloseNotebookDocumentParams struct { // line 4085 } // The parameters sent in a close text document notification -type DidCloseTextDocumentParams struct { // line 4527 +type DidCloseTextDocumentParams struct { // The document that was closed. TextDocument TextDocumentIdentifier `json:"textDocument"` } @@ -1083,7 +1089,7 @@ type DidCloseTextDocumentParams struct { // line 4527 // The params sent in an open notebook document notification. // // @since 3.17.0 -type DidOpenNotebookDocumentParams struct { // line 4021 +type DidOpenNotebookDocumentParams struct { // The notebook document that got opened. NotebookDocument NotebookDocument `json:"notebookDocument"` // The text documents that represent the content @@ -1092,7 +1098,7 @@ type DidOpenNotebookDocumentParams struct { // line 4021 } // The parameters sent in an open text document notification -type DidOpenTextDocumentParams struct { // line 4468 +type DidOpenTextDocumentParams struct { // The document that was opened. TextDocument TextDocumentItem `json:"textDocument"` } @@ -1100,37 +1106,37 @@ type DidOpenTextDocumentParams struct { // line 4468 // The params sent in a save notebook document notification. // // @since 3.17.0 -type DidSaveNotebookDocumentParams struct { // line 4070 +type DidSaveNotebookDocumentParams struct { // The notebook document that got saved. NotebookDocument NotebookDocumentIdentifier `json:"notebookDocument"` } // The parameters sent in a save text document notification -type DidSaveTextDocumentParams struct { // line 4541 +type DidSaveTextDocumentParams struct { // The document that was saved. TextDocument TextDocumentIdentifier `json:"textDocument"` // Optional the content when saved. Depends on the includeText value // when the save notification was requested. Text *string `json:"text,omitempty"` } -type DocumentColorClientCapabilities struct { // line 12240 +type DocumentColorClientCapabilities struct { // Whether implementation supports dynamic registration. If this is set to `true` // the client supports the new `DocumentColorRegistrationOptions` return value // for the corresponding server capability as well. DynamicRegistration bool `json:"dynamicRegistration,omitempty"` } -type DocumentColorOptions struct { // line 6707 +type DocumentColorOptions struct { WorkDoneProgressOptions } // Parameters for a {@link DocumentColorRequest}. -type DocumentColorParams struct { // line 2288 +type DocumentColorParams struct { // The text document. TextDocument TextDocumentIdentifier `json:"textDocument"` WorkDoneProgressParams PartialResultParams } -type DocumentColorRegistrationOptions struct { // line 2334 +type DocumentColorRegistrationOptions struct { TextDocumentRegistrationOptions DocumentColorOptions StaticRegistrationOptions @@ -1139,7 +1145,7 @@ type DocumentColorRegistrationOptions struct { // line 2334 // Parameters of the document diagnostic request. // // @since 3.17.0 -type DocumentDiagnosticParams struct { // line 3841 +type DocumentDiagnosticParams struct { // The text document. TextDocument TextDocumentIdentifier `json:"textDocument"` // The additional identifier provided during registration. @@ -1153,11 +1159,12 @@ type DocumentDiagnosticReport = Or_DocumentDiagnosticReport // (alias) line 1390 // The document diagnostic report kinds. // // @since 3.17.0 -type DocumentDiagnosticReportKind string // line 13115 +type DocumentDiagnosticReportKind string + // A partial result for a document diagnostic report. // // @since 3.17.0 -type DocumentDiagnosticReportPartialResult struct { // line 3884 +type DocumentDiagnosticReportPartialResult struct { RelatedDocuments map[DocumentURI]interface{} `json:"relatedDocuments"` } @@ -1167,18 +1174,18 @@ type DocumentDiagnosticReportPartialResult struct { // line 3884 // @since 3.17.0 - proposed support for NotebookCellTextDocumentFilter. type DocumentFilter = Or_DocumentFilter // (alias) line 14508 // Client capabilities of a {@link DocumentFormattingRequest}. -type DocumentFormattingClientCapabilities struct { // line 12254 +type DocumentFormattingClientCapabilities struct { // Whether formatting supports dynamic registration. DynamicRegistration bool `json:"dynamicRegistration,omitempty"` } // Provider options for a {@link DocumentFormattingRequest}. -type DocumentFormattingOptions struct { // line 9539 +type DocumentFormattingOptions struct { WorkDoneProgressOptions } // The parameters of a {@link DocumentFormattingRequest}. -type DocumentFormattingParams struct { // line 5922 +type DocumentFormattingParams struct { // The document to format. TextDocument TextDocumentIdentifier `json:"textDocument"` // The format options. @@ -1187,7 +1194,7 @@ type DocumentFormattingParams struct { // line 5922 } // Registration options for a {@link DocumentFormattingRequest}. -type DocumentFormattingRegistrationOptions struct { // line 5950 +type DocumentFormattingRegistrationOptions struct { TextDocumentRegistrationOptions DocumentFormattingOptions } @@ -1195,7 +1202,7 @@ type DocumentFormattingRegistrationOptions struct { // line 5950 // A document highlight is a range inside a text document which deserves // special attention. Usually a document highlight is visualized by changing // the background color of its range. -type DocumentHighlight struct { // line 5314 +type DocumentHighlight struct { // The range this highlight applies to. Range Range `json:"range"` // The highlight kind, default is {@link DocumentHighlightKind.Text text}. @@ -1203,34 +1210,35 @@ type DocumentHighlight struct { // line 5314 } // Client Capabilities for a {@link DocumentHighlightRequest}. -type DocumentHighlightClientCapabilities struct { // line 11989 +type DocumentHighlightClientCapabilities struct { // Whether document highlight supports dynamic registration. DynamicRegistration bool `json:"dynamicRegistration,omitempty"` } // A document highlight kind. -type DocumentHighlightKind uint32 // line 13694 +type DocumentHighlightKind uint32 + // Provider options for a {@link DocumentHighlightRequest}. -type DocumentHighlightOptions struct { // line 9273 +type DocumentHighlightOptions struct { WorkDoneProgressOptions } // Parameters for a {@link DocumentHighlightRequest}. -type DocumentHighlightParams struct { // line 5293 +type DocumentHighlightParams struct { TextDocumentPositionParams WorkDoneProgressParams PartialResultParams } // Registration options for a {@link DocumentHighlightRequest}. -type DocumentHighlightRegistrationOptions struct { // line 5337 +type DocumentHighlightRegistrationOptions struct { TextDocumentRegistrationOptions DocumentHighlightOptions } // A document link is a range in a text document that links to an internal or external resource, like another // text document or a web site. -type DocumentLink struct { // line 5865 +type DocumentLink struct { // The range this link applies to. Range Range `json:"range"` // The uri this link points to. If missing a resolve request is sent later. @@ -1249,7 +1257,7 @@ type DocumentLink struct { // line 5865 } // The client capabilities of a {@link DocumentLinkRequest}. -type DocumentLinkClientCapabilities struct { // line 12215 +type DocumentLinkClientCapabilities struct { // Whether document link supports dynamic registration. DynamicRegistration bool `json:"dynamicRegistration,omitempty"` // Whether the client supports the `tooltip` property on `DocumentLink`. @@ -1259,14 +1267,14 @@ type DocumentLinkClientCapabilities struct { // line 12215 } // Provider options for a {@link DocumentLinkRequest}. -type DocumentLinkOptions struct { // line 9466 +type DocumentLinkOptions struct { // Document links have a resolve provider as well. ResolveProvider bool `json:"resolveProvider,omitempty"` WorkDoneProgressOptions } // The parameters of a {@link DocumentLinkRequest}. -type DocumentLinkParams struct { // line 5841 +type DocumentLinkParams struct { // The document to provide document links for. TextDocument TextDocumentIdentifier `json:"textDocument"` WorkDoneProgressParams @@ -1274,19 +1282,19 @@ type DocumentLinkParams struct { // line 5841 } // Registration options for a {@link DocumentLinkRequest}. -type DocumentLinkRegistrationOptions struct { // line 5907 +type DocumentLinkRegistrationOptions struct { TextDocumentRegistrationOptions DocumentLinkOptions } // Client capabilities of a {@link DocumentOnTypeFormattingRequest}. -type DocumentOnTypeFormattingClientCapabilities struct { // line 12295 +type DocumentOnTypeFormattingClientCapabilities struct { // Whether on type formatting supports dynamic registration. DynamicRegistration bool `json:"dynamicRegistration,omitempty"` } // Provider options for a {@link DocumentOnTypeFormattingRequest}. -type DocumentOnTypeFormattingOptions struct { // line 9573 +type DocumentOnTypeFormattingOptions struct { // A character on which formatting should be triggered, like `{`. FirstTriggerCharacter string `json:"firstTriggerCharacter"` // More trigger characters. @@ -1294,7 +1302,7 @@ type DocumentOnTypeFormattingOptions struct { // line 9573 } // The parameters of a {@link DocumentOnTypeFormattingRequest}. -type DocumentOnTypeFormattingParams struct { // line 6057 +type DocumentOnTypeFormattingParams struct { // The document to format. TextDocument TextDocumentIdentifier `json:"textDocument"` // The position around which the on type formatting should happen. @@ -1311,13 +1319,13 @@ type DocumentOnTypeFormattingParams struct { // line 6057 } // Registration options for a {@link DocumentOnTypeFormattingRequest}. -type DocumentOnTypeFormattingRegistrationOptions struct { // line 6095 +type DocumentOnTypeFormattingRegistrationOptions struct { TextDocumentRegistrationOptions DocumentOnTypeFormattingOptions } // Client capabilities of a {@link DocumentRangeFormattingRequest}. -type DocumentRangeFormattingClientCapabilities struct { // line 12269 +type DocumentRangeFormattingClientCapabilities struct { // Whether range formatting supports dynamic registration. DynamicRegistration bool `json:"dynamicRegistration,omitempty"` // Whether the client supports formatting multiple ranges at once. @@ -1328,7 +1336,7 @@ type DocumentRangeFormattingClientCapabilities struct { // line 12269 } // Provider options for a {@link DocumentRangeFormattingRequest}. -type DocumentRangeFormattingOptions struct { // line 9550 +type DocumentRangeFormattingOptions struct { // Whether the server supports formatting multiple ranges at once. // // @since 3.18.0 @@ -1338,7 +1346,7 @@ type DocumentRangeFormattingOptions struct { // line 9550 } // The parameters of a {@link DocumentRangeFormattingRequest}. -type DocumentRangeFormattingParams struct { // line 5965 +type DocumentRangeFormattingParams struct { // The document to format. TextDocument TextDocumentIdentifier `json:"textDocument"` // The range to format @@ -1349,7 +1357,7 @@ type DocumentRangeFormattingParams struct { // line 5965 } // Registration options for a {@link DocumentRangeFormattingRequest}. -type DocumentRangeFormattingRegistrationOptions struct { // line 6001 +type DocumentRangeFormattingRegistrationOptions struct { TextDocumentRegistrationOptions DocumentRangeFormattingOptions } @@ -1358,7 +1366,7 @@ type DocumentRangeFormattingRegistrationOptions struct { // line 6001 // // @since 3.18.0 // @proposed -type DocumentRangesFormattingParams struct { // line 6016 +type DocumentRangesFormattingParams struct { // The document to format. TextDocument TextDocumentIdentifier `json:"textDocument"` // The ranges to format @@ -1378,7 +1386,7 @@ type DocumentSelector = []DocumentFilter // (alias) line 14363 // that appear in a document. Document symbols can be hierarchical and they // have two ranges: one that encloses its definition and one that points to // its most interesting range, e.g. the range of an identifier. -type DocumentSymbol struct { // line 5406 +type DocumentSymbol struct { // The name of this symbol. Will be displayed in the user interface and therefore must not be // an empty string or a string only consisting of white spaces. Name string `json:"name"` @@ -1406,7 +1414,7 @@ type DocumentSymbol struct { // line 5406 } // Client Capabilities for a {@link DocumentSymbolRequest}. -type DocumentSymbolClientCapabilities struct { // line 12004 +type DocumentSymbolClientCapabilities struct { // Whether document symbol supports dynamic registration. DynamicRegistration bool `json:"dynamicRegistration,omitempty"` // Specific capabilities for the `SymbolKind` in the @@ -1428,7 +1436,7 @@ type DocumentSymbolClientCapabilities struct { // line 12004 } // Provider options for a {@link DocumentSymbolRequest}. -type DocumentSymbolOptions struct { // line 9328 +type DocumentSymbolOptions struct { // A human-readable string that is shown when multiple outlines trees // are shown for the same document. // @@ -1438,7 +1446,7 @@ type DocumentSymbolOptions struct { // line 9328 } // Parameters for a {@link DocumentSymbolRequest}. -type DocumentSymbolParams struct { // line 5352 +type DocumentSymbolParams struct { // The text document. TextDocument TextDocumentIdentifier `json:"textDocument"` WorkDoneProgressParams @@ -1446,29 +1454,30 @@ type DocumentSymbolParams struct { // line 5352 } // Registration options for a {@link DocumentSymbolRequest}. -type DocumentSymbolRegistrationOptions struct { // line 5488 +type DocumentSymbolRegistrationOptions struct { TextDocumentRegistrationOptions DocumentSymbolOptions } type DocumentURI string // Predefined error codes. -type ErrorCodes int32 // line 13136 +type ErrorCodes int32 + // The client capabilities of a {@link ExecuteCommandRequest}. -type ExecuteCommandClientCapabilities struct { // line 11327 +type ExecuteCommandClientCapabilities struct { // Execute command supports dynamic registration. DynamicRegistration bool `json:"dynamicRegistration,omitempty"` } // The server capabilities of a {@link ExecuteCommandRequest}. -type ExecuteCommandOptions struct { // line 9621 +type ExecuteCommandOptions struct { // The commands to be executed on the server Commands []string `json:"commands"` WorkDoneProgressOptions } // The parameters of a {@link ExecuteCommandRequest}. -type ExecuteCommandParams struct { // line 6177 +type ExecuteCommandParams struct { // The identifier of the actual command handler. Command string `json:"command"` // Arguments that the command should be invoked with. @@ -1477,10 +1486,10 @@ type ExecuteCommandParams struct { // line 6177 } // Registration options for a {@link ExecuteCommandRequest}. -type ExecuteCommandRegistrationOptions struct { // line 6209 +type ExecuteCommandRegistrationOptions struct { ExecuteCommandOptions } -type ExecutionSummary struct { // line 10516 +type ExecutionSummary struct { // A strict monotonically increasing value // indicating the execution order of a cell // inside a notebook. @@ -1491,7 +1500,7 @@ type ExecutionSummary struct { // line 10516 } // created for Literal (Lit_CodeActionClientCapabilities_codeActionLiteralSupport_codeActionKind) -type FCodeActionKindPCodeActionLiteralSupport struct { // line 12107 +type FCodeActionKindPCodeActionLiteralSupport struct { // The code action kind values the client supports. When this // property exists the client also guarantees that it will // handle values outside its set gracefully and falls back @@ -1500,25 +1509,25 @@ type FCodeActionKindPCodeActionLiteralSupport struct { // line 12107 } // created for Literal (Lit_CompletionList_itemDefaults_editRange_Item1) -type FEditRangePItemDefaults struct { // line 4972 +type FEditRangePItemDefaults struct { Insert Range `json:"insert"` Replace Range `json:"replace"` } // created for Literal (Lit_SemanticTokensClientCapabilities_requests_full_Item1) -type FFullPRequests struct { // line 12581 +type FFullPRequests struct { // The client will send the `textDocument/semanticTokens/full/delta` request if // the server provides a corresponding handler. Delta bool `json:"delta"` } // created for Literal (Lit_CompletionClientCapabilities_completionItem_insertTextModeSupport) -type FInsertTextModeSupportPCompletionItem struct { // line 11660 +type FInsertTextModeSupportPCompletionItem struct { ValueSet []InsertTextMode `json:"valueSet"` } // created for Literal (Lit_SignatureHelpClientCapabilities_signatureInformation_parameterInformation) -type FParameterInformationPSignatureInformation struct { // line 11826 +type FParameterInformationPSignatureInformation struct { // The client supports processing label offsets instead of a // simple label string. // @@ -1527,17 +1536,17 @@ type FParameterInformationPSignatureInformation struct { // line 11826 } // created for Literal (Lit_SemanticTokensClientCapabilities_requests_range_Item1) -type FRangePRequests struct { // line 12561 +type FRangePRequests struct { } // created for Literal (Lit_CompletionClientCapabilities_completionItem_resolveSupport) -type FResolveSupportPCompletionItem struct { // line 11636 +type FResolveSupportPCompletionItem struct { // The properties that a client can resolve lazily. Properties []string `json:"properties"` } // created for Literal (Lit_NotebookDocumentChangeEvent_cells_structure) -type FStructurePCells struct { // line 7723 +type FStructurePCells struct { // The change to the cell array. Array NotebookCellArrayChange `json:"array"` // Additional opened cell text documents. @@ -1547,17 +1556,19 @@ type FStructurePCells struct { // line 7723 } // created for Literal (Lit_CompletionClientCapabilities_completionItem_tagSupport) -type FTagSupportPCompletionItem struct { // line 11602 +type FTagSupportPCompletionItem struct { // The tags supported by the client. ValueSet []CompletionItemTag `json:"valueSet"` } -type FailureHandlingKind string // line 14108 +type FailureHandlingKind string + // The file event type -type FileChangeType uint32 // line 13869 +type FileChangeType uint32 + // Represents information on a file/folder create. // // @since 3.16.0 -type FileCreate struct { // line 6898 +type FileCreate struct { // A file:// URI for the location of the file/folder being created. URI string `json:"uri"` } @@ -1565,13 +1576,13 @@ type FileCreate struct { // line 6898 // Represents information on a file/folder delete. // // @since 3.16.0 -type FileDelete struct { // line 7147 +type FileDelete struct { // A file:// URI for the location of the file/folder being deleted. URI string `json:"uri"` } // An event describing a file change. -type FileEvent struct { // line 8798 +type FileEvent struct { // The file's uri. URI DocumentURI `json:"uri"` // The change type. @@ -1584,7 +1595,7 @@ type FileEvent struct { // line 8798 // like renaming a file in the UI. // // @since 3.16.0 -type FileOperationClientCapabilities struct { // line 11374 +type FileOperationClientCapabilities struct { // Whether the client supports dynamic registration for file requests/notifications. DynamicRegistration bool `json:"dynamicRegistration,omitempty"` // The client has support for sending didCreateFiles notifications. @@ -1605,7 +1616,7 @@ type FileOperationClientCapabilities struct { // line 11374 // the server is interested in receiving. // // @since 3.16.0 -type FileOperationFilter struct { // line 7100 +type FileOperationFilter struct { // A Uri scheme like `file` or `untitled`. Scheme string `json:"scheme,omitempty"` // The actual file operation pattern. @@ -1615,7 +1626,7 @@ type FileOperationFilter struct { // line 7100 // Options for notifications/requests for user operations on files. // // @since 3.16.0 -type FileOperationOptions struct { // line 10319 +type FileOperationOptions struct { // The server is interested in receiving didCreateFiles notifications. DidCreate *FileOperationRegistrationOptions `json:"didCreate,omitempty"` // The server is interested in receiving willCreateFiles requests. @@ -1634,7 +1645,7 @@ type FileOperationOptions struct { // line 10319 // the server is interested in receiving. // // @since 3.16.0 -type FileOperationPattern struct { // line 9819 +type FileOperationPattern struct { // The glob pattern to match. Glob patterns can have the following syntax: // // - `*` to match one or more characters in a path segment @@ -1656,11 +1667,12 @@ type FileOperationPattern struct { // line 9819 // both. // // @since 3.16.0 -type FileOperationPatternKind string // line 14042 +type FileOperationPatternKind string + // Matching options for the file operation pattern. // // @since 3.16.0 -type FileOperationPatternOptions struct { // line 10500 +type FileOperationPatternOptions struct { // The pattern should be matched ignoring casing. IgnoreCase bool `json:"ignoreCase,omitempty"` } @@ -1668,7 +1680,7 @@ type FileOperationPatternOptions struct { // line 10500 // The options to register for file operations. // // @since 3.16.0 -type FileOperationRegistrationOptions struct { // line 3337 +type FileOperationRegistrationOptions struct { // The actual filters. Filters []FileOperationFilter `json:"filters"` } @@ -1676,13 +1688,13 @@ type FileOperationRegistrationOptions struct { // line 3337 // Represents information on a file/folder rename. // // @since 3.16.0 -type FileRename struct { // line 7124 +type FileRename struct { // A file:// URI for the original location of the file/folder being renamed. OldURI string `json:"oldUri"` // A file:// URI for the new location of the file/folder being renamed. NewURI string `json:"newUri"` } -type FileSystemWatcher struct { // line 8820 +type FileSystemWatcher struct { // The glob pattern to watch. See {@link GlobPattern glob pattern} for more detail. // // @since 3.17.0 support for relative patterns. @@ -1695,7 +1707,7 @@ type FileSystemWatcher struct { // line 8820 // Represents a folding range. To be valid, start and end line must be bigger than zero and smaller // than the number of lines in the document. Clients are free to ignore invalid ranges. -type FoldingRange struct { // line 2488 +type FoldingRange struct { // The zero-based start line of the range to fold. The folded area starts after the line's last character. // To be valid, the end must be zero or larger and smaller than the number of lines in the document. StartLine uint32 `json:"startLine"` @@ -1717,7 +1729,7 @@ type FoldingRange struct { // line 2488 // @since 3.17.0 CollapsedText string `json:"collapsedText,omitempty"` } -type FoldingRangeClientCapabilities struct { // line 12354 +type FoldingRangeClientCapabilities struct { // Whether implementation supports dynamic registration for folding range // providers. If this is set to `true` the client supports the new // `FoldingRangeRegistrationOptions` return value for the corresponding @@ -1742,26 +1754,26 @@ type FoldingRangeClientCapabilities struct { // line 12354 } // A set of predefined range kinds. -type FoldingRangeKind string // line 13208 -type FoldingRangeOptions struct { // line 6717 +type FoldingRangeKind string +type FoldingRangeOptions struct { WorkDoneProgressOptions } // Parameters for a {@link FoldingRangeRequest}. -type FoldingRangeParams struct { // line 2464 +type FoldingRangeParams struct { // The text document. TextDocument TextDocumentIdentifier `json:"textDocument"` WorkDoneProgressParams PartialResultParams } -type FoldingRangeRegistrationOptions struct { // line 2547 +type FoldingRangeRegistrationOptions struct { TextDocumentRegistrationOptions FoldingRangeOptions StaticRegistrationOptions } // Value-object describing what options formatting should use. -type FormattingOptions struct { // line 9487 +type FormattingOptions struct { // Size of a tab in spaces. TabSize uint32 `json:"tabSize"` // Prefer spaces over tabs. @@ -1783,7 +1795,7 @@ type FormattingOptions struct { // line 9487 // A diagnostic report with a full set of problems. // // @since 3.17.0 -type FullDocumentDiagnosticReport struct { // line 7471 +type FullDocumentDiagnosticReport struct { // A full document diagnostic report. Kind string `json:"kind"` // An optional result id. If provided it will @@ -1797,7 +1809,7 @@ type FullDocumentDiagnosticReport struct { // line 7471 // General client capabilities. // // @since 3.16.0 -type GeneralClientCapabilities struct { // line 11029 +type GeneralClientCapabilities struct { // Client capability that signals how the client // handles stale requests (e.g. a request // for which the client will not process the response @@ -1839,14 +1851,14 @@ type GeneralClientCapabilities struct { // line 11029 // @since 3.17.0 type GlobPattern = string // (alias) line 14542 // The result of a hover request. -type Hover struct { // line 5081 +type Hover struct { // The hover's content Contents MarkupContent `json:"contents"` // An optional range inside the text document that is used to // visualize the hover, e.g. by changing the background color. Range Range `json:"range,omitempty"` } -type HoverClientCapabilities struct { // line 11767 +type HoverClientCapabilities struct { // Whether hover supports dynamic registration. DynamicRegistration bool `json:"dynamicRegistration,omitempty"` // Client supports the following content formats for the content @@ -1855,24 +1867,24 @@ type HoverClientCapabilities struct { // line 11767 } // Hover options. -type HoverOptions struct { // line 9094 +type HoverOptions struct { WorkDoneProgressOptions } // Parameters for a {@link HoverRequest}. -type HoverParams struct { // line 5064 +type HoverParams struct { TextDocumentPositionParams WorkDoneProgressParams } // Registration options for a {@link HoverRequest}. -type HoverRegistrationOptions struct { // line 5120 +type HoverRegistrationOptions struct { TextDocumentRegistrationOptions HoverOptions } // @since 3.6.0 -type ImplementationClientCapabilities struct { // line 11948 +type ImplementationClientCapabilities struct { // Whether implementation supports dynamic registration. If this is set to `true` // the client supports the new `ImplementationRegistrationOptions` return value // for the corresponding server capability as well. @@ -1882,15 +1894,15 @@ type ImplementationClientCapabilities struct { // line 11948 // @since 3.14.0 LinkSupport bool `json:"linkSupport,omitempty"` } -type ImplementationOptions struct { // line 6569 +type ImplementationOptions struct { WorkDoneProgressOptions } -type ImplementationParams struct { // line 2136 +type ImplementationParams struct { TextDocumentPositionParams WorkDoneProgressParams PartialResultParams } -type ImplementationRegistrationOptions struct { // line 2176 +type ImplementationRegistrationOptions struct { TextDocumentRegistrationOptions ImplementationOptions StaticRegistrationOptions @@ -1898,20 +1910,20 @@ type ImplementationRegistrationOptions struct { // line 2176 // The data type of the ResponseError if the // initialize request fails. -type InitializeError struct { // line 4321 +type InitializeError struct { // Indicates whether the client execute the following retry logic: // (1) show the message provided by the ResponseError to the user // (2) user selects retry or cancel // (3) if user selected retry the initialize method is sent again. Retry bool `json:"retry"` } -type InitializeParams struct { // line 4263 +type InitializeParams struct { XInitializeParams WorkspaceFoldersInitializeParams } // The result returned from an initialize request. -type InitializeResult struct { // line 4277 +type InitializeResult struct { // The capabilities the language server provides. Capabilities ServerCapabilities `json:"capabilities"` // Information about the server. @@ -1919,13 +1931,13 @@ type InitializeResult struct { // line 4277 // @since 3.15.0 ServerInfo *PServerInfoMsg_initialize `json:"serverInfo,omitempty"` } -type InitializedParams struct { // line 4335 +type InitializedParams struct { } // Inlay hint information. // // @since 3.17.0 -type InlayHint struct { // line 3718 +type InlayHint struct { // The position of this hint. Position Position `json:"position"` // The label of this hint. A human readable string or an array of @@ -1964,7 +1976,7 @@ type InlayHint struct { // line 3718 // Inlay hint client capabilities. // // @since 3.17.0 -type InlayHintClientCapabilities struct { // line 12745 +type InlayHintClientCapabilities struct { // Whether inlay hints support dynamic registration. DynamicRegistration bool `json:"dynamicRegistration,omitempty"` // Indicates which properties a client can resolve lazily on an inlay @@ -1975,12 +1987,13 @@ type InlayHintClientCapabilities struct { // line 12745 // Inlay hint kinds. // // @since 3.17.0 -type InlayHintKind uint32 // line 13426 +type InlayHintKind uint32 + // An inlay hint label part allows for interactive and composite labels // of inlay hints. // // @since 3.17.0 -type InlayHintLabelPart struct { // line 7298 +type InlayHintLabelPart struct { // The value of this label part. Value string `json:"value"` // The tooltip text when you hover over this label part. Depending on @@ -2009,7 +2022,7 @@ type InlayHintLabelPart struct { // line 7298 // Inlay hint options used during static registration. // // @since 3.17.0 -type InlayHintOptions struct { // line 7371 +type InlayHintOptions struct { // The server provides support to resolve additional // information for an inlay hint item. ResolveProvider bool `json:"resolveProvider,omitempty"` @@ -2019,7 +2032,7 @@ type InlayHintOptions struct { // line 7371 // A parameter literal used in inlay hint requests. // // @since 3.17.0 -type InlayHintParams struct { // line 3689 +type InlayHintParams struct { // The text document. TextDocument TextDocumentIdentifier `json:"textDocument"` // The document range for which inlay hints should be computed. @@ -2030,7 +2043,7 @@ type InlayHintParams struct { // line 3689 // Inlay hint options used during static or dynamic registration. // // @since 3.17.0 -type InlayHintRegistrationOptions struct { // line 3819 +type InlayHintRegistrationOptions struct { InlayHintOptions TextDocumentRegistrationOptions StaticRegistrationOptions @@ -2039,7 +2052,7 @@ type InlayHintRegistrationOptions struct { // line 3819 // Client workspace capabilities specific to inlay hints. // // @since 3.17.0 -type InlayHintWorkspaceClientCapabilities struct { // line 11460 +type InlayHintWorkspaceClientCapabilities struct { // Whether the client implementation supports a refresh request sent from // the server to the client. // @@ -2054,7 +2067,7 @@ type InlayHintWorkspaceClientCapabilities struct { // line 11460 // // @since 3.18.0 // @proposed -type InlineCompletionClientCapabilities struct { // line 12809 +type InlineCompletionClientCapabilities struct { // Whether implementation supports dynamic registration for inline completion providers. DynamicRegistration bool `json:"dynamicRegistration,omitempty"` } @@ -2063,7 +2076,7 @@ type InlineCompletionClientCapabilities struct { // line 12809 // // @since 3.18.0 // @proposed -type InlineCompletionContext struct { // line 7833 +type InlineCompletionContext struct { // Describes how the inline completion was triggered. TriggerKind InlineCompletionTriggerKind `json:"triggerKind"` // Provides information about the currently selected item in the autocomplete widget if it is visible. @@ -2074,7 +2087,7 @@ type InlineCompletionContext struct { // line 7833 // // @since 3.18.0 // @proposed -type InlineCompletionItem struct { // line 4158 +type InlineCompletionItem struct { // The text to replace the range with. Must be set. InsertText Or_InlineCompletionItem_insertText `json:"insertText"` // A text that is used to decide if this inline completion should be shown. When `falsy` the {@link InlineCompletionItem.insertText} is used. @@ -2089,7 +2102,7 @@ type InlineCompletionItem struct { // line 4158 // // @since 3.18.0 // @proposed -type InlineCompletionList struct { // line 4139 +type InlineCompletionList struct { // The inline completion items Items []InlineCompletionItem `json:"items"` } @@ -2098,7 +2111,7 @@ type InlineCompletionList struct { // line 4139 // // @since 3.18.0 // @proposed -type InlineCompletionOptions struct { // line 7882 +type InlineCompletionOptions struct { WorkDoneProgressOptions } @@ -2106,7 +2119,7 @@ type InlineCompletionOptions struct { // line 7882 // // @since 3.18.0 // @proposed -type InlineCompletionParams struct { // line 4111 +type InlineCompletionParams struct { // Additional information about the context in which inline completions were // requested. Context InlineCompletionContext `json:"context"` @@ -2118,7 +2131,7 @@ type InlineCompletionParams struct { // line 4111 // // @since 3.18.0 // @proposed -type InlineCompletionRegistrationOptions struct { // line 4210 +type InlineCompletionRegistrationOptions struct { InlineCompletionOptions TextDocumentRegistrationOptions StaticRegistrationOptions @@ -2128,7 +2141,8 @@ type InlineCompletionRegistrationOptions struct { // line 4210 // // @since 3.18.0 // @proposed -type InlineCompletionTriggerKind uint32 // line 13820 +type InlineCompletionTriggerKind uint32 + // Inline value information can be provided by different means: // // - directly as a text value (class InlineValueText). @@ -2142,13 +2156,13 @@ type InlineValue = Or_InlineValue // (alias) line 14276 // Client capabilities specific to inline values. // // @since 3.17.0 -type InlineValueClientCapabilities struct { // line 12729 +type InlineValueClientCapabilities struct { // Whether implementation supports dynamic registration for inline value providers. DynamicRegistration bool `json:"dynamicRegistration,omitempty"` } // @since 3.17.0 -type InlineValueContext struct { // line 7184 +type InlineValueContext struct { // The stack frame (as a DAP Id) where the execution has stopped. FrameID int32 `json:"frameId"` // The document range where execution has stopped. @@ -2161,7 +2175,7 @@ type InlineValueContext struct { // line 7184 // An optional expression can be used to override the extracted expression. // // @since 3.17.0 -type InlineValueEvaluatableExpression struct { // line 7262 +type InlineValueEvaluatableExpression struct { // The document range for which the inline value applies. // The range is used to extract the evaluatable expression from the underlying document. Range Range `json:"range"` @@ -2172,14 +2186,14 @@ type InlineValueEvaluatableExpression struct { // line 7262 // Inline value options used during static registration. // // @since 3.17.0 -type InlineValueOptions struct { // line 7286 +type InlineValueOptions struct { WorkDoneProgressOptions } // A parameter literal used in inline value requests. // // @since 3.17.0 -type InlineValueParams struct { // line 3630 +type InlineValueParams struct { // The text document. TextDocument TextDocumentIdentifier `json:"textDocument"` // The document range for which inline values should be computed. @@ -2193,7 +2207,7 @@ type InlineValueParams struct { // line 3630 // Inline value options used during static or dynamic registration. // // @since 3.17.0 -type InlineValueRegistrationOptions struct { // line 3667 +type InlineValueRegistrationOptions struct { InlineValueOptions TextDocumentRegistrationOptions StaticRegistrationOptions @@ -2202,7 +2216,7 @@ type InlineValueRegistrationOptions struct { // line 3667 // Provide inline value as text. // // @since 3.17.0 -type InlineValueText struct { // line 7207 +type InlineValueText struct { // The document range for which the inline value applies. Range Range `json:"range"` // The text of the inline value. @@ -2214,7 +2228,7 @@ type InlineValueText struct { // line 7207 // An optional variable name can be used to override the extracted name. // // @since 3.17.0 -type InlineValueVariableLookup struct { // line 7230 +type InlineValueVariableLookup struct { // The document range for which the inline value applies. // The range is used to extract the variable name from the underlying document. Range Range `json:"range"` @@ -2227,7 +2241,7 @@ type InlineValueVariableLookup struct { // line 7230 // Client workspace capabilities specific to inline values. // // @since 3.17.0 -type InlineValueWorkspaceClientCapabilities struct { // line 11444 +type InlineValueWorkspaceClientCapabilities struct { // Whether the client implementation supports a refresh request sent from the // server to the client. // @@ -2241,7 +2255,7 @@ type InlineValueWorkspaceClientCapabilities struct { // line 11444 // A special text edit to provide an insert and a replace operation. // // @since 3.16.0 -type InsertReplaceEdit struct { // line 8994 +type InsertReplaceEdit struct { // The string to be inserted. NewText string `json:"newText"` // The range if the insert is requested @@ -2252,38 +2266,40 @@ type InsertReplaceEdit struct { // line 8994 // Defines whether the insert text in a completion item should be interpreted as // plain text or a snippet. -type InsertTextFormat uint32 // line 13653 +type InsertTextFormat uint32 + // How whitespace and indentation is handled during completion // item insertion. // // @since 3.16.0 -type InsertTextMode uint32 // line 13673 +type InsertTextMode uint32 type LSPAny = interface{} // LSP arrays. // @since 3.17.0 type LSPArray = []interface{} // (alias) line 14194 -type LSPErrorCodes int32 // line 13176 +type LSPErrorCodes int32 + // LSP object definition. // @since 3.17.0 type LSPObject = map[string]LSPAny // (alias) line 14526 // Client capabilities for the linked editing range request. // // @since 3.16.0 -type LinkedEditingRangeClientCapabilities struct { // line 12681 +type LinkedEditingRangeClientCapabilities struct { // Whether implementation supports dynamic registration. If this is set to `true` // the client supports the new `(TextDocumentRegistrationOptions & StaticRegistrationOptions)` // return value for the corresponding server capability as well. DynamicRegistration bool `json:"dynamicRegistration,omitempty"` } -type LinkedEditingRangeOptions struct { // line 6888 +type LinkedEditingRangeOptions struct { WorkDoneProgressOptions } -type LinkedEditingRangeParams struct { // line 3185 +type LinkedEditingRangeParams struct { TextDocumentPositionParams WorkDoneProgressParams } -type LinkedEditingRangeRegistrationOptions struct { // line 3228 +type LinkedEditingRangeRegistrationOptions struct { TextDocumentRegistrationOptions LinkedEditingRangeOptions StaticRegistrationOptions @@ -2292,7 +2308,7 @@ type LinkedEditingRangeRegistrationOptions struct { // line 3228 // The result of a linked editing range request. // // @since 3.16.0 -type LinkedEditingRanges struct { // line 3201 +type LinkedEditingRanges struct { // A list of ranges that can be edited together. The ranges must have // identical length and contain identical text content. The ranges cannot overlap. Ranges []Range `json:"ranges"` @@ -2303,13 +2319,13 @@ type LinkedEditingRanges struct { // line 3201 } // created for Literal (Lit_NotebookDocumentChangeEvent_cells_textContent_Elem) -type Lit_NotebookDocumentChangeEvent_cells_textContent_Elem struct { // line 7781 +type Lit_NotebookDocumentChangeEvent_cells_textContent_Elem struct { Document VersionedTextDocumentIdentifier `json:"document"` Changes []TextDocumentContentChangeEvent `json:"changes"` } // created for Literal (Lit_NotebookDocumentFilter_Item1) -type Lit_NotebookDocumentFilter_Item1 struct { // line 14708 +type Lit_NotebookDocumentFilter_Item1 struct { // The type of the enclosing notebook. NotebookType string `json:"notebookType,omitempty"` // A Uri {@link Uri.scheme scheme}, like `file` or `untitled`. @@ -2319,7 +2335,7 @@ type Lit_NotebookDocumentFilter_Item1 struct { // line 14708 } // created for Literal (Lit_NotebookDocumentFilter_Item2) -type Lit_NotebookDocumentFilter_Item2 struct { // line 14741 +type Lit_NotebookDocumentFilter_Item2 struct { // The type of the enclosing notebook. NotebookType string `json:"notebookType,omitempty"` // A Uri {@link Uri.scheme scheme}, like `file` or `untitled`. @@ -2329,12 +2345,12 @@ type Lit_NotebookDocumentFilter_Item2 struct { // line 14741 } // created for Literal (Lit_NotebookDocumentSyncOptions_notebookSelector_Elem_Item0_cells_Elem) -type Lit_NotebookDocumentSyncOptions_notebookSelector_Elem_Item0_cells_Elem struct { // line 10185 +type Lit_NotebookDocumentSyncOptions_notebookSelector_Elem_Item0_cells_Elem struct { Language string `json:"language"` } // created for Literal (Lit_NotebookDocumentSyncOptions_notebookSelector_Elem_Item1) -type Lit_NotebookDocumentSyncOptions_notebookSelector_Elem_Item1 struct { // line 10206 +type Lit_NotebookDocumentSyncOptions_notebookSelector_Elem_Item1 struct { // The notebook to be synced If a string // value is provided it matches against the // notebook type. '*' matches every notebook. @@ -2344,23 +2360,23 @@ type Lit_NotebookDocumentSyncOptions_notebookSelector_Elem_Item1 struct { // lin } // created for Literal (Lit_NotebookDocumentSyncOptions_notebookSelector_Elem_Item1_cells_Elem) -type Lit_NotebookDocumentSyncOptions_notebookSelector_Elem_Item1_cells_Elem struct { // line 10232 +type Lit_NotebookDocumentSyncOptions_notebookSelector_Elem_Item1_cells_Elem struct { Language string `json:"language"` } // created for Literal (Lit_PrepareRenameResult_Item2) -type Lit_PrepareRenameResult_Item2 struct { // line 14347 +type Lit_PrepareRenameResult_Item2 struct { DefaultBehavior bool `json:"defaultBehavior"` } // created for Literal (Lit_TextDocumentContentChangeEvent_Item1) -type Lit_TextDocumentContentChangeEvent_Item1 struct { // line 14455 +type Lit_TextDocumentContentChangeEvent_Item1 struct { // The new text of the whole document. Text string `json:"text"` } // created for Literal (Lit_TextDocumentFilter_Item2) -type Lit_TextDocumentFilter_Item2 struct { // line 14632 +type Lit_TextDocumentFilter_Item2 struct { // A language id, like `typescript`. Language string `json:"language,omitempty"` // A Uri {@link Uri.scheme scheme}, like `file` or `untitled`. @@ -2371,14 +2387,14 @@ type Lit_TextDocumentFilter_Item2 struct { // line 14632 // Represents a location inside a resource, such as a line // inside a text file. -type Location struct { // line 2156 +type Location struct { URI DocumentURI `json:"uri"` Range Range `json:"range"` } // Represents the connection of two locations. Provides additional metadata over normal {@link Location locations}, // including an origin range. -type LocationLink struct { // line 6508 +type LocationLink struct { // Span of the origin of this link. // // Used as the underlined span for mouse interaction. Defaults to the word range at @@ -2396,13 +2412,13 @@ type LocationLink struct { // line 6508 } // The log message parameters. -type LogMessageParams struct { // line 4446 +type LogMessageParams struct { // The message type. See {@link MessageType} Type MessageType `json:"type"` // The actual message. Message string `json:"message"` } -type LogTraceParams struct { // line 6395 +type LogTraceParams struct { Message string `json:"message"` Verbose string `json:"verbose,omitempty"` } @@ -2410,7 +2426,7 @@ type LogTraceParams struct { // line 6395 // Client capabilities specific to the used markdown parser. // // @since 3.16.0 -type MarkdownClientCapabilities struct { // line 12917 +type MarkdownClientCapabilities struct { // The name of the parser. Parser string `json:"parser"` // The version of the parser. @@ -2459,7 +2475,7 @@ type MarkedString = Or_MarkedString // (alias) line 14473 // // *Please Note* that clients might sanitize the return markdown. A client could decide to // remove HTML from the markdown to avoid script execution. -type MarkupContent struct { // line 7349 +type MarkupContent struct { // The type of the Markup Kind MarkupKind `json:"kind"` // The content itself @@ -2471,18 +2487,19 @@ type MarkupContent struct { // line 7349 // // Please note that `MarkupKinds` must not start with a `$`. This kinds // are reserved for internal usage. -type MarkupKind string // line 13800 -type MessageActionItem struct { // line 4433 +type MarkupKind string +type MessageActionItem struct { // A short title like 'Retry', 'Open Log' etc. Title string `json:"title"` } // The message type -type MessageType uint32 // line 13447 +type MessageType uint32 + // Moniker definition to match LSIF 0.5 moniker definition. // // @since 3.16.0 -type Moniker struct { // line 3411 +type Moniker struct { // The scheme of the moniker. For example tsc or .Net Scheme string `json:"scheme"` // The identifier of the moniker. The value is opaque in LSIF however @@ -2497,7 +2514,7 @@ type Moniker struct { // line 3411 // Client capabilities specific to the moniker request. // // @since 3.16.0 -type MonikerClientCapabilities struct { // line 12697 +type MonikerClientCapabilities struct { // Whether moniker supports dynamic registration. If this is set to `true` // the client supports the new `MonikerRegistrationOptions` return value // for the corresponding server capability as well. @@ -2507,28 +2524,28 @@ type MonikerClientCapabilities struct { // line 12697 // The moniker kind. // // @since 3.16.0 -type MonikerKind string // line 13400 -type MonikerOptions struct { // line 7162 +type MonikerKind string +type MonikerOptions struct { WorkDoneProgressOptions } -type MonikerParams struct { // line 3391 +type MonikerParams struct { TextDocumentPositionParams WorkDoneProgressParams PartialResultParams } -type MonikerRegistrationOptions struct { // line 3451 +type MonikerRegistrationOptions struct { TextDocumentRegistrationOptions MonikerOptions } // created for Literal (Lit_MarkedString_Item1) -type Msg_MarkedString struct { // line 14483 +type Msg_MarkedString struct { Language string `json:"language"` Value string `json:"value"` } // created for Literal (Lit_NotebookDocumentFilter_Item0) -type Msg_NotebookDocumentFilter struct { // line 14675 +type Msg_NotebookDocumentFilter struct { // The type of the enclosing notebook. NotebookType string `json:"notebookType"` // A Uri {@link Uri.scheme scheme}, like `file` or `untitled`. @@ -2538,13 +2555,13 @@ type Msg_NotebookDocumentFilter struct { // line 14675 } // created for Literal (Lit_PrepareRenameResult_Item1) -type Msg_PrepareRename2Gn struct { // line 14326 +type Msg_PrepareRename2Gn struct { Range Range `json:"range"` Placeholder string `json:"placeholder"` } // created for Literal (Lit_TextDocumentContentChangeEvent_Item0) -type Msg_TextDocumentContentChangeEvent struct { // line 14423 +type Msg_TextDocumentContentChangeEvent struct { // The range of the document that changed. Range *Range `json:"range"` // The optional length of the range that got replaced. @@ -2556,7 +2573,7 @@ type Msg_TextDocumentContentChangeEvent struct { // line 14423 } // created for Literal (Lit_TextDocumentFilter_Item1) -type Msg_TextDocumentFilter struct { // line 14599 +type Msg_TextDocumentFilter struct { // A language id, like `typescript`. Language string `json:"language,omitempty"` // A Uri {@link Uri.scheme scheme}, like `file` or `untitled`. @@ -2566,7 +2583,7 @@ type Msg_TextDocumentFilter struct { // line 14599 } // created for Literal (Lit__InitializeParams_clientInfo) -type Msg_XInitializeParams_clientInfo struct { // line 7971 +type Msg_XInitializeParams_clientInfo struct { // The name of the client as defined by the client. Name string `json:"name"` // The client's version as defined by the client. @@ -2580,7 +2597,7 @@ type Msg_XInitializeParams_clientInfo struct { // line 7971 // notebook cell or the cell's text document. // // @since 3.17.0 -type NotebookCell struct { // line 9928 +type NotebookCell struct { // The cell's kind Kind NotebookCellKind `json:"kind"` // The URI of the cell's text document @@ -2599,7 +2616,7 @@ type NotebookCell struct { // line 9928 // array from state S to S'. // // @since 3.17.0 -type NotebookCellArrayChange struct { // line 9969 +type NotebookCellArrayChange struct { // The start oftest of the cell that changed. Start uint32 `json:"start"` // The deleted cells @@ -2611,12 +2628,13 @@ type NotebookCellArrayChange struct { // line 9969 // A notebook cell kind. // // @since 3.17.0 -type NotebookCellKind uint32 // line 14063 +type NotebookCellKind uint32 + // A notebook cell text document filter denotes a cell text // document by different properties. // // @since 3.17.0 -type NotebookCellTextDocumentFilter struct { // line 10467 +type NotebookCellTextDocumentFilter struct { // A filter that matches against the notebook // containing the notebook cell. If a string // value is provided it matches against the @@ -2632,7 +2650,7 @@ type NotebookCellTextDocumentFilter struct { // line 10467 // A notebook document. // // @since 3.17.0 -type NotebookDocument struct { // line 7590 +type NotebookDocument struct { // The notebook document's uri. URI URI `json:"uri"` // The type of the notebook. @@ -2652,7 +2670,7 @@ type NotebookDocument struct { // line 7590 // A change event for a notebook document. // // @since 3.17.0 -type NotebookDocumentChangeEvent struct { // line 7702 +type NotebookDocumentChangeEvent struct { // The changed meta data if any. // // Note: should always be an object literal (e.g. LSPObject) @@ -2664,7 +2682,7 @@ type NotebookDocumentChangeEvent struct { // line 7702 // Capabilities specific to the notebook document support. // // @since 3.17.0 -type NotebookDocumentClientCapabilities struct { // line 10978 +type NotebookDocumentClientCapabilities struct { // Capabilities specific to notebook document synchronization // // @since 3.17.0 @@ -2680,7 +2698,7 @@ type NotebookDocumentFilter = Msg_NotebookDocumentFilter // (alias) line 14669 // A literal to identify a notebook document in the client. // // @since 3.17.0 -type NotebookDocumentIdentifier struct { // line 7818 +type NotebookDocumentIdentifier struct { // The notebook document's uri. URI URI `json:"uri"` } @@ -2688,7 +2706,7 @@ type NotebookDocumentIdentifier struct { // line 7818 // Notebook specific client capabilities. // // @since 3.17.0 -type NotebookDocumentSyncClientCapabilities struct { // line 12826 +type NotebookDocumentSyncClientCapabilities struct { // Whether implementation supports dynamic registration. If this is // set to `true` the client supports the new // `(TextDocumentRegistrationOptions & StaticRegistrationOptions)` @@ -2711,7 +2729,7 @@ type NotebookDocumentSyncClientCapabilities struct { // line 12826 // cell will be synced. // // @since 3.17.0 -type NotebookDocumentSyncOptions struct { // line 10149 +type NotebookDocumentSyncOptions struct { // The notebooks to be synced NotebookSelector []PNotebookSelectorPNotebookDocumentSync `json:"notebookSelector"` // Whether save notification should be forwarded to @@ -2722,13 +2740,13 @@ type NotebookDocumentSyncOptions struct { // line 10149 // Registration options specific to a notebook. // // @since 3.17.0 -type NotebookDocumentSyncRegistrationOptions struct { // line 10269 +type NotebookDocumentSyncRegistrationOptions struct { NotebookDocumentSyncOptions StaticRegistrationOptions } // A text document identifier to optionally denote a specific version of a text document. -type OptionalVersionedTextDocumentIdentifier struct { // line 9673 +type OptionalVersionedTextDocumentIdentifier struct { // The version number of this document. If a versioned text document identifier // is sent from the server to the client and the file is not open in the editor // (the server has not received an open notification before) the server can send @@ -2739,322 +2757,322 @@ type OptionalVersionedTextDocumentIdentifier struct { // line 9673 } // created for Or [FEditRangePItemDefaults Range] -type OrFEditRangePItemDefaults struct { // line 4965 +type OrFEditRangePItemDefaults struct { Value interface{} `json:"value"` } // created for Or [NotebookDocumentFilter string] -type OrFNotebookPNotebookSelector struct { // line 10166 +type OrFNotebookPNotebookSelector struct { Value interface{} `json:"value"` } // created for Or [Location PLocationMsg_workspace_symbol] -type OrPLocation_workspace_symbol struct { // line 5716 +type OrPLocation_workspace_symbol struct { Value interface{} `json:"value"` } // created for Or [[]string string] -type OrPSection_workspace_didChangeConfiguration struct { // line 4359 +type OrPSection_workspace_didChangeConfiguration struct { Value interface{} `json:"value"` } // created for Or [MarkupContent string] -type OrPTooltipPLabel struct { // line 7312 +type OrPTooltipPLabel struct { Value interface{} `json:"value"` } // created for Or [MarkupContent string] -type OrPTooltip_textDocument_inlayHint struct { // line 3773 +type OrPTooltip_textDocument_inlayHint struct { Value interface{} `json:"value"` } // created for Or [int32 string] -type Or_CancelParams_id struct { // line 6421 +type Or_CancelParams_id struct { Value interface{} `json:"value"` } // created for Or [MarkupContent string] -type Or_CompletionItem_documentation struct { // line 4778 +type Or_CompletionItem_documentation struct { Value interface{} `json:"value"` } // created for Or [InsertReplaceEdit TextEdit] -type Or_CompletionItem_textEdit struct { // line 4861 +type Or_CompletionItem_textEdit struct { Value interface{} `json:"value"` } // created for Or [Location []Location] -type Or_Definition struct { // line 14169 +type Or_Definition struct { Value interface{} `json:"value"` } // created for Or [int32 string] -type Or_Diagnostic_code struct { // line 8866 +type Or_Diagnostic_code struct { Value interface{} `json:"value"` } // created for Or [RelatedFullDocumentDiagnosticReport RelatedUnchangedDocumentDiagnosticReport] -type Or_DocumentDiagnosticReport struct { // line 14301 +type Or_DocumentDiagnosticReport struct { Value interface{} `json:"value"` } // created for Or [FullDocumentDiagnosticReport UnchangedDocumentDiagnosticReport] -type Or_DocumentDiagnosticReportPartialResult_relatedDocuments_Value struct { // line 3896 +type Or_DocumentDiagnosticReportPartialResult_relatedDocuments_Value struct { Value interface{} `json:"value"` } // created for Or [NotebookCellTextDocumentFilter TextDocumentFilter] -type Or_DocumentFilter struct { // line 14511 +type Or_DocumentFilter struct { Value interface{} `json:"value"` } // created for Or [MarkedString MarkupContent []MarkedString] -type Or_Hover_contents struct { // line 5087 +type Or_Hover_contents struct { Value interface{} `json:"value"` } // created for Or [[]InlayHintLabelPart string] -type Or_InlayHint_label struct { // line 3732 +type Or_InlayHint_label struct { Value interface{} `json:"value"` } // created for Or [StringValue string] -type Or_InlineCompletionItem_insertText struct { // line 4164 +type Or_InlineCompletionItem_insertText struct { Value interface{} `json:"value"` } // created for Or [InlineValueEvaluatableExpression InlineValueText InlineValueVariableLookup] -type Or_InlineValue struct { // line 14279 +type Or_InlineValue struct { Value interface{} `json:"value"` } // created for Or [Msg_MarkedString string] -type Or_MarkedString struct { // line 14476 +type Or_MarkedString struct { Value interface{} `json:"value"` } // created for Or [NotebookDocumentFilter string] -type Or_NotebookCellTextDocumentFilter_notebook struct { // line 10473 +type Or_NotebookCellTextDocumentFilter_notebook struct { Value interface{} `json:"value"` } // created for Or [NotebookDocumentFilter string] -type Or_NotebookDocumentSyncOptions_notebookSelector_Elem_Item1_notebook struct { // line 10212 +type Or_NotebookDocumentSyncOptions_notebookSelector_Elem_Item1_notebook struct { Value interface{} `json:"value"` } // created for Or [FullDocumentDiagnosticReport UnchangedDocumentDiagnosticReport] -type Or_RelatedFullDocumentDiagnosticReport_relatedDocuments_Value struct { // line 7405 +type Or_RelatedFullDocumentDiagnosticReport_relatedDocuments_Value struct { Value interface{} `json:"value"` } // created for Or [FullDocumentDiagnosticReport UnchangedDocumentDiagnosticReport] -type Or_RelatedUnchangedDocumentDiagnosticReport_relatedDocuments_Value struct { // line 7444 +type Or_RelatedUnchangedDocumentDiagnosticReport_relatedDocuments_Value struct { Value interface{} `json:"value"` } // created for Or [URI WorkspaceFolder] -type Or_RelativePattern_baseUri struct { // line 11107 +type Or_RelativePattern_baseUri struct { Value interface{} `json:"value"` } // created for Or [CodeAction Command] -type Or_Result_textDocument_codeAction_Item0_Elem struct { // line 1414 +type Or_Result_textDocument_codeAction_Item0_Elem struct { Value interface{} `json:"value"` } // created for Or [InlineCompletionList []InlineCompletionItem] -type Or_Result_textDocument_inlineCompletion struct { // line 981 +type Or_Result_textDocument_inlineCompletion struct { Value interface{} `json:"value"` } // created for Or [FFullPRequests bool] -type Or_SemanticTokensClientCapabilities_requests_full struct { // line 12574 +type Or_SemanticTokensClientCapabilities_requests_full struct { Value interface{} `json:"value"` } // created for Or [FRangePRequests bool] -type Or_SemanticTokensClientCapabilities_requests_range struct { // line 12554 +type Or_SemanticTokensClientCapabilities_requests_range struct { Value interface{} `json:"value"` } // created for Or [PFullESemanticTokensOptions bool] -type Or_SemanticTokensOptions_full struct { // line 6816 +type Or_SemanticTokensOptions_full struct { Value interface{} `json:"value"` } // created for Or [PRangeESemanticTokensOptions bool] -type Or_SemanticTokensOptions_range struct { // line 6796 +type Or_SemanticTokensOptions_range struct { Value interface{} `json:"value"` } // created for Or [CallHierarchyOptions CallHierarchyRegistrationOptions bool] -type Or_ServerCapabilities_callHierarchyProvider struct { // line 8526 +type Or_ServerCapabilities_callHierarchyProvider struct { Value interface{} `json:"value"` } // created for Or [CodeActionOptions bool] -type Or_ServerCapabilities_codeActionProvider struct { // line 8334 +type Or_ServerCapabilities_codeActionProvider struct { Value interface{} `json:"value"` } // created for Or [DocumentColorOptions DocumentColorRegistrationOptions bool] -type Or_ServerCapabilities_colorProvider struct { // line 8370 +type Or_ServerCapabilities_colorProvider struct { Value interface{} `json:"value"` } // created for Or [DeclarationOptions DeclarationRegistrationOptions bool] -type Or_ServerCapabilities_declarationProvider struct { // line 8196 +type Or_ServerCapabilities_declarationProvider struct { Value interface{} `json:"value"` } // created for Or [DefinitionOptions bool] -type Or_ServerCapabilities_definitionProvider struct { // line 8218 +type Or_ServerCapabilities_definitionProvider struct { Value interface{} `json:"value"` } // created for Or [DiagnosticOptions DiagnosticRegistrationOptions] -type Or_ServerCapabilities_diagnosticProvider struct { // line 8683 +type Or_ServerCapabilities_diagnosticProvider struct { Value interface{} `json:"value"` } // created for Or [DocumentFormattingOptions bool] -type Or_ServerCapabilities_documentFormattingProvider struct { // line 8410 +type Or_ServerCapabilities_documentFormattingProvider struct { Value interface{} `json:"value"` } // created for Or [DocumentHighlightOptions bool] -type Or_ServerCapabilities_documentHighlightProvider struct { // line 8298 +type Or_ServerCapabilities_documentHighlightProvider struct { Value interface{} `json:"value"` } // created for Or [DocumentRangeFormattingOptions bool] -type Or_ServerCapabilities_documentRangeFormattingProvider struct { // line 8428 +type Or_ServerCapabilities_documentRangeFormattingProvider struct { Value interface{} `json:"value"` } // created for Or [DocumentSymbolOptions bool] -type Or_ServerCapabilities_documentSymbolProvider struct { // line 8316 +type Or_ServerCapabilities_documentSymbolProvider struct { Value interface{} `json:"value"` } // created for Or [FoldingRangeOptions FoldingRangeRegistrationOptions bool] -type Or_ServerCapabilities_foldingRangeProvider struct { // line 8473 +type Or_ServerCapabilities_foldingRangeProvider struct { Value interface{} `json:"value"` } // created for Or [HoverOptions bool] -type Or_ServerCapabilities_hoverProvider struct { // line 8169 +type Or_ServerCapabilities_hoverProvider struct { Value interface{} `json:"value"` } // created for Or [ImplementationOptions ImplementationRegistrationOptions bool] -type Or_ServerCapabilities_implementationProvider struct { // line 8258 +type Or_ServerCapabilities_implementationProvider struct { Value interface{} `json:"value"` } // created for Or [InlayHintOptions InlayHintRegistrationOptions bool] -type Or_ServerCapabilities_inlayHintProvider struct { // line 8660 +type Or_ServerCapabilities_inlayHintProvider struct { Value interface{} `json:"value"` } // created for Or [InlineCompletionOptions bool] -type Or_ServerCapabilities_inlineCompletionProvider struct { // line 8702 +type Or_ServerCapabilities_inlineCompletionProvider struct { Value interface{} `json:"value"` } // created for Or [InlineValueOptions InlineValueRegistrationOptions bool] -type Or_ServerCapabilities_inlineValueProvider struct { // line 8637 +type Or_ServerCapabilities_inlineValueProvider struct { Value interface{} `json:"value"` } // created for Or [LinkedEditingRangeOptions LinkedEditingRangeRegistrationOptions bool] -type Or_ServerCapabilities_linkedEditingRangeProvider struct { // line 8549 +type Or_ServerCapabilities_linkedEditingRangeProvider struct { Value interface{} `json:"value"` } // created for Or [MonikerOptions MonikerRegistrationOptions bool] -type Or_ServerCapabilities_monikerProvider struct { // line 8591 +type Or_ServerCapabilities_monikerProvider struct { Value interface{} `json:"value"` } // created for Or [NotebookDocumentSyncOptions NotebookDocumentSyncRegistrationOptions] -type Or_ServerCapabilities_notebookDocumentSync struct { // line 8141 +type Or_ServerCapabilities_notebookDocumentSync struct { Value interface{} `json:"value"` } // created for Or [ReferenceOptions bool] -type Or_ServerCapabilities_referencesProvider struct { // line 8280 +type Or_ServerCapabilities_referencesProvider struct { Value interface{} `json:"value"` } // created for Or [RenameOptions bool] -type Or_ServerCapabilities_renameProvider struct { // line 8455 +type Or_ServerCapabilities_renameProvider struct { Value interface{} `json:"value"` } // created for Or [SelectionRangeOptions SelectionRangeRegistrationOptions bool] -type Or_ServerCapabilities_selectionRangeProvider struct { // line 8495 +type Or_ServerCapabilities_selectionRangeProvider struct { Value interface{} `json:"value"` } // created for Or [SemanticTokensOptions SemanticTokensRegistrationOptions] -type Or_ServerCapabilities_semanticTokensProvider struct { // line 8572 +type Or_ServerCapabilities_semanticTokensProvider struct { Value interface{} `json:"value"` } // created for Or [TextDocumentSyncKind TextDocumentSyncOptions] -type Or_ServerCapabilities_textDocumentSync struct { // line 8123 +type Or_ServerCapabilities_textDocumentSync struct { Value interface{} `json:"value"` } // created for Or [TypeDefinitionOptions TypeDefinitionRegistrationOptions bool] -type Or_ServerCapabilities_typeDefinitionProvider struct { // line 8236 +type Or_ServerCapabilities_typeDefinitionProvider struct { Value interface{} `json:"value"` } // created for Or [TypeHierarchyOptions TypeHierarchyRegistrationOptions bool] -type Or_ServerCapabilities_typeHierarchyProvider struct { // line 8614 +type Or_ServerCapabilities_typeHierarchyProvider struct { Value interface{} `json:"value"` } // created for Or [WorkspaceSymbolOptions bool] -type Or_ServerCapabilities_workspaceSymbolProvider struct { // line 8392 +type Or_ServerCapabilities_workspaceSymbolProvider struct { Value interface{} `json:"value"` } // created for Or [MarkupContent string] -type Or_SignatureInformation_documentation struct { // line 9160 +type Or_SignatureInformation_documentation struct { Value interface{} `json:"value"` } // created for Or [AnnotatedTextEdit TextEdit] -type Or_TextDocumentEdit_edits_Elem struct { // line 6929 +type Or_TextDocumentEdit_edits_Elem struct { Value interface{} `json:"value"` } // created for Or [SaveOptions bool] -type Or_TextDocumentSyncOptions_save struct { // line 10132 +type Or_TextDocumentSyncOptions_save struct { Value interface{} `json:"value"` } // created for Or [WorkspaceFullDocumentDiagnosticReport WorkspaceUnchangedDocumentDiagnosticReport] -type Or_WorkspaceDocumentDiagnosticReport struct { // line 14402 +type Or_WorkspaceDocumentDiagnosticReport struct { Value interface{} `json:"value"` } // created for Or [CreateFile DeleteFile RenameFile TextDocumentEdit] -type Or_WorkspaceEdit_documentChanges_Elem struct { // line 3293 +type Or_WorkspaceEdit_documentChanges_Elem struct { Value interface{} `json:"value"` } // created for Or [Declaration []DeclarationLink] -type Or_textDocument_declaration struct { // line 249 +type Or_textDocument_declaration struct { Value interface{} `json:"value"` } // created for Literal (Lit_NotebookDocumentChangeEvent_cells) -type PCellsPChange struct { // line 7717 +type PCellsPChange struct { // Changes to the cell structure to add or // remove cells. Structure *FStructurePCells `json:"structure,omitempty"` @@ -3066,7 +3084,7 @@ type PCellsPChange struct { // line 7717 } // created for Literal (Lit_WorkspaceEditClientCapabilities_changeAnnotationSupport) -type PChangeAnnotationSupportPWorkspaceEdit struct { // line 11181 +type PChangeAnnotationSupportPWorkspaceEdit struct { // Whether the client groups edits with equal labels into tree nodes, // for instance all edits labelled with "Changes in Strings" would // be a tree node. @@ -3074,14 +3092,14 @@ type PChangeAnnotationSupportPWorkspaceEdit struct { // line 11181 } // created for Literal (Lit_CodeActionClientCapabilities_codeActionLiteralSupport) -type PCodeActionLiteralSupportPCodeAction struct { // line 12101 +type PCodeActionLiteralSupportPCodeAction struct { // The code action kind is support with the following value // set. CodeActionKind FCodeActionKindPCodeActionLiteralSupport `json:"codeActionKind"` } // created for Literal (Lit_CompletionClientCapabilities_completionItemKind) -type PCompletionItemKindPCompletion struct { // line 11699 +type PCompletionItemKindPCompletion struct { // The completion item kind values the client supports. When this // property exists the client also guarantees that it will // handle values outside its set gracefully and falls back @@ -3094,7 +3112,7 @@ type PCompletionItemKindPCompletion struct { // line 11699 } // created for Literal (Lit_CompletionClientCapabilities_completionItem) -type PCompletionItemPCompletion struct { // line 11548 +type PCompletionItemPCompletion struct { // Client supports snippets as insert text. // // A snippet can define tab stops and placeholders with `$1`, `$2` @@ -3143,7 +3161,7 @@ type PCompletionItemPCompletion struct { // line 11548 } // created for Literal (Lit_CompletionOptions_completionItem) -type PCompletionItemPCompletionProvider struct { // line 9065 +type PCompletionItemPCompletionProvider struct { // The server has support for completion item label // details (see also `CompletionItemLabelDetails`) when // receiving a completion item in a resolve call. @@ -3153,7 +3171,7 @@ type PCompletionItemPCompletionProvider struct { // line 9065 } // created for Literal (Lit_CompletionClientCapabilities_completionList) -type PCompletionListPCompletion struct { // line 11741 +type PCompletionListPCompletion struct { // The client supports the following itemDefaults on // a completion list. // @@ -3166,7 +3184,7 @@ type PCompletionListPCompletion struct { // line 11741 } // created for Literal (Lit_CodeAction_disabled) -type PDisabledMsg_textDocument_codeAction struct { // line 5622 +type PDisabledMsg_textDocument_codeAction struct { // Human readable description of why the code action is currently disabled. // // This is displayed in the code actions UI. @@ -3174,7 +3192,7 @@ type PDisabledMsg_textDocument_codeAction struct { // line 5622 } // created for Literal (Lit_FoldingRangeClientCapabilities_foldingRangeKind) -type PFoldingRangeKindPFoldingRange struct { // line 12387 +type PFoldingRangeKindPFoldingRange struct { // The folding range kind values the client supports. When this // property exists the client also guarantees that it will // handle values outside its set gracefully and falls back @@ -3183,7 +3201,7 @@ type PFoldingRangeKindPFoldingRange struct { // line 12387 } // created for Literal (Lit_FoldingRangeClientCapabilities_foldingRange) -type PFoldingRangePFoldingRange struct { // line 12412 +type PFoldingRangePFoldingRange struct { // If set, the client signals that it supports setting collapsedText on // folding ranges to display custom labels instead of the default text. // @@ -3192,13 +3210,13 @@ type PFoldingRangePFoldingRange struct { // line 12412 } // created for Literal (Lit_SemanticTokensOptions_full_Item1) -type PFullESemanticTokensOptions struct { // line 6823 +type PFullESemanticTokensOptions struct { // The server supports deltas for full documents. Delta bool `json:"delta"` } // created for Literal (Lit_CompletionList_itemDefaults) -type PItemDefaultsMsg_textDocument_completion struct { // line 4946 +type PItemDefaultsMsg_textDocument_completion struct { // A default commit character set. // // @since 3.17.0 @@ -3222,12 +3240,12 @@ type PItemDefaultsMsg_textDocument_completion struct { // line 4946 } // created for Literal (Lit_WorkspaceSymbol_location_Item1) -type PLocationMsg_workspace_symbol struct { // line 5723 +type PLocationMsg_workspace_symbol struct { URI DocumentURI `json:"uri"` } // created for Literal (Lit_ShowMessageRequestClientCapabilities_messageActionItem) -type PMessageActionItemPShowMessage struct { // line 12857 +type PMessageActionItemPShowMessage struct { // Whether the client supports additional attributes which // are preserved and send back to the server in the // request's response. @@ -3235,7 +3253,7 @@ type PMessageActionItemPShowMessage struct { // line 12857 } // created for Literal (Lit_NotebookDocumentSyncOptions_notebookSelector_Elem_Item0) -type PNotebookSelectorPNotebookDocumentSync struct { // line 10160 +type PNotebookSelectorPNotebookDocumentSync struct { // The notebook to be synced If a string // value is provided it matches against the // notebook type. '*' matches every notebook. @@ -3245,11 +3263,11 @@ type PNotebookSelectorPNotebookDocumentSync struct { // line 10160 } // created for Literal (Lit_SemanticTokensOptions_range_Item1) -type PRangeESemanticTokensOptions struct { // line 6803 +type PRangeESemanticTokensOptions struct { } // created for Literal (Lit_SemanticTokensClientCapabilities_requests) -type PRequestsPSemanticTokens struct { // line 12548 +type PRequestsPSemanticTokens struct { // The client will send the `textDocument/semanticTokens/range` request if // the server provides a corresponding handler. Range Or_SemanticTokensClientCapabilities_requests_range `json:"range"` @@ -3259,26 +3277,26 @@ type PRequestsPSemanticTokens struct { // line 12548 } // created for Literal (Lit_CodeActionClientCapabilities_resolveSupport) -type PResolveSupportPCodeAction struct { // line 12166 +type PResolveSupportPCodeAction struct { // The properties that a client can resolve lazily. Properties []string `json:"properties"` } // created for Literal (Lit_InlayHintClientCapabilities_resolveSupport) -type PResolveSupportPInlayHint struct { // line 12760 +type PResolveSupportPInlayHint struct { // The properties that a client can resolve lazily. Properties []string `json:"properties"` } // created for Literal (Lit_WorkspaceSymbolClientCapabilities_resolveSupport) -type PResolveSupportPSymbol struct { // line 11303 +type PResolveSupportPSymbol struct { // The properties that a client can resolve lazily. Usually // `location.range` Properties []string `json:"properties"` } // created for Literal (Lit_InitializeResult_serverInfo) -type PServerInfoMsg_initialize struct { // line 4291 +type PServerInfoMsg_initialize struct { // The name of the server as defined by the server. Name string `json:"name"` // The server's version as defined by the server. @@ -3286,7 +3304,7 @@ type PServerInfoMsg_initialize struct { // line 4291 } // created for Literal (Lit_SignatureHelpClientCapabilities_signatureInformation) -type PSignatureInformationPSignatureHelp struct { // line 11808 +type PSignatureInformationPSignatureHelp struct { // Client supports the following content formats for the documentation // property. The order describes the preferred format of the client. DocumentationFormat []MarkupKind `json:"documentationFormat,omitempty"` @@ -3300,7 +3318,7 @@ type PSignatureInformationPSignatureHelp struct { // line 11808 } // created for Literal (Lit_GeneralClientCapabilities_staleRequestSupport) -type PStaleRequestSupportPGeneral struct { // line 11035 +type PStaleRequestSupportPGeneral struct { // The client will actively cancel the request. Cancel bool `json:"cancel"` // The list of requests for which the client @@ -3310,7 +3328,7 @@ type PStaleRequestSupportPGeneral struct { // line 11035 } // created for Literal (Lit_DocumentSymbolClientCapabilities_symbolKind) -type PSymbolKindPDocumentSymbol struct { // line 12019 +type PSymbolKindPDocumentSymbol struct { // The symbol kind values the client supports. When this // property exists the client also guarantees that it will // handle values outside its set gracefully and falls back @@ -3323,7 +3341,7 @@ type PSymbolKindPDocumentSymbol struct { // line 12019 } // created for Literal (Lit_WorkspaceSymbolClientCapabilities_symbolKind) -type PSymbolKindPSymbol struct { // line 11255 +type PSymbolKindPSymbol struct { // The symbol kind values the client supports. When this // property exists the client also guarantees that it will // handle values outside its set gracefully and falls back @@ -3336,35 +3354,35 @@ type PSymbolKindPSymbol struct { // line 11255 } // created for Literal (Lit_DocumentSymbolClientCapabilities_tagSupport) -type PTagSupportPDocumentSymbol struct { // line 12052 +type PTagSupportPDocumentSymbol struct { // The tags supported by the client. ValueSet []SymbolTag `json:"valueSet"` } // created for Literal (Lit_PublishDiagnosticsClientCapabilities_tagSupport) -type PTagSupportPPublishDiagnostics struct { // line 12463 +type PTagSupportPPublishDiagnostics struct { // The tags supported by the client. ValueSet []DiagnosticTag `json:"valueSet"` } // created for Literal (Lit_WorkspaceSymbolClientCapabilities_tagSupport) -type PTagSupportPSymbol struct { // line 11279 +type PTagSupportPSymbol struct { // The tags supported by the client. ValueSet []SymbolTag `json:"valueSet"` } // The parameters of a configuration request. -type ParamConfiguration struct { // line 2272 +type ParamConfiguration struct { Items []ConfigurationItem `json:"items"` } -type ParamInitialize struct { // line 4263 +type ParamInitialize struct { XInitializeParams WorkspaceFoldersInitializeParams } // Represents a parameter of a callable-signature. A parameter can // have a label and a doc-comment. -type ParameterInformation struct { // line 10417 +type ParameterInformation struct { // The label of this parameter information. // // Either a string or an inclusive start and exclusive end offsets within its containing @@ -3378,7 +3396,7 @@ type ParameterInformation struct { // line 10417 // in the UI but can be omitted. Documentation string `json:"documentation,omitempty"` } -type PartialResultParams struct { // line 6494 +type PartialResultParams struct { // An optional token that a server can use to report partial results (e.g. streaming) to // the client. PartialResultToken *ProgressToken `json:"partialResultToken,omitempty"` @@ -3422,7 +3440,7 @@ type Pattern = string // (alias) line 14778 // that denotes `\r|\n` or `\n|` where `|` represents the character offset. // // @since 3.17.0 - support for negotiated position encoding. -type Position struct { // line 6737 +type Position struct { // Line position in a document (zero-based). // // If a line number is greater than the number of lines in a document, it defaults back to the number of lines in the document. @@ -3441,18 +3459,19 @@ type Position struct { // line 6737 // A set of predefined position encoding kinds. // // @since 3.17.0 -type PositionEncodingKind string // line 13842 +type PositionEncodingKind string type PrepareRename2Gn = Msg_PrepareRename2Gn // (alias) line 13927 -type PrepareRenameParams struct { // line 6161 +type PrepareRenameParams struct { TextDocumentPositionParams WorkDoneProgressParams } type PrepareRenameResult = Msg_PrepareRename2Gn // (alias) line 13927 -type PrepareSupportDefaultBehavior uint32 // line 14137 +type PrepareSupportDefaultBehavior uint32 + // A previous result id in a workspace pull request. // // @since 3.17.0 -type PreviousResultID struct { // line 7567 +type PreviousResultID struct { // The URI for which the client knowns a // result id. URI DocumentURI `json:"uri"` @@ -3463,14 +3482,14 @@ type PreviousResultID struct { // line 7567 // A previous result id in a workspace pull request. // // @since 3.17.0 -type PreviousResultId struct { // line 7567 +type PreviousResultId struct { // The URI for which the client knowns a // result id. URI DocumentURI `json:"uri"` // The value of the previous result id. Value string `json:"value"` } -type ProgressParams struct { // line 6437 +type ProgressParams struct { // The progress token provided by the client or server. Token ProgressToken `json:"token"` // The progress data. @@ -3478,7 +3497,7 @@ type ProgressParams struct { // line 6437 } type ProgressToken = interface{} // (alias) line 14375 // The publish diagnostic client capabilities. -type PublishDiagnosticsClientCapabilities struct { // line 12448 +type PublishDiagnosticsClientCapabilities struct { // Whether the clients accepts diagnostics with related information. RelatedInformation bool `json:"relatedInformation,omitempty"` // Client supports the tag property to provide meta data about a diagnostic. @@ -3504,7 +3523,7 @@ type PublishDiagnosticsClientCapabilities struct { // line 12448 } // The publish diagnostic notification's parameters. -type PublishDiagnosticsParams struct { // line 4657 +type PublishDiagnosticsParams struct { // The URI for which diagnostic information is reported. URI DocumentURI `json:"uri"` // Optional the version number of the document the diagnostics are published for. @@ -3528,7 +3547,7 @@ type PublishDiagnosticsParams struct { // line 4657 // } // // ``` -type Range struct { // line 6547 +type Range struct { // The range's start position. Start Position `json:"start"` // The range's end position. @@ -3536,25 +3555,25 @@ type Range struct { // line 6547 } // Client Capabilities for a {@link ReferencesRequest}. -type ReferenceClientCapabilities struct { // line 11974 +type ReferenceClientCapabilities struct { // Whether references supports dynamic registration. DynamicRegistration bool `json:"dynamicRegistration,omitempty"` } // Value-object that contains additional information when // requesting references. -type ReferenceContext struct { // line 9248 +type ReferenceContext struct { // Include the declaration of the current symbol. IncludeDeclaration bool `json:"includeDeclaration"` } // Reference options. -type ReferenceOptions struct { // line 9262 +type ReferenceOptions struct { WorkDoneProgressOptions } // Parameters for a {@link ReferencesRequest}. -type ReferenceParams struct { // line 5249 +type ReferenceParams struct { Context ReferenceContext `json:"context"` TextDocumentPositionParams WorkDoneProgressParams @@ -3562,13 +3581,13 @@ type ReferenceParams struct { // line 5249 } // Registration options for a {@link ReferencesRequest}. -type ReferenceRegistrationOptions struct { // line 5278 +type ReferenceRegistrationOptions struct { TextDocumentRegistrationOptions ReferenceOptions } // General parameters to register for a notification or to register a provider. -type Registration struct { // line 7895 +type Registration struct { // The id used to register the request. The id can be used to deregister // the request again. ID string `json:"id"` @@ -3577,14 +3596,14 @@ type Registration struct { // line 7895 // Options necessary for the registration. RegisterOptions interface{} `json:"registerOptions,omitempty"` } -type RegistrationParams struct { // line 4233 +type RegistrationParams struct { Registrations []Registration `json:"registrations"` } // Client capabilities specific to regular expressions. // // @since 3.16.0 -type RegularExpressionsClientCapabilities struct { // line 12893 +type RegularExpressionsClientCapabilities struct { // The engine's name. Engine string `json:"engine"` // The engine's version. @@ -3594,7 +3613,7 @@ type RegularExpressionsClientCapabilities struct { // line 12893 // A full diagnostic report with a set of related documents. // // @since 3.17.0 -type RelatedFullDocumentDiagnosticReport struct { // line 7393 +type RelatedFullDocumentDiagnosticReport struct { // Diagnostics of related documents. This information is useful // in programming languages where code in a file A can generate // diagnostics in a file B which A depends on. An example of @@ -3609,7 +3628,7 @@ type RelatedFullDocumentDiagnosticReport struct { // line 7393 // An unchanged diagnostic report with a set of related documents. // // @since 3.17.0 -type RelatedUnchangedDocumentDiagnosticReport struct { // line 7432 +type RelatedUnchangedDocumentDiagnosticReport struct { // Diagnostics of related documents. This information is useful // in programming languages where code in a file A can generate // diagnostics in a file B which A depends on. An example of @@ -3626,14 +3645,14 @@ type RelatedUnchangedDocumentDiagnosticReport struct { // line 7432 // folder root, but it can be another absolute URI as well. // // @since 3.17.0 -type RelativePattern struct { // line 11101 +type RelativePattern struct { // A workspace folder or a base URI to which this pattern will be matched // against relatively. BaseURI Or_RelativePattern_baseUri `json:"baseUri"` // The actual glob pattern; Pattern Pattern `json:"pattern"` } -type RenameClientCapabilities struct { // line 12310 +type RenameClientCapabilities struct { // Whether rename supports dynamic registration. DynamicRegistration bool `json:"dynamicRegistration,omitempty"` // Client supports testing for validity of rename operations @@ -3659,7 +3678,7 @@ type RenameClientCapabilities struct { // line 12310 } // Rename file operation -type RenameFile struct { // line 6985 +type RenameFile struct { // A rename Kind string `json:"kind"` // The old (existing) location. @@ -3672,7 +3691,7 @@ type RenameFile struct { // line 6985 } // Rename file options -type RenameFileOptions struct { // line 9771 +type RenameFileOptions struct { // Overwrite target if existing. Overwrite wins over `ignoreIfExists` Overwrite bool `json:"overwrite,omitempty"` // Ignores if target exists. @@ -3683,14 +3702,14 @@ type RenameFileOptions struct { // line 9771 // files. // // @since 3.16.0 -type RenameFilesParams struct { // line 3355 +type RenameFilesParams struct { // An array of all files/folders renamed in this operation. When a folder is renamed, only // the folder will be included, and not its children. Files []FileRename `json:"files"` } // Provider options for a {@link RenameRequest}. -type RenameOptions struct { // line 9599 +type RenameOptions struct { // Renames should be checked and tested before being executed. // // @since version 3.12.0 @@ -3699,7 +3718,7 @@ type RenameOptions struct { // line 9599 } // The parameters of a {@link RenameRequest}. -type RenameParams struct { // line 6110 +type RenameParams struct { // The document to rename. TextDocument TextDocumentIdentifier `json:"textDocument"` // The position at which this request was sent. @@ -3712,13 +3731,13 @@ type RenameParams struct { // line 6110 } // Registration options for a {@link RenameRequest}. -type RenameRegistrationOptions struct { // line 6146 +type RenameRegistrationOptions struct { TextDocumentRegistrationOptions RenameOptions } // A generic resource operation. -type ResourceOperation struct { // line 9723 +type ResourceOperation struct { // The resource operation kind. Kind string `json:"kind"` // An optional annotation identifier describing the operation. @@ -3726,9 +3745,10 @@ type ResourceOperation struct { // line 9723 // @since 3.16.0 AnnotationID *ChangeAnnotationIdentifier `json:"annotationId,omitempty"` } -type ResourceOperationKind string // line 14084 +type ResourceOperationKind string + // Save options. -type SaveOptions struct { // line 8783 +type SaveOptions struct { // The client is supposed to include the content on save. IncludeText bool `json:"includeText,omitempty"` } @@ -3737,7 +3757,7 @@ type SaveOptions struct { // line 8783 // // @since 3.18.0 // @proposed -type SelectedCompletionInfo struct { // line 10004 +type SelectedCompletionInfo struct { // The range that will be replaced if this completion item is accepted. Range Range `json:"range"` // The text the range will be replaced with if this completion is accepted. @@ -3746,24 +3766,24 @@ type SelectedCompletionInfo struct { // line 10004 // A selection range represents a part of a selection hierarchy. A selection range // may have a parent selection range that contains it. -type SelectionRange struct { // line 2642 +type SelectionRange struct { // The {@link Range range} of this selection range. Range Range `json:"range"` // The parent selection range containing this range. Therefore `parent.range` must contain `this.range`. Parent *SelectionRange `json:"parent,omitempty"` } -type SelectionRangeClientCapabilities struct { // line 12434 +type SelectionRangeClientCapabilities struct { // Whether implementation supports dynamic registration for selection range providers. If this is set to `true` // the client supports the new `SelectionRangeRegistrationOptions` return value for the corresponding server // capability as well. DynamicRegistration bool `json:"dynamicRegistration,omitempty"` } -type SelectionRangeOptions struct { // line 6760 +type SelectionRangeOptions struct { WorkDoneProgressOptions } // A parameter literal used in selection range requests. -type SelectionRangeParams struct { // line 2607 +type SelectionRangeParams struct { // The text document. TextDocument TextDocumentIdentifier `json:"textDocument"` // The positions inside the text document. @@ -3771,7 +3791,7 @@ type SelectionRangeParams struct { // line 2607 WorkDoneProgressParams PartialResultParams } -type SelectionRangeRegistrationOptions struct { // line 2665 +type SelectionRangeRegistrationOptions struct { SelectionRangeOptions TextDocumentRegistrationOptions StaticRegistrationOptions @@ -3782,15 +3802,17 @@ type SelectionRangeRegistrationOptions struct { // line 2665 // corresponding client capabilities. // // @since 3.16.0 -type SemanticTokenModifiers string // line 13063 +type SemanticTokenModifiers string + // A set of predefined token types. This set is not fixed // an clients can specify additional token types via the // corresponding client capabilities. // // @since 3.16.0 -type SemanticTokenTypes string // line 12956 +type SemanticTokenTypes string + // @since 3.16.0 -type SemanticTokens struct { // line 2953 +type SemanticTokens struct { // An optional result id. If provided and clients support delta updating // the client will include the result id in the next semantic token request. // A server can then instead of computing all semantic tokens again simply @@ -3801,7 +3823,7 @@ type SemanticTokens struct { // line 2953 } // @since 3.16.0 -type SemanticTokensClientCapabilities struct { // line 12533 +type SemanticTokensClientCapabilities struct { // Whether implementation supports dynamic registration. If this is set to `true` // the client supports the new `(TextDocumentRegistrationOptions & StaticRegistrationOptions)` // return value for the corresponding server capability as well. @@ -3846,14 +3868,14 @@ type SemanticTokensClientCapabilities struct { // line 12533 } // @since 3.16.0 -type SemanticTokensDelta struct { // line 3052 +type SemanticTokensDelta struct { ResultID string `json:"resultId,omitempty"` // The semantic token edits to transform a previous result into a new result. Edits []SemanticTokensEdit `json:"edits"` } // @since 3.16.0 -type SemanticTokensDeltaParams struct { // line 3019 +type SemanticTokensDeltaParams struct { // The text document. TextDocument TextDocumentIdentifier `json:"textDocument"` // The result id of a previous response. The result Id can either point to a full response @@ -3864,12 +3886,12 @@ type SemanticTokensDeltaParams struct { // line 3019 } // @since 3.16.0 -type SemanticTokensDeltaPartialResult struct { // line 3078 +type SemanticTokensDeltaPartialResult struct { Edits []SemanticTokensEdit `json:"edits"` } // @since 3.16.0 -type SemanticTokensEdit struct { // line 6853 +type SemanticTokensEdit struct { // The start offset of the edit. Start uint32 `json:"start"` // The count of elements to remove. @@ -3879,7 +3901,7 @@ type SemanticTokensEdit struct { // line 6853 } // @since 3.16.0 -type SemanticTokensLegend struct { // line 9644 +type SemanticTokensLegend struct { // The token types a server uses. TokenTypes []string `json:"tokenTypes"` // The token modifiers a server uses. @@ -3887,7 +3909,7 @@ type SemanticTokensLegend struct { // line 9644 } // @since 3.16.0 -type SemanticTokensOptions struct { // line 6782 +type SemanticTokensOptions struct { // The legend used by the server Legend SemanticTokensLegend `json:"legend"` // Server supports providing semantic tokens for a specific range @@ -3899,7 +3921,7 @@ type SemanticTokensOptions struct { // line 6782 } // @since 3.16.0 -type SemanticTokensParams struct { // line 2928 +type SemanticTokensParams struct { // The text document. TextDocument TextDocumentIdentifier `json:"textDocument"` WorkDoneProgressParams @@ -3907,12 +3929,12 @@ type SemanticTokensParams struct { // line 2928 } // @since 3.16.0 -type SemanticTokensPartialResult struct { // line 2980 +type SemanticTokensPartialResult struct { Data []uint32 `json:"data"` } // @since 3.16.0 -type SemanticTokensRangeParams struct { // line 3095 +type SemanticTokensRangeParams struct { // The text document. TextDocument TextDocumentIdentifier `json:"textDocument"` // The range the semantic tokens are requested for. @@ -3922,14 +3944,14 @@ type SemanticTokensRangeParams struct { // line 3095 } // @since 3.16.0 -type SemanticTokensRegistrationOptions struct { // line 2997 +type SemanticTokensRegistrationOptions struct { TextDocumentRegistrationOptions SemanticTokensOptions StaticRegistrationOptions } // @since 3.16.0 -type SemanticTokensWorkspaceClientCapabilities struct { // line 11342 +type SemanticTokensWorkspaceClientCapabilities struct { // Whether the client implementation supports a refresh request sent from // the server to the client. // @@ -3942,7 +3964,7 @@ type SemanticTokensWorkspaceClientCapabilities struct { // line 11342 // Defines the capabilities provided by a language // server. -type ServerCapabilities struct { // line 8107 +type ServerCapabilities struct { // The position encoding the server picked from the encodings offered // by the client via the client capability `general.positionEncodings`. // @@ -4051,14 +4073,14 @@ type ServerCapabilities struct { // line 8107 // Experimental server capabilities. Experimental interface{} `json:"experimental,omitempty"` } -type SetTraceParams struct { // line 6383 +type SetTraceParams struct { Value TraceValues `json:"value"` } // Client capabilities for the showDocument request. // // @since 3.16.0 -type ShowDocumentClientCapabilities struct { // line 12878 +type ShowDocumentClientCapabilities struct { // The client has support for the showDocument // request. Support bool `json:"support"` @@ -4067,7 +4089,7 @@ type ShowDocumentClientCapabilities struct { // line 12878 // Params to show a resource in the UI. // // @since 3.16.0 -type ShowDocumentParams struct { // line 3128 +type ShowDocumentParams struct { // The uri to show. URI URI `json:"uri"` // Indicates to show the resource in an external program. @@ -4089,13 +4111,13 @@ type ShowDocumentParams struct { // line 3128 // The result of a showDocument request. // // @since 3.16.0 -type ShowDocumentResult struct { // line 3170 +type ShowDocumentResult struct { // A boolean indicating if the show was successful. Success bool `json:"success"` } // The parameters of a notification message. -type ShowMessageParams struct { // line 4378 +type ShowMessageParams struct { // The message type. See {@link MessageType} Type MessageType `json:"type"` // The actual message. @@ -4103,11 +4125,11 @@ type ShowMessageParams struct { // line 4378 } // Show message request client capabilities -type ShowMessageRequestClientCapabilities struct { // line 12851 +type ShowMessageRequestClientCapabilities struct { // Capabilities specific to the `MessageActionItem` type. MessageActionItem *PMessageActionItemPShowMessage `json:"messageActionItem,omitempty"` } -type ShowMessageRequestParams struct { // line 4400 +type ShowMessageRequestParams struct { // The message type. See {@link MessageType} Type MessageType `json:"type"` // The actual message. @@ -4119,7 +4141,7 @@ type ShowMessageRequestParams struct { // line 4400 // Signature help represents the signature of something // callable. There can be multiple signature but only one // active and only one active parameter. -type SignatureHelp struct { // line 5163 +type SignatureHelp struct { // One or more signatures. Signatures []SignatureInformation `json:"signatures"` // The active signature. If omitted or the value lies outside the @@ -4143,7 +4165,7 @@ type SignatureHelp struct { // line 5163 } // Client Capabilities for a {@link SignatureHelpRequest}. -type SignatureHelpClientCapabilities struct { // line 11793 +type SignatureHelpClientCapabilities struct { // Whether signature help supports dynamic registration. DynamicRegistration bool `json:"dynamicRegistration,omitempty"` // The client supports the following `SignatureInformation` @@ -4161,7 +4183,7 @@ type SignatureHelpClientCapabilities struct { // line 11793 // Additional information about the context in which a signature help request was triggered. // // @since 3.15.0 -type SignatureHelpContext struct { // line 9105 +type SignatureHelpContext struct { // Action that caused signature help to be triggered. TriggerKind SignatureHelpTriggerKind `json:"triggerKind"` // Character that caused signature help to be triggered. @@ -4181,7 +4203,7 @@ type SignatureHelpContext struct { // line 9105 } // Server Capabilities for a {@link SignatureHelpRequest}. -type SignatureHelpOptions struct { // line 9200 +type SignatureHelpOptions struct { // List of characters that trigger signature help automatically. TriggerCharacters []string `json:"triggerCharacters,omitempty"` // List of characters that re-trigger signature help. @@ -4195,7 +4217,7 @@ type SignatureHelpOptions struct { // line 9200 } // Parameters for a {@link SignatureHelpRequest}. -type SignatureHelpParams struct { // line 5135 +type SignatureHelpParams struct { // The signature help context. This is only available if the client specifies // to send this using the client capability `textDocument.signatureHelp.contextSupport === true` // @@ -4206,7 +4228,7 @@ type SignatureHelpParams struct { // line 5135 } // Registration options for a {@link SignatureHelpRequest}. -type SignatureHelpRegistrationOptions struct { // line 5198 +type SignatureHelpRegistrationOptions struct { TextDocumentRegistrationOptions SignatureHelpOptions } @@ -4214,11 +4236,12 @@ type SignatureHelpRegistrationOptions struct { // line 5198 // How a signature help was triggered. // // @since 3.15.0 -type SignatureHelpTriggerKind uint32 // line 13995 +type SignatureHelpTriggerKind uint32 + // Represents the signature of something callable. A signature // can have a label, like a function-name, a doc-comment, and // a set of parameters. -type SignatureInformation struct { // line 9146 +type SignatureInformation struct { // The label of this signature. Will be shown in // the UI. Label string `json:"label"` @@ -4237,7 +4260,7 @@ type SignatureInformation struct { // line 9146 // Static registration options to be returned in the initialize // request. -type StaticRegistrationOptions struct { // line 6579 +type StaticRegistrationOptions struct { // The id used to register the request. The id can be used to deregister // the request again. See also Registration#id. ID string `json:"id,omitempty"` @@ -4253,7 +4276,7 @@ type StaticRegistrationOptions struct { // line 6579 // // @since 3.18.0 // @proposed -type StringValue struct { // line 7858 +type StringValue struct { // The kind of string value. Kind string `json:"kind"` // The snippet string. @@ -4262,7 +4285,7 @@ type StringValue struct { // line 7858 // Represents information about programming constructs like variables, classes, // interfaces etc. -type SymbolInformation struct { // line 5376 +type SymbolInformation struct { // extends BaseSymbolInformation // Indicates if this symbol is deprecated. // @@ -4294,20 +4317,22 @@ type SymbolInformation struct { // line 5376 } // A symbol kind. -type SymbolKind uint32 // line 13234 +type SymbolKind uint32 + // Symbol tags are extra annotations that tweak the rendering of a symbol. // // @since 3.16 -type SymbolTag uint32 // line 13348 +type SymbolTag uint32 + // Describe options to be used when registered for text document change events. -type TextDocumentChangeRegistrationOptions struct { // line 4507 +type TextDocumentChangeRegistrationOptions struct { // How documents are synced to the server. SyncKind TextDocumentSyncKind `json:"syncKind"` TextDocumentRegistrationOptions } // Text document specific client capabilities. -type TextDocumentClientCapabilities struct { // line 10677 +type TextDocumentClientCapabilities struct { // Defines which synchronization capabilities the client supports. Synchronization *TextDocumentSyncClientCapabilities `json:"synchronization,omitempty"` // Capabilities specific to the `textDocument/completion` request. @@ -4411,7 +4436,7 @@ type TextDocumentContentChangeEvent = Msg_TextDocumentContentChangeEvent // (ali // on a document version Si and after they are applied move the document to version Si+1. // So the creator of a TextDocumentEdit doesn't need to sort the array of edits or do any // kind of ordering. However the edits must be non overlapping. -type TextDocumentEdit struct { // line 6913 +type TextDocumentEdit struct { // The text document to change. TextDocument OptionalVersionedTextDocumentIdentifier `json:"textDocument"` // The edits to be applied. @@ -4440,14 +4465,14 @@ type TextDocumentEdit struct { // line 6913 // @since 3.17.0 type TextDocumentFilter = Msg_TextDocumentFilter // (alias) line 14560 // A literal to identify a text document in the client. -type TextDocumentIdentifier struct { // line 6655 +type TextDocumentIdentifier struct { // The text document's uri. URI DocumentURI `json:"uri"` } // An item to transfer a text document from the client to the // server. -type TextDocumentItem struct { // line 7641 +type TextDocumentItem struct { // The text document's uri. URI DocumentURI `json:"uri"` // The text document's language identifier. @@ -4461,7 +4486,7 @@ type TextDocumentItem struct { // line 7641 // A parameter literal used in requests to pass a text document and a position inside that // document. -type TextDocumentPositionParams struct { // line 6458 +type TextDocumentPositionParams struct { // The text document. TextDocument TextDocumentIdentifier `json:"textDocument"` // The position inside the text document. @@ -4469,20 +4494,21 @@ type TextDocumentPositionParams struct { // line 6458 } // General text document registration options. -type TextDocumentRegistrationOptions struct { // line 2441 +type TextDocumentRegistrationOptions struct { // A document selector to identify the scope of the registration. If set to null // the document selector provided on the client side will be used. DocumentSelector DocumentSelector `json:"documentSelector"` } // Represents reasons why a text document is saved. -type TextDocumentSaveReason uint32 // line 13502 +type TextDocumentSaveReason uint32 + // Save registration options. -type TextDocumentSaveRegistrationOptions struct { // line 4564 +type TextDocumentSaveRegistrationOptions struct { TextDocumentRegistrationOptions SaveOptions } -type TextDocumentSyncClientCapabilities struct { // line 11492 +type TextDocumentSyncClientCapabilities struct { // Whether text document synchronization supports dynamic registration. DynamicRegistration bool `json:"dynamicRegistration,omitempty"` // The client supports sending will save notifications. @@ -4497,8 +4523,8 @@ type TextDocumentSyncClientCapabilities struct { // line 11492 // Defines how the host (editor) should sync // document changes to the language server. -type TextDocumentSyncKind uint32 // line 13477 -type TextDocumentSyncOptions struct { // line 10090 +type TextDocumentSyncKind uint32 +type TextDocumentSyncOptions struct { // Open and close notifications are sent to the server. If omitted open close notification should not // be sent. OpenClose bool `json:"openClose,omitempty"` @@ -4517,7 +4543,7 @@ type TextDocumentSyncOptions struct { // line 10090 } // A text edit applicable to a text document. -type TextEdit struct { // line 4601 +type TextEdit struct { // The range of the text document to be manipulated. To insert // text into a document create a range where start === end. Range Range `json:"range"` @@ -4525,10 +4551,11 @@ type TextEdit struct { // line 4601 // empty string. NewText string `json:"newText"` } -type TokenFormat string // line 14151 -type TraceValues string // line 13776 +type TokenFormat string +type TraceValues string + // Since 3.6.0 -type TypeDefinitionClientCapabilities struct { // line 11924 +type TypeDefinitionClientCapabilities struct { // Whether implementation supports dynamic registration. If this is set to `true` // the client supports the new `TypeDefinitionRegistrationOptions` return value // for the corresponding server capability as well. @@ -4538,22 +4565,22 @@ type TypeDefinitionClientCapabilities struct { // line 11924 // Since 3.14.0 LinkSupport bool `json:"linkSupport,omitempty"` } -type TypeDefinitionOptions struct { // line 6594 +type TypeDefinitionOptions struct { WorkDoneProgressOptions } -type TypeDefinitionParams struct { // line 2196 +type TypeDefinitionParams struct { TextDocumentPositionParams WorkDoneProgressParams PartialResultParams } -type TypeDefinitionRegistrationOptions struct { // line 2216 +type TypeDefinitionRegistrationOptions struct { TextDocumentRegistrationOptions TypeDefinitionOptions StaticRegistrationOptions } // @since 3.17.0 -type TypeHierarchyClientCapabilities struct { // line 12713 +type TypeHierarchyClientCapabilities struct { // Whether implementation supports dynamic registration. If this is set to `true` // the client supports the new `(TextDocumentRegistrationOptions & StaticRegistrationOptions)` // return value for the corresponding server capability as well. @@ -4561,7 +4588,7 @@ type TypeHierarchyClientCapabilities struct { // line 12713 } // @since 3.17.0 -type TypeHierarchyItem struct { // line 3483 +type TypeHierarchyItem struct { // The name of this item. Name string `json:"name"` // The kind of this item. @@ -4589,14 +4616,14 @@ type TypeHierarchyItem struct { // line 3483 // Type hierarchy options used during static registration. // // @since 3.17.0 -type TypeHierarchyOptions struct { // line 7172 +type TypeHierarchyOptions struct { WorkDoneProgressOptions } // The parameter of a `textDocument/prepareTypeHierarchy` request. // // @since 3.17.0 -type TypeHierarchyPrepareParams struct { // line 3465 +type TypeHierarchyPrepareParams struct { TextDocumentPositionParams WorkDoneProgressParams } @@ -4604,7 +4631,7 @@ type TypeHierarchyPrepareParams struct { // line 3465 // Type hierarchy options used during static or dynamic registration. // // @since 3.17.0 -type TypeHierarchyRegistrationOptions struct { // line 3560 +type TypeHierarchyRegistrationOptions struct { TextDocumentRegistrationOptions TypeHierarchyOptions StaticRegistrationOptions @@ -4613,7 +4640,7 @@ type TypeHierarchyRegistrationOptions struct { // line 3560 // The parameter of a `typeHierarchy/subtypes` request. // // @since 3.17.0 -type TypeHierarchySubtypesParams struct { // line 3606 +type TypeHierarchySubtypesParams struct { Item TypeHierarchyItem `json:"item"` WorkDoneProgressParams PartialResultParams @@ -4622,14 +4649,14 @@ type TypeHierarchySubtypesParams struct { // line 3606 // The parameter of a `typeHierarchy/supertypes` request. // // @since 3.17.0 -type TypeHierarchySupertypesParams struct { // line 3582 +type TypeHierarchySupertypesParams struct { Item TypeHierarchyItem `json:"item"` WorkDoneProgressParams PartialResultParams } // created for Tuple -type UIntCommaUInt struct { // line 10430 +type UIntCommaUInt struct { Fld0 uint32 `json:"fld0"` Fld1 uint32 `json:"fld1"` } @@ -4639,7 +4666,7 @@ type URI = string // report is still accurate. // // @since 3.17.0 -type UnchangedDocumentDiagnosticReport struct { // line 7506 +type UnchangedDocumentDiagnosticReport struct { // A document diagnostic report indicating // no changes to the last result. A server can // only return `unchanged` if result ids are @@ -4653,23 +4680,24 @@ type UnchangedDocumentDiagnosticReport struct { // line 7506 // Moniker uniqueness level to define scope of the moniker. // // @since 3.16.0 -type UniquenessLevel string // line 13364 +type UniquenessLevel string + // General parameters to unregister a request or notification. -type Unregistration struct { // line 7926 +type Unregistration struct { // The id used to unregister the request or notification. Usually an id // provided during the register request. ID string `json:"id"` // The method to unregister for. Method string `json:"method"` } -type UnregistrationParams struct { // line 4248 +type UnregistrationParams struct { Unregisterations []Unregistration `json:"unregisterations"` } // A versioned notebook document identifier. // // @since 3.17.0 -type VersionedNotebookDocumentIdentifier struct { // line 7679 +type VersionedNotebookDocumentIdentifier struct { // The version number of this notebook document. Version int32 `json:"version"` // The notebook document's uri. @@ -4677,19 +4705,19 @@ type VersionedNotebookDocumentIdentifier struct { // line 7679 } // A text document identifier to denote a specific version of a text document. -type VersionedTextDocumentIdentifier struct { // line 8763 +type VersionedTextDocumentIdentifier struct { // The version number of this document. Version int32 `json:"version"` TextDocumentIdentifier } -type WatchKind = uint32 // line 13505// The parameters sent in a will save text document notification. -type WillSaveTextDocumentParams struct { // line 4579 +type WatchKind = uint32 // line 13505// The parameters sent in a will save text document notification. +type WillSaveTextDocumentParams struct { // The document that will be saved. TextDocument TextDocumentIdentifier `json:"textDocument"` // The 'TextDocumentSaveReason'. Reason TextDocumentSaveReason `json:"reason"` } -type WindowClientCapabilities struct { // line 10994 +type WindowClientCapabilities struct { // It indicates whether the client supports server initiated // progress using the `window/workDoneProgress/create` request. // @@ -4709,7 +4737,7 @@ type WindowClientCapabilities struct { // line 10994 // @since 3.16.0 ShowDocument *ShowDocumentClientCapabilities `json:"showDocument,omitempty"` } -type WorkDoneProgressBegin struct { // line 6276 +type WorkDoneProgressBegin struct { Kind string `json:"kind"` // Mandatory title of the progress operation. Used to briefly inform about // the kind of operation being performed. @@ -4734,34 +4762,34 @@ type WorkDoneProgressBegin struct { // line 6276 // that are not following this rule. The value range is [0, 100]. Percentage uint32 `json:"percentage,omitempty"` } -type WorkDoneProgressCancelParams struct { // line 2698 +type WorkDoneProgressCancelParams struct { // The token to be used to report progress. Token ProgressToken `json:"token"` } -type WorkDoneProgressCreateParams struct { // line 2685 +type WorkDoneProgressCreateParams struct { // The token to be used to report progress. Token ProgressToken `json:"token"` } -type WorkDoneProgressEnd struct { // line 6362 +type WorkDoneProgressEnd struct { Kind string `json:"kind"` // Optional, a final message indicating to for example indicate the outcome // of the operation. Message string `json:"message,omitempty"` } -type WorkDoneProgressOptions struct { // line 2428 +type WorkDoneProgressOptions struct { WorkDoneProgress bool `json:"workDoneProgress,omitempty"` } // created for And -type WorkDoneProgressOptionsAndTextDocumentRegistrationOptions struct { // line 196 +type WorkDoneProgressOptionsAndTextDocumentRegistrationOptions struct { WorkDoneProgressOptions TextDocumentRegistrationOptions } -type WorkDoneProgressParams struct { // line 6480 +type WorkDoneProgressParams struct { // An optional token that a server can use to report work done progress. WorkDoneToken ProgressToken `json:"workDoneToken,omitempty"` } -type WorkDoneProgressReport struct { // line 6323 +type WorkDoneProgressReport struct { Kind string `json:"kind"` // Controls enablement state of a cancel button. // @@ -4784,7 +4812,7 @@ type WorkDoneProgressReport struct { // line 6323 } // created for Literal (Lit_ServerCapabilities_workspace) -type Workspace6Gn struct { // line 8722 +type Workspace6Gn struct { // The server supports workspace folder. // // @since 3.6.0 @@ -4796,7 +4824,7 @@ type Workspace6Gn struct { // line 8722 } // Workspace specific client capabilities. -type WorkspaceClientCapabilities struct { // line 10538 +type WorkspaceClientCapabilities struct { // The client supports applying batch edits // to the workspace by supporting the request // 'workspace/applyEdit' @@ -4853,7 +4881,7 @@ type WorkspaceClientCapabilities struct { // line 10538 // Parameters of the workspace diagnostic request. // // @since 3.17.0 -type WorkspaceDiagnosticParams struct { // line 3950 +type WorkspaceDiagnosticParams struct { // The additional identifier provided during registration. Identifier string `json:"identifier,omitempty"` // The currently known diagnostic reports with their @@ -4866,14 +4894,14 @@ type WorkspaceDiagnosticParams struct { // line 3950 // A workspace diagnostic report. // // @since 3.17.0 -type WorkspaceDiagnosticReport struct { // line 3987 +type WorkspaceDiagnosticReport struct { Items []WorkspaceDocumentDiagnosticReport `json:"items"` } // A partial result for a workspace diagnostic report. // // @since 3.17.0 -type WorkspaceDiagnosticReportPartialResult struct { // line 4004 +type WorkspaceDiagnosticReportPartialResult struct { Items []WorkspaceDocumentDiagnosticReport `json:"items"` } @@ -4893,7 +4921,7 @@ type WorkspaceDocumentDiagnosticReport = Or_WorkspaceDocumentDiagnosticReport // // An invalid sequence (e.g. (1) delete file a.txt and (2) insert text into file a.txt) will // cause failure of the operation. How the client recovers from the failure is described by // the client capability: `workspace.workspaceEdit.failureHandling` -type WorkspaceEdit struct { // line 3266 +type WorkspaceEdit struct { // Holds changes to existing resources. Changes map[DocumentURI][]TextEdit `json:"changes,omitempty"` // Depending on the client capability `workspace.workspaceEdit.resourceOperations` document changes @@ -4915,7 +4943,7 @@ type WorkspaceEdit struct { // line 3266 // @since 3.16.0 ChangeAnnotations map[ChangeAnnotationIdentifier]ChangeAnnotation `json:"changeAnnotations,omitempty"` } -type WorkspaceEditClientCapabilities struct { // line 11133 +type WorkspaceEditClientCapabilities struct { // The client supports versioned document changes in `WorkspaceEdit`s DocumentChanges bool `json:"documentChanges,omitempty"` // The resource operations the client supports. Clients should at least @@ -4944,14 +4972,14 @@ type WorkspaceEditClientCapabilities struct { // line 11133 } // A workspace folder inside a client. -type WorkspaceFolder struct { // line 2236 +type WorkspaceFolder struct { // The associated URI for this workspace folder. URI URI `json:"uri"` // The name of the workspace folder. Used to refer to this // workspace folder in the user interface. Name string `json:"name"` } -type WorkspaceFolders5Gn struct { // line 10287 +type WorkspaceFolders5Gn struct { // The server has support for workspace folders Supported bool `json:"supported,omitempty"` // Whether the server wants to receive workspace folder @@ -4965,13 +4993,13 @@ type WorkspaceFolders5Gn struct { // line 10287 } // The workspace folder change event. -type WorkspaceFoldersChangeEvent struct { // line 6604 +type WorkspaceFoldersChangeEvent struct { // The array of added workspace folders Added []WorkspaceFolder `json:"added"` // The array of the removed workspace folders Removed []WorkspaceFolder `json:"removed"` } -type WorkspaceFoldersInitializeParams struct { // line 8080 +type WorkspaceFoldersInitializeParams struct { // The workspace folders configured in the client when the server starts. // // This property is only available if the client supports workspace folders. @@ -4981,7 +5009,7 @@ type WorkspaceFoldersInitializeParams struct { // line 8080 // @since 3.6.0 WorkspaceFolders []WorkspaceFolder `json:"workspaceFolders,omitempty"` } -type WorkspaceFoldersServerCapabilities struct { // line 10287 +type WorkspaceFoldersServerCapabilities struct { // The server has support for workspace folders Supported bool `json:"supported,omitempty"` // Whether the server wants to receive workspace folder @@ -4997,7 +5025,7 @@ type WorkspaceFoldersServerCapabilities struct { // line 10287 // A full document diagnostic report for a workspace diagnostic result. // // @since 3.17.0 -type WorkspaceFullDocumentDiagnosticReport struct { // line 9852 +type WorkspaceFullDocumentDiagnosticReport struct { // The URI for which diagnostic information is reported. URI DocumentURI `json:"uri"` // The version number for which the diagnostics are reported. @@ -5011,7 +5039,7 @@ type WorkspaceFullDocumentDiagnosticReport struct { // line 9852 // See also SymbolInformation. // // @since 3.17.0 -type WorkspaceSymbol struct { // line 5710 +type WorkspaceSymbol struct { // The location of the symbol. Whether a server is allowed to // return a location without a range depends on the client // capability `workspace.symbol.resolveSupport`. @@ -5025,7 +5053,7 @@ type WorkspaceSymbol struct { // line 5710 } // Client capabilities for a {@link WorkspaceSymbolRequest}. -type WorkspaceSymbolClientCapabilities struct { // line 11240 +type WorkspaceSymbolClientCapabilities struct { // Symbol request supports dynamic registration. DynamicRegistration bool `json:"dynamicRegistration,omitempty"` // Specific capabilities for the `SymbolKind` in the `workspace/symbol` request. @@ -5044,7 +5072,7 @@ type WorkspaceSymbolClientCapabilities struct { // line 11240 } // Server capabilities for a {@link WorkspaceSymbolRequest}. -type WorkspaceSymbolOptions struct { // line 9423 +type WorkspaceSymbolOptions struct { // The server provides support to resolve additional // information for a workspace symbol. // @@ -5054,7 +5082,7 @@ type WorkspaceSymbolOptions struct { // line 9423 } // The parameters of a {@link WorkspaceSymbolRequest}. -type WorkspaceSymbolParams struct { // line 5686 +type WorkspaceSymbolParams struct { // A query string to filter symbols by. Clients may send an empty // string here to request all symbols. Query string `json:"query"` @@ -5063,14 +5091,14 @@ type WorkspaceSymbolParams struct { // line 5686 } // Registration options for a {@link WorkspaceSymbolRequest}. -type WorkspaceSymbolRegistrationOptions struct { // line 5759 +type WorkspaceSymbolRegistrationOptions struct { WorkspaceSymbolOptions } // An unchanged document diagnostic report for a workspace diagnostic result. // // @since 3.17.0 -type WorkspaceUnchangedDocumentDiagnosticReport struct { // line 9890 +type WorkspaceUnchangedDocumentDiagnosticReport struct { // The URI for which diagnostic information is reported. URI DocumentURI `json:"uri"` // The version number for which the diagnostics are reported. @@ -5080,7 +5108,7 @@ type WorkspaceUnchangedDocumentDiagnosticReport struct { // line 9890 } // The initialize parameters -type XInitializeParams struct { // line 7948 +type XInitializeParams struct { // The process Id of the parent process that started // the server. // @@ -5121,7 +5149,7 @@ type XInitializeParams struct { // line 7948 } // The initialize parameters -type _InitializeParams struct { // line 7948 +type _InitializeParams struct { // The process Id of the parent process that started // the server. // @@ -5164,11 +5192,11 @@ type _InitializeParams struct { // line 7948 const ( // A set of predefined code action kinds // Empty kind. - Empty CodeActionKind = "" // line 13726 + Empty CodeActionKind = "" // Base kind for quickfix actions: 'quickfix' - QuickFix CodeActionKind = "quickfix" // line 13731 + QuickFix CodeActionKind = "quickfix" // Base kind for refactoring actions: 'refactor' - Refactor CodeActionKind = "refactor" // line 13736 + Refactor CodeActionKind = "refactor" // Base kind for refactoring extraction actions: 'refactor.extract' // // Example extract actions: @@ -5179,7 +5207,7 @@ const ( // - Extract variable // - Extract interface from class // - ... - RefactorExtract CodeActionKind = "refactor.extract" // line 13741 + RefactorExtract CodeActionKind = "refactor.extract" // Base kind for refactoring inline actions: 'refactor.inline' // // Example inline actions: @@ -5189,7 +5217,7 @@ const ( // - Inline variable // - Inline constant // - ... - RefactorInline CodeActionKind = "refactor.inline" // line 13746 + RefactorInline CodeActionKind = "refactor.inline" // Base kind for refactoring rewrite actions: 'refactor.rewrite' // // Example rewrite actions: @@ -5201,80 +5229,80 @@ const ( // - Make method static // - Move method to base class // - ... - RefactorRewrite CodeActionKind = "refactor.rewrite" // line 13751 + RefactorRewrite CodeActionKind = "refactor.rewrite" // Base kind for source actions: `source` // // Source code actions apply to the entire file. - Source CodeActionKind = "source" // line 13756 + Source CodeActionKind = "source" // Base kind for an organize imports source action: `source.organizeImports` - SourceOrganizeImports CodeActionKind = "source.organizeImports" // line 13761 + SourceOrganizeImports CodeActionKind = "source.organizeImports" // Base kind for auto-fix source actions: `source.fixAll`. // // Fix all actions automatically fix errors that have a clear fix that do not require user input. // They should not suppress errors or perform unsafe fixes such as generating new types or classes. // // @since 3.15.0 - SourceFixAll CodeActionKind = "source.fixAll" // line 13766 + SourceFixAll CodeActionKind = "source.fixAll" // The reason why code actions were requested. // // @since 3.17.0 // Code actions were explicitly requested by the user or by an extension. - CodeActionInvoked CodeActionTriggerKind = 1 // line 14028 + CodeActionInvoked CodeActionTriggerKind = 1 // Code actions were requested automatically. // // This typically happens when current selection in a file changes, but can // also be triggered when file content changes. - CodeActionAutomatic CodeActionTriggerKind = 2 // line 14033 + CodeActionAutomatic CodeActionTriggerKind = 2 // The kind of a completion entry. - TextCompletion CompletionItemKind = 1 // line 13534 - MethodCompletion CompletionItemKind = 2 // line 13538 - FunctionCompletion CompletionItemKind = 3 // line 13542 - ConstructorCompletion CompletionItemKind = 4 // line 13546 - FieldCompletion CompletionItemKind = 5 // line 13550 - VariableCompletion CompletionItemKind = 6 // line 13554 - ClassCompletion CompletionItemKind = 7 // line 13558 - InterfaceCompletion CompletionItemKind = 8 // line 13562 - ModuleCompletion CompletionItemKind = 9 // line 13566 - PropertyCompletion CompletionItemKind = 10 // line 13570 - UnitCompletion CompletionItemKind = 11 // line 13574 - ValueCompletion CompletionItemKind = 12 // line 13578 - EnumCompletion CompletionItemKind = 13 // line 13582 - KeywordCompletion CompletionItemKind = 14 // line 13586 - SnippetCompletion CompletionItemKind = 15 // line 13590 - ColorCompletion CompletionItemKind = 16 // line 13594 - FileCompletion CompletionItemKind = 17 // line 13598 - ReferenceCompletion CompletionItemKind = 18 // line 13602 - FolderCompletion CompletionItemKind = 19 // line 13606 - EnumMemberCompletion CompletionItemKind = 20 // line 13610 - ConstantCompletion CompletionItemKind = 21 // line 13614 - StructCompletion CompletionItemKind = 22 // line 13618 - EventCompletion CompletionItemKind = 23 // line 13622 - OperatorCompletion CompletionItemKind = 24 // line 13626 - TypeParameterCompletion CompletionItemKind = 25 // line 13630 + TextCompletion CompletionItemKind = 1 + MethodCompletion CompletionItemKind = 2 + FunctionCompletion CompletionItemKind = 3 + ConstructorCompletion CompletionItemKind = 4 + FieldCompletion CompletionItemKind = 5 + VariableCompletion CompletionItemKind = 6 + ClassCompletion CompletionItemKind = 7 + InterfaceCompletion CompletionItemKind = 8 + ModuleCompletion CompletionItemKind = 9 + PropertyCompletion CompletionItemKind = 10 + UnitCompletion CompletionItemKind = 11 + ValueCompletion CompletionItemKind = 12 + EnumCompletion CompletionItemKind = 13 + KeywordCompletion CompletionItemKind = 14 + SnippetCompletion CompletionItemKind = 15 + ColorCompletion CompletionItemKind = 16 + FileCompletion CompletionItemKind = 17 + ReferenceCompletion CompletionItemKind = 18 + FolderCompletion CompletionItemKind = 19 + EnumMemberCompletion CompletionItemKind = 20 + ConstantCompletion CompletionItemKind = 21 + StructCompletion CompletionItemKind = 22 + EventCompletion CompletionItemKind = 23 + OperatorCompletion CompletionItemKind = 24 + TypeParameterCompletion CompletionItemKind = 25 // Completion item tags are extra annotations that tweak the rendering of a completion // item. // // @since 3.15.0 // Render a completion as obsolete, usually using a strike-out. - ComplDeprecated CompletionItemTag = 1 // line 13644 + ComplDeprecated CompletionItemTag = 1 // How a completion was triggered // Completion was triggered by typing an identifier (24x7 code // complete), manual invocation (e.g Ctrl+Space) or via API. - Invoked CompletionTriggerKind = 1 // line 13977 + Invoked CompletionTriggerKind = 1 // Completion was triggered by a trigger character specified by // the `triggerCharacters` properties of the `CompletionRegistrationOptions`. - TriggerCharacter CompletionTriggerKind = 2 // line 13982 + TriggerCharacter CompletionTriggerKind = 2 // Completion was re-triggered as current completion list is incomplete - TriggerForIncompleteCompletions CompletionTriggerKind = 3 // line 13987 + TriggerForIncompleteCompletions CompletionTriggerKind = 3 // The diagnostic's severity. // Reports an error. - SeverityError DiagnosticSeverity = 1 // line 13926 + SeverityError DiagnosticSeverity = 1 // Reports a warning. - SeverityWarning DiagnosticSeverity = 2 // line 13931 + SeverityWarning DiagnosticSeverity = 2 // Reports an information. - SeverityInformation DiagnosticSeverity = 3 // line 13936 + SeverityInformation DiagnosticSeverity = 3 // Reports a hint. - SeverityHint DiagnosticSeverity = 4 // line 13941 + SeverityHint DiagnosticSeverity = 4 // The diagnostic tags. // // @since 3.15.0 @@ -5282,91 +5310,91 @@ const ( // // Clients are allowed to render diagnostics with this tag faded out instead of having // an error squiggle. - Unnecessary DiagnosticTag = 1 // line 13956 + Unnecessary DiagnosticTag = 1 // Deprecated or obsolete code. // // Clients are allowed to rendered diagnostics with this tag strike through. - Deprecated DiagnosticTag = 2 // line 13961 + Deprecated DiagnosticTag = 2 // The document diagnostic report kinds. // // @since 3.17.0 // A diagnostic report with a full // set of problems. - DiagnosticFull DocumentDiagnosticReportKind = "full" // line 13122 + DiagnosticFull DocumentDiagnosticReportKind = "full" // A report indicating that the last // returned report is still accurate. - DiagnosticUnchanged DocumentDiagnosticReportKind = "unchanged" // line 13127 + DiagnosticUnchanged DocumentDiagnosticReportKind = "unchanged" // A document highlight kind. // A textual occurrence. - Text DocumentHighlightKind = 1 // line 13701 + Text DocumentHighlightKind = 1 // Read-access of a symbol, like reading a variable. - Read DocumentHighlightKind = 2 // line 13706 + Read DocumentHighlightKind = 2 // Write-access of a symbol, like writing to a variable. - Write DocumentHighlightKind = 3 // line 13711 + Write DocumentHighlightKind = 3 // Predefined error codes. - ParseError ErrorCodes = -32700 // line 13143 - InvalidRequest ErrorCodes = -32600 // line 13147 - MethodNotFound ErrorCodes = -32601 // line 13151 - InvalidParams ErrorCodes = -32602 // line 13155 - InternalError ErrorCodes = -32603 // line 13159 + ParseError ErrorCodes = -32700 + InvalidRequest ErrorCodes = -32600 + MethodNotFound ErrorCodes = -32601 + InvalidParams ErrorCodes = -32602 + InternalError ErrorCodes = -32603 // Error code indicating that a server received a notification or // request before the server has received the `initialize` request. - ServerNotInitialized ErrorCodes = -32002 // line 13163 - UnknownErrorCode ErrorCodes = -32001 // line 13168 + ServerNotInitialized ErrorCodes = -32002 + UnknownErrorCode ErrorCodes = -32001 // Applying the workspace change is simply aborted if one of the changes provided // fails. All operations executed before the failing operation stay executed. - Abort FailureHandlingKind = "abort" // line 14115 + Abort FailureHandlingKind = "abort" // All operations are executed transactional. That means they either all // succeed or no changes at all are applied to the workspace. - Transactional FailureHandlingKind = "transactional" // line 14120 + Transactional FailureHandlingKind = "transactional" // If the workspace edit contains only textual file changes they are executed transactional. // If resource changes (create, rename or delete file) are part of the change the failure // handling strategy is abort. - TextOnlyTransactional FailureHandlingKind = "textOnlyTransactional" // line 14125 + TextOnlyTransactional FailureHandlingKind = "textOnlyTransactional" // The client tries to undo the operations already executed. But there is no // guarantee that this is succeeding. - Undo FailureHandlingKind = "undo" // line 14130 + Undo FailureHandlingKind = "undo" // The file event type // The file got created. - Created FileChangeType = 1 // line 13876 + Created FileChangeType = 1 // The file got changed. - Changed FileChangeType = 2 // line 13881 + Changed FileChangeType = 2 // The file got deleted. - Deleted FileChangeType = 3 // line 13886 + Deleted FileChangeType = 3 // A pattern kind describing if a glob pattern matches a file a folder or // both. // // @since 3.16.0 // The pattern matches a file only. - FilePattern FileOperationPatternKind = "file" // line 14049 + FilePattern FileOperationPatternKind = "file" // The pattern matches a folder only. - FolderPattern FileOperationPatternKind = "folder" // line 14054 + FolderPattern FileOperationPatternKind = "folder" // A set of predefined range kinds. // Folding range for a comment - Comment FoldingRangeKind = "comment" // line 13215 + Comment FoldingRangeKind = "comment" // Folding range for an import or include - Imports FoldingRangeKind = "imports" // line 13220 + Imports FoldingRangeKind = "imports" // Folding range for a region (e.g. `#region`) - Region FoldingRangeKind = "region" // line 13225 + Region FoldingRangeKind = "region" // Inlay hint kinds. // // @since 3.17.0 // An inlay hint that for a type annotation. - Type InlayHintKind = 1 // line 13433 + Type InlayHintKind = 1 // An inlay hint that is for a parameter. - Parameter InlayHintKind = 2 // line 13438 + Parameter InlayHintKind = 2 // Describes how an {@link InlineCompletionItemProvider inline completion provider} was triggered. // // @since 3.18.0 // @proposed // Completion was triggered explicitly by a user gesture. - InlineInvoked InlineCompletionTriggerKind = 0 // line 13827 + InlineInvoked InlineCompletionTriggerKind = 0 // Completion was triggered automatically while editing. - InlineAutomatic InlineCompletionTriggerKind = 1 // line 13832 + InlineAutomatic InlineCompletionTriggerKind = 1 // Defines whether the insert text in a completion item should be interpreted as // plain text or a snippet. // The primary text to be inserted is treated as a plain string. - PlainTextTextFormat InsertTextFormat = 1 // line 13660 + PlainTextTextFormat InsertTextFormat = 1 // The primary text to be inserted is treated as a snippet. // // A snippet can define tab stops and placeholders with `$1`, `$2` @@ -5375,7 +5403,7 @@ const ( // that is typing in one will update others too. // // See also: https://microsoft.github.io/language-server-protocol/specifications/specification-current/#snippet_syntax - SnippetTextFormat InsertTextFormat = 2 // line 13665 + SnippetTextFormat InsertTextFormat = 2 // How whitespace and indentation is handled during completion // item insertion. // @@ -5385,7 +5413,7 @@ const ( // inserted using the indentation defined in the string value. // The client will not apply any kind of adjustments to the // string. - AsIs InsertTextMode = 1 // line 13680 + AsIs InsertTextMode = 1 // The editor adjusts leading whitespace of new lines so that // they match the indentation up to the cursor of the line for // which the item is accepted. @@ -5393,20 +5421,20 @@ const ( // Consider a line like this: <2tabs><3tabs>foo. Accepting a // multi line completion item is indented using 2 tabs and all // following lines inserted will be indented using 2 tabs as well. - AdjustIndentation InsertTextMode = 2 // line 13685 + AdjustIndentation InsertTextMode = 2 // A request failed but it was syntactically correct, e.g the // method name was known and the parameters were valid. The error // message should contain human readable information about why // the request failed. // // @since 3.17.0 - RequestFailed LSPErrorCodes = -32803 // line 13183 + RequestFailed LSPErrorCodes = -32803 // The server cancelled the request. This error code should // only be used for requests that explicitly support being // server cancellable. // // @since 3.17.0 - ServerCancelled LSPErrorCodes = -32802 // line 13189 + ServerCancelled LSPErrorCodes = -32802 // The server detected that the content of a document got // modified outside normal conditions. A server should // NOT send this error code if it detects a content change @@ -5415,200 +5443,200 @@ const ( // // If a client decides that a result is not of any use anymore // the client should cancel the request. - ContentModified LSPErrorCodes = -32801 // line 13195 + ContentModified LSPErrorCodes = -32801 // The client has canceled a request and a server as detected // the cancel. - RequestCancelled LSPErrorCodes = -32800 // line 13200 + RequestCancelled LSPErrorCodes = -32800 // Describes the content type that a client supports in various // result literals like `Hover`, `ParameterInfo` or `CompletionItem`. // // Please note that `MarkupKinds` must not start with a `$`. This kinds // are reserved for internal usage. // Plain text is supported as a content format - PlainText MarkupKind = "plaintext" // line 13807 + PlainText MarkupKind = "plaintext" // Markdown is supported as a content format - Markdown MarkupKind = "markdown" // line 13812 + Markdown MarkupKind = "markdown" // The message type // An error message. - Error MessageType = 1 // line 13454 + Error MessageType = 1 // A warning message. - Warning MessageType = 2 // line 13459 + Warning MessageType = 2 // An information message. - Info MessageType = 3 // line 13464 + Info MessageType = 3 // A log message. - Log MessageType = 4 // line 13469 + Log MessageType = 4 // The moniker kind. // // @since 3.16.0 // The moniker represent a symbol that is imported into a project - Import MonikerKind = "import" // line 13407 + Import MonikerKind = "import" // The moniker represents a symbol that is exported from a project - Export MonikerKind = "export" // line 13412 + Export MonikerKind = "export" // The moniker represents a symbol that is local to a project (e.g. a local // variable of a function, a class not visible outside the project, ...) - Local MonikerKind = "local" // line 13417 + Local MonikerKind = "local" // A notebook cell kind. // // @since 3.17.0 // A markup-cell is formatted source that is used for display. - Markup NotebookCellKind = 1 // line 14070 + Markup NotebookCellKind = 1 // A code-cell is source code. - Code NotebookCellKind = 2 // line 14075 + Code NotebookCellKind = 2 // A set of predefined position encoding kinds. // // @since 3.17.0 // Character offsets count UTF-8 code units (e.g. bytes). - UTF8 PositionEncodingKind = "utf-8" // line 13849 + UTF8 PositionEncodingKind = "utf-8" // Character offsets count UTF-16 code units. // // This is the default and must always be supported // by servers - UTF16 PositionEncodingKind = "utf-16" // line 13854 + UTF16 PositionEncodingKind = "utf-16" // Character offsets count UTF-32 code units. // // Implementation note: these are the same as Unicode codepoints, // so this `PositionEncodingKind` may also be used for an // encoding-agnostic representation of character offsets. - UTF32 PositionEncodingKind = "utf-32" // line 13859 + UTF32 PositionEncodingKind = "utf-32" // The client's default behavior is to select the identifier // according the to language's syntax rule. - Identifier PrepareSupportDefaultBehavior = 1 // line 14144 + Identifier PrepareSupportDefaultBehavior = 1 // Supports creating new files and folders. - Create ResourceOperationKind = "create" // line 14091 + Create ResourceOperationKind = "create" // Supports renaming existing files and folders. - Rename ResourceOperationKind = "rename" // line 14096 + Rename ResourceOperationKind = "rename" // Supports deleting existing files and folders. - Delete ResourceOperationKind = "delete" // line 14101 + Delete ResourceOperationKind = "delete" // A set of predefined token modifiers. This set is not fixed // an clients can specify additional token types via the // corresponding client capabilities. // // @since 3.16.0 - ModDeclaration SemanticTokenModifiers = "declaration" // line 13070 - ModDefinition SemanticTokenModifiers = "definition" // line 13074 - ModReadonly SemanticTokenModifiers = "readonly" // line 13078 - ModStatic SemanticTokenModifiers = "static" // line 13082 - ModDeprecated SemanticTokenModifiers = "deprecated" // line 13086 - ModAbstract SemanticTokenModifiers = "abstract" // line 13090 - ModAsync SemanticTokenModifiers = "async" // line 13094 - ModModification SemanticTokenModifiers = "modification" // line 13098 - ModDocumentation SemanticTokenModifiers = "documentation" // line 13102 - ModDefaultLibrary SemanticTokenModifiers = "defaultLibrary" // line 13106 + ModDeclaration SemanticTokenModifiers = "declaration" + ModDefinition SemanticTokenModifiers = "definition" + ModReadonly SemanticTokenModifiers = "readonly" + ModStatic SemanticTokenModifiers = "static" + ModDeprecated SemanticTokenModifiers = "deprecated" + ModAbstract SemanticTokenModifiers = "abstract" + ModAsync SemanticTokenModifiers = "async" + ModModification SemanticTokenModifiers = "modification" + ModDocumentation SemanticTokenModifiers = "documentation" + ModDefaultLibrary SemanticTokenModifiers = "defaultLibrary" // A set of predefined token types. This set is not fixed // an clients can specify additional token types via the // corresponding client capabilities. // // @since 3.16.0 - NamespaceType SemanticTokenTypes = "namespace" // line 12963 + NamespaceType SemanticTokenTypes = "namespace" // Represents a generic type. Acts as a fallback for types which can't be mapped to // a specific type like class or enum. - TypeType SemanticTokenTypes = "type" // line 12967 - ClassType SemanticTokenTypes = "class" // line 12972 - EnumType SemanticTokenTypes = "enum" // line 12976 - InterfaceType SemanticTokenTypes = "interface" // line 12980 - StructType SemanticTokenTypes = "struct" // line 12984 - TypeParameterType SemanticTokenTypes = "typeParameter" // line 12988 - ParameterType SemanticTokenTypes = "parameter" // line 12992 - VariableType SemanticTokenTypes = "variable" // line 12996 - PropertyType SemanticTokenTypes = "property" // line 13000 - EnumMemberType SemanticTokenTypes = "enumMember" // line 13004 - EventType SemanticTokenTypes = "event" // line 13008 - FunctionType SemanticTokenTypes = "function" // line 13012 - MethodType SemanticTokenTypes = "method" // line 13016 - MacroType SemanticTokenTypes = "macro" // line 13020 - KeywordType SemanticTokenTypes = "keyword" // line 13024 - ModifierType SemanticTokenTypes = "modifier" // line 13028 - CommentType SemanticTokenTypes = "comment" // line 13032 - StringType SemanticTokenTypes = "string" // line 13036 - NumberType SemanticTokenTypes = "number" // line 13040 - RegexpType SemanticTokenTypes = "regexp" // line 13044 - OperatorType SemanticTokenTypes = "operator" // line 13048 + TypeType SemanticTokenTypes = "type" + ClassType SemanticTokenTypes = "class" + EnumType SemanticTokenTypes = "enum" + InterfaceType SemanticTokenTypes = "interface" + StructType SemanticTokenTypes = "struct" + TypeParameterType SemanticTokenTypes = "typeParameter" + ParameterType SemanticTokenTypes = "parameter" + VariableType SemanticTokenTypes = "variable" + PropertyType SemanticTokenTypes = "property" + EnumMemberType SemanticTokenTypes = "enumMember" + EventType SemanticTokenTypes = "event" + FunctionType SemanticTokenTypes = "function" + MethodType SemanticTokenTypes = "method" + MacroType SemanticTokenTypes = "macro" + KeywordType SemanticTokenTypes = "keyword" + ModifierType SemanticTokenTypes = "modifier" + CommentType SemanticTokenTypes = "comment" + StringType SemanticTokenTypes = "string" + NumberType SemanticTokenTypes = "number" + RegexpType SemanticTokenTypes = "regexp" + OperatorType SemanticTokenTypes = "operator" // @since 3.17.0 - DecoratorType SemanticTokenTypes = "decorator" // line 13052 + DecoratorType SemanticTokenTypes = "decorator" // How a signature help was triggered. // // @since 3.15.0 // Signature help was invoked manually by the user or by a command. - SigInvoked SignatureHelpTriggerKind = 1 // line 14002 + SigInvoked SignatureHelpTriggerKind = 1 // Signature help was triggered by a trigger character. - SigTriggerCharacter SignatureHelpTriggerKind = 2 // line 14007 + SigTriggerCharacter SignatureHelpTriggerKind = 2 // Signature help was triggered by the cursor moving or by the document content changing. - SigContentChange SignatureHelpTriggerKind = 3 // line 14012 + SigContentChange SignatureHelpTriggerKind = 3 // A symbol kind. - File SymbolKind = 1 // line 13241 - Module SymbolKind = 2 // line 13245 - Namespace SymbolKind = 3 // line 13249 - Package SymbolKind = 4 // line 13253 - Class SymbolKind = 5 // line 13257 - Method SymbolKind = 6 // line 13261 - Property SymbolKind = 7 // line 13265 - Field SymbolKind = 8 // line 13269 - Constructor SymbolKind = 9 // line 13273 - Enum SymbolKind = 10 // line 13277 - Interface SymbolKind = 11 // line 13281 - Function SymbolKind = 12 // line 13285 - Variable SymbolKind = 13 // line 13289 - Constant SymbolKind = 14 // line 13293 - String SymbolKind = 15 // line 13297 - Number SymbolKind = 16 // line 13301 - Boolean SymbolKind = 17 // line 13305 - Array SymbolKind = 18 // line 13309 - Object SymbolKind = 19 // line 13313 - Key SymbolKind = 20 // line 13317 - Null SymbolKind = 21 // line 13321 - EnumMember SymbolKind = 22 // line 13325 - Struct SymbolKind = 23 // line 13329 - Event SymbolKind = 24 // line 13333 - Operator SymbolKind = 25 // line 13337 - TypeParameter SymbolKind = 26 // line 13341 + File SymbolKind = 1 + Module SymbolKind = 2 + Namespace SymbolKind = 3 + Package SymbolKind = 4 + Class SymbolKind = 5 + Method SymbolKind = 6 + Property SymbolKind = 7 + Field SymbolKind = 8 + Constructor SymbolKind = 9 + Enum SymbolKind = 10 + Interface SymbolKind = 11 + Function SymbolKind = 12 + Variable SymbolKind = 13 + Constant SymbolKind = 14 + String SymbolKind = 15 + Number SymbolKind = 16 + Boolean SymbolKind = 17 + Array SymbolKind = 18 + Object SymbolKind = 19 + Key SymbolKind = 20 + Null SymbolKind = 21 + EnumMember SymbolKind = 22 + Struct SymbolKind = 23 + Event SymbolKind = 24 + Operator SymbolKind = 25 + TypeParameter SymbolKind = 26 // Symbol tags are extra annotations that tweak the rendering of a symbol. // // @since 3.16 // Render a symbol as obsolete, usually using a strike-out. - DeprecatedSymbol SymbolTag = 1 // line 13355 + DeprecatedSymbol SymbolTag = 1 // Represents reasons why a text document is saved. // Manually triggered, e.g. by the user pressing save, by starting debugging, // or by an API call. - Manual TextDocumentSaveReason = 1 // line 13509 + Manual TextDocumentSaveReason = 1 // Automatic after a delay. - AfterDelay TextDocumentSaveReason = 2 // line 13514 + AfterDelay TextDocumentSaveReason = 2 // When the editor lost focus. - FocusOut TextDocumentSaveReason = 3 // line 13519 + FocusOut TextDocumentSaveReason = 3 // Defines how the host (editor) should sync // document changes to the language server. // Documents should not be synced at all. - None TextDocumentSyncKind = 0 // line 13484 + None TextDocumentSyncKind = 0 // Documents are synced by always sending the full content // of the document. - Full TextDocumentSyncKind = 1 // line 13489 + Full TextDocumentSyncKind = 1 // Documents are synced by sending the full content on open. // After that only incremental updates to the document are // send. - Incremental TextDocumentSyncKind = 2 // line 13494 - Relative TokenFormat = "relative" // line 14158 + Incremental TextDocumentSyncKind = 2 + Relative TokenFormat = "relative" // Turn tracing off. - Off TraceValues = "off" // line 13783 + Off TraceValues = "off" // Trace messages only. - Messages TraceValues = "messages" // line 13788 + Messages TraceValues = "messages" // Verbose message tracing. - Verbose TraceValues = "verbose" // line 13793 + Verbose TraceValues = "verbose" // Moniker uniqueness level to define scope of the moniker. // // @since 3.16.0 // The moniker is only unique inside a document - Document UniquenessLevel = "document" // line 13371 + Document UniquenessLevel = "document" // The moniker is unique inside a project for which a dump got created - Project UniquenessLevel = "project" // line 13376 + Project UniquenessLevel = "project" // The moniker is unique inside the group to which a project belongs - Group UniquenessLevel = "group" // line 13381 + Group UniquenessLevel = "group" // The moniker is unique inside the moniker scheme. - Scheme UniquenessLevel = "scheme" // line 13386 + Scheme UniquenessLevel = "scheme" // The moniker is globally unique - Global UniquenessLevel = "global" // line 13391 + Global UniquenessLevel = "global" // Interested in create events. - WatchCreate WatchKind = 1 // line 13901 + WatchCreate WatchKind = 1 // Interested in change events - WatchChange WatchKind = 2 // line 13906 + WatchChange WatchKind = 2 // Interested in delete events - WatchDelete WatchKind = 4 // line 13911 + WatchDelete WatchKind = 4 ) From 59fd05da6bc129e362716f84f412deececf95d37 Mon Sep 17 00:00:00 2001 From: Robert Findley Date: Wed, 9 Aug 2023 13:28:07 -0400 Subject: [PATCH 006/178] go/types/objectpath: remove use of linkname for gopls back doors Use internal variables as back doors for gopls into the objectpath package, rather than linkname. Using linkname breaks x/tools vendoring. See golang/go#61443 for background as to why this back door is necessary. Change-Id: Iabf6e825d169ac1c4080dc326eccc661eaae7ec6 Reviewed-on: https://go-review.googlesource.com/c/tools/+/517737 TryBot-Result: Gopher Robot Run-TryBot: Robert Findley gopls-CI: kokoro Reviewed-by: Bryan Mills --- go/types/objectpath/objectpath.go | 21 ++++++++++++--------- internal/facts/facts.go | 2 +- internal/typesinternal/objectpath.go | 24 ++++++++++++++++++++++++ internal/typesinternal/types.go | 16 ---------------- 4 files changed, 37 insertions(+), 26 deletions(-) create mode 100644 internal/typesinternal/objectpath.go diff --git a/go/types/objectpath/objectpath.go b/go/types/objectpath/objectpath.go index c725d839ba1..fa5834baf72 100644 --- a/go/types/objectpath/objectpath.go +++ b/go/types/objectpath/objectpath.go @@ -32,6 +32,7 @@ import ( _ "unsafe" "golang.org/x/tools/internal/typeparams" + "golang.org/x/tools/internal/typesinternal" ) // A Path is an opaque name that identifies a types.Object @@ -127,12 +128,15 @@ type Encoder struct { skipMethodSorting bool } -// Exposed to gopls via golang.org/x/tools/internal/typesinternal -// TODO(golang/go#61443): eliminate this parameter one way or the other. +// Expose back doors so that gopls can avoid method sorting, which can dominate +// analysis on certain repositories. // -//go:linkname skipMethodSorting -func skipMethodSorting(enc *Encoder) { - enc.skipMethodSorting = true +// TODO(golang/go#61443): remove this. +func init() { + typesinternal.SkipEncoderMethodSorting = func(enc interface{}) { + enc.(*Encoder).skipMethodSorting = true + } + typesinternal.ObjectpathObject = object } // For returns the path to an object relative to its package, @@ -572,17 +576,16 @@ func findTypeParam(obj types.Object, list *typeparams.TypeParamList, path []byte // Object returns the object denoted by path p within the package pkg. func Object(pkg *types.Package, p Path) (types.Object, error) { - return object(pkg, p, false) + return object(pkg, string(p), false) } // Note: the skipMethodSorting parameter must match the value of // Encoder.skipMethodSorting used during encoding. -func object(pkg *types.Package, p Path, skipMethodSorting bool) (types.Object, error) { - if p == "" { +func object(pkg *types.Package, pathstr string, skipMethodSorting bool) (types.Object, error) { + if pathstr == "" { return nil, fmt.Errorf("empty path") } - pathstr := string(p) var pkgobj, suffix string if dot := strings.IndexByte(pathstr, opType); dot < 0 { pkgobj = pathstr diff --git a/internal/facts/facts.go b/internal/facts/facts.go index 44c0605db27..ec11d5e0af1 100644 --- a/internal/facts/facts.go +++ b/internal/facts/facts.go @@ -247,7 +247,7 @@ func (d *Decoder) Decode(skipMethodSorting bool, read func(pkgPath string) ([]by key := key{pkg: factPkg, t: reflect.TypeOf(f.Fact)} if f.Object != "" { // object fact - obj, err := typesinternal.ObjectpathObject(factPkg, f.Object, skipMethodSorting) + obj, err := typesinternal.ObjectpathObject(factPkg, string(f.Object), skipMethodSorting) if err != nil { // (most likely due to unexported object) // TODO(adonovan): audit for other possibilities. diff --git a/internal/typesinternal/objectpath.go b/internal/typesinternal/objectpath.go new file mode 100644 index 00000000000..5e96e895573 --- /dev/null +++ b/internal/typesinternal/objectpath.go @@ -0,0 +1,24 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package typesinternal + +import "go/types" + +// This file contains back doors that allow gopls to avoid method sorting when +// using the objectpath package. +// +// This is performance-critical in certain repositories, but changing the +// behavior of the objectpath package is still being discussed in +// golang/go#61443. If we decide to remove the sorting in objectpath we can +// simply delete these back doors. Otherwise, we should add a new API to +// objectpath that allows controlling the sorting. + +// SkipEncoderMethodSorting marks enc (which must be an *objectpath.Encoder) as +// not requiring sorted methods. +var SkipEncoderMethodSorting func(enc interface{}) + +// ObjectpathObject is like objectpath.Object, but allows suppressing method +// sorting. +var ObjectpathObject func(pkg *types.Package, p string, skipMethodSorting bool) (types.Object, error) diff --git a/internal/typesinternal/types.go b/internal/typesinternal/types.go index 66e8b099bd6..ce7d4351b22 100644 --- a/internal/typesinternal/types.go +++ b/internal/typesinternal/types.go @@ -11,8 +11,6 @@ import ( "go/types" "reflect" "unsafe" - - "golang.org/x/tools/go/types/objectpath" ) func SetUsesCgo(conf *types.Config) bool { @@ -52,17 +50,3 @@ func ReadGo116ErrorData(err types.Error) (code ErrorCode, start, end token.Pos, } var SetGoVersion = func(conf *types.Config, version string) bool { return false } - -// SkipEncoderMethodSorting marks the encoder as not requiring sorted methods, -// as an optimization for gopls (which guarantees the order of parsed source files). -// -// TODO(golang/go#61443): eliminate this parameter one way or the other. -// -//go:linkname SkipEncoderMethodSorting golang.org/x/tools/go/types/objectpath.skipMethodSorting -func SkipEncoderMethodSorting(enc *objectpath.Encoder) - -// ObjectpathObject is like objectpath.Object, but allows suppressing method -// sorting (which is not necessary for gopls). -// -//go:linkname ObjectpathObject golang.org/x/tools/go/types/objectpath.object -func ObjectpathObject(pkg *types.Package, p objectpath.Path, skipMethodSorting bool) (types.Object, error) From 50271875f277c5104301803c328b53f544fde6ac Mon Sep 17 00:00:00 2001 From: Robert Findley Date: Wed, 9 Aug 2023 12:01:10 -0400 Subject: [PATCH 007/178] gopls/internal/regtest/bench: enable oracle benchmarks with -short This will cause them to appear on the gopls performance dashboard. Change-Id: I20ff0f2c095d377c2059870674e9718187d7c218 Reviewed-on: https://go-review.googlesource.com/c/tools/+/517736 Reviewed-by: Hyang-Ah Hana Kim Run-TryBot: Robert Findley TryBot-Result: Gopher Robot gopls-CI: kokoro --- gopls/internal/regtest/bench/repo_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/gopls/internal/regtest/bench/repo_test.go b/gopls/internal/regtest/bench/repo_test.go index c3b8b3bace9..3a4575e65c4 100644 --- a/gopls/internal/regtest/bench/repo_test.go +++ b/gopls/internal/regtest/bench/repo_test.go @@ -65,6 +65,7 @@ var repos = map[string]*repo{ name: "oracle", url: "https://github.com/oracle/oci-go-sdk.git", commit: "v65.43.0", + short: true, inDir: flag.String("oracle_dir", "", "if set, reuse this directory as oracle/oci-go-sdk@v65.43.0"), }, From 053d3c442498df7fd8850c4f3b7345d2928c981d Mon Sep 17 00:00:00 2001 From: "Hana (Hyang-Ah) Kim" Date: Fri, 4 Aug 2023 19:15:12 -0400 Subject: [PATCH 008/178] gopls/telemetry: test that telemetry counters are written Change-Id: Iceb8406cf3290180690f29bcba9f2fe6019285ad Reviewed-on: https://go-review.googlesource.com/c/tools/+/517135 TryBot-Result: Gopher Robot Run-TryBot: Hyang-Ah Hana Kim Reviewed-by: Robert Findley --- gopls/go.mod | 2 +- gopls/go.sum | 4 ++ gopls/internal/bug/bug.go | 5 +- gopls/internal/telemetry/telemetry_test.go | 82 ++++++++++++++++++++++ 4 files changed, 90 insertions(+), 3 deletions(-) create mode 100644 gopls/internal/telemetry/telemetry_test.go diff --git a/gopls/go.mod b/gopls/go.mod index f28b5bdc19b..1d6fdd3d6b7 100644 --- a/gopls/go.mod +++ b/gopls/go.mod @@ -10,7 +10,7 @@ require ( golang.org/x/mod v0.12.0 golang.org/x/sync v0.3.0 golang.org/x/sys v0.11.0 - golang.org/x/telemetry v0.0.0-20230728182230-e84a26264b60 + golang.org/x/telemetry v0.0.0-20230808152233-a65b40c0fdb0 golang.org/x/text v0.12.0 golang.org/x/tools v0.6.0 golang.org/x/vuln v0.0.0-20230110180137-6ad3e3d07815 diff --git a/gopls/go.sum b/gopls/go.sum index 3745b42d613..39157d6e719 100644 --- a/gopls/go.sum +++ b/gopls/go.sum @@ -77,6 +77,10 @@ golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/telemetry v0.0.0-20230728182230-e84a26264b60 h1:OCiXqf7/gdoaS7dKppAtPxi783Ke/JIb+r20ZYGiEFg= golang.org/x/telemetry v0.0.0-20230728182230-e84a26264b60/go.mod h1:kO7uNSGGmqCHII6C0TYfaLwSBIfcyhj53//nu0+Fy4A= +golang.org/x/telemetry v0.0.0-20230803164656-36ff770d3d6b h1:FZUooIb6Dx+mzx9n5mi6wmY/xpUZ4U1ffUVX1DCsuSs= +golang.org/x/telemetry v0.0.0-20230803164656-36ff770d3d6b/go.mod h1:kO7uNSGGmqCHII6C0TYfaLwSBIfcyhj53//nu0+Fy4A= +golang.org/x/telemetry v0.0.0-20230808152233-a65b40c0fdb0 h1:ZB9hzIbPBkRCCOVWOmfZEI5f6YiTbRAq6LK2x/StgiU= +golang.org/x/telemetry v0.0.0-20230808152233-a65b40c0fdb0/go.mod h1:kO7uNSGGmqCHII6C0TYfaLwSBIfcyhj53//nu0+Fy4A= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= diff --git a/gopls/internal/bug/bug.go b/gopls/internal/bug/bug.go index 7331ba8c85c..7c290b0cd27 100644 --- a/gopls/internal/bug/bug.go +++ b/gopls/internal/bug/bug.go @@ -65,7 +65,8 @@ func Report(description string) { report(description) } -var bugReport = counter.NewStack("gopls/bug", 16) +// BugReportCount is a telemetry counter that tracks # of bug reports. +var BugReportCount = counter.NewStack("gopls/bug", 16) func report(description string) { _, file, line, ok := runtime.Caller(2) // all exported reporting functions call report directly @@ -102,7 +103,7 @@ func report(description string) { mu.Unlock() if newBug { - bugReport.Inc() + BugReportCount.Inc() } // Call the handlers outside the critical section since a // handler may itself fail and call bug.Report. Since handlers diff --git a/gopls/internal/telemetry/telemetry_test.go b/gopls/internal/telemetry/telemetry_test.go new file mode 100644 index 00000000000..7951a419225 --- /dev/null +++ b/gopls/internal/telemetry/telemetry_test.go @@ -0,0 +1,82 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 && !openbsd && !js && !wasip1 && !solaris && !android && !386 +// +build go1.21,!openbsd,!js,!wasip1,!solaris,!android,!386 + +package telemetry_test + +import ( + "os" + "strconv" + "strings" + "testing" + + "golang.org/x/telemetry/counter" + "golang.org/x/telemetry/counter/countertest" // requires go1.21+ + "golang.org/x/tools/gopls/internal/bug" + "golang.org/x/tools/gopls/internal/hooks" + . "golang.org/x/tools/gopls/internal/lsp/regtest" +) + +func TestMain(m *testing.M) { + tmp, err := os.MkdirTemp("", "gopls-telemetry-test") + if err != nil { + panic(err) + } + countertest.Open(tmp) + defer os.RemoveAll(tmp) + Main(m, hooks.Options) +} + +func TestTelemetry(t *testing.T) { + var ( + goversion = "" + editor = "vscode" // We set ClientName("Visual Studio Code") below. + ) + + // Verify that a properly configured session gets notified of a bug on the + // server. + WithOptions( + Modes(Default), // must be in-process to receive the bug report below + Settings{"showBugReports": true}, + ClientName("Visual Studio Code"), + ).Run(t, "", func(t *testing.T, env *Env) { + goversion = strconv.Itoa(env.GoVersion()) + const desc = "got a bug" + bug.Report(desc) // want a stack counter with the trace starting from here. + env.Await(ShownMessage(desc)) + }) + + // gopls/editor:client + // gopls/goversion:1.x + for _, c := range []*counter.Counter{ + counter.New("gopls/client:" + editor), + counter.New("gopls/goversion:1." + goversion), + } { + count, err := countertest.ReadCounter(c) + if err != nil || count != 1 { + t.Errorf("ReadCounter(%q) = (%v, %v), want (1, nil)", c.Name(), count, err) + } + } + + // gopls/bug + bugcount := bug.BugReportCount + counts, err := countertest.ReadStackCounter(bugcount) + if err != nil { + t.Fatalf("ReadStackCounter(bugreportcount) failed - %v", err) + } + if len(counts) != 1 || !hasEntry(counts, t.Name(), 1) { + t.Errorf("read stackcounter(%q) = (%#v, %v), want one entry", "gopls/bug", counts, err) + } +} + +func hasEntry(counts map[string]uint64, pattern string, want uint64) bool { + for k, v := range counts { + if strings.Contains(k, pattern) && v == want { + return true + } + } + return false +} From 47c5305d201d9d9d835b5d6a50ef5c1ec3f7b121 Mon Sep 17 00:00:00 2001 From: Robert Findley Date: Thu, 10 Aug 2023 09:18:18 -0400 Subject: [PATCH 009/178] gopls/internal/regtest/bench: skip oracle didSave test The oracle change test is in a generated package (intentionally, because this is a very large package). This causes didSave to fail, because we don't support code actions on generated files. As a result, our benchmark results are failing to be uploaded. Change-Id: Ibd0f45d85c52c3806e22e333c259a47052e8860d Reviewed-on: https://go-review.googlesource.com/c/tools/+/518215 Reviewed-by: Hyang-Ah Hana Kim TryBot-Result: Gopher Robot gopls-CI: kokoro Run-TryBot: Robert Findley --- .../internal/regtest/bench/didchange_test.go | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/gopls/internal/regtest/bench/didchange_test.go b/gopls/internal/regtest/bench/didchange_test.go index 27856f3031e..56da0ae7a68 100644 --- a/gopls/internal/regtest/bench/didchange_test.go +++ b/gopls/internal/regtest/bench/didchange_test.go @@ -20,19 +20,20 @@ import ( var editID int64 = time.Now().UnixNano() type changeTest struct { - repo string - file string + repo string + file string + canSave bool } var didChangeTests = []changeTest{ - {"google-cloud-go", "internal/annotate.go"}, - {"istio", "pkg/fuzz/util.go"}, - {"kubernetes", "pkg/controller/lookup_cache.go"}, - {"kuma", "api/generic/insights.go"}, - {"oracle", "dataintegration/data_type.go"}, // diagnoseSave fails because this package is generated - {"pkgsite", "internal/frontend/server.go"}, - {"starlark", "starlark/eval.go"}, - {"tools", "internal/lsp/cache/snapshot.go"}, + {"google-cloud-go", "internal/annotate.go", true}, + {"istio", "pkg/fuzz/util.go", true}, + {"kubernetes", "pkg/controller/lookup_cache.go", true}, + {"kuma", "api/generic/insights.go", true}, + {"oracle", "dataintegration/data_type.go", false}, // diagnoseSave fails because this package is generated + {"pkgsite", "internal/frontend/server.go", true}, + {"starlark", "starlark/eval.go", true}, + {"tools", "internal/lsp/cache/snapshot.go", true}, } // BenchmarkDidChange benchmarks modifications of a single file by making @@ -89,6 +90,9 @@ func BenchmarkDiagnoseSave(b *testing.B) { // await the resulting diagnostics pass. If save is set, the file is also saved. func runChangeDiagnosticsBenchmark(b *testing.B, test changeTest, save bool, operation string) { b.Run(test.repo, func(b *testing.B) { + if !test.canSave { + b.Skipf("skipping as %s cannot be saved", test.file) + } sharedEnv := getRepo(b, test.repo).sharedEnv(b) config := fake.EditorConfig{ Env: map[string]string{ From 6b4d1de19b45d239410b68335312e2aa04528c17 Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Thu, 13 Jul 2023 14:37:00 -0400 Subject: [PATCH 010/178] gopls/internal/lsp: avoid duplicate type checking following invalidation Following a keystroke, it is common to compute both diagnostics and completion results. For small packages, this sometimes results in redundant work, but not enough to significantly affect benchmarks. However, for very large packages where type checking takes >100ms, these two operations always run in parallel recomputing the same shared state. This is made clear in the oracle completion benchmark. Fix this by guarding type checking with a mutex, and slightly delaying initial diagnostics to yield to other operations (though because diagnostics will also recompute shared, it doesn't matter too much which operation acquires the mutex first). For golang/go#61207 Change-Id: I761aef9c66ebdd54fab8c61605c42d82a8f412cc Reviewed-on: https://go-review.googlesource.com/c/tools/+/511435 gopls-CI: kokoro Run-TryBot: Robert Findley TryBot-Result: Gopher Robot Reviewed-by: Hyang-Ah Hana Kim --- gopls/internal/lsp/cache/check.go | 3 +++ gopls/internal/lsp/cache/snapshot.go | 12 ++++++++++++ gopls/internal/lsp/diagnostics.go | 18 ++++++++++++++++++ 3 files changed, 33 insertions(+) diff --git a/gopls/internal/lsp/cache/check.go b/gopls/internal/lsp/cache/check.go index 50ce8955014..1b9d08c2ae6 100644 --- a/gopls/internal/lsp/cache/check.go +++ b/gopls/internal/lsp/cache/check.go @@ -323,6 +323,9 @@ type ( // // Both pre and post may be called concurrently. func (s *snapshot) forEachPackage(ctx context.Context, ids []PackageID, pre preTypeCheck, post postTypeCheck) error { + s.typeCheckMu.Lock() + defer s.typeCheckMu.Unlock() + ctx, done := event.Start(ctx, "cache.forEachPackage", tag.PackageCount.Of(len(ids))) defer done() diff --git a/gopls/internal/lsp/cache/snapshot.go b/gopls/internal/lsp/cache/snapshot.go index 863e488c4e3..167c56d57cf 100644 --- a/gopls/internal/lsp/cache/snapshot.go +++ b/gopls/internal/lsp/cache/snapshot.go @@ -184,6 +184,18 @@ type snapshot struct { // detect ignored files. ignoreFilterOnce sync.Once ignoreFilter *ignoreFilter + + // typeCheckMu guards type checking. + // + // Only one type checking pass should be running at a given time, for two reasons: + // 1. type checking batches are optimized to use all available processors. + // Generally speaking, running two type checking batches serially is about + // as fast as running them in parallel. + // 2. type checking produces cached artifacts that may be re-used by the + // next type-checking batch: the shared import graph and the set of + // active packages. Running type checking batches in parallel after an + // invalidation can cause redundant calculation of this shared state. + typeCheckMu sync.Mutex } var globalSnapshotID uint64 diff --git a/gopls/internal/lsp/diagnostics.go b/gopls/internal/lsp/diagnostics.go index 69c9aeb3da7..dbc163aa529 100644 --- a/gopls/internal/lsp/diagnostics.go +++ b/gopls/internal/lsp/diagnostics.go @@ -188,9 +188,27 @@ func (s *Server) diagnoseSnapshot(snapshot source.Snapshot, changedURIs []span.U // file modifications. // // The second phase runs after the delay, and does everything. + // + // We wait a brief delay before the first phase, to allow higher priority + // work such as autocompletion to acquire the type checking mutex (though + // typically both diagnosing changed files and performing autocompletion + // will be doing the same work: recomputing active packages). + const minDelay = 20 * time.Millisecond + select { + case <-time.After(minDelay): + case <-ctx.Done(): + return + } + s.diagnoseChangedFiles(ctx, snapshot, changedURIs, onDisk) s.publishDiagnostics(ctx, false, snapshot) + if delay < minDelay { + delay = 0 + } else { + delay -= minDelay + } + select { case <-time.After(delay): case <-ctx.Done(): From 6290d8a967be4ebd02218f5c53858de30b9107f3 Mon Sep 17 00:00:00 2001 From: Tim King Date: Fri, 11 Aug 2023 13:55:54 -0700 Subject: [PATCH 011/178] go/analysis/passes/copylock: ignore parens on rhs Treat parenthesized RHS expressions the same as non-parenthesized RHS expressions. Fixes golang/go#61962 Change-Id: I42474ede7dccf5a694a484f96d000ef8d80da8d3 Reviewed-on: https://go-review.googlesource.com/c/tools/+/518935 Run-TryBot: Tim King TryBot-Result: Gopher Robot Commit-Queue: Tim King Reviewed-by: Robert Findley gopls-CI: kokoro --- go/analysis/passes/copylock/copylock.go | 4 +++- .../passes/copylock/testdata/src/a/copylock.go | 11 +++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/go/analysis/passes/copylock/copylock.go b/go/analysis/passes/copylock/copylock.go index ff2b41ac4aa..ec7727de769 100644 --- a/go/analysis/passes/copylock/copylock.go +++ b/go/analysis/passes/copylock/copylock.go @@ -223,6 +223,8 @@ func (path typePath) String() string { } func lockPathRhs(pass *analysis.Pass, x ast.Expr) typePath { + x = analysisutil.Unparen(x) // ignore parens on rhs + if _, ok := x.(*ast.CompositeLit); ok { return nil } @@ -231,7 +233,7 @@ func lockPathRhs(pass *analysis.Pass, x ast.Expr) typePath { return nil } if star, ok := x.(*ast.StarExpr); ok { - if _, ok := star.X.(*ast.CallExpr); ok { + if _, ok := analysisutil.Unparen(star.X).(*ast.CallExpr); ok { // A call may return a pointer to a zero value. return nil } diff --git a/go/analysis/passes/copylock/testdata/src/a/copylock.go b/go/analysis/passes/copylock/testdata/src/a/copylock.go index 4ab66dca1f6..2f0f8136628 100644 --- a/go/analysis/passes/copylock/testdata/src/a/copylock.go +++ b/go/analysis/passes/copylock/testdata/src/a/copylock.go @@ -34,6 +34,9 @@ func OkFunc() { xx := struct{ L *sync.Mutex }{ L: new(sync.Mutex), } + + var pz = (sync.Mutex{}) + pw := (sync.Mutex{}) } type Tlock struct { @@ -214,3 +217,11 @@ func AtomicTypesCheck() { vP := &vX vZ := &atomic.Value{} } + +// PointerRhsCheck checks that exceptions are made for pointer return values of +// function calls. These may be zero initialized so they are considered OK. +func PointerRhsCheck() { + newMutex := func() *sync.Mutex { return new(sync.Mutex) } + d := *newMutex() + pd := *(newMutex()) +} From f4c86275ba5b5d07dd732ccaacc1ed80ec49c81b Mon Sep 17 00:00:00 2001 From: Robert Findley Date: Fri, 11 Aug 2023 20:11:07 -0400 Subject: [PATCH 012/178] gopls: fix raciness related to TestOrphanedFiles While investigating the high rate of flakiness in TestOrphanedFiles, I found two problems: - The change to use Await(...) rather than AfterChange(..), which was intended to *avoid* flakes due to asynchronous log messages, actually increased flakiness. After that change, the test can sometimes proceed to make additional edits before change processing completes, so that the load succeeds but the snapshot is cloned before the resulting state can be written (sigh...). - If the load succeeds, we should proceed to update state even if the context is cancelled. Writing the state is not expensive. Fixes golang/go#61521 Change-Id: I4e70526a0108013c66d378da966289a7c2f5dbe2 Reviewed-on: https://go-review.googlesource.com/c/tools/+/518975 TryBot-Result: Gopher Robot Run-TryBot: Robert Findley Reviewed-by: Alan Donovan gopls-CI: kokoro --- gopls/internal/lsp/cache/snapshot.go | 23 +++++++++++-------- .../regtest/diagnostics/diagnostics_test.go | 9 +++++++- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/gopls/internal/lsp/cache/snapshot.go b/gopls/internal/lsp/cache/snapshot.go index 167c56d57cf..2c2f79bf25d 100644 --- a/gopls/internal/lsp/cache/snapshot.go +++ b/gopls/internal/lsp/cache/snapshot.go @@ -755,12 +755,22 @@ func (s *snapshot) MetadataForFile(ctx context.Context, uri span.URI) ([]*source scope := fileLoadScope(uri) err := s.load(ctx, false, scope) - // Guard against failed loads due to context cancellation. // // Return the context error here as the current operation is no longer // valid. - if ctxErr := ctx.Err(); ctxErr != nil { - return nil, ctxErr + if err != nil { + // Guard against failed loads due to context cancellation. We don't want + // to mark loads as completed if they failed due to context cancellation. + if ctx.Err() != nil { + return nil, ctx.Err() + } + + // Don't return an error here, as we may still return stale IDs. + // Furthermore, the result of MetadataForFile should be consistent upon + // subsequent calls, even if the file is marked as unloadable. + if !errors.Is(err, errNoPackages) { + event.Error(ctx, "MetadataForFile", err) + } } // We must clear scopes after loading. @@ -769,13 +779,6 @@ func (s *snapshot) MetadataForFile(ctx context.Context, uri span.URI) ([]*source // packages as loaded. We could do this from snapshot.load and avoid // raciness. s.clearShouldLoad(scope) - - // Don't return an error here, as we may still return stale IDs. - // Furthermore, the result of MetadataForFile should be consistent upon - // subsequent calls, even if the file is marked as unloadable. - if err != nil && !errors.Is(err, errNoPackages) { - event.Error(ctx, "MetadataForFile", err) - } } // Retrieve the metadata. diff --git a/gopls/internal/regtest/diagnostics/diagnostics_test.go b/gopls/internal/regtest/diagnostics/diagnostics_test.go index 623cd724cec..8066b7502c2 100644 --- a/gopls/internal/regtest/diagnostics/diagnostics_test.go +++ b/gopls/internal/regtest/diagnostics/diagnostics_test.go @@ -1307,7 +1307,14 @@ func _() { env.OpenFile("a/a_exclude.go") loadOnce := LogMatching(protocol.Info, "query=.*file=.*a_exclude.go", 1, false) - env.Await(loadOnce) // can't use OnceMet or AfterChange as logs are async + + // can't use OnceMet or AfterChange as logs are async + env.Await(loadOnce) + // ...but ensure that the change has been fully processed before editing. + // Otherwise, there may be a race where the snapshot is cloned before all + // state changes resulting from the load have been processed + // (golang/go#61521). + env.AfterChange() // Check that orphaned files are not reloaded, by making a change in // a.go file and confirming that the workspace diagnosis did not reload From fa12f34b4218307705bf0365ab7df7c119b3653a Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Tue, 8 Aug 2023 23:31:25 +0700 Subject: [PATCH 013/178] go/packages: avoid unnecessary dependency on StdSizes Nothing ever promised SizesFor would return a *StdSizes. Updates golang/go#61035 Change-Id: Ib54a7c9d4898cd435e87aea32067a9cfa6975367 Reviewed-on: https://go-review.googlesource.com/c/tools/+/516917 Auto-Submit: Cuong Manh Le Reviewed-by: Russ Cox TryBot-Result: Gopher Robot gopls-CI: kokoro Run-TryBot: Cuong Manh Le Reviewed-by: Michael Matloob --- go/internal/packagesdriver/sizes.go | 11 +++++------ go/packages/golist.go | 9 ++++----- go/packages/packages.go | 8 +++++--- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/go/internal/packagesdriver/sizes.go b/go/internal/packagesdriver/sizes.go index 18a002f82a1..0454cdd78e5 100644 --- a/go/internal/packagesdriver/sizes.go +++ b/go/internal/packagesdriver/sizes.go @@ -8,7 +8,6 @@ package packagesdriver import ( "context" "fmt" - "go/types" "strings" "golang.org/x/tools/internal/gocommand" @@ -16,7 +15,7 @@ import ( var debug = false -func GetSizesGolist(ctx context.Context, inv gocommand.Invocation, gocmdRunner *gocommand.Runner) (types.Sizes, error) { +func GetSizesForArgsGolist(ctx context.Context, inv gocommand.Invocation, gocmdRunner *gocommand.Runner) (string, string, error) { inv.Verb = "list" inv.Args = []string{"-f", "{{context.GOARCH}} {{context.Compiler}}", "--", "unsafe"} stdout, stderr, friendlyErr, rawErr := gocmdRunner.RunRaw(ctx, inv) @@ -29,21 +28,21 @@ func GetSizesGolist(ctx context.Context, inv gocommand.Invocation, gocmdRunner * inv.Args = []string{"GOARCH"} envout, enverr := gocmdRunner.Run(ctx, inv) if enverr != nil { - return nil, enverr + return "", "", enverr } goarch = strings.TrimSpace(envout.String()) compiler = "gc" } else { - return nil, friendlyErr + return "", "", friendlyErr } } else { fields := strings.Fields(stdout.String()) if len(fields) < 2 { - return nil, fmt.Errorf("could not parse GOARCH and Go compiler in format \" \":\nstdout: <<%s>>\nstderr: <<%s>>", + return "", "", fmt.Errorf("could not parse GOARCH and Go compiler in format \" \":\nstdout: <<%s>>\nstderr: <<%s>>", stdout.String(), stderr.String()) } goarch = fields[0] compiler = fields[1] } - return types.SizesFor(compiler, goarch), nil + return compiler, goarch, nil } diff --git a/go/packages/golist.go b/go/packages/golist.go index 58230038a7c..b5de9cf9f21 100644 --- a/go/packages/golist.go +++ b/go/packages/golist.go @@ -9,7 +9,6 @@ import ( "context" "encoding/json" "fmt" - "go/types" "io/ioutil" "log" "os" @@ -153,10 +152,10 @@ func goListDriver(cfg *Config, patterns ...string) (*driverResponse, error) { if cfg.Mode&NeedTypesSizes != 0 || cfg.Mode&NeedTypes != 0 { sizeswg.Add(1) go func() { - var sizes types.Sizes - sizes, sizeserr = packagesdriver.GetSizesGolist(ctx, state.cfgInvocation(), cfg.gocmdRunner) - // types.SizesFor always returns nil or a *types.StdSizes. - response.dr.Sizes, _ = sizes.(*types.StdSizes) + compiler, arch, err := packagesdriver.GetSizesForArgsGolist(ctx, state.cfgInvocation(), cfg.gocmdRunner) + sizeserr = err + response.dr.Compiler = compiler + response.dr.Arch = arch sizeswg.Done() }() } diff --git a/go/packages/packages.go b/go/packages/packages.go index da1a27eea62..124a6fe143b 100644 --- a/go/packages/packages.go +++ b/go/packages/packages.go @@ -220,8 +220,10 @@ type driverResponse struct { // lists of multiple drivers, go/packages will fall back to the next driver. NotHandled bool - // Sizes, if not nil, is the types.Sizes to use when type checking. - Sizes *types.StdSizes + // Compiler and Arch are the arguments pass of types.SizesFor + // to get a types.Sizes to use when type checking. + Compiler string + Arch string // Roots is the set of package IDs that make up the root packages. // We have to encode this separately because when we encode a single package @@ -262,7 +264,7 @@ func Load(cfg *Config, patterns ...string) ([]*Package, error) { if err != nil { return nil, err } - l.sizes = response.Sizes + l.sizes = types.SizesFor(response.Compiler, response.Arch) return l.refine(response) } From 74c255bcf846b936dc569874e235e62f888c3727 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Tue, 15 Aug 2023 00:39:53 +0700 Subject: [PATCH 014/178] gopls/internal/lsp/cache: avoid dependency on StdSizes Same as CL 516917, but for gopls. Updates golang/go#61035 Change-Id: Id6cc1d84f7cac06e95a1fb151a7c3f9a8ef25302 Reviewed-on: https://go-review.googlesource.com/c/tools/+/519295 gopls-CI: kokoro Reviewed-by: Robert Findley Reviewed-by: Hyang-Ah Hana Kim Run-TryBot: Cuong Manh Le Auto-Submit: Cuong Manh Le TryBot-Result: Gopher Robot --- gopls/internal/lsp/cache/analysis.go | 7 +++---- gopls/internal/lsp/cache/check.go | 5 +++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/gopls/internal/lsp/cache/analysis.go b/gopls/internal/lsp/cache/analysis.go index dd15843bc19..c5f6f7293f4 100644 --- a/gopls/internal/lsp/cache/analysis.go +++ b/gopls/internal/lsp/cache/analysis.go @@ -707,10 +707,9 @@ func (an *analysisNode) cacheKey() [sha256.Size]byte { // uses those fields, we account for them by hashing vdeps. // type sizes - // This assertion is safe, but if a black-box implementation - // is ever needed, record Sizeof(*int) and Alignof(int64). - sz := m.TypesSizes.(*types.StdSizes) - fmt.Fprintf(hasher, "sizes: %d %d\n", sz.WordSize, sz.MaxAlign) + wordSize := an.m.TypesSizes.Sizeof(types.Typ[types.Int]) + maxAlign := an.m.TypesSizes.Alignof(types.NewPointer(types.Typ[types.Int64])) + fmt.Fprintf(hasher, "sizes: %d %d\n", wordSize, maxAlign) // metadata errors: used for 'compiles' field fmt.Fprintf(hasher, "errors: %d", len(m.Errors)) diff --git a/gopls/internal/lsp/cache/check.go b/gopls/internal/lsp/cache/check.go index 1b9d08c2ae6..74404af98c1 100644 --- a/gopls/internal/lsp/cache/check.go +++ b/gopls/internal/lsp/cache/check.go @@ -1401,8 +1401,9 @@ func localPackageKey(inputs typeCheckInputs) source.Hash { } // types sizes - sz := inputs.sizes.(*types.StdSizes) - fmt.Fprintf(hasher, "sizes: %d %d\n", sz.WordSize, sz.MaxAlign) + wordSize := inputs.sizes.Sizeof(types.Typ[types.Int]) + maxAlign := inputs.sizes.Alignof(types.NewPointer(types.Typ[types.Int64])) + fmt.Fprintf(hasher, "sizes: %d %d\n", wordSize, maxAlign) fmt.Fprintf(hasher, "relatedInformation: %t\n", inputs.relatedInformation) fmt.Fprintf(hasher, "linkTarget: %s\n", inputs.linkTarget) From 64e92489801f992edea1c405beab858d2f239064 Mon Sep 17 00:00:00 2001 From: Nicholas Cheng Date: Fri, 4 Aug 2023 23:39:58 +0800 Subject: [PATCH 015/178] gopls/internal/lsp/source/completion: add ifnotnil postfix snippet The ifnotnil postfix snippet applies to any nil-able expression and expands it to `if expr != nil`. Fixes golang/go#61763 Change-Id: Ia2fc6d8fa0b07b4ab4193a5b23adcfeee225eadf Reviewed-on: https://go-review.googlesource.com/c/tools/+/516015 TryBot-Result: Gopher Robot Auto-Submit: Robert Findley Run-TryBot: Robert Findley Reviewed-by: Hyang-Ah Hana Kim Reviewed-by: Robert Findley --- .../lsp/source/completion/postfix_snippets.go | 9 ++ .../internal/lsp/testdata/snippets/postfix.go | 3 +- .../completion/postfix_snippet_test.go | 126 ++++++++++++++++++ 3 files changed, 137 insertions(+), 1 deletion(-) diff --git a/gopls/internal/lsp/source/completion/postfix_snippets.go b/gopls/internal/lsp/source/completion/postfix_snippets.go index c1582e6b379..a10004993b2 100644 --- a/gopls/internal/lsp/source/completion/postfix_snippets.go +++ b/gopls/internal/lsp/source/completion/postfix_snippets.go @@ -194,6 +194,14 @@ for {{.VarName .ElemType "e"}} := range {{.X}} { body: `{{if and (eq .Kind "slice") (eq (.TypeName .ElemType) "string") -}} {{.Import "strings"}}.Join({{.X}}, "{{.Cursor}}") {{- end}}`, +}, { + label: "ifnotnil", + details: "if expr != nil", + body: `{{if and (or (eq .Kind "pointer") (eq .Kind "chan") (eq .Kind "signature") (eq .Kind "interface") (eq .Kind "map") (eq .Kind "slice")) .StmtOK -}} +if {{.X}} != nil {{"{"}} + {{.Cursor}} +{{"}"}} +{{- end}}`, }} // Cursor indicates where the client's cursor should end up after the @@ -211,6 +219,7 @@ func (a *postfixTmplArgs) Import(path string) (string, error) { return "", fmt.Errorf("couldn't import %q: %w", path, err) } a.edits = append(a.edits, edits...) + return name, nil } diff --git a/gopls/internal/lsp/testdata/snippets/postfix.go b/gopls/internal/lsp/testdata/snippets/postfix.go index d29694e835f..78a091ada5c 100644 --- a/gopls/internal/lsp/testdata/snippets/postfix.go +++ b/gopls/internal/lsp/testdata/snippets/postfix.go @@ -34,9 +34,10 @@ func _() { /* reverse! */ //@item(postfixReverse, "reverse!", "reverse slice", "snippet") /* sort! */ //@item(postfixSort, "sort!", "sort.Slice()", "snippet") /* var! */ //@item(postfixVar, "var!", "assign to variable", "snippet") + /* ifnotnil! */ //@item(postfixIfNotNil, "ifnotnil!", "if expr != nil", "snippet") var foo []int - foo. //@complete(" //", postfixAppend, postfixCopy, postfixLast, postfixPrint, postfixRange, postfixReverse, postfixSort, postfixVar) + foo. //@complete(" //", postfixAppend, postfixCopy, postfixIfNotNil, postfixLast, postfixPrint, postfixRange, postfixReverse, postfixSort, postfixVar) foo = nil } diff --git a/gopls/internal/regtest/completion/postfix_snippet_test.go b/gopls/internal/regtest/completion/postfix_snippet_test.go index df69703ee26..bfaa8f664f4 100644 --- a/gopls/internal/regtest/completion/postfix_snippet_test.go +++ b/gopls/internal/regtest/completion/postfix_snippet_test.go @@ -430,6 +430,132 @@ func foo() string { return strings.Join(x, "$0") }`, }, + { + name: "if not nil interface", + before: ` +package foo + +func _() { + var foo error + foo.ifnotnil +} +`, + after: ` +package foo + +func _() { + var foo error + if foo != nil { + $0 +} +} +`, + }, + { + name: "if not nil pointer", + before: ` +package foo + +func _() { + var foo *int + foo.ifnotnil +} +`, + after: ` +package foo + +func _() { + var foo *int + if foo != nil { + $0 +} +} +`, + }, + { + name: "if not nil slice", + before: ` +package foo + +func _() { + var foo []int + foo.ifnotnil +} +`, + after: ` +package foo + +func _() { + var foo []int + if foo != nil { + $0 +} +} +`, + }, + { + name: "if not nil map", + before: ` +package foo + +func _() { + var foo map[string]any + foo.ifnotnil +} +`, + after: ` +package foo + +func _() { + var foo map[string]any + if foo != nil { + $0 +} +} +`, + }, + { + name: "if not nil channel", + before: ` +package foo + +func _() { + var foo chan int + foo.ifnotnil +} +`, + after: ` +package foo + +func _() { + var foo chan int + if foo != nil { + $0 +} +} +`, + }, + { + name: "if not nil function", + before: ` +package foo + +func _() { + var foo func() + foo.ifnotnil +} +`, + after: ` +package foo + +func _() { + var foo func() + if foo != nil { + $0 +} +} +`, + }, } r := WithOptions( From a80931dd2e6932a5eab323e4d8a8e56b359d40d4 Mon Sep 17 00:00:00 2001 From: Peter Weinberger Date: Tue, 15 Aug 2023 12:46:20 -0400 Subject: [PATCH 016/178] gopls/telemetry: accept vscode-insiders as a known editor Also, for unknown editors, increment a local counter in addition to gopls/client:other Change-Id: Id689343502f62b1479497eca81b0ef3944898d8c Reviewed-on: https://go-review.googlesource.com/c/tools/+/519735 TryBot-Result: Gopher Robot gopls-CI: kokoro Reviewed-by: Hyang-Ah Hana Kim Run-TryBot: Peter Weinberger --- gopls/internal/telemetry/telemetry.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/gopls/internal/telemetry/telemetry.go b/gopls/internal/telemetry/telemetry.go index 62c89b7610d..d3a9f6388a2 100644 --- a/gopls/internal/telemetry/telemetry.go +++ b/gopls/internal/telemetry/telemetry.go @@ -27,6 +27,8 @@ func RecordClientInfo(params *protocol.ParamInitialize) { switch params.ClientInfo.Name { case "Visual Studio Code": client = "gopls/client:vscode" + case "Visual Studio Code - Insiders": + client = "gopls/client:vscode-insiders" case "VSCodium": client = "gopls/client:vscodium" case "code-server": @@ -47,6 +49,10 @@ func RecordClientInfo(params *protocol.ParamInitialize) { case "Sublime Text LSP": // https://github.com/sublimelsp/LSP/blob/e608f878e7e9dd34aabe4ff0462540fadcd88fcc/plugin/core/sessions.py#L493 client = "gopls/client:sublimetext" + default: + // at least accumulate the client name locally + counter.New(fmt.Sprintf("gopls/client-other:%s", params.ClientInfo.Name)).Inc() + // but also record client:other } } counter.Inc(client) From 0286c389ddab9cb064c3e90e23c998a1be240c40 Mon Sep 17 00:00:00 2001 From: Robert Findley Date: Wed, 16 Aug 2023 11:45:04 -0400 Subject: [PATCH 017/178] gopls/internal/lsp: switch default diff to "new" As the next step in the diff migration, stop running both diffs by default. For golang/go#52967 Change-Id: I4b1a434d13f4349feabc5eb0daf3b9e8d56f99aa Reviewed-on: https://go-review.googlesource.com/c/tools/+/520135 TryBot-Result: Gopher Robot Reviewed-by: Peter Weinberger Run-TryBot: Robert Findley gopls-CI: kokoro --- gopls/internal/lsp/source/options.go | 2 +- gopls/internal/lsp/tests/tests.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gopls/internal/lsp/source/options.go b/gopls/internal/lsp/source/options.go index c2e3223e6c1..45899e62969 100644 --- a/gopls/internal/lsp/source/options.go +++ b/gopls/internal/lsp/source/options.go @@ -171,7 +171,7 @@ func DefaultOptions() *Options { CompletionDocumentation: true, DeepCompletion: true, ChattyDiagnostics: true, - NewDiff: "both", + NewDiff: "new", SubdirWatchPatterns: SubdirWatchPatternsAuto, ReportAnalysisProgressAfter: 5 * time.Second, }, diff --git a/gopls/internal/lsp/tests/tests.go b/gopls/internal/lsp/tests/tests.go index 9ab114e4834..65e9c0e9532 100644 --- a/gopls/internal/lsp/tests/tests.go +++ b/gopls/internal/lsp/tests/tests.go @@ -246,7 +246,7 @@ func DefaultOptions(o *source.Options) { o.CompletionBudget = time.Minute o.HierarchicalDocumentSymbolSupport = true o.SemanticTokens = true - o.InternalOptions.NewDiff = "both" + o.InternalOptions.NewDiff = "new" } func RunTests(t *testing.T, dataDir string, includeMultiModule bool, f func(*testing.T, *Data)) { From b225aa0590e71ba82bf4bb7696430681ced04c53 Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Wed, 16 Aug 2023 11:58:10 -0700 Subject: [PATCH 018/178] internal/typeparams: run go generate Change-Id: I732afe9ff8c4451ead84cd1a429ccd47b781938e Reviewed-on: https://go-review.googlesource.com/c/tools/+/520176 Auto-Submit: Ian Lance Taylor Reviewed-by: Robert Findley Run-TryBot: Ian Lance Taylor gopls-CI: kokoro Reviewed-by: Ian Lance Taylor Run-TryBot: Ian Lance Taylor TryBot-Result: Gopher Robot --- internal/typeparams/termlist.go | 2 +- internal/typeparams/typeterm.go | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/internal/typeparams/termlist.go b/internal/typeparams/termlist.go index 933106a23dd..cbd12f80131 100644 --- a/internal/typeparams/termlist.go +++ b/internal/typeparams/termlist.go @@ -30,7 +30,7 @@ func (xl termlist) String() string { var buf bytes.Buffer for i, x := range xl { if i > 0 { - buf.WriteString(" ∪ ") + buf.WriteString(" | ") } buf.WriteString(x.String()) } diff --git a/internal/typeparams/typeterm.go b/internal/typeparams/typeterm.go index 7ddee28d987..7350bb702a1 100644 --- a/internal/typeparams/typeterm.go +++ b/internal/typeparams/typeterm.go @@ -10,11 +10,10 @@ import "go/types" // A term describes elementary type sets: // -// ∅: (*term)(nil) == ∅ // set of no types (empty set) -// 𝓤: &term{} == 𝓤 // set of all types (𝓤niverse) -// T: &term{false, T} == {T} // set of type T -// ~t: &term{true, t} == {t' | under(t') == t} // set of types with underlying type t -// +// ∅: (*term)(nil) == ∅ // set of no types (empty set) +// 𝓤: &term{} == 𝓤 // set of all types (𝓤niverse) +// T: &term{false, T} == {T} // set of type T +// ~t: &term{true, t} == {t' | under(t') == t} // set of types with underlying type t type term struct { tilde bool // valid if typ != nil typ types.Type From 9f2e103e08d2581745e79a50295df3816fc19f9e Mon Sep 17 00:00:00 2001 From: cui fliter Date: Wed, 16 Aug 2023 23:12:51 +0800 Subject: [PATCH 019/178] all: gofmt format Change-Id: I8419bdaab5164388de846ccd942af1914ef944b7 Reviewed-on: https://go-review.googlesource.com/c/tools/+/520075 gopls-CI: kokoro Run-TryBot: shuang cui TryBot-Result: Gopher Robot Run-TryBot: Ian Lance Taylor Reviewed-by: Dmitri Shuralyov Auto-Submit: Ian Lance Taylor Reviewed-by: Ian Lance Taylor Reviewed-by: Dmitri Shuralyov Auto-Submit: Dmitri Shuralyov --- gopls/internal/lsp/command/gen/gen.go | 2 +- gopls/internal/lsp/lsprpc/goenv.go | 2 +- internal/diff/lcs/labels.go | 2 +- internal/typeparams/coretype.go | 8 ++++---- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/gopls/internal/lsp/command/gen/gen.go b/gopls/internal/lsp/command/gen/gen.go index 29428699ee6..b3f89c8d773 100644 --- a/gopls/internal/lsp/command/gen/gen.go +++ b/gopls/internal/lsp/command/gen/gen.go @@ -12,8 +12,8 @@ import ( "go/types" "text/template" - "golang.org/x/tools/internal/imports" "golang.org/x/tools/gopls/internal/lsp/command/commandmeta" + "golang.org/x/tools/internal/imports" ) const src = `// Copyright 2021 The Go Authors. All rights reserved. diff --git a/gopls/internal/lsp/lsprpc/goenv.go b/gopls/internal/lsp/lsprpc/goenv.go index c316ea07c70..b7717844f17 100644 --- a/gopls/internal/lsp/lsprpc/goenv.go +++ b/gopls/internal/lsp/lsprpc/goenv.go @@ -10,10 +10,10 @@ import ( "fmt" "os" + "golang.org/x/tools/gopls/internal/lsp/protocol" "golang.org/x/tools/internal/event" "golang.org/x/tools/internal/gocommand" jsonrpc2_v2 "golang.org/x/tools/internal/jsonrpc2_v2" - "golang.org/x/tools/gopls/internal/lsp/protocol" ) func GoEnvMiddleware() (Middleware, error) { diff --git a/internal/diff/lcs/labels.go b/internal/diff/lcs/labels.go index 0689f1ed700..504913d1da3 100644 --- a/internal/diff/lcs/labels.go +++ b/internal/diff/lcs/labels.go @@ -8,7 +8,7 @@ import ( "fmt" ) -// For each D, vec[D] has length D+1, +// For each D, vec[D] has length D+1, // and the label for (D, k) is stored in vec[D][(D+k)/2]. type label struct { vec [][]int diff --git a/internal/typeparams/coretype.go b/internal/typeparams/coretype.go index 993135ec90e..71248209ee5 100644 --- a/internal/typeparams/coretype.go +++ b/internal/typeparams/coretype.go @@ -81,13 +81,13 @@ func CoreType(T types.Type) types.Type { // restrictions may be arbitrarily complex. For example, consider the // following: // -// type A interface{ ~string|~[]byte } +// type A interface{ ~string|~[]byte } // -// type B interface{ int|string } +// type B interface{ int|string } // -// type C interface { ~string|~int } +// type C interface { ~string|~int } // -// type T[P interface{ A|B; C }] int +// type T[P interface{ A|B; C }] int // // In this example, the structural type restriction of P is ~string|int: A|B // expands to ~string|~[]byte|int|string, which reduces to ~string|~[]byte|int, From 1517d1a3ba600175b642a4ae8106f0484516d3d9 Mon Sep 17 00:00:00 2001 From: Robert Findley Date: Fri, 11 Aug 2023 19:35:44 -0400 Subject: [PATCH 020/178] gopls/internal/lsp/source: fix renaming instantiated fields Correctly associate instantiated functions and fields with their origin during renaming. Fixes golang/go#61640 Change-Id: I819ffe303a2b1c35810d5b3c2d71fa5f4231a0c4 Reviewed-on: https://go-review.googlesource.com/c/tools/+/518897 TryBot-Result: Gopher Robot Run-TryBot: Robert Findley Reviewed-by: Hyang-Ah Hana Kim gopls-CI: kokoro --- gopls/internal/lsp/source/origin.go | 26 ++++++++++ gopls/internal/lsp/source/origin_119.go | 33 +++++++++++++ gopls/internal/lsp/source/references.go | 13 +---- gopls/internal/lsp/source/rename.go | 8 +--- .../marker/testdata/rename/issue61640.txt | 47 +++++++++++++++++++ 5 files changed, 109 insertions(+), 18 deletions(-) create mode 100644 gopls/internal/lsp/source/origin.go create mode 100644 gopls/internal/lsp/source/origin_119.go create mode 100644 gopls/internal/regtest/marker/testdata/rename/issue61640.txt diff --git a/gopls/internal/lsp/source/origin.go b/gopls/internal/lsp/source/origin.go new file mode 100644 index 00000000000..8ee467e844e --- /dev/null +++ b/gopls/internal/lsp/source/origin.go @@ -0,0 +1,26 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !go1.19 +// +build !go1.19 + +package source + +import "go/types" + +// containsOrigin reports whether the provided object set contains an object +// with the same origin as the provided obj (which may be a synthetic object +// created during instantiation). +func containsOrigin(objSet map[types.Object]bool, obj types.Object) bool { + if obj == nil { + return objSet[obj] + } + // In Go 1.18, we can't use the types.Var.Origin and types.Func.Origin methods. + for target := range objSet { + if target.Pkg() == obj.Pkg() && target.Pos() == obj.Pos() && target.Name() == obj.Name() { + return true + } + } + return false +} diff --git a/gopls/internal/lsp/source/origin_119.go b/gopls/internal/lsp/source/origin_119.go new file mode 100644 index 00000000000..a249ce4b1c5 --- /dev/null +++ b/gopls/internal/lsp/source/origin_119.go @@ -0,0 +1,33 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.19 +// +build go1.19 + +package source + +import "go/types" + +// containsOrigin reports whether the provided object set contains an object +// with the same origin as the provided obj (which may be a synthetic object +// created during instantiation). +func containsOrigin(objSet map[types.Object]bool, obj types.Object) bool { + objOrigin := origin(obj) + for target := range objSet { + if origin(target) == objOrigin { + return true + } + } + return false +} + +func origin(obj types.Object) types.Object { + switch obj := obj.(type) { + case *types.Var: + return obj.Origin() + case *types.Func: + return obj.Origin() + } + return obj +} diff --git a/gopls/internal/lsp/source/references.go b/gopls/internal/lsp/source/references.go index 3d923e44702..46459dcbec4 100644 --- a/gopls/internal/lsp/source/references.go +++ b/gopls/internal/lsp/source/references.go @@ -580,10 +580,8 @@ func localReferences(pkg Package, targets map[types.Object]bool, correspond bool // matches reports whether obj either is or corresponds to a target. // (Correspondence is defined as usual for interface methods.) matches := func(obj types.Object) bool { - for target := range targets { - if equalOrigin(obj, target) { - return true - } + if containsOrigin(targets, obj) { + return true } if methodRecvs != nil && obj.Name() == methodName { if orecv := effectiveReceiver(obj); orecv != nil { @@ -611,13 +609,6 @@ func localReferences(pkg Package, targets map[types.Object]bool, correspond bool return nil } -// equalOrigin reports whether obj1 and obj2 have equivalent origin object. -// This may be the case even if obj1 != obj2, if one or both of them is -// instantiated. -func equalOrigin(obj1, obj2 types.Object) bool { - return obj1.Pkg() == obj2.Pkg() && obj1.Pos() == obj2.Pos() && obj1.Name() == obj2.Name() -} - // effectiveReceiver returns the effective receiver type for method-set // comparisons for obj, if it is a method, or nil otherwise. func effectiveReceiver(obj types.Object) types.Type { diff --git a/gopls/internal/lsp/source/rename.go b/gopls/internal/lsp/source/rename.go index c1db0e5fd5d..eb1fdce622f 100644 --- a/gopls/internal/lsp/source/rename.go +++ b/gopls/internal/lsp/source/rename.go @@ -1054,13 +1054,7 @@ func (r *renamer) update() (map[span.URI][]diff.Edit, error) { // shouldUpdate reports whether obj is one of (or an // instantiation of one of) the target objects. shouldUpdate := func(obj types.Object) bool { - if r.objsToUpdate[obj] { - return true - } - if fn, ok := obj.(*types.Func); ok && r.objsToUpdate[funcOrigin(fn)] { - return true - } - return false + return containsOrigin(r.objsToUpdate, obj) } // Find all identifiers in the package that define or use a diff --git a/gopls/internal/regtest/marker/testdata/rename/issue61640.txt b/gopls/internal/regtest/marker/testdata/rename/issue61640.txt new file mode 100644 index 00000000000..91c2b76933d --- /dev/null +++ b/gopls/internal/regtest/marker/testdata/rename/issue61640.txt @@ -0,0 +1,47 @@ +This test verifies that gopls can rename instantiated fields. + +-- flags -- +-min_go=go1.18 + +-- a.go -- +package a + +// This file is adapted from the example in the issue. + +type builder[S ~[]int] struct { + elements S //@rename("elements", elements2, OneToTwo) +} + +type BuilderImpl[S ~[]int] struct{ builder[S] } + +func NewBuilderImpl[S ~[]int](name string) *BuilderImpl[S] { + impl := &BuilderImpl[S]{ + builder[S]{ + elements: S{}, + }, + } + + _ = impl.elements + return impl +} +-- @OneToTwo/a.go -- +package a + +// This file is adapted from the example in the issue. + +type builder[S ~[]int] struct { + elements2 S //@rename("elements", elements2, OneToTwo) +} + +type BuilderImpl[S ~[]int] struct{ builder[S] } + +func NewBuilderImpl[S ~[]int](name string) *BuilderImpl[S] { + impl := &BuilderImpl[S]{ + builder[S]{ + elements2: S{}, + }, + } + + _ = impl.elements2 + return impl +} From a46a10facf8bb162fcff4ab3f22b28ca51004048 Mon Sep 17 00:00:00 2001 From: toad Date: Fri, 18 Aug 2023 05:57:27 +0000 Subject: [PATCH 021/178] gopls: stubbed methods shouldn't qualify the current package Fixes golang/go#61830 Change-Id: I41aff8177b57d64a684cd11ff78aa7c4f00ce36c GitHub-Last-Rev: a6677458679d57df22a1ffff293333f53f2a9603 GitHub-Pull-Request: golang/tools#447 Reviewed-on: https://go-review.googlesource.com/c/tools/+/516758 Reviewed-by: Robert Findley Run-TryBot: t hepudds gopls-CI: kokoro Reviewed-by: Hyang-Ah Hana Kim TryBot-Result: Gopher Robot --- gopls/internal/lsp/source/stub.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/gopls/internal/lsp/source/stub.go b/gopls/internal/lsp/source/stub.go index b7b2292e4e3..1f886796d92 100644 --- a/gopls/internal/lsp/source/stub.go +++ b/gopls/internal/lsp/source/stub.go @@ -125,6 +125,12 @@ func stub(ctx context.Context, snapshot Snapshot, si *stubmethods.StubInfo) (*to var newImports []newImport // for AddNamedImport qual := func(pkg *types.Package) string { // TODO(adonovan): don't ignore vendor prefix. + // + // Ignore the current package import. + if pkg.Path() == conc.Pkg().Path() { + return "" + } + importPath := ImportPath(pkg.Path()) name, ok := importEnv[importPath] if !ok { From 5913c0262260b745759d930367e090380ecde6b9 Mon Sep 17 00:00:00 2001 From: Robert Findley Date: Fri, 11 Aug 2023 16:42:53 -0400 Subject: [PATCH 022/178] gopls/internal/lsp/source: add a test for local types stubbing For golang/go#61830 Change-Id: Ic30cc528ecfff299a1fc90a17b4e7ccbf5249a8a Reviewed-on: https://go-review.googlesource.com/c/tools/+/518915 Auto-Submit: Robert Findley TryBot-Result: Gopher Robot Run-TryBot: Robert Findley gopls-CI: kokoro Reviewed-by: Peter Weinberger --- .../testdata/stubmethods/issue61830.txt | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 gopls/internal/regtest/marker/testdata/stubmethods/issue61830.txt diff --git a/gopls/internal/regtest/marker/testdata/stubmethods/issue61830.txt b/gopls/internal/regtest/marker/testdata/stubmethods/issue61830.txt new file mode 100644 index 00000000000..43633557d89 --- /dev/null +++ b/gopls/internal/regtest/marker/testdata/stubmethods/issue61830.txt @@ -0,0 +1,36 @@ +This test verifies that method stubbing qualifies types relative to the current +package. + +-- p.go -- +package p + +import "io" + +type B struct{} + +type I interface { + M(io.Reader, B) +} + +type A struct{} + +var _ I = &A{} //@suggestedfix(re"&A..", re"missing method M", "quickfix", stub) +-- @stub/p.go -- +package p + +import "io" + +type B struct{} + +type I interface { + M(io.Reader, B) +} + +type A struct{} + +// M implements I. +func (*A) M(io.Reader, B) { + panic("unimplemented") +} + +var _ I = &A{} //@suggestedfix(re"&A..", re"missing method M", "quickfix", stub) From 9425c2e70961cb8c2a3b5764658fb70e03d194e4 Mon Sep 17 00:00:00 2001 From: Robert Findley Date: Fri, 11 Aug 2023 19:04:09 -0400 Subject: [PATCH 023/178] gopls/internal/lsp/source: fix implementations query on error type Refactor so that implementations queries do not require local packages for the queried object. Fixes golang/go#43655 Change-Id: I1bf87c236cefbffff53fa2b2c1a11c2cfb96b763 Reviewed-on: https://go-review.googlesource.com/c/tools/+/518896 TryBot-Result: Gopher Robot Run-TryBot: Robert Findley Reviewed-by: Hyang-Ah Hana Kim gopls-CI: kokoro --- gopls/internal/lsp/source/implementation.go | 112 ++++++++---------- .../testdata/implementation/issue43655.txt | 22 ++++ 2 files changed, 69 insertions(+), 65 deletions(-) create mode 100644 gopls/internal/regtest/marker/testdata/implementation/issue43655.txt diff --git a/gopls/internal/lsp/source/implementation.go b/gopls/internal/lsp/source/implementation.go index 25beccf6e1d..d9eb814099b 100644 --- a/gopls/internal/lsp/source/implementation.go +++ b/gopls/internal/lsp/source/implementation.go @@ -17,6 +17,7 @@ import ( "sync" "golang.org/x/sync/errgroup" + "golang.org/x/tools/gopls/internal/bug" "golang.org/x/tools/gopls/internal/lsp/protocol" "golang.org/x/tools/gopls/internal/lsp/safetoken" "golang.org/x/tools/gopls/internal/lsp/source/methodsets" @@ -70,60 +71,36 @@ func Implementation(ctx context.Context, snapshot Snapshot, f FileHandle, pp pro } func implementations(ctx context.Context, snapshot Snapshot, fh FileHandle, pp protocol.Position) ([]protocol.Location, error) { - - // Type-check the query package, find the query identifier, - // and locate the type or method declaration it refers to. - declPosn, err := typeDeclPosition(ctx, snapshot, fh.URI(), pp) - if err != nil { - return nil, err - } - - // Type-check the declaring package (incl. variants) for use - // by the "local" search, which uses type information to - // enumerate all types within the package that satisfy the - // query type, even those defined local to a function. - declURI := span.URIFromPath(declPosn.Filename) - declMetas, err := snapshot.MetadataForFile(ctx, declURI) - if err != nil { - return nil, err - } - RemoveIntermediateTestVariants(&declMetas) - if len(declMetas) == 0 { - return nil, fmt.Errorf("no packages for file %s", declURI) - } - ids := make([]PackageID, len(declMetas)) - for i, m := range declMetas { - ids[i] = m.ID - } - localPkgs, err := snapshot.TypeCheck(ctx, ids...) + obj, pkg, err := implementsObj(ctx, snapshot, fh.URI(), pp) if err != nil { return nil, err } - // The narrowest package will do, since the local search is based - // on position and the global search is based on fingerprint. - // (Neither is based on object identity.) - declPkg := localPkgs[0] - declFile, err := declPkg.File(declURI) - if err != nil { - return nil, err // "can't happen" - } - // Find declaration of corresponding object - // in this package based on (URI, offset). - pos, err := safetoken.Pos(declFile.Tok, declPosn.Offset) - if err != nil { - return nil, err - } - // TODO(adonovan): simplify: use objectsAt? - path := pathEnclosingObjNode(declFile.File, pos) - if path == nil { - return nil, ErrNoIdentFound // checked earlier - } - id, ok := path[0].(*ast.Ident) - if !ok { - return nil, ErrNoIdentFound // checked earlier + var localPkgs []Package + if obj.Pos().IsValid() { // no local package for error or error.Error + declPosn := safetoken.StartPosition(pkg.FileSet(), obj.Pos()) + // Type-check the declaring package (incl. variants) for use + // by the "local" search, which uses type information to + // enumerate all types within the package that satisfy the + // query type, even those defined local to a function. + declURI := span.URIFromPath(declPosn.Filename) + declMetas, err := snapshot.MetadataForFile(ctx, declURI) + if err != nil { + return nil, err + } + RemoveIntermediateTestVariants(&declMetas) + if len(declMetas) == 0 { + return nil, fmt.Errorf("no packages for file %s", declURI) + } + ids := make([]PackageID, len(declMetas)) + for i, m := range declMetas { + ids[i] = m.ID + } + localPkgs, err = snapshot.TypeCheck(ctx, ids...) + if err != nil { + return nil, err + } } - obj := declPkg.GetTypesInfo().ObjectOf(id) // may be nil // Is the selected identifier a type name or method? // (For methods, report the corresponding method names.) @@ -140,7 +117,7 @@ func implementations(ctx context.Context, snapshot Snapshot, fh FileHandle, pp p } } if queryType == nil { - return nil, fmt.Errorf("%s is not a type or method", id.Name) + return nil, bug.Errorf("%s is not a type or method", obj.Name()) // should have been handled by implementsObj } // Compute the method-set fingerprint used as a key to the global search. @@ -166,8 +143,13 @@ func implementations(ctx context.Context, snapshot Snapshot, fh FileHandle, pp p } RemoveIntermediateTestVariants(&globalMetas) globalIDs := make([]PackageID, 0, len(globalMetas)) + + var pkgPath PackagePath + if obj.Pkg() != nil { // nil for error + pkgPath = PackagePath(obj.Pkg().Path()) + } for _, m := range globalMetas { - if m.PkgPath == declPkg.Metadata().PkgPath { + if m.PkgPath == pkgPath { continue // declaring package is handled by local implementation } globalIDs = append(globalIDs, m.ID) @@ -241,18 +223,19 @@ func offsetToLocation(ctx context.Context, snapshot Snapshot, filename string, s return m.OffsetLocation(start, end) } -// typeDeclPosition returns the position of the declaration of the -// type (or one of its methods) referred to at (uri, ppos). -func typeDeclPosition(ctx context.Context, snapshot Snapshot, uri span.URI, ppos protocol.Position) (token.Position, error) { - var noPosn token.Position - +// implementsObj returns the object to query for implementations, which is a +// type name or method. +// +// The returned Package is the narrowest package containing ppos, which is the +// package using the resulting obj but not necessarily the declaring package. +func implementsObj(ctx context.Context, snapshot Snapshot, uri span.URI, ppos protocol.Position) (types.Object, Package, error) { pkg, pgf, err := NarrowestPackageForFile(ctx, snapshot, uri) if err != nil { - return noPosn, err + return nil, nil, err } pos, err := pgf.PositionPos(ppos) if err != nil { - return noPosn, err + return nil, nil, err } // This function inherits the limitation of its predecessor in @@ -267,11 +250,11 @@ func typeDeclPosition(ctx context.Context, snapshot Snapshot, uri span.URI, ppos // TODO(adonovan): simplify: use objectsAt? path := pathEnclosingObjNode(pgf.File, pos) if path == nil { - return noPosn, ErrNoIdentFound + return nil, nil, ErrNoIdentFound } id, ok := path[0].(*ast.Ident) if !ok { - return noPosn, ErrNoIdentFound + return nil, nil, ErrNoIdentFound } // Is the object a type or method? Reject other kinds. @@ -287,18 +270,17 @@ func typeDeclPosition(ctx context.Context, snapshot Snapshot, uri span.URI, ppos // ok case *types.Func: if obj.Type().(*types.Signature).Recv() == nil { - return noPosn, fmt.Errorf("%s is a function, not a method", id.Name) + return nil, nil, fmt.Errorf("%s is a function, not a method", id.Name) } case nil: - return noPosn, fmt.Errorf("%s denotes unknown object", id.Name) + return nil, nil, fmt.Errorf("%s denotes unknown object", id.Name) default: // e.g. *types.Var -> "var". kind := strings.ToLower(strings.TrimPrefix(reflect.TypeOf(obj).String(), "*types.")) - return noPosn, fmt.Errorf("%s is a %s, not a type", id.Name, kind) + return nil, nil, fmt.Errorf("%s is a %s, not a type", id.Name, kind) } - declPosn := safetoken.StartPosition(pkg.FileSet(), obj.Pos()) - return declPosn, nil + return obj, pkg, nil } // localImplementations searches within pkg for declarations of all diff --git a/gopls/internal/regtest/marker/testdata/implementation/issue43655.txt b/gopls/internal/regtest/marker/testdata/implementation/issue43655.txt new file mode 100644 index 00000000000..a7f1d57f80d --- /dev/null +++ b/gopls/internal/regtest/marker/testdata/implementation/issue43655.txt @@ -0,0 +1,22 @@ +This test verifies that we fine implementations of the built-in error interface. + +-- go.mod -- +module example.com +go 1.12 + +-- p.go -- +package p + +type errA struct{ error } //@loc(errA, "errA") + +type errB struct{} //@loc(errB, "errB") +func (errB) Error() string{ return "" } //@loc(errBError, "Error") + +type notAnError struct{} +func (notAnError) Error() int { return 0 } + +func _() { + var _ error //@implementation("error", errA, errB) + var a errA + _ = a.Error //@implementation("Error", errBError) +} From ff7f2b9c4a76157aac48a61ae2943caa78a8c972 Mon Sep 17 00:00:00 2001 From: Robert Findley Date: Fri, 18 Aug 2023 12:54:26 -0400 Subject: [PATCH 024/178] gopls/internal/lsp/source completion: don't suggest untyped conversions During candidate inference, it is possible that the implicit type of the completion is untyped (as would be the case for an operand in an untyped constant expression). In this case, when we've determined that a candidate must be converted, use the default type rather than suggesting a conversion to e.g. "untyped float". Notably, untyped consts of different kinds are considered compatible, so this conversion logic does not apply to them. While at it, express the test in the new marker framework, by adding an 'acceptcompletion' marker. Fixes golang/go#62141 Change-Id: I81fa7b61f54d141084ac02ee863076355e4effc4 Reviewed-on: https://go-review.googlesource.com/c/tools/+/520876 Reviewed-by: Alan Donovan Run-TryBot: Robert Findley TryBot-Result: Gopher Robot gopls-CI: kokoro --- gopls/internal/lsp/regtest/marker.go | 72 +++++++++++++++---- .../internal/lsp/source/completion/format.go | 9 ++- .../marker/testdata/completion/issue62141.txt | 39 ++++++++++ 3 files changed, 104 insertions(+), 16 deletions(-) create mode 100644 gopls/internal/regtest/marker/testdata/completion/issue62141.txt diff --git a/gopls/internal/lsp/regtest/marker.go b/gopls/internal/lsp/regtest/marker.go index 6ae306b3d4e..1c364bdc952 100644 --- a/gopls/internal/lsp/regtest/marker.go +++ b/gopls/internal/lsp/regtest/marker.go @@ -132,6 +132,10 @@ var update = flag.Bool("update", false, "if set, update test data during marker // // The following markers are supported within marker tests: // +// - acceptcompletion(location, label, golden): specifies that accepting the +// completion candidate produced at the given location with provided label +// results in the given golden state. +// // - codeaction(kind, start, end, golden): specifies a codeaction to request // for the given range. To support multi-line ranges, the range is defined // to be between start.Start and end.End. The golden directory contains @@ -546,21 +550,22 @@ arity: // Marker funcs should not mutate the test environment (e.g. via opening files // or applying edits in the editor). var markerFuncs = map[string]markerFunc{ - "codeaction": makeMarkerFunc(codeActionMarker), - "codeactionerr": makeMarkerFunc(codeActionErrMarker), - "complete": makeMarkerFunc(completeMarker), - "def": makeMarkerFunc(defMarker), - "diag": makeMarkerFunc(diagMarker), - "hover": makeMarkerFunc(hoverMarker), - "format": makeMarkerFunc(formatMarker), - "implementation": makeMarkerFunc(implementationMarker), - "loc": makeMarkerFunc(locMarker), - "rename": makeMarkerFunc(renameMarker), - "renameerr": makeMarkerFunc(renameErrMarker), - "suggestedfix": makeMarkerFunc(suggestedfixMarker), - "symbol": makeMarkerFunc(symbolMarker), - "refs": makeMarkerFunc(refsMarker), - "workspacesymbol": makeMarkerFunc(workspaceSymbolMarker), + "acceptcompletion": makeMarkerFunc(acceptCompletionMarker), + "codeaction": makeMarkerFunc(codeActionMarker), + "codeactionerr": makeMarkerFunc(codeActionErrMarker), + "complete": makeMarkerFunc(completeMarker), + "def": makeMarkerFunc(defMarker), + "diag": makeMarkerFunc(diagMarker), + "hover": makeMarkerFunc(hoverMarker), + "format": makeMarkerFunc(formatMarker), + "implementation": makeMarkerFunc(implementationMarker), + "loc": makeMarkerFunc(locMarker), + "rename": makeMarkerFunc(renameMarker), + "renameerr": makeMarkerFunc(renameErrMarker), + "suggestedfix": makeMarkerFunc(suggestedfixMarker), + "symbol": makeMarkerFunc(symbolMarker), + "refs": makeMarkerFunc(refsMarker), + "workspacesymbol": makeMarkerFunc(workspaceSymbolMarker), } // markerTest holds all the test data extracted from a test txtar archive. @@ -1294,6 +1299,43 @@ func completeMarker(mark marker, src protocol.Location, want ...string) { } } +// acceptCompletionMarker implements the @acceptCompletion marker, running +// textDocument/completion at the given src location and accepting the +// candidate with the given label. The resulting source must match the provided +// golden content. +func acceptCompletionMarker(mark marker, src protocol.Location, label string, golden *Golden) { + list := mark.run.env.Completion(src) + var selected *protocol.CompletionItem + for _, item := range list.Items { + if item.Label == label { + selected = &item + break + } + } + if selected == nil { + mark.errorf("Completion(...) did not return an item labeled %q", label) + return + } + filename := mark.run.env.Sandbox.Workdir.URIToPath(mark.uri()) + mapper, err := mark.run.env.Editor.Mapper(filename) + if err != nil { + mark.errorf("Editor.Mapper(%s) failed: %v", filename, err) + return + } + + patched, _, err := source.ApplyProtocolEdits(mapper, append([]protocol.TextEdit{ + *selected.TextEdit, + }, selected.AdditionalTextEdits...)) + + if err != nil { + mark.errorf("ApplyProtocolEdits failed: %v", err) + return + } + changes := map[string][]byte{filename: patched} + // Check the file state. + checkChangedFiles(mark, changes, golden) +} + // defMarker implements the @def marker, running textDocument/definition at // the given src location and asserting that there is exactly one resulting // location, matching dst. diff --git a/gopls/internal/lsp/source/completion/format.go b/gopls/internal/lsp/source/completion/format.go index c2d2c0bc035..848f52d3132 100644 --- a/gopls/internal/lsp/source/completion/format.go +++ b/gopls/internal/lsp/source/completion/format.go @@ -183,11 +183,18 @@ Suffixes: if cand.convertTo != nil { typeName := types.TypeString(cand.convertTo, c.qf) - switch cand.convertTo.(type) { + switch t := cand.convertTo.(type) { // We need extra parens when casting to these types. For example, // we need "(*int)(foo)", not "*int(foo)". case *types.Pointer, *types.Signature: typeName = "(" + typeName + ")" + case *types.Basic: + // If the types are incompatible (as determined by typeMatches), then we + // must need a conversion here. However, if the target type is untyped, + // don't suggest converting to e.g. "untyped float" (golang/go#62141). + if t.Info()&types.IsUntyped != 0 { + typeName = types.TypeString(types.Default(cand.convertTo), c.qf) + } } prefix = typeName + "(" + prefix diff --git a/gopls/internal/regtest/marker/testdata/completion/issue62141.txt b/gopls/internal/regtest/marker/testdata/completion/issue62141.txt new file mode 100644 index 00000000000..877e59d0b7c --- /dev/null +++ b/gopls/internal/regtest/marker/testdata/completion/issue62141.txt @@ -0,0 +1,39 @@ +This test checks that we don't suggest completion to an untyped conversion such +as "untyped float(abcdef)". + +-- main.go -- +package main + +func main() { + abcdef := 32 //@diag("abcdef", re"not used") + x := 1.0 / abcd //@acceptcompletion(re"abcd()", "abcdef", int), diag("x", re"not used"), diag("abcd", re"(undefined|undeclared)") + + // Verify that we don't suggest converting compatible untyped constants. + const untypedConst = 42 + y := 1.1 / untypedC //@acceptcompletion(re"untypedC()", "untypedConst", untyped), diag("y", re"not used"), diag("untypedC", re"(undefined|undeclared)") +} + +-- @int/main.go -- +package main + +func main() { + abcdef := 32 //@diag("abcdef", re"not used") + x := 1.0 / float64(abcdef) //@acceptcompletion(re"abcd()", "abcdef", int), diag("x", re"not used"), diag("abcd", re"(undefined|undeclared)") + + // Verify that we don't suggest converting compatible untyped constants. + const untypedConst = 42 + y := 1.1 / untypedC //@acceptcompletion(re"untypedC()", "untypedConst", untyped), diag("y", re"not used"), diag("untypedC", re"(undefined|undeclared)") +} + +-- @untyped/main.go -- +package main + +func main() { + abcdef := 32 //@diag("abcdef", re"not used") + x := 1.0 / abcd //@acceptcompletion(re"abcd()", "abcdef", int), diag("x", re"not used"), diag("abcd", re"(undefined|undeclared)") + + // Verify that we don't suggest converting compatible untyped constants. + const untypedConst = 42 + y := 1.1 / untypedConst //@acceptcompletion(re"untypedC()", "untypedConst", untyped), diag("y", re"not used"), diag("untypedC", re"(undefined|undeclared)") +} + From c38e6b06affdeb1fe20214d082f17cbf01b2e20f Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Mon, 21 Aug 2023 13:15:21 -0400 Subject: [PATCH 025/178] gopls/internal/lsp: add testing support for the new zero builtin Update tests that fail at CL 520336. For now, disable the test to avoid failures while the corresponding change is being merged in the Go repo. It is easier to skip at 1.21 as well, due to the summary file. For golang/go#61372 Change-Id: I454a1b05947079a7ff908efc07599fb36f841912 Reviewed-on: https://go-review.googlesource.com/c/tools/+/521119 Run-TryBot: Robert Findley TryBot-Result: Gopher Robot gopls-CI: kokoro Reviewed-by: Robert Griesemer --- .../internal/lsp/testdata/builtins/builtin_go121.go | 4 ++-- .../internal/lsp/testdata/builtins/builtin_go122.go | 8 ++++++++ gopls/internal/lsp/testdata/builtins/builtins.go | 1 + .../internal/lsp/testdata/summary_go1.21.txt.golden | 2 +- gopls/internal/lsp/tests/util_go122.go | 12 ++++++++++++ 5 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 gopls/internal/lsp/testdata/builtins/builtin_go122.go create mode 100644 gopls/internal/lsp/tests/util_go122.go diff --git a/gopls/internal/lsp/testdata/builtins/builtin_go121.go b/gopls/internal/lsp/testdata/builtins/builtin_go121.go index a52d168636e..14f59def9ac 100644 --- a/gopls/internal/lsp/testdata/builtins/builtin_go121.go +++ b/gopls/internal/lsp/testdata/builtins/builtin_go121.go @@ -1,5 +1,5 @@ -//go:build go1.21 -// +build go1.21 +//go:build go1.21 && !go1.22 && ignore +// +build go1.21,!go1.22,ignore package builtins diff --git a/gopls/internal/lsp/testdata/builtins/builtin_go122.go b/gopls/internal/lsp/testdata/builtins/builtin_go122.go new file mode 100644 index 00000000000..f799c1225a1 --- /dev/null +++ b/gopls/internal/lsp/testdata/builtins/builtin_go122.go @@ -0,0 +1,8 @@ +//go:build go1.22 && ignore +// +build go1.22,ignore + +package builtins + +func _() { + //@complete("", any, append, bool, byte, cap, clear, close, comparable, complex, complex128, complex64, copy, delete, error, _false, float32, float64, imag, int, int16, int32, int64, int8, len, make, max, min, new, panic, print, println, real, recover, rune, string, _true, uint, uint16, uint32, uint64, uint8, uintptr, zero, _nil) +} diff --git a/gopls/internal/lsp/testdata/builtins/builtins.go b/gopls/internal/lsp/testdata/builtins/builtins.go index 2e3361c7e6d..a6450362a78 100644 --- a/gopls/internal/lsp/testdata/builtins/builtins.go +++ b/gopls/internal/lsp/testdata/builtins/builtins.go @@ -47,3 +47,4 @@ package builtins /* uint64 */ //@item(uint64, "uint64", "", "type") /* uint8 */ //@item(uint8, "uint8", "", "type") /* uintptr */ //@item(uintptr, "uintptr", "", "type") +/* zero */ //@item(zero, "zero", "", "var") diff --git a/gopls/internal/lsp/testdata/summary_go1.21.txt.golden b/gopls/internal/lsp/testdata/summary_go1.21.txt.golden index 8d6a32bb986..619c25ba757 100644 --- a/gopls/internal/lsp/testdata/summary_go1.21.txt.golden +++ b/gopls/internal/lsp/testdata/summary_go1.21.txt.golden @@ -1,7 +1,7 @@ -- summary -- CallHierarchyCount = 2 CodeLensCount = 5 -CompletionsCount = 264 +CompletionsCount = 263 CompletionSnippetCount = 115 UnimportedCompletionsCount = 5 DeepCompletionsCount = 5 diff --git a/gopls/internal/lsp/tests/util_go122.go b/gopls/internal/lsp/tests/util_go122.go new file mode 100644 index 00000000000..90ae029766a --- /dev/null +++ b/gopls/internal/lsp/tests/util_go122.go @@ -0,0 +1,12 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.22 +// +build go1.22 + +package tests + +func init() { + builtins["zero"] = true +} From 6eca6dfbb247d138eb92cbbdcb5c2723727693e3 Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Mon, 21 Aug 2023 15:59:15 -0400 Subject: [PATCH 026/178] gopls/doc: include stderr output from 'go list' in pkgDir errors For golang/go#62195. Change-Id: I2be66e7f951cf1b0b800e4aa1423dc4989b7d27a Reviewed-on: https://go-review.googlesource.com/c/tools/+/521124 Auto-Submit: Bryan Mills TryBot-Result: Gopher Robot Reviewed-by: Hyang-Ah Hana Kim Run-TryBot: Bryan Mills gopls-CI: kokoro --- gopls/doc/generate.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/gopls/doc/generate.go b/gopls/doc/generate.go index f7e69972897..51987f6a7b0 100644 --- a/gopls/doc/generate.go +++ b/gopls/doc/generate.go @@ -85,9 +85,13 @@ func doMain(write bool) (bool, error) { // pkgDir returns the directory corresponding to the import path pkgPath. func pkgDir(pkgPath string) (string, error) { - out, err := exec.Command("go", "list", "-f", "{{.Dir}}", pkgPath).Output() + cmd := exec.Command("go", "list", "-f", "{{.Dir}}", pkgPath) + out, err := cmd.Output() if err != nil { - return "", err + if ee, _ := err.(*exec.ExitError); ee != nil && len(ee.Stderr) > 0 { + return "", fmt.Errorf("%v: %w\n%s", cmd, err, ee.Stderr) + } + return "", fmt.Errorf("%v: %w", cmd, err) } return strings.TrimSpace(string(out)), nil } From 2e5fc54113b393d44cdf1e156747c3bcdd1f5b16 Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Tue, 22 Aug 2023 15:32:54 -0400 Subject: [PATCH 027/178] gopls/internal/regtest/codelens: avoid a race in TestUpgradeCodelens Avoid (but do not fix) a race in TestUpgradeCodelens: because upgrade and vuln diagnostics access state on the View, they race with asynchronous diagnostics, which assume that diagnostics are idempotent. Diagnostics _should_ be idempotent: we should create a new snapshot when the view state changes (and inject that state into the snapshot). But that is a larger change, and this CL will confirm the hypothesis above if the flakes go away. For golang/go#58750 Change-Id: If827aab0ae187c5c377d830d76caf626b51bc3bc Reviewed-on: https://go-review.googlesource.com/c/tools/+/521895 TryBot-Result: Gopher Robot Run-TryBot: Robert Findley gopls-CI: kokoro Reviewed-by: Hyang-Ah Hana Kim --- gopls/internal/regtest/codelens/codelens_test.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/gopls/internal/regtest/codelens/codelens_test.go b/gopls/internal/regtest/codelens/codelens_test.go index 8f718855f66..b72e598c913 100644 --- a/gopls/internal/regtest/codelens/codelens_test.go +++ b/gopls/internal/regtest/codelens/codelens_test.go @@ -199,13 +199,25 @@ require golang.org/x/hello v1.2.3 } for _, vendoring := range []bool{false, true} { t.Run(fmt.Sprintf("Upgrade individual dependency vendoring=%v", vendoring), func(t *testing.T) { - WithOptions(ProxyFiles(proxyWithLatest)).Run(t, shouldUpdateDep, func(t *testing.T, env *Env) { + WithOptions( + ProxyFiles(proxyWithLatest), + ).Run(t, shouldUpdateDep, func(t *testing.T, env *Env) { if vendoring { env.RunGoCommandInDirWithEnv("a", []string{"GOWORK=off"}, "mod", "vendor") } env.AfterChange() env.OpenFile("a/go.mod") env.OpenFile("b/go.mod") + + // Await the diagnostics resulting from opening the modfiles, because + // otherwise they may cause races when running asynchronously to the + // explicit re-diagnosing below. + // + // TODO(golang/go#58750): there is still a race here, inherent to + // accessing state on the View; we should create a new snapshot when + // the view diagnostics change. + env.AfterChange() + env.ExecuteCodeLensCommand("a/go.mod", command.CheckUpgrades, nil) d := &protocol.PublishDiagnosticsParams{} env.OnceMet( From 8fd71c085fded07ec79a34b4f6e16f53cc56d8a7 Mon Sep 17 00:00:00 2001 From: Robert Findley Date: Fri, 11 Aug 2023 20:52:39 -0400 Subject: [PATCH 028/178] gopls/internal/lsp/mod: remove TestModfileRemainsUnchanged Testdata for this test includes a go.mod file, which is problematic for a number of reasons (e.g. running from the mod cache; go work use -r). Replace it with a regtest. But really we should just delete the "tempModfile" setting (golang/go#61970). Fixes golang/go#57784 Change-Id: I79726c6106f3118d021a8f9ef52f385f1393d4a8 Reviewed-on: https://go-review.googlesource.com/c/tools/+/518976 gopls-CI: kokoro TryBot-Result: Gopher Robot Reviewed-by: Peter Weinberger Reviewed-by: Bryan Mills Run-TryBot: Robert Findley --- gopls/internal/lsp/mod/mod_test.go | 59 ------------------- .../lsp/mod/testdata/unchanged/go.mod | 1 - .../lsp/mod/testdata/unchanged/main.go | 6 -- .../regtest/modfile/tempmodfile_test.go | 41 +++++++++++++ 4 files changed, 41 insertions(+), 66 deletions(-) delete mode 100644 gopls/internal/lsp/mod/mod_test.go delete mode 100644 gopls/internal/lsp/mod/testdata/unchanged/go.mod delete mode 100644 gopls/internal/lsp/mod/testdata/unchanged/main.go create mode 100644 gopls/internal/regtest/modfile/tempmodfile_test.go diff --git a/gopls/internal/lsp/mod/mod_test.go b/gopls/internal/lsp/mod/mod_test.go deleted file mode 100644 index 4ec3067e4ac..00000000000 --- a/gopls/internal/lsp/mod/mod_test.go +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2019 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package mod - -import ( - "io/ioutil" - "os" - "path/filepath" - "testing" - - "golang.org/x/tools/gopls/internal/lsp/cache" - "golang.org/x/tools/gopls/internal/lsp/source" - "golang.org/x/tools/gopls/internal/lsp/tests" - "golang.org/x/tools/gopls/internal/span" - "golang.org/x/tools/internal/testenv" -) - -func TestMain(m *testing.M) { - testenv.ExitIfSmallMachine() - os.Exit(m.Run()) -} - -func TestModfileRemainsUnchanged(t *testing.T) { - testenv.NeedsExec(t) - - ctx := tests.Context(t) - session := cache.NewSession(ctx, cache.New(nil), nil) - options := source.DefaultOptions().Clone() - tests.DefaultOptions(options) - options.TempModfile = true - options.Env = map[string]string{"GOPACKAGESDRIVER": "off", "GOROOT": ""} - - // Make sure to copy the test directory to a temporary directory so we do not - // modify the test code or add go.sum files when we run the tests. - folder, err := tests.CopyFolderToTempDir(filepath.Join("testdata", "unchanged")) - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(folder) - - before, err := ioutil.ReadFile(filepath.Join(folder, "go.mod")) - if err != nil { - t.Fatal(err) - } - _, _, release, err := session.NewView(ctx, "diagnostics_test", span.URIFromPath(folder), options) - if err != nil { - t.Fatal(err) - } - release() - after, err := ioutil.ReadFile(filepath.Join(folder, "go.mod")) - if err != nil { - t.Fatal(err) - } - if string(before) != string(after) { - t.Errorf("the real go.mod file was changed even when tempModfile=true") - } -} diff --git a/gopls/internal/lsp/mod/testdata/unchanged/go.mod b/gopls/internal/lsp/mod/testdata/unchanged/go.mod deleted file mode 100644 index e3d13cebe54..00000000000 --- a/gopls/internal/lsp/mod/testdata/unchanged/go.mod +++ /dev/null @@ -1 +0,0 @@ -module unchanged diff --git a/gopls/internal/lsp/mod/testdata/unchanged/main.go b/gopls/internal/lsp/mod/testdata/unchanged/main.go deleted file mode 100644 index b258445f438..00000000000 --- a/gopls/internal/lsp/mod/testdata/unchanged/main.go +++ /dev/null @@ -1,6 +0,0 @@ -// Package unchanged does something -package unchanged - -func Yo() { - println("yo") -} diff --git a/gopls/internal/regtest/modfile/tempmodfile_test.go b/gopls/internal/regtest/modfile/tempmodfile_test.go new file mode 100644 index 00000000000..8b0926ab422 --- /dev/null +++ b/gopls/internal/regtest/modfile/tempmodfile_test.go @@ -0,0 +1,41 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package modfile + +import ( + "testing" + + . "golang.org/x/tools/gopls/internal/lsp/regtest" +) + +// This test replaces an older, problematic test (golang/go#57784). But it has +// been a long time since the go command would mutate go.mod files. +// +// TODO(golang/go#61970): the tempModfile setting should be removed entirely. +func TestTempModfileUnchanged(t *testing.T) { + // badMod has a go.mod file that is missing a go directive. + const badMod = ` +-- go.mod -- +module badmod.test/p +-- p.go -- +package p +` + + WithOptions( + Modes(Default), // no reason to test this with a remote gopls + ProxyFiles(workspaceProxy), + Settings{ + "tempModfile": true, + }, + ).Run(t, badMod, func(t *testing.T, env *Env) { + env.OpenFile("p.go") + env.AfterChange() + want := "module badmod.test/p\n" + got := env.ReadWorkspaceFile("go.mod") + if got != want { + t.Errorf("go.mod content:\n%s\nwant:\n%s", got, want) + } + }) +} From 7b7b9a1334c937269bb0ead596bbeae7bac108b1 Mon Sep 17 00:00:00 2001 From: "Hana (Hyang-Ah) Kim" Date: Tue, 15 Aug 2023 20:45:03 -0400 Subject: [PATCH 029/178] gopls/internal/telemetry: write counter unconditionally and run upload Upload is still behind GOPLS_TELEMETRY_EXP env var. Change-Id: I3da6daf50310b05a328f1f8ec920b7295c7f8961 Reviewed-on: https://go-review.googlesource.com/c/tools/+/519875 Run-TryBot: Hyang-Ah Hana Kim Auto-Submit: Hyang-Ah Hana Kim TryBot-Result: Gopher Robot Reviewed-by: Peter Weinberger Reviewed-by: Robert Findley --- gopls/go.mod | 2 +- gopls/go.sum | 8 ++------ gopls/internal/telemetry/telemetry.go | 16 ++++++++++++++-- gopls/internal/telemetry/telemetry_go118.go | 19 +++++++++++++++++++ 4 files changed, 36 insertions(+), 9 deletions(-) create mode 100644 gopls/internal/telemetry/telemetry_go118.go diff --git a/gopls/go.mod b/gopls/go.mod index 1d6fdd3d6b7..77e4f9037bd 100644 --- a/gopls/go.mod +++ b/gopls/go.mod @@ -10,7 +10,7 @@ require ( golang.org/x/mod v0.12.0 golang.org/x/sync v0.3.0 golang.org/x/sys v0.11.0 - golang.org/x/telemetry v0.0.0-20230808152233-a65b40c0fdb0 + golang.org/x/telemetry v0.0.0-20230821195303-8cbe8f819be9 golang.org/x/text v0.12.0 golang.org/x/tools v0.6.0 golang.org/x/vuln v0.0.0-20230110180137-6ad3e3d07815 diff --git a/gopls/go.sum b/gopls/go.sum index 39157d6e719..d60043d5878 100644 --- a/gopls/go.sum +++ b/gopls/go.sum @@ -75,12 +75,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/telemetry v0.0.0-20230728182230-e84a26264b60 h1:OCiXqf7/gdoaS7dKppAtPxi783Ke/JIb+r20ZYGiEFg= -golang.org/x/telemetry v0.0.0-20230728182230-e84a26264b60/go.mod h1:kO7uNSGGmqCHII6C0TYfaLwSBIfcyhj53//nu0+Fy4A= -golang.org/x/telemetry v0.0.0-20230803164656-36ff770d3d6b h1:FZUooIb6Dx+mzx9n5mi6wmY/xpUZ4U1ffUVX1DCsuSs= -golang.org/x/telemetry v0.0.0-20230803164656-36ff770d3d6b/go.mod h1:kO7uNSGGmqCHII6C0TYfaLwSBIfcyhj53//nu0+Fy4A= -golang.org/x/telemetry v0.0.0-20230808152233-a65b40c0fdb0 h1:ZB9hzIbPBkRCCOVWOmfZEI5f6YiTbRAq6LK2x/StgiU= -golang.org/x/telemetry v0.0.0-20230808152233-a65b40c0fdb0/go.mod h1:kO7uNSGGmqCHII6C0TYfaLwSBIfcyhj53//nu0+Fy4A= +golang.org/x/telemetry v0.0.0-20230821195303-8cbe8f819be9 h1:gOhDu2Op3LqXSw1YAR9iRbKBQK70Q3xgXf/N+jDh2k8= +golang.org/x/telemetry v0.0.0-20230821195303-8cbe8f819be9/go.mod h1:kO7uNSGGmqCHII6C0TYfaLwSBIfcyhj53//nu0+Fy4A= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= diff --git a/gopls/internal/telemetry/telemetry.go b/gopls/internal/telemetry/telemetry.go index d3a9f6388a2..067e4f8e6aa 100644 --- a/gopls/internal/telemetry/telemetry.go +++ b/gopls/internal/telemetry/telemetry.go @@ -2,24 +2,36 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build go1.19 +// +build go1.19 + package telemetry import ( "fmt" "os" + "time" "golang.org/x/telemetry/counter" + "golang.org/x/telemetry/upload" "golang.org/x/tools/gopls/internal/lsp/protocol" ) // Start starts telemetry instrumentation. func Start() { + counter.Open() if os.Getenv("GOPLS_TELEMETRY_EXP") != "" { - counter.Open() - // TODO: add upload logic. + go packAndUpload() } } +func packAndUpload() { + start := time.Now() + upload.Run(nil) + elapsed := time.Since(start) + time.AfterFunc(24*time.Hour-elapsed, packAndUpload) +} + // RecordClientInfo records gopls client info. func RecordClientInfo(params *protocol.ParamInitialize) { client := "gopls/client:other" diff --git a/gopls/internal/telemetry/telemetry_go118.go b/gopls/internal/telemetry/telemetry_go118.go new file mode 100644 index 00000000000..b0c1197cb77 --- /dev/null +++ b/gopls/internal/telemetry/telemetry_go118.go @@ -0,0 +1,19 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !go1.19 +// +build !go1.19 + +package telemetry + +import "golang.org/x/tools/gopls/internal/lsp/protocol" + +func Start() { +} + +func RecordClientInfo(params *protocol.ParamInitialize) { +} + +func RecordViewGoVersion(x int) { +} From 0a044c0b5b3c2becb2e8ef5d92acca07f6ab8b60 Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Wed, 23 Aug 2023 11:40:04 -0400 Subject: [PATCH 030/178] gopls/doc: update tests that fail on Kokoro to run at Go 1.21+ TestGenerated and TestLicenses both fail on Kokoro, for unknown reasons, and only need to be run on one Go version. Change-Id: I49950c7b6f34747ee94960f5cc0860cd348b391e Reviewed-on: https://go-review.googlesource.com/c/tools/+/522235 gopls-CI: kokoro Run-TryBot: Robert Findley TryBot-Result: Gopher Robot Reviewed-by: Bryan Mills --- gopls/doc/generate_test.go | 6 +++--- gopls/internal/hooks/licenses_test.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/gopls/doc/generate_test.go b/gopls/doc/generate_test.go index 44e6041721d..6e1c23b94db 100644 --- a/gopls/doc/generate_test.go +++ b/gopls/doc/generate_test.go @@ -12,9 +12,9 @@ import ( func TestGenerated(t *testing.T) { testenv.NeedsGoPackages(t) - // This test fails on 1.18 Kokoro for unknown reasons; in any case, it - // suffices to run this test on any builder. - testenv.NeedsGo1Point(t, 19) + // This test fails on Kokoro, for unknown reasons, so must be run only on TryBots. + // In any case, it suffices to run this test on any builder. + testenv.NeedsGo1Point(t, 21) testenv.NeedsLocalXTools(t) diff --git a/gopls/internal/hooks/licenses_test.go b/gopls/internal/hooks/licenses_test.go index a7853cd5f66..609f05a606c 100644 --- a/gopls/internal/hooks/licenses_test.go +++ b/gopls/internal/hooks/licenses_test.go @@ -18,7 +18,7 @@ func TestLicenses(t *testing.T) { // License text differs for older Go versions because staticcheck or gofumpt // isn't supported for those versions, and this fails for unknown, unrelated // reasons on Kokoro legacy CI. - testenv.NeedsGo1Point(t, 19) + testenv.NeedsGo1Point(t, 21) if runtime.GOOS != "linux" && runtime.GOOS != "darwin" { t.Skip("generating licenses only works on Unixes") From c28af0abbd9a213421dca5a03f6de8c93830263a Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Thu, 24 Aug 2023 11:23:10 -0400 Subject: [PATCH 031/178] gopls/internal/lsp/debug: remove hard-coded version Just use main module information for producing the gopls version, saving us a CL during the release process. This means that we won't see a version if gopls is built in GOPATH mode, but we probably should support this installation mode (if it even works...). Change-Id: Ib964b2dd6b2cf202805507a47f04b5077f4d24c3 Reviewed-on: https://go-review.googlesource.com/c/tools/+/522184 gopls-CI: kokoro Reviewed-by: Hyang-Ah Hana Kim Run-TryBot: Robert Findley TryBot-Result: Gopher Robot --- gopls/internal/lsp/cmd/stats.go | 2 +- .../internal/lsp/cmd/test/integration_test.go | 2 +- gopls/internal/lsp/debug/buildinfo_go1.12.go | 29 -------------- gopls/internal/lsp/debug/buildinfo_go1.18.go | 19 ---------- gopls/internal/lsp/debug/info.go | 38 ++++++++++--------- gopls/internal/lsp/debug/info_test.go | 7 ++-- 6 files changed, 26 insertions(+), 71 deletions(-) delete mode 100644 gopls/internal/lsp/debug/buildinfo_go1.12.go delete mode 100644 gopls/internal/lsp/debug/buildinfo_go1.18.go diff --git a/gopls/internal/lsp/cmd/stats.go b/gopls/internal/lsp/cmd/stats.go index 4986107134e..4e339f1c543 100644 --- a/gopls/internal/lsp/cmd/stats.go +++ b/gopls/internal/lsp/cmd/stats.go @@ -74,7 +74,7 @@ func (s *stats) Run(ctx context.Context, args ...string) error { GOARCH: runtime.GOARCH, GOPLSCACHE: os.Getenv("GOPLSCACHE"), GoVersion: runtime.Version(), - GoplsVersion: debug.Version, + GoplsVersion: debug.Version(), } opts := s.app.options diff --git a/gopls/internal/lsp/cmd/test/integration_test.go b/gopls/internal/lsp/cmd/test/integration_test.go index 5c694d070b8..b7b6d9bd86c 100644 --- a/gopls/internal/lsp/cmd/test/integration_test.go +++ b/gopls/internal/lsp/cmd/test/integration_test.go @@ -55,7 +55,7 @@ func TestVersion(t *testing.T) { tree := writeTree(t, "") // There's not much we can robustly assert about the actual version. - const want = debug.Version // e.g. "master" + want := debug.Version() // e.g. "master" // basic { diff --git a/gopls/internal/lsp/debug/buildinfo_go1.12.go b/gopls/internal/lsp/debug/buildinfo_go1.12.go deleted file mode 100644 index 2f360dbfc70..00000000000 --- a/gopls/internal/lsp/debug/buildinfo_go1.12.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build !go1.18 -// +build !go1.18 - -package debug - -import ( - "runtime" - "runtime/debug" -) - -type BuildInfo struct { - debug.BuildInfo - GoVersion string // Version of Go that produced this binary -} - -func readBuildInfo() (*BuildInfo, bool) { - rinfo, ok := debug.ReadBuildInfo() - if !ok { - return nil, false - } - return &BuildInfo{ - GoVersion: runtime.Version(), - BuildInfo: *rinfo, - }, true -} diff --git a/gopls/internal/lsp/debug/buildinfo_go1.18.go b/gopls/internal/lsp/debug/buildinfo_go1.18.go deleted file mode 100644 index 4121c4bc9cd..00000000000 --- a/gopls/internal/lsp/debug/buildinfo_go1.18.go +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build go1.18 -// +build go1.18 - -package debug - -import ( - "runtime/debug" -) - -type BuildInfo debug.BuildInfo - -func readBuildInfo() (*BuildInfo, bool) { - info, ok := debug.ReadBuildInfo() - return (*BuildInfo)(info), ok -} diff --git a/gopls/internal/lsp/debug/info.go b/gopls/internal/lsp/debug/info.go index 5ce23fc2f59..34e6dd4e2b1 100644 --- a/gopls/internal/lsp/debug/info.go +++ b/gopls/internal/lsp/debug/info.go @@ -30,12 +30,19 @@ const ( ) // Version is a manually-updated mechanism for tracking versions. -const Version = "master" +func Version() string { + if info, ok := debug.ReadBuildInfo(); ok { + if info.Main.Version != "" { + return info.Main.Version + } + } + return "(unknown)" +} // ServerVersion is the format used by gopls to report its version to the // client. This format is structured so that the client can parse it easily. type ServerVersion struct { - *BuildInfo + *debug.BuildInfo Version string } @@ -43,23 +50,18 @@ type ServerVersion struct { // built in module mode, we return a GOPATH-specific message with the // hardcoded version. func VersionInfo() *ServerVersion { - if info, ok := readBuildInfo(); ok { - return getVersion(info) - } - buildInfo := &BuildInfo{} - // go1.17 or earlier, part of s.BuildInfo are embedded fields. - buildInfo.Path = "gopls, built in GOPATH mode" - buildInfo.GoVersion = runtime.Version() - return &ServerVersion{ - Version: Version, - BuildInfo: buildInfo, + if info, ok := debug.ReadBuildInfo(); ok { + return &ServerVersion{ + Version: Version(), + BuildInfo: info, + } } -} - -func getVersion(info *BuildInfo) *ServerVersion { return &ServerVersion{ - Version: Version, - BuildInfo: info, + Version: Version(), + BuildInfo: &debug.BuildInfo{ + Path: "gopls, built in GOPATH mode", + GoVersion: runtime.Version(), + }, } } @@ -125,7 +127,7 @@ func section(w io.Writer, mode PrintMode, title string, body func()) { } func printBuildInfo(w io.Writer, info *ServerVersion, verbose bool, mode PrintMode) { - fmt.Fprintf(w, "%v %v\n", info.Path, Version) + fmt.Fprintf(w, "%v %v\n", info.Path, Version()) printModuleInfo(w, info.Main, mode) if !verbose { return diff --git a/gopls/internal/lsp/debug/info_test.go b/gopls/internal/lsp/debug/info_test.go index 5a536284193..3bc9290c157 100644 --- a/gopls/internal/lsp/debug/info_test.go +++ b/gopls/internal/lsp/debug/info_test.go @@ -27,7 +27,7 @@ func TestPrintVersionInfoJSON(t *testing.T) { if g, w := got.GoVersion, runtime.Version(); g != w { t.Errorf("go version = %v, want %v", g, w) } - if g, w := got.Version, Version; g != w { + if g, w := got.Version, Version(); g != w { t.Errorf("gopls version = %v, want %v", g, w) } // Other fields of BuildInfo may not be available during test. @@ -41,7 +41,8 @@ func TestPrintVersionInfoPlainText(t *testing.T) { res := buf.Bytes() // Other fields of BuildInfo may not be available during test. - if !bytes.Contains(res, []byte(Version)) || !bytes.Contains(res, []byte(runtime.Version())) { - t.Errorf("plaintext output = %q,\nwant (version: %v, go: %v)", res, Version, runtime.Version()) + wantGoplsVersion, wantGoVersion := Version(), runtime.Version() + if !bytes.Contains(res, []byte(wantGoplsVersion)) || !bytes.Contains(res, []byte(wantGoVersion)) { + t.Errorf("plaintext output = %q,\nwant (version: %v, go: %v)", res, wantGoplsVersion, wantGoVersion) } } From 39bfef48d3b68fca9707aae7b9961202bc76ad55 Mon Sep 17 00:00:00 2001 From: "Hana (Hyang-Ah) Kim" Date: Thu, 24 Aug 2023 14:11:07 -0400 Subject: [PATCH 032/178] gopls: update x/telemetry dependency This picks up the fix for timestamp timezone issue https://go.dev/cl/521795 Change-Id: Ica486a89cca7992fa60ab97e2b3f192c9ce7dd16 Reviewed-on: https://go-review.googlesource.com/c/tools/+/522186 Reviewed-by: Peter Weinberger TryBot-Result: Gopher Robot Run-TryBot: Hyang-Ah Hana Kim gopls-CI: kokoro --- gopls/go.mod | 2 +- gopls/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/gopls/go.mod b/gopls/go.mod index 77e4f9037bd..f1c3aa130ae 100644 --- a/gopls/go.mod +++ b/gopls/go.mod @@ -10,7 +10,7 @@ require ( golang.org/x/mod v0.12.0 golang.org/x/sync v0.3.0 golang.org/x/sys v0.11.0 - golang.org/x/telemetry v0.0.0-20230821195303-8cbe8f819be9 + golang.org/x/telemetry v0.0.0-20230822160736-17171dbf1d76 golang.org/x/text v0.12.0 golang.org/x/tools v0.6.0 golang.org/x/vuln v0.0.0-20230110180137-6ad3e3d07815 diff --git a/gopls/go.sum b/gopls/go.sum index d60043d5878..4085d484b6c 100644 --- a/gopls/go.sum +++ b/gopls/go.sum @@ -75,8 +75,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/telemetry v0.0.0-20230821195303-8cbe8f819be9 h1:gOhDu2Op3LqXSw1YAR9iRbKBQK70Q3xgXf/N+jDh2k8= -golang.org/x/telemetry v0.0.0-20230821195303-8cbe8f819be9/go.mod h1:kO7uNSGGmqCHII6C0TYfaLwSBIfcyhj53//nu0+Fy4A= +golang.org/x/telemetry v0.0.0-20230822160736-17171dbf1d76 h1:Lv25uIMpljmSMN0+GCC+xgiC/4ikIdKMkQfw/EVq2Nk= +golang.org/x/telemetry v0.0.0-20230822160736-17171dbf1d76/go.mod h1:kO7uNSGGmqCHII6C0TYfaLwSBIfcyhj53//nu0+Fy4A= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= From 7e1bfe8bc98ca8e94b7853eca0993bc9ee369101 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Sun, 16 Jul 2023 13:14:42 -0400 Subject: [PATCH 033/178] go/analysis/unitchecker: Example of separate analysis This change adds an Example that demonstrates the principle of separate analysis by having one program (the manager) visit the import graph in postorder, invoking a second program (the worker) as a child process once per package. The manager is analogous to go vet, and the worker to the vettool. Because unitchecker is closely coupled to go vet, both in its config file, and in assumptions about where and how facts and types are produced, this change parameterizes unitchecker to illustrate that not just facts but types too may be produced by the worker, without the need for the compiler. This may be a simpler and more efficient design for (say) a distributed analysis system. There are no changes to the public API yet (a proposal will follow) so the example can't yet serve directly as the basis for a new implementation, but in the meantime it should at least illuminate what is involved and, perhaps, how unitchecker can be easily forked and adapted. The changes to unitchecker should not affect the behavior of existing programs. Change-Id: I4ae414ccde91853c77b8a45771dd6e5cfc015173 Reviewed-on: https://go-review.googlesource.com/c/tools/+/510215 TryBot-Result: Gopher Robot Reviewed-by: Robert Findley Auto-Submit: Alan Donovan Run-TryBot: Alan Donovan --- go/analysis/unitchecker/export_test.go | 26 ++ go/analysis/unitchecker/separate_test.go | 302 ++++++++++++++++++++ go/analysis/unitchecker/unitchecker.go | 111 ++++--- go/analysis/unitchecker/unitchecker_test.go | 3 + 4 files changed, 398 insertions(+), 44 deletions(-) create mode 100644 go/analysis/unitchecker/export_test.go create mode 100644 go/analysis/unitchecker/separate_test.go diff --git a/go/analysis/unitchecker/export_test.go b/go/analysis/unitchecker/export_test.go new file mode 100644 index 00000000000..04eacc47576 --- /dev/null +++ b/go/analysis/unitchecker/export_test.go @@ -0,0 +1,26 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package unitchecker + +import ( + "go/token" + "go/types" +) + +// This file exposes various internal hooks to the separate_test. +// +// TODO(adonovan): expose a public API to unitchecker that doesn't +// rely on details of JSON .cfg files or enshrine I/O decisions or +// assumptions about how "go vet" locates things. Ideally the new Run +// function would accept an interface, and a Config file would be just +// one way--the go vet way--to implement it. + +func SetTypeImportExport( + MakeTypesImporter func(*Config, *token.FileSet) types.Importer, + ExportTypes func(*Config, *token.FileSet, *types.Package) error, +) { + makeTypesImporter = MakeTypesImporter + exportTypes = ExportTypes +} diff --git a/go/analysis/unitchecker/separate_test.go b/go/analysis/unitchecker/separate_test.go new file mode 100644 index 00000000000..12d91043d09 --- /dev/null +++ b/go/analysis/unitchecker/separate_test.go @@ -0,0 +1,302 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package unitchecker_test + +// This file illustrates separate analysis with an example. + +import ( + "bytes" + "encoding/json" + "fmt" + "go/token" + "go/types" + "io" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + "sync/atomic" + + "golang.org/x/tools/go/analysis/passes/printf" + "golang.org/x/tools/go/analysis/unitchecker" + "golang.org/x/tools/go/gcexportdata" + "golang.org/x/tools/go/packages" + "golang.org/x/tools/txtar" +) + +// ExampleSeparateAnalysis demonstrates the principle of separate +// analysis, the distribution of units of type-checking and analysis +// work across several processes, using serialized summaries to +// communicate between them. +// +// It uses two different kinds of task, "manager" and "worker": +// +// - The manager computes the graph of package dependencies, and makes +// a request to the worker for each package. It does not parse, +// type-check, or analyze Go code. It is analogous "go vet". +// +// - The worker, which contains the Analyzers, reads each request, +// loads, parses, and type-checks the files of one package, +// applies all necessary analyzers to the package, then writes +// its results to a file. It is a unitchecker-based driver, +// analogous to the program specified by go vet -vettool= flag. +// +// In practice these would be separate executables, but for simplicity +// of this example they are provided by one executable in two +// different modes: the Example function is the manager, and the same +// executable invoked with ENTRYPOINT=worker is the worker. +// (See TestIntegration for how this happens.) +func ExampleSeparateAnalysis() { + // src is an archive containing a module with a printf mistake. + const src = ` +-- go.mod -- +module separate +go 1.18 + +-- main/main.go -- +package main + +import "separate/lib" + +func main() { + lib.MyPrintf("%s", 123) +} + +-- lib/lib.go -- +package lib + +import "fmt" + +func MyPrintf(format string, args ...any) { + fmt.Printf(format, args...) +} +` + + // Expand archive into tmp tree. + tmpdir, err := os.MkdirTemp("", "SeparateAnalysis") + if err != nil { + log.Fatal(err) + } + if err := extractTxtar(txtar.Parse([]byte(src)), tmpdir); err != nil { + log.Fatal(err) + } + + // Load metadata for the main package and all its dependencies. + cfg := &packages.Config{ + Mode: packages.NeedName | packages.NeedFiles | packages.NeedCompiledGoFiles | packages.NeedImports | packages.NeedModule, + Dir: tmpdir, + Env: append(os.Environ(), + "GOPROXY=off", // disable network + "GOWORK=off", // an ambient GOWORK value would break package loading + ), + } + pkgs, err := packages.Load(cfg, "separate/main") + if err != nil { + log.Fatal(err) + } + // Stop if any package had a metadata error. + if packages.PrintErrors(pkgs) > 0 { + os.Exit(1) + } + + // Now we have loaded the import graph, + // let's begin the proper work of the manager. + + // Gather root packages. They will get all analyzers, + // whereas dependencies get only the subset that + // produce facts or are required by them. + roots := make(map[*packages.Package]bool) + for _, pkg := range pkgs { + roots[pkg] = true + } + + // nextID generates sequence numbers for each unit of work. + // We use it to create names of temporary files. + var nextID atomic.Int32 + + // Visit all packages in postorder: dependencies first. + // TODO(adonovan): opt: use parallel postorder. + packages.Visit(pkgs, nil, func(pkg *packages.Package) { + if pkg.PkgPath == "unsafe" { + return + } + + // Choose a unique prefix for temporary files + // (.cfg .types .facts) produced by this package. + // We stow it in an otherwise unused field of + // Package so it can be accessed by our importers. + prefix := fmt.Sprintf("%s/%d", tmpdir, nextID.Add(1)) + pkg.ExportFile = prefix + + // Construct the request to the worker. + var ( + importMap = make(map[string]string) + packageFile = make(map[string]string) + packageVetx = make(map[string]string) + ) + for importPath, dep := range pkg.Imports { + importMap[importPath] = dep.PkgPath + if depPrefix := dep.ExportFile; depPrefix != "" { // skip "unsafe" + packageFile[dep.PkgPath] = depPrefix + ".types" + packageVetx[dep.PkgPath] = depPrefix + ".facts" + } + } + cfg := unitchecker.Config{ + ID: pkg.ID, + ImportPath: pkg.PkgPath, + GoFiles: pkg.CompiledGoFiles, + NonGoFiles: pkg.OtherFiles, + IgnoredFiles: pkg.IgnoredFiles, + ImportMap: importMap, + PackageFile: packageFile, + PackageVetx: packageVetx, + VetxOnly: !roots[pkg], + VetxOutput: prefix + ".facts", + } + if pkg.Module != nil { + if v := pkg.Module.GoVersion; v != "" { + cfg.GoVersion = "go" + v + } + } + + // Write the JSON configuration message to a file. + cfgData, err := json.Marshal(cfg) + if err != nil { + log.Fatal(err) + } + cfgFile := prefix + ".cfg" + if err := os.WriteFile(cfgFile, cfgData, 0666); err != nil { + log.Fatal(err) + } + + // Send the request to the worker. + cmd := exec.Command(os.Args[0], "-json", cfgFile) + cmd.Stderr = os.Stderr + cmd.Stdout = new(bytes.Buffer) + cmd.Env = append(os.Environ(), "ENTRYPOINT=worker") + if err := cmd.Run(); err != nil { + log.Fatal(err) + } + + // Parse JSON output and print plainly. + dec := json.NewDecoder(cmd.Stdout.(io.Reader)) + for { + type jsonDiagnostic struct { + Posn string `json:"posn"` + Message string `json:"message"` + } + // 'results' maps Package.Path -> Analyzer.Name -> diagnostics + var results map[string]map[string][]jsonDiagnostic + if err := dec.Decode(&results); err != nil { + if err == io.EOF { + break + } + log.Fatal(err) + } + for _, result := range results { + for analyzer, diags := range result { + for _, diag := range diags { + rel := strings.ReplaceAll(diag.Posn, tmpdir, "") + rel = filepath.ToSlash(rel) + fmt.Printf("%s: [%s] %s\n", + rel, analyzer, diag.Message) + } + } + } + } + }) + + // Observe that the example produces a fact-based diagnostic + // from separate analysis of "main", "lib", and "fmt": + + // Output: + // /main/main.go:6:2: [printf] separate/lib.MyPrintf format %s has arg 123 of wrong type int +} + +// -- worker process -- + +// worker is the main entry point for a unitchecker-based driver +// with only a single analyzer, for illustration. +func worker() { + // Currently the unitchecker API doesn't allow clients to + // control exactly how and where fact and type information + // is produced and consumed. + // + // So, for example, it assumes that type information has + // already been produced by the compiler, which is true when + // running under "go vet", but isn't necessary. It may be more + // convenient and efficient for a distributed analysis system + // if the worker generates both of them, which is the approach + // taken in this example; they could even be saved as two + // sections of a single file. + // + // Consequently, this test currently needs special access to + // private hooks in unitchecker to control how and where facts + // and types are produced and consumed. In due course this + // will become a respectable public API. In the meantime, it + // should at least serve as a demonstration of how one could + // fork unitchecker to achieve separate analysis without go vet. + unitchecker.SetTypeImportExport(makeTypesImporter, exportTypes) + + unitchecker.Main(printf.Analyzer) +} + +func makeTypesImporter(cfg *unitchecker.Config, fset *token.FileSet) types.Importer { + imports := make(map[string]*types.Package) + return importerFunc(func(importPath string) (*types.Package, error) { + // Resolve import path to package path (vendoring, etc) + path, ok := cfg.ImportMap[importPath] + if !ok { + return nil, fmt.Errorf("can't resolve import %q", path) + } + if path == "unsafe" { + return types.Unsafe, nil + } + + // Find, read, and decode file containing type information. + file, ok := cfg.PackageFile[path] + if !ok { + return nil, fmt.Errorf("no package file for %q", path) + } + f, err := os.Open(file) + if err != nil { + return nil, err + } + defer f.Close() // ignore error + return gcexportdata.Read(f, fset, imports, path) + }) +} + +func exportTypes(cfg *unitchecker.Config, fset *token.FileSet, pkg *types.Package) error { + var out bytes.Buffer + if err := gcexportdata.Write(&out, fset, pkg); err != nil { + return err + } + typesFile := strings.TrimSuffix(cfg.VetxOutput, ".facts") + ".types" + return os.WriteFile(typesFile, out.Bytes(), 0666) +} + +// -- helpers -- + +type importerFunc func(path string) (*types.Package, error) + +func (f importerFunc) Import(path string) (*types.Package, error) { return f(path) } + +// extractTxtar writes each archive file to the corresponding location beneath dir. +// +// TODO(adonovan): move this to txtar package, we need it all the time (#61386). +func extractTxtar(ar *txtar.Archive, dir string) error { + for _, file := range ar.Files { + name := filepath.Join(dir, file.Name) + if err := os.MkdirAll(filepath.Dir(name), 0777); err != nil { + return err + } + if err := os.WriteFile(name, file.Data, 0666); err != nil { + return err + } + } + return nil +} diff --git a/go/analysis/unitchecker/unitchecker.go b/go/analysis/unitchecker/unitchecker.go index 88527d7a8e2..4ff45feb4ce 100644 --- a/go/analysis/unitchecker/unitchecker.go +++ b/go/analysis/unitchecker/unitchecker.go @@ -38,7 +38,6 @@ import ( "go/token" "go/types" "io" - "io/ioutil" "log" "os" "path/filepath" @@ -59,19 +58,19 @@ import ( // whose name ends with ".cfg". type Config struct { ID string // e.g. "fmt [fmt.test]" - Compiler string - Dir string - ImportPath string + Compiler string // gc or gccgo, provided to MakeImporter + Dir string // (unused) + ImportPath string // package path GoVersion string // minimum required Go version, such as "go1.21.0" GoFiles []string NonGoFiles []string IgnoredFiles []string - ImportMap map[string]string - PackageFile map[string]string - Standard map[string]bool - PackageVetx map[string]string - VetxOnly bool - VetxOutput string + ImportMap map[string]string // maps import path to package path + PackageFile map[string]string // maps package path to file of type information + Standard map[string]bool // package belongs to standard library + PackageVetx map[string]string // maps package path to file of fact information + VetxOnly bool // run analysis only for facts, not diagnostics + VetxOutput string // where to write file of fact information SucceedOnTypecheckFailure bool } @@ -167,7 +166,7 @@ func Run(configFile string, analyzers []*analysis.Analyzer) { } func readConfig(filename string) (*Config, error) { - data, err := ioutil.ReadFile(filename) + data, err := os.ReadFile(filename) if err != nil { return nil, err } @@ -184,6 +183,55 @@ func readConfig(filename string) (*Config, error) { return cfg, nil } +type factImporter = func(pkgPath string) ([]byte, error) + +// These four hook variables are a proof of concept of a future +// parameterization of a unitchecker API that allows the client to +// determine how and where facts and types are produced and consumed. +// (Note that the eventual API will likely be quite different.) +// +// The defaults honor a Config in a manner compatible with 'go vet'. +var ( + makeTypesImporter = func(cfg *Config, fset *token.FileSet) types.Importer { + return importer.ForCompiler(fset, cfg.Compiler, func(importPath string) (io.ReadCloser, error) { + // Resolve import path to package path (vendoring, etc) + path, ok := cfg.ImportMap[importPath] + if !ok { + return nil, fmt.Errorf("can't resolve import %q", path) + } + + // path is a resolved package path, not an import path. + file, ok := cfg.PackageFile[path] + if !ok { + if cfg.Compiler == "gccgo" && cfg.Standard[path] { + return nil, nil // fall back to default gccgo lookup + } + return nil, fmt.Errorf("no package file for %q", path) + } + return os.Open(file) + }) + } + + exportTypes = func(*Config, *token.FileSet, *types.Package) error { + // By default this is a no-op, because "go vet" + // makes the compiler produce type information. + return nil + } + + makeFactImporter = func(cfg *Config) factImporter { + return func(pkgPath string) ([]byte, error) { + if vetx, ok := cfg.PackageVetx[pkgPath]; ok { + return os.ReadFile(vetx) + } + return nil, nil // no .vetx file, no facts + } + } + + exportFacts = func(cfg *Config, data []byte) error { + return os.WriteFile(cfg.VetxOutput, data, 0666) + } +) + func run(fset *token.FileSet, cfg *Config, analyzers []*analysis.Analyzer) ([]result, error) { // Load, parse, typecheck. var files []*ast.File @@ -199,27 +247,9 @@ func run(fset *token.FileSet, cfg *Config, analyzers []*analysis.Analyzer) ([]re } files = append(files, f) } - compilerImporter := importer.ForCompiler(fset, cfg.Compiler, func(path string) (io.ReadCloser, error) { - // path is a resolved package path, not an import path. - file, ok := cfg.PackageFile[path] - if !ok { - if cfg.Compiler == "gccgo" && cfg.Standard[path] { - return nil, nil // fall back to default gccgo lookup - } - return nil, fmt.Errorf("no package file for %q", path) - } - return os.Open(file) - }) - importer := importerFunc(func(importPath string) (*types.Package, error) { - path, ok := cfg.ImportMap[importPath] // resolve vendoring, etc - if !ok { - return nil, fmt.Errorf("can't resolve import %q", path) - } - return compilerImporter.Import(path) - }) tc := &types.Config{ - Importer: importer, - Sizes: types.SizesFor("gc", build.Default.GOARCH), // assume gccgo ≡ gc? + Importer: makeTypesImporter(cfg, fset), + Sizes: types.SizesFor("gc", build.Default.GOARCH), // TODO(adonovan): use cfg.Compiler GoVersion: cfg.GoVersion, } info := &types.Info{ @@ -288,13 +318,7 @@ func run(fset *token.FileSet, cfg *Config, analyzers []*analysis.Analyzer) ([]re analyzers = filtered // Read facts from imported packages. - read := func(pkgPath string) ([]byte, error) { - if vetx, ok := cfg.PackageVetx[pkgPath]; ok { - return ioutil.ReadFile(vetx) - } - return nil, nil // no .vetx file, no facts - } - facts, err := facts.NewDecoder(pkg).Decode(false, read) + facts, err := facts.NewDecoder(pkg).Decode(false, makeFactImporter(cfg)) if err != nil { return nil, err } @@ -394,8 +418,11 @@ func run(fset *token.FileSet, cfg *Config, analyzers []*analysis.Analyzer) ([]re } data := facts.Encode(false) - if err := ioutil.WriteFile(cfg.VetxOutput, data, 0666); err != nil { - return nil, fmt.Errorf("failed to write analysis facts: %v", err) + if err := exportFacts(cfg, data); err != nil { + return nil, fmt.Errorf("failed to export analysis facts: %v", err) + } + if err := exportTypes(cfg, fset, pkg); err != nil { + return nil, fmt.Errorf("failed to export type information: %v", err) } return results, nil @@ -406,7 +433,3 @@ type result struct { diagnostics []analysis.Diagnostic err error } - -type importerFunc func(path string) (*types.Package, error) - -func (f importerFunc) Import(path string) (*types.Package, error) { return f(path) } diff --git a/go/analysis/unitchecker/unitchecker_test.go b/go/analysis/unitchecker/unitchecker_test.go index 1ed001247c6..270a3582ccf 100644 --- a/go/analysis/unitchecker/unitchecker_test.go +++ b/go/analysis/unitchecker/unitchecker_test.go @@ -29,6 +29,9 @@ func TestMain(m *testing.M) { case "minivet": minivet() panic("unreachable") + case "worker": + worker() // see ExampleSeparateAnalysis + panic("unreachable") } // test process From c51e3f732f19410479f1bafb6f8a2b0f693a96a7 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Tue, 18 Jul 2023 19:18:17 -0400 Subject: [PATCH 034/178] gopls/internal/lsp/frob: make API generic Change-Id: Ia3f2c68cec6588fa23f35d1a125d9f602767f375 Reviewed-on: https://go-review.googlesource.com/c/tools/+/510378 Run-TryBot: Alan Donovan TryBot-Result: Gopher Robot Reviewed-by: Robert Findley Auto-Submit: Alan Donovan --- gopls/internal/lsp/cache/analysis.go | 6 +- gopls/internal/lsp/frob/frob.go | 88 +++++-------------- gopls/internal/lsp/frob/frob_test.go | 4 +- .../lsp/source/methodsets/methodsets.go | 3 +- gopls/internal/lsp/source/typerefs/refs.go | 3 +- gopls/internal/lsp/source/xrefs/xrefs.go | 3 +- 6 files changed, 30 insertions(+), 77 deletions(-) diff --git a/gopls/internal/lsp/cache/analysis.go b/gopls/internal/lsp/cache/analysis.go index c5f6f7293f4..1cfa84dba59 100644 --- a/gopls/internal/lsp/cache/analysis.go +++ b/gopls/internal/lsp/cache/analysis.go @@ -1403,14 +1403,12 @@ func mustEncode(x interface{}) []byte { return buf.Bytes() } -// var analyzeSummaryCodec = frob.For[*analyzeSummary]() -var analyzeSummaryCodec = frob.CodecFor117(new(*analyzeSummary)) +var analyzeSummaryCodec = frob.CodecFor[*analyzeSummary]() // -- data types for serialization of analysis.Diagnostic and source.Diagnostic -- // (The name says gob but we use frob.) -// var diagnosticsCodec = frob.For[[]gobDiagnostic]() -var diagnosticsCodec = frob.CodecFor117(new([]gobDiagnostic)) +var diagnosticsCodec = frob.CodecFor[[]gobDiagnostic]() type gobDiagnostic struct { Location protocol.Location diff --git a/gopls/internal/lsp/frob/frob.go b/gopls/internal/lsp/frob/frob.go index 5582ebee0df..e3abcfeb277 100644 --- a/gopls/internal/lsp/frob/frob.go +++ b/gopls/internal/lsp/frob/frob.go @@ -3,11 +3,18 @@ // license that can be found in the LICENSE file. // Package frob is a fast restricted object encoder/decoder in the -// spirit of gob. Restrictions include: +// spirit of encoding/gob. // -// - Interface values are not supported. This avoids the need for +// As with gob, types that recursively contain functions, +// channels, and unsafe.Pointers cannot encoded, but frob has these +// additional restrictions: +// +// - Interface values are not supported; this avoids the need for // the encoding to describe types. // +// - Types that recursively contain private struct fields are not +// permitted. +// // - The encoding is unspecified and subject to change, so the encoder // and decoder must exactly agree on their implementation and on the // definitions of the target types. @@ -33,38 +40,20 @@ import ( "sync" ) -// Use CodecFor117(new(T)) to create a codec for values of type T. -// Then call Encode(T) and Decode(data, *T). -// This is a placeholder for the forthcoming generic API -- see below. -// CodecFor117 panics if type T is unsuitable. -func CodecFor117(x any) Codec { +// A Codec[T] is an immutable encoder and decoder for values of type T. +type Codec[T any] struct{ frob *frob } + +// CodecFor[T] returns a codec for values of type T. +// It panics if type T is unsuitable. +func CodecFor[T any]() Codec[T] { frobsMu.Lock() defer frobsMu.Unlock() - return Codec{frobFor(reflect.TypeOf(x).Elem())} + return Codec[T]{frobFor(reflect.TypeOf((*T)(nil)).Elem())} } -type any = interface{} - -// A Codec is an immutable encoder and decoder for values of a particular type. -type Codec struct{ *frob } - -// TODO(adonovan): after go1.18, enable this generic interface. -/* - -// CodecFor[T] returns a codec for values of type T. -// -// For panics if the type recursively contains members of unsupported -// types: functions, channels, interfaces, unsafe.Pointer. -func CodecFor[T any]() Codec[T] { return For117((*T)(nil)) } - -// A Codec[T] is an immutable encoder and decoder for values of type T. -type Codec[T any] struct{ frob *frob } - func (codec Codec[T]) Encode(v T) []byte { return codec.frob.Encode(v) } func (codec Codec[T]) Decode(data []byte, ptr *T) { codec.frob.Decode(data, ptr) } -*/ - var ( frobsMu sync.Mutex frobs = make(map[reflect.Type]*frob) @@ -106,7 +95,7 @@ func frobFor(t reflect.Type) *frob { case reflect.Array, reflect.Slice, - reflect.Ptr: // TODO(adonovan): after go1.18, use Pointer + reflect.Pointer: fr.addElem(fr.t.Elem()) case reflect.Map: @@ -224,7 +213,7 @@ func (fr *frob) encode(out *writer, v reflect.Value) { } } - case reflect.Ptr: // TODO(adonovan): after go1.18, use Pointer + case reflect.Pointer: if v.IsNil() { out.uint8(0) } else { @@ -337,7 +326,7 @@ func (fr *frob) decode(in *reader, addr reflect.Value) { kzero := reflect.Zero(kfrob.t) vzero := reflect.Zero(vfrob.t) for i := 0; i < len; i++ { - // TODO(adonovan): after go1.18, use SetZero. + // TODO(adonovan): use SetZero from go1.20. // k.SetZero() // v.SetZero() k.Set(kzero) @@ -348,7 +337,7 @@ func (fr *frob) decode(in *reader, addr reflect.Value) { } } - case reflect.Ptr: // TODO(adonovan): after go1.18, use Pointer + case reflect.Pointer: isNil := in.uint8() == 0 if !isNil { ptr := reflect.New(fr.elems[0].t) @@ -409,38 +398,7 @@ func (r *reader) bytes(n int) []byte { type writer struct{ data []byte } func (w *writer) uint8(v uint8) { w.data = append(w.data, v) } -func (w *writer) uint16(v uint16) { w.data = appendUint16(w.data, v) } -func (w *writer) uint32(v uint32) { w.data = appendUint32(w.data, v) } -func (w *writer) uint64(v uint64) { w.data = appendUint64(w.data, v) } +func (w *writer) uint16(v uint16) { w.data = le.AppendUint16(w.data, v) } +func (w *writer) uint32(v uint32) { w.data = le.AppendUint32(w.data, v) } +func (w *writer) uint64(v uint64) { w.data = le.AppendUint64(w.data, v) } func (w *writer) bytes(v []byte) { w.data = append(w.data, v...) } - -// TODO(adonovan): delete these as in go1.18 they are methods on LittleEndian: - -func appendUint16(b []byte, v uint16) []byte { - return append(b, - byte(v), - byte(v>>8), - ) -} - -func appendUint32(b []byte, v uint32) []byte { - return append(b, - byte(v), - byte(v>>8), - byte(v>>16), - byte(v>>24), - ) -} - -func appendUint64(b []byte, v uint64) []byte { - return append(b, - byte(v), - byte(v>>8), - byte(v>>16), - byte(v>>24), - byte(v>>32), - byte(v>>40), - byte(v>>48), - byte(v>>56), - ) -} diff --git a/gopls/internal/lsp/frob/frob_test.go b/gopls/internal/lsp/frob/frob_test.go index d2a9f2a5bc7..892f18b0eec 100644 --- a/gopls/internal/lsp/frob/frob_test.go +++ b/gopls/internal/lsp/frob/frob_test.go @@ -19,7 +19,7 @@ func TestBasics(t *testing.T) { C *Basics D map[string]int } - codec := frob.CodecFor117(new(Basics)) + codec := frob.CodecFor[Basics]() s1, s2 := "hello", "world" x := Basics{ @@ -55,7 +55,7 @@ func TestInts(t *testing.T) { C64 complex64 C128 complex128 } - codec := frob.CodecFor117(new(Ints)) + codec := frob.CodecFor[Ints]() // maxima max1 := Ints{ diff --git a/gopls/internal/lsp/source/methodsets/methodsets.go b/gopls/internal/lsp/source/methodsets/methodsets.go index 1ade7402421..d934c3c6907 100644 --- a/gopls/internal/lsp/source/methodsets/methodsets.go +++ b/gopls/internal/lsp/source/methodsets/methodsets.go @@ -455,8 +455,7 @@ func fingerprint(method *types.Func) (string, bool) { // -- serial format of index -- // (The name says gob but in fact we use frob.) -// var packageCodec = frob.For[gobPackage]() -var packageCodec = frob.CodecFor117(new(gobPackage)) +var packageCodec = frob.CodecFor[gobPackage]() // A gobPackage records the method set of each package-level type for a single package. type gobPackage struct { diff --git a/gopls/internal/lsp/source/typerefs/refs.go b/gopls/internal/lsp/source/typerefs/refs.go index 2f6b1d92ee4..9adbb88fe4c 100644 --- a/gopls/internal/lsp/source/typerefs/refs.go +++ b/gopls/internal/lsp/source/typerefs/refs.go @@ -738,8 +738,7 @@ func assert(cond bool, msg string) { // -- serialization -- // (The name says gob but in fact we use frob.) -// var classesCodec = frob.For[gobClasses]() -var classesCodec = frob.CodecFor117(new(gobClasses)) +var classesCodec = frob.CodecFor[gobClasses]() type gobClasses struct { Strings []string // table of strings (PackageIDs and names) diff --git a/gopls/internal/lsp/source/xrefs/xrefs.go b/gopls/internal/lsp/source/xrefs/xrefs.go index 0a8d5741157..88f76b1eb64 100644 --- a/gopls/internal/lsp/source/xrefs/xrefs.go +++ b/gopls/internal/lsp/source/xrefs/xrefs.go @@ -172,8 +172,7 @@ func Lookup(m *source.Metadata, data []byte, targets map[source.PackagePath]map[ // The gobRef.Range field is the obvious place to begin. // (The name says gob but in fact we use frob.) -// var packageCodec = frob.For[[]*gobPackage]() -var packageCodec = frob.CodecFor117(new([]*gobPackage)) +var packageCodec = frob.CodecFor[[]*gobPackage]() // A gobPackage records the set of outgoing references from the index // package to symbols defined in a dependency package. From 5fb106a913f107c9bfe4fc1fa589721d73b799a6 Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Thu, 24 Aug 2023 15:52:54 -0400 Subject: [PATCH 035/178] internal/testenv: simplify 'go build' support check We have tried various approaches in testenv.hasTool to detect whether 'go build' works without actually building a binary. However, they all turn out to be fairly fragile. If cgo is enabled then we need a C linker, but if the platform requires external linking then cgo must be enabled too. Instead of trying to second-guess whether 'go build' will actually work, let's just ask it to build a binary and see whether it succeeds. Hopefully this appreach will be simple enough in practice. This parallels the approach taken in CL 492979. Fixes golang/go#62268. Updates golang/go#46330. Change-Id: I9fd85fb73d291306e7bfb2f3ef6cf75a09cb6bed Reviewed-on: https://go-review.googlesource.com/c/tools/+/522795 Run-TryBot: Bryan Mills Auto-Submit: Bryan Mills TryBot-Result: Gopher Robot Reviewed-by: Michael Matloob gopls-CI: kokoro --- .../internal/versiontest/version_test.go | 7 +++ internal/testenv/testenv.go | 62 +++++++++++-------- 2 files changed, 42 insertions(+), 27 deletions(-) diff --git a/go/analysis/internal/versiontest/version_test.go b/go/analysis/internal/versiontest/version_test.go index 45eef8b89d2..43c52f565f7 100644 --- a/go/analysis/internal/versiontest/version_test.go +++ b/go/analysis/internal/versiontest/version_test.go @@ -20,6 +20,7 @@ import ( "golang.org/x/tools/go/analysis/analysistest" "golang.org/x/tools/go/analysis/multichecker" "golang.org/x/tools/go/analysis/singlechecker" + "golang.org/x/tools/internal/testenv" ) var analyzer = &analysis.Analyzer{ @@ -60,6 +61,8 @@ func TestAnalysistest(t *testing.T) { } func TestMultichecker(t *testing.T) { + testenv.NeedsGoPackages(t) + exe, err := os.Executable() if err != nil { t.Fatal(err) @@ -74,6 +77,8 @@ func TestMultichecker(t *testing.T) { } func TestSinglechecker(t *testing.T) { + testenv.NeedsGoPackages(t) + exe, err := os.Executable() if err != nil { t.Fatal(err) @@ -88,6 +93,8 @@ func TestSinglechecker(t *testing.T) { } func TestVettool(t *testing.T) { + testenv.NeedsGoPackages(t) + exe, err := os.Executable() if err != nil { t.Fatal(err) diff --git a/internal/testenv/testenv.go b/internal/testenv/testenv.go index 9b01888adaa..0fe217b3c16 100644 --- a/internal/testenv/testenv.go +++ b/internal/testenv/testenv.go @@ -42,7 +42,7 @@ func packageMainIsDevel() bool { return info.Main.Version == "(devel)" } -var checkGoGoroot struct { +var checkGoBuild struct { once sync.Once err error } @@ -79,40 +79,48 @@ func hasTool(tool string) error { } case "go": - checkGoGoroot.once.Do(func() { - // Ensure that the 'go' command found by exec.LookPath is from the correct - // GOROOT. Otherwise, 'some/path/go test ./...' will test against some - // version of the 'go' binary other than 'some/path/go', which is almost - // certainly not what the user intended. - out, err := exec.Command(tool, "env", "GOROOT").CombinedOutput() - if err != nil { - checkGoGoroot.err = err - return + checkGoBuild.once.Do(func() { + if runtime.GOROOT() != "" { + // Ensure that the 'go' command found by exec.LookPath is from the correct + // GOROOT. Otherwise, 'some/path/go test ./...' will test against some + // version of the 'go' binary other than 'some/path/go', which is almost + // certainly not what the user intended. + out, err := exec.Command(tool, "env", "GOROOT").CombinedOutput() + if err != nil { + checkGoBuild.err = err + return + } + GOROOT := strings.TrimSpace(string(out)) + if GOROOT != runtime.GOROOT() { + checkGoBuild.err = fmt.Errorf("'go env GOROOT' does not match runtime.GOROOT:\n\tgo env: %s\n\tGOROOT: %s", GOROOT, runtime.GOROOT()) + return + } } - GOROOT := strings.TrimSpace(string(out)) - if GOROOT != runtime.GOROOT() { - checkGoGoroot.err = fmt.Errorf("'go env GOROOT' does not match runtime.GOROOT:\n\tgo env: %s\n\tGOROOT: %s", GOROOT, runtime.GOROOT()) + + dir, err := os.MkdirTemp("", "testenv-*") + if err != nil { + checkGoBuild.err = err return } + defer os.RemoveAll(dir) - // Also ensure that that GOROOT includes a compiler: 'go' commands - // don't in general work without it, and some builders - // (such as android-amd64-emu) seem to lack it in the test environment. - cmd := exec.Command(tool, "tool", "-n", "compile") - stderr := new(bytes.Buffer) - stderr.Write([]byte("\n")) - cmd.Stderr = stderr - out, err = cmd.Output() - if err != nil { - checkGoGoroot.err = fmt.Errorf("%v: %v%s", cmd, err, stderr) + mainGo := filepath.Join(dir, "main.go") + if err := os.WriteFile(mainGo, []byte("package main\nfunc main() {}\n"), 0644); err != nil { + checkGoBuild.err = err return } - if _, err := exec.LookPath(string(bytes.TrimSpace(out))); err != nil { - checkGoGoroot.err = err + cmd := exec.Command("go", "build", "-o", os.DevNull, mainGo) + cmd.Dir = dir + if out, err := cmd.CombinedOutput(); err != nil { + if len(out) > 0 { + checkGoBuild.err = fmt.Errorf("%v: %v\n%s", cmd, err, out) + } else { + checkGoBuild.err = fmt.Errorf("%v: %v", cmd, err) + } } }) - if checkGoGoroot.err != nil { - return checkGoGoroot.err + if checkGoBuild.err != nil { + return checkGoBuild.err } case "diff": From 2191a27a6dc5c03f6eb47dc56c86f29441e2a63c Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Fri, 25 Aug 2023 15:04:40 -0400 Subject: [PATCH 036/178] gopls/internal/lsp/frob: fix build breakage CL 510378 fixed only one of the three cases of prematurely using new API that was called out during its code review, but I didn't notice this because I used AutoSubmit, which circumvents the Kokoro check of older toolchains. Change-Id: I723b94149a6b3fb3d05ec667910e4a14e2303a9a Reviewed-on: https://go-review.googlesource.com/c/tools/+/523075 TryBot-Result: Gopher Robot Run-TryBot: Alan Donovan gopls-CI: kokoro Reviewed-by: Robert Findley --- gopls/internal/lsp/frob/frob.go | 43 ++++++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/gopls/internal/lsp/frob/frob.go b/gopls/internal/lsp/frob/frob.go index e3abcfeb277..57f1ef5014c 100644 --- a/gopls/internal/lsp/frob/frob.go +++ b/gopls/internal/lsp/frob/frob.go @@ -95,7 +95,7 @@ func frobFor(t reflect.Type) *frob { case reflect.Array, reflect.Slice, - reflect.Pointer: + reflect.Ptr: // TODO(adonovan): after go1.18, use Pointer fr.addElem(fr.t.Elem()) case reflect.Map: @@ -213,7 +213,7 @@ func (fr *frob) encode(out *writer, v reflect.Value) { } } - case reflect.Pointer: + case reflect.Ptr: // TODO(adonovan): after go1.18, use Pointer if v.IsNil() { out.uint8(0) } else { @@ -337,7 +337,7 @@ func (fr *frob) decode(in *reader, addr reflect.Value) { } } - case reflect.Pointer: + case reflect.Ptr: // TODO(adonovan): after go1.18, use Pointer isNil := in.uint8() == 0 if !isNil { ptr := reflect.New(fr.elems[0].t) @@ -398,7 +398,38 @@ func (r *reader) bytes(n int) []byte { type writer struct{ data []byte } func (w *writer) uint8(v uint8) { w.data = append(w.data, v) } -func (w *writer) uint16(v uint16) { w.data = le.AppendUint16(w.data, v) } -func (w *writer) uint32(v uint32) { w.data = le.AppendUint32(w.data, v) } -func (w *writer) uint64(v uint64) { w.data = le.AppendUint64(w.data, v) } +func (w *writer) uint16(v uint16) { w.data = appendUint16(w.data, v) } +func (w *writer) uint32(v uint32) { w.data = appendUint32(w.data, v) } +func (w *writer) uint64(v uint64) { w.data = appendUint64(w.data, v) } func (w *writer) bytes(v []byte) { w.data = append(w.data, v...) } + +// TODO(adonovan): delete these as in go1.19 they are methods on LittleEndian: + +func appendUint16(b []byte, v uint16) []byte { + return append(b, + byte(v), + byte(v>>8), + ) +} + +func appendUint32(b []byte, v uint32) []byte { + return append(b, + byte(v), + byte(v>>8), + byte(v>>16), + byte(v>>24), + ) +} + +func appendUint64(b []byte, v uint64) []byte { + return append(b, + byte(v), + byte(v>>8), + byte(v>>16), + byte(v>>24), + byte(v>>32), + byte(v>>40), + byte(v>>48), + byte(v>>56), + ) +} From cd226032e97018fa8bea48ad36ca5f0ff04b5edd Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Fri, 25 Aug 2023 17:54:11 -0400 Subject: [PATCH 037/178] go/analysis/unitchecker: NeedGoPackages in ExampleSeparateAnalysis This example test uses go/packages and thus the go command, so it fails on some builders. Unfortunately testenv.hasTool is unexported, and testenv.NeedsTool et al require a testing.T, so we have to convert this example into a test. (This is an alternative approach to CL 523076.) Fixes golang/go#62291 Change-Id: If821464f6d1e82c79a0dd85bfd5fc6e4f0f98d6a Reviewed-on: https://go-review.googlesource.com/c/tools/+/523077 gopls-CI: kokoro Reviewed-by: Bryan Mills Run-TryBot: Alan Donovan TryBot-Result: Gopher Robot --- go/analysis/unitchecker/separate_test.go | 49 ++++++++++++++---------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/go/analysis/unitchecker/separate_test.go b/go/analysis/unitchecker/separate_test.go index 12d91043d09..37e74e481ec 100644 --- a/go/analysis/unitchecker/separate_test.go +++ b/go/analysis/unitchecker/separate_test.go @@ -13,21 +13,21 @@ import ( "go/token" "go/types" "io" - "log" "os" - "os/exec" "path/filepath" "strings" "sync/atomic" + "testing" "golang.org/x/tools/go/analysis/passes/printf" "golang.org/x/tools/go/analysis/unitchecker" "golang.org/x/tools/go/gcexportdata" "golang.org/x/tools/go/packages" + "golang.org/x/tools/internal/testenv" "golang.org/x/tools/txtar" ) -// ExampleSeparateAnalysis demonstrates the principle of separate +// TestExampleSeparateAnalysis demonstrates the principle of separate // analysis, the distribution of units of type-checking and analysis // work across several processes, using serialized summaries to // communicate between them. @@ -49,7 +49,12 @@ import ( // different modes: the Example function is the manager, and the same // executable invoked with ENTRYPOINT=worker is the worker. // (See TestIntegration for how this happens.) -func ExampleSeparateAnalysis() { +// +// Unfortunately this can't be a true Example because of the skip, +// which requires a testing.T. +func TestExampleSeparateAnalysis(t *testing.T) { + testenv.NeedsGoPackages(t) + // src is an archive containing a module with a printf mistake. const src = ` -- go.mod -- @@ -76,12 +81,9 @@ func MyPrintf(format string, args ...any) { ` // Expand archive into tmp tree. - tmpdir, err := os.MkdirTemp("", "SeparateAnalysis") - if err != nil { - log.Fatal(err) - } + tmpdir := t.TempDir() if err := extractTxtar(txtar.Parse([]byte(src)), tmpdir); err != nil { - log.Fatal(err) + t.Fatal(err) } // Load metadata for the main package and all its dependencies. @@ -92,14 +94,15 @@ func MyPrintf(format string, args ...any) { "GOPROXY=off", // disable network "GOWORK=off", // an ambient GOWORK value would break package loading ), + Logf: t.Logf, } pkgs, err := packages.Load(cfg, "separate/main") if err != nil { - log.Fatal(err) + t.Fatal(err) } // Stop if any package had a metadata error. if packages.PrintErrors(pkgs) > 0 { - os.Exit(1) + t.Fatal("there were errors among loaded packages") } // Now we have loaded the import graph, @@ -117,6 +120,8 @@ func MyPrintf(format string, args ...any) { // We use it to create names of temporary files. var nextID atomic.Int32 + var allDiagnostics []string + // Visit all packages in postorder: dependencies first. // TODO(adonovan): opt: use parallel postorder. packages.Visit(pkgs, nil, func(pkg *packages.Package) { @@ -165,23 +170,23 @@ func MyPrintf(format string, args ...any) { // Write the JSON configuration message to a file. cfgData, err := json.Marshal(cfg) if err != nil { - log.Fatal(err) + t.Fatalf("internal error in json.Marshal: %v", err) } cfgFile := prefix + ".cfg" if err := os.WriteFile(cfgFile, cfgData, 0666); err != nil { - log.Fatal(err) + t.Fatal(err) } // Send the request to the worker. - cmd := exec.Command(os.Args[0], "-json", cfgFile) + cmd := testenv.Command(t, os.Args[0], "-json", cfgFile) cmd.Stderr = os.Stderr cmd.Stdout = new(bytes.Buffer) cmd.Env = append(os.Environ(), "ENTRYPOINT=worker") if err := cmd.Run(); err != nil { - log.Fatal(err) + t.Fatal(err) } - // Parse JSON output and print plainly. + // Parse JSON output and gather in allDiagnostics. dec := json.NewDecoder(cmd.Stdout.(io.Reader)) for { type jsonDiagnostic struct { @@ -194,15 +199,15 @@ func MyPrintf(format string, args ...any) { if err == io.EOF { break } - log.Fatal(err) + t.Fatalf("internal error decoding JSON: %v", err) } for _, result := range results { for analyzer, diags := range result { for _, diag := range diags { rel := strings.ReplaceAll(diag.Posn, tmpdir, "") rel = filepath.ToSlash(rel) - fmt.Printf("%s: [%s] %s\n", - rel, analyzer, diag.Message) + msg := fmt.Sprintf("%s: [%s] %s", rel, analyzer, diag.Message) + allDiagnostics = append(allDiagnostics, msg) } } } @@ -212,8 +217,10 @@ func MyPrintf(format string, args ...any) { // Observe that the example produces a fact-based diagnostic // from separate analysis of "main", "lib", and "fmt": - // Output: - // /main/main.go:6:2: [printf] separate/lib.MyPrintf format %s has arg 123 of wrong type int + const want = `/main/main.go:6:2: [printf] separate/lib.MyPrintf format %s has arg 123 of wrong type int` + if got := strings.Join(allDiagnostics, "\n"); got != want { + t.Errorf("Got: %s\nWant: %s", got, want) + } } // -- worker process -- From 2926c1f403f31a0c8d554caa3ce81fa1130663ee Mon Sep 17 00:00:00 2001 From: "Hana (Hyang-Ah) Kim" Date: Mon, 28 Aug 2023 16:52:14 -0400 Subject: [PATCH 038/178] gopls/internal/telemetry: log timestamp when test fails We observed a couple of test failures. My current hypothesis is that the bug is in how countertest.ReadCounter and countertest.ReadStackCounter read the data. They try to access the counter values by reading the contents of the mapped file on disk and parsing them with counter.Parse. They call 'file.rotate' before reading the files to ensure the file is mapped. However, if file.rotate ends up actually rotating the underlying file (due to crossing the UTC day boundary), the counter data may be flushed out and reset. As we are moving to 'week' instead of 'day' for file rotation, I don't know how frequently we will encounter this issue though if this was the root cause. Log the timestamp when the test fails, to evaluate the hypothesis. For golang/go#62161 Change-Id: If00b0788b6927640a979d17f8fe888232d27ea09 Reviewed-on: https://go-review.googlesource.com/c/tools/+/523795 Run-TryBot: Hyang-Ah Hana Kim TryBot-Result: Gopher Robot gopls-CI: kokoro Reviewed-by: Jamal Carvalho --- gopls/internal/telemetry/telemetry_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gopls/internal/telemetry/telemetry_test.go b/gopls/internal/telemetry/telemetry_test.go index 7951a419225..93751bff1d8 100644 --- a/gopls/internal/telemetry/telemetry_test.go +++ b/gopls/internal/telemetry/telemetry_test.go @@ -12,6 +12,7 @@ import ( "strconv" "strings" "testing" + "time" "golang.org/x/telemetry/counter" "golang.org/x/telemetry/counter/countertest" // requires go1.21+ @@ -58,6 +59,7 @@ func TestTelemetry(t *testing.T) { count, err := countertest.ReadCounter(c) if err != nil || count != 1 { t.Errorf("ReadCounter(%q) = (%v, %v), want (1, nil)", c.Name(), count, err) + t.Logf("Current timestamp = %v", time.Now().UTC()) } } @@ -69,6 +71,7 @@ func TestTelemetry(t *testing.T) { } if len(counts) != 1 || !hasEntry(counts, t.Name(), 1) { t.Errorf("read stackcounter(%q) = (%#v, %v), want one entry", "gopls/bug", counts, err) + t.Logf("Current timestamp = %v", time.Now().UTC()) } } From edda81f8e90fb13ea6d6227bde3d4f8f0f65d90c Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Sun, 16 Jul 2023 09:43:14 -0400 Subject: [PATCH 039/178] internal/refactor/inline: an inliner for Go source This change creates at new (internal) package that implements an inlining algorithm for Go functions, and an analyzer in the go/analysis framework that uses it to perform automatic inlining of calls to specially annotated ("inlineme") functions. Run this command to invoke the analyzer and apply any suggested fixes to the source tree: $ go run ./internal/refactor/inline/analyzer/main.go -fix packages... The package is intended for use both in interactive tools such as gopls and batch tools such as the analyzer just mentioned and the tool proposed in the attached issue. As noted in the code comments, correct inlining is a surprisingly tricky problem, so for now we primarily address the most general case in which a call f(args...) has the function name f replaced by a literal copy of the called function (func (...) {...})(args...). Only in the simplest of special cases is the call itself eliminated by replacing it with the function body. There is much further work to do by careful analysis of cases. The processing of the callee function occurs first, and results in a serializable summary of the callee that can be used for a later call to Inline, possibly from a different process, thus enabling "separate analysis" pipelines using the analysis.Fact mechanism. Recommended reviewing order: - callee.go, inline.go - inline_test.go, testdata/*txtar - analyzer/... Updates golang/go#32816 Change-Id: If28e43a6ba9ab92639276c5b50b5a89a3b0c54c4 Reviewed-on: https://go-review.googlesource.com/c/tools/+/519715 TryBot-Result: Gopher Robot Run-TryBot: Alan Donovan Auto-Submit: Alan Donovan Reviewed-by: Robert Findley --- internal/refactor/inline/analyzer/analyzer.go | 161 ++++ .../refactor/inline/analyzer/analyzer_test.go | 16 + internal/refactor/inline/analyzer/main.go | 19 + .../inline/analyzer/testdata/src/a/a.go | 16 + .../analyzer/testdata/src/a/a.go.golden | 16 + .../inline/analyzer/testdata/src/b/b.go | 9 + .../analyzer/testdata/src/b/b.go.golden | 9 + internal/refactor/inline/callee.go | 350 +++++++++ internal/refactor/inline/inline.go | 688 ++++++++++++++++++ internal/refactor/inline/inline_test.go | 322 ++++++++ .../refactor/inline/testdata/basic-err.txtar | 24 + .../inline/testdata/basic-literal.txtar | 19 + .../inline/testdata/basic-reduce.txtar | 19 + .../refactor/inline/testdata/comments.txtar | 56 ++ .../refactor/inline/testdata/crosspkg.txtar | 77 ++ .../refactor/inline/testdata/dotimport.txtar | 35 + .../refactor/inline/testdata/err-basic.txtar | 30 + .../inline/testdata/err-shadow-builtin.txtar | 36 + .../inline/testdata/err-shadow-pkg.txtar | 36 + .../inline/testdata/err-unexported.txtar | 31 + .../refactor/inline/testdata/exprstmt.txtar | 99 +++ .../inline/testdata/import-shadow.txtar | 41 ++ .../refactor/inline/testdata/internal.txtar | 29 + .../refactor/inline/testdata/method.txtar | 104 +++ internal/refactor/inline/testdata/n-ary.txtar | 79 ++ .../inline/testdata/revdotimport.txtar | 43 ++ 26 files changed, 2364 insertions(+) create mode 100644 internal/refactor/inline/analyzer/analyzer.go create mode 100644 internal/refactor/inline/analyzer/analyzer_test.go create mode 100644 internal/refactor/inline/analyzer/main.go create mode 100644 internal/refactor/inline/analyzer/testdata/src/a/a.go create mode 100644 internal/refactor/inline/analyzer/testdata/src/a/a.go.golden create mode 100644 internal/refactor/inline/analyzer/testdata/src/b/b.go create mode 100644 internal/refactor/inline/analyzer/testdata/src/b/b.go.golden create mode 100644 internal/refactor/inline/callee.go create mode 100644 internal/refactor/inline/inline.go create mode 100644 internal/refactor/inline/inline_test.go create mode 100644 internal/refactor/inline/testdata/basic-err.txtar create mode 100644 internal/refactor/inline/testdata/basic-literal.txtar create mode 100644 internal/refactor/inline/testdata/basic-reduce.txtar create mode 100644 internal/refactor/inline/testdata/comments.txtar create mode 100644 internal/refactor/inline/testdata/crosspkg.txtar create mode 100644 internal/refactor/inline/testdata/dotimport.txtar create mode 100644 internal/refactor/inline/testdata/err-basic.txtar create mode 100644 internal/refactor/inline/testdata/err-shadow-builtin.txtar create mode 100644 internal/refactor/inline/testdata/err-shadow-pkg.txtar create mode 100644 internal/refactor/inline/testdata/err-unexported.txtar create mode 100644 internal/refactor/inline/testdata/exprstmt.txtar create mode 100644 internal/refactor/inline/testdata/import-shadow.txtar create mode 100644 internal/refactor/inline/testdata/internal.txtar create mode 100644 internal/refactor/inline/testdata/method.txtar create mode 100644 internal/refactor/inline/testdata/n-ary.txtar create mode 100644 internal/refactor/inline/testdata/revdotimport.txtar diff --git a/internal/refactor/inline/analyzer/analyzer.go b/internal/refactor/inline/analyzer/analyzer.go new file mode 100644 index 00000000000..2356fa484e7 --- /dev/null +++ b/internal/refactor/inline/analyzer/analyzer.go @@ -0,0 +1,161 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package analyzer + +import ( + "fmt" + "go/ast" + "go/token" + "go/types" + "os" + "strings" + + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/analysis/passes/inspect" + "golang.org/x/tools/go/ast/inspector" + "golang.org/x/tools/go/types/typeutil" + "golang.org/x/tools/internal/diff" + "golang.org/x/tools/internal/refactor/inline" +) + +const Doc = `inline calls to functions with "inlineme" doc comment` + +var Analyzer = &analysis.Analyzer{ + Name: "inline", + Doc: Doc, + URL: "https://pkg.go.dev/golang.org/x/tools/internal/refactor/inline/analyzer", + Run: run, + FactTypes: []analysis.Fact{new(inlineMeFact)}, + Requires: []*analysis.Analyzer{inspect.Analyzer}, +} + +func run(pass *analysis.Pass) (interface{}, error) { + // Memoize repeated calls for same file. + // TODO(adonovan): the analysis.Pass should abstract this (#62292) + // as the driver may not be reading directly from the file system. + fileContent := make(map[string][]byte) + readFile := func(node ast.Node) ([]byte, error) { + filename := pass.Fset.File(node.Pos()).Name() + content, ok := fileContent[filename] + if !ok { + var err error + content, err = os.ReadFile(filename) + if err != nil { + return nil, err + } + fileContent[filename] = content + } + return content, nil + } + + // Pass 1: find functions annotated with an "inlineme" + // comment, and export a fact for each one. + inlinable := make(map[*types.Func]*inline.Callee) // memoization of fact import (nil => no fact) + for _, file := range pass.Files { + for _, decl := range file.Decls { + if decl, ok := decl.(*ast.FuncDecl); ok { + // TODO(adonovan): this is just a placeholder. + // Use the precise go:fix syntax in the proposal. + // Beware that //go: comments are treated specially + // by (*ast.CommentGroup).Text(). + // TODO(adonovan): alternatively, consider using + // the universal annotation mechanism sketched in + // https://go.dev/cl/489835 (which doesn't yet have + // a proper proposal). + if strings.Contains(decl.Doc.Text(), "inlineme") { + content, err := readFile(file) + if err != nil { + pass.Reportf(decl.Doc.Pos(), "invalid inlining candidate: cannot read source file: %v", err) + continue + } + callee, err := inline.AnalyzeCallee(pass.Fset, pass.Pkg, pass.TypesInfo, decl, content) + if err != nil { + pass.Reportf(decl.Doc.Pos(), "invalid inlining candidate: %v", err) + continue + } + fn := pass.TypesInfo.Defs[decl.Name].(*types.Func) + pass.ExportObjectFact(fn, &inlineMeFact{callee}) + inlinable[fn] = callee + } + } + } + } + + // Pass 2. Inline each static call to an inlinable function. + inspect := pass.ResultOf[inspect.Analyzer].(*inspector.Inspector) + nodeFilter := []ast.Node{ + (*ast.File)(nil), + (*ast.CallExpr)(nil), + } + var currentFile *ast.File + inspect.Preorder(nodeFilter, func(n ast.Node) { + if file, ok := n.(*ast.File); ok { + currentFile = file + return + } + call := n.(*ast.CallExpr) + if fn := typeutil.StaticCallee(pass.TypesInfo, call); fn != nil { + // Inlinable? + callee, ok := inlinable[fn] + if !ok { + var fact inlineMeFact + if pass.ImportObjectFact(fn, &fact) { + callee = fact.callee + inlinable[fn] = callee + } + } + if callee == nil { + return // nope + } + + // Inline the call. + content, err := readFile(call) + if err != nil { + pass.Reportf(call.Lparen, "invalid inlining candidate: cannot read source file: %v", err) + return + } + caller := &inline.Caller{ + Fset: pass.Fset, + Types: pass.Pkg, + Info: pass.TypesInfo, + File: currentFile, + Call: call, + Content: content, + } + got, err := inline.Inline(caller, callee) + if err != nil { + pass.Reportf(call.Lparen, "%v", err) + return + } + + // Suggest the "fix". + var textEdits []analysis.TextEdit + for _, edit := range diff.Bytes(content, got) { + textEdits = append(textEdits, analysis.TextEdit{ + Pos: currentFile.FileStart + token.Pos(edit.Start), + End: currentFile.FileStart + token.Pos(edit.End), + NewText: []byte(edit.New), + }) + } + msg := fmt.Sprintf("inline call of %v", callee) + pass.Report(analysis.Diagnostic{ + Pos: call.Pos(), + End: call.End(), + Message: msg, + SuggestedFixes: []analysis.SuggestedFix{{ + Message: msg, + TextEdits: textEdits, + }}, + }) + } + }) + + return nil, nil +} + +type inlineMeFact struct{ callee *inline.Callee } + +func (f *inlineMeFact) String() string { return "inlineme " + f.callee.String() } +func (*inlineMeFact) AFact() {} diff --git a/internal/refactor/inline/analyzer/analyzer_test.go b/internal/refactor/inline/analyzer/analyzer_test.go new file mode 100644 index 00000000000..5ad85cfb821 --- /dev/null +++ b/internal/refactor/inline/analyzer/analyzer_test.go @@ -0,0 +1,16 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package analyzer_test + +import ( + "testing" + + "golang.org/x/tools/go/analysis/analysistest" + inlineanalyzer "golang.org/x/tools/internal/refactor/inline/analyzer" +) + +func TestAnalyzer(t *testing.T) { + analysistest.RunWithSuggestedFixes(t, analysistest.TestData(), inlineanalyzer.Analyzer, "a", "b") +} diff --git a/internal/refactor/inline/analyzer/main.go b/internal/refactor/inline/analyzer/main.go new file mode 100644 index 00000000000..4be223a80d6 --- /dev/null +++ b/internal/refactor/inline/analyzer/main.go @@ -0,0 +1,19 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build ignore +// +build ignore + +// The inline command applies the inliner to the specified packages of +// Go source code. Run with: +// +// $ go run ./internal/refactor/inline/analyzer/main.go -fix packages... +package main + +import ( + "golang.org/x/tools/go/analysis/singlechecker" + inlineanalyzer "golang.org/x/tools/internal/refactor/inline/analyzer" +) + +func main() { singlechecker.Main(inlineanalyzer.Analyzer) } diff --git a/internal/refactor/inline/analyzer/testdata/src/a/a.go b/internal/refactor/inline/analyzer/testdata/src/a/a.go new file mode 100644 index 00000000000..e661515b7c7 --- /dev/null +++ b/internal/refactor/inline/analyzer/testdata/src/a/a.go @@ -0,0 +1,16 @@ +package a + +func f() { + One() // want `inline call of a.One` + new(T).Two() // want `inline call of \(a.T\).Two` +} + +type T struct{} + +// inlineme +func One() int { return one } // want One:`inlineme a.One` + +const one = 1 + +// inlineme +func (T) Two() int { return 2 } // want Two:`inlineme \(a.T\).Two` diff --git a/internal/refactor/inline/analyzer/testdata/src/a/a.go.golden b/internal/refactor/inline/analyzer/testdata/src/a/a.go.golden new file mode 100644 index 00000000000..fe9877b69c1 --- /dev/null +++ b/internal/refactor/inline/analyzer/testdata/src/a/a.go.golden @@ -0,0 +1,16 @@ +package a + +func f() { + _ = one // want `inline call of a.One` + func(_ T) int { return 2 }(*new(T)) // want `inline call of \(a.T\).Two` +} + +type T struct{} + +// inlineme +func One() int { return one } // want One:`inlineme a.One` + +const one = 1 + +// inlineme +func (T) Two() int { return 2 } // want Two:`inlineme \(a.T\).Two` diff --git a/internal/refactor/inline/analyzer/testdata/src/b/b.go b/internal/refactor/inline/analyzer/testdata/src/b/b.go new file mode 100644 index 00000000000..069e670d51e --- /dev/null +++ b/internal/refactor/inline/analyzer/testdata/src/b/b.go @@ -0,0 +1,9 @@ +package b + +import "a" + +func f() { + a.One() // want `cannot inline call to a.One because body refers to non-exported one` + + new(a.T).Two() // want `inline call of \(a.T\).Two` +} diff --git a/internal/refactor/inline/analyzer/testdata/src/b/b.go.golden b/internal/refactor/inline/analyzer/testdata/src/b/b.go.golden new file mode 100644 index 00000000000..61b7bd9b349 --- /dev/null +++ b/internal/refactor/inline/analyzer/testdata/src/b/b.go.golden @@ -0,0 +1,9 @@ +package b + +import "a" + +func f() { + a.One() // want `cannot inline call to a.One because body refers to non-exported one` + + func(_ a.T) int { return 2 }(*new(a.T)) // want `inline call of \(a.T\).Two` +} diff --git a/internal/refactor/inline/callee.go b/internal/refactor/inline/callee.go new file mode 100644 index 00000000000..291971cf6d8 --- /dev/null +++ b/internal/refactor/inline/callee.go @@ -0,0 +1,350 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package inline + +// This file defines the analysis of the callee function. + +import ( + "bytes" + "encoding/gob" + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + + "golang.org/x/tools/go/ast/astutil" + "golang.org/x/tools/go/types/typeutil" + "golang.org/x/tools/internal/typeparams" +) + +// A Callee holds information about an inlinable function. Gob-serializable. +type Callee struct { + impl gobCallee +} + +func (callee *Callee) String() string { return callee.impl.Name } + +type gobCallee struct { + Content []byte // file content, compacted to a single func decl + + // syntax derived from compacted Content (not serialized) + fset *token.FileSet + decl *ast.FuncDecl + + // results of type analysis (does not reach go/types data structures) + PkgPath string // package path of declaring package + Name string // user-friendly name for error messages + Unexported []string // names of free objects that are unexported + FreeRefs []freeRef // locations of references to free objects + FreeObjs []object // descriptions of free objects + BodyIsReturnExpr bool // function body is "return expr(s)" + ValidForCallStmt bool // => bodyIsReturnExpr and sole expr is f() or <-ch + NumResults int // number of results (according to type, not ast.FieldList) +} + +// A freeRef records a reference to a free object. Gob-serializable. +type freeRef struct { + Start, End int // Callee.content[start:end] is extent of the reference + Object int // index into Callee.freeObjs +} + +// An object abstracts a free types.Object referenced by the callee. Gob-serializable. +type object struct { + Name string // Object.Name() + Kind string // one of {var,func,const,type,pkgname,nil,builtin} + PkgPath string // pkgpath of object (or of imported package if kind="pkgname") + ValidPos bool // Object.Pos().IsValid() +} + +func (callee *gobCallee) offset(pos token.Pos) int { return offsetOf(callee.fset, pos) } + +// AnalyzeCallee analyzes a function that is a candidate for inlining +// and returns a Callee that describes it. The Callee object, which is +// serializable, can be passed to one or more subsequent calls to +// Inline, each with a different Caller. +// +// This design allows separate analysis of callers and callees in the +// golang.org/x/tools/go/analysis framework: the inlining information +// about a callee can be recorded as a "fact". +func AnalyzeCallee(fset *token.FileSet, pkg *types.Package, info *types.Info, decl *ast.FuncDecl, content []byte) (*Callee, error) { + + // The client is expected to have determined that the callee + // is a function with a declaration (not a built-in or var). + fn := info.Defs[decl.Name].(*types.Func) + sig := fn.Type().(*types.Signature) + + // Create user-friendly name ("pkg.Func" or "(pkg.T).Method") + var name string + if sig.Recv() == nil { + name = fmt.Sprintf("%s.%s", fn.Pkg().Name(), fn.Name()) + } else { + name = fmt.Sprintf("(%s).%s", types.TypeString(sig.Recv().Type(), (*types.Package).Name), fn.Name()) + } + + if decl.Body == nil { + return nil, fmt.Errorf("cannot inline function %s as it has no body", name) + } + + // TODO(adonovan): support inlining of instantiated generic + // functions by replacing each occurrence of a type parameter + // T by its instantiating type argument (e.g. int). We'll need + // to wrap the instantiating type in parens when it's not an + // ident or qualified ident to prevent "if x == struct{}" + // parsing ambiguity, or "T(x)" where T = "*int" or "func()" + // from misparsing. + if decl.Type.TypeParams != nil { + return nil, fmt.Errorf("cannot inline generic function %s: type parameters are not yet supported", name) + } + + // Record the location of all free references in the callee body. + var ( + freeObjIndex = make(map[types.Object]int) + freeObjs []object + freeRefs []freeRef // free refs that may need renaming + unexported []string // free refs to unexported objects, for later error checks + ) + var visit func(n ast.Node) bool + visit = func(n ast.Node) bool { + switch n := n.(type) { + case *ast.SelectorExpr: + // Check selections of free fields/methods. + if sel, ok := info.Selections[n]; ok && + !within(sel.Obj().Pos(), decl) && + !n.Sel.IsExported() { + sym := fmt.Sprintf("(%s).%s", info.TypeOf(n.X), n.Sel.Name) + unexported = append(unexported, sym) + } + + // Don't recur into SelectorExpr.Sel. + visit(n.X) + return false + + case *ast.CompositeLit: + // Check for struct literals that refer to unexported fields, + // whether keyed or unkeyed. (Logic assumes well-typedness.) + litType := deref(info.TypeOf(n)) + if s, ok := typeparams.CoreType(litType).(*types.Struct); ok { + for i, elt := range n.Elts { + var field *types.Var + var value ast.Expr + if kv, ok := elt.(*ast.KeyValueExpr); ok { + field = info.Uses[kv.Key.(*ast.Ident)].(*types.Var) + value = kv.Value + } else { + field = s.Field(i) + value = elt + } + if !within(field.Pos(), decl) && !field.Exported() { + sym := fmt.Sprintf("(%s).%s", litType, field.Name()) + unexported = append(unexported, sym) + } + + // Don't recur into KeyValueExpr.Key. + visit(value) + } + return false + } + + case *ast.Ident: + if obj, ok := info.Uses[n]; ok { + // Methods and fields are handled by SelectorExpr and CompositeLit. + if isField(obj) || isMethod(obj) { + panic(obj) + } + // Inv: id is a lexical reference. + + // A reference to an unexported package-level declaration + // cannot be inlined into another package. + if !n.IsExported() && + obj.Pkg() != nil && obj.Parent() == obj.Pkg().Scope() { + unexported = append(unexported, n.Name) + } + + // Record free reference. + if !within(obj.Pos(), decl) { + objidx, ok := freeObjIndex[obj] + if !ok { + objidx = len(freeObjIndex) + var pkgpath string + if pkgname, ok := obj.(*types.PkgName); ok { + pkgpath = pkgname.Imported().Path() + } else if obj.Pkg() != nil { + pkgpath = obj.Pkg().Path() + } + freeObjs = append(freeObjs, object{ + Name: obj.Name(), + Kind: objectKind(obj), + PkgPath: pkgpath, + ValidPos: obj.Pos().IsValid(), + }) + freeObjIndex[obj] = objidx + } + freeRefs = append(freeRefs, freeRef{ + Start: offsetOf(fset, n.Pos()), + End: offsetOf(fset, n.End()), + Object: objidx, + }) + } + } + } + return true + } + ast.Inspect(decl, visit) + + // Analyze callee body for "return results" form, where + // results is one or more expressions or an n-ary call. + validForCallStmt := false + bodyIsReturnExpr := decl.Type.Results != nil && len(decl.Type.Results.List) > 0 && + len(decl.Body.List) == 1 && + is[*ast.ReturnStmt](decl.Body.List[0]) && + len(decl.Body.List[0].(*ast.ReturnStmt).Results) > 0 + if bodyIsReturnExpr { + ret := decl.Body.List[0].(*ast.ReturnStmt) + + // Ascertain whether the results expression(s) + // would be safe to inline as a standalone statement. + // (This is true only for a single call or receive expression.) + validForCallStmt = func() bool { + if len(ret.Results) == 1 { + switch expr := astutil.Unparen(ret.Results[0]).(type) { + case *ast.CallExpr: // f(x) + callee := typeutil.Callee(info, expr) + if callee == nil { + return false // conversion T(x) + } + + // The only non-void built-in functions that may be + // called as a statement are copy and recover + // (though arguably a call to recover should never + // be inlined as that changes its behavior). + if builtin, ok := callee.(*types.Builtin); ok { + return builtin.Name() == "copy" || + builtin.Name() == "recover" + } + + return true // ordinary call f() + + case *ast.UnaryExpr: // <-x + return expr.Op == token.ARROW // channel receive <-ch + } + } + + // No other expressions are valid statements. + return false + }() + } + + // As a space optimization, we don't retain the complete + // callee file content; all we need is "package _; func f() { ... }". + // This reduces the size of analysis facts. + // + // The FileSet file/line info is no longer meaningful + // and should not be used in error messages. + // But the FileSet offsets are valid w.r.t. the content. + // + // (For ease of debugging we could insert a //line directive after + // the package decl but it seems more trouble than it's worth.) + { + start, end := offsetOf(fset, decl.Pos()), offsetOf(fset, decl.End()) + + var compact bytes.Buffer + compact.WriteString("package _\n") + compact.Write(content[start:end]) + content = compact.Bytes() + + // Re-parse the compacted content. + var err error + decl, err = parseCompact(fset, content) + if err != nil { + return nil, err + } + + // (content, decl) are now updated. + + // Adjust the freeRefs offsets. + delta := int(offsetOf(fset, decl.Pos()) - start) + for i := range freeRefs { + freeRefs[i].Start += delta + freeRefs[i].End += delta + } + } + + return &Callee{gobCallee{ + Content: content, + fset: fset, + decl: decl, + PkgPath: pkg.Path(), + Name: name, + Unexported: unexported, + FreeObjs: freeObjs, + FreeRefs: freeRefs, + BodyIsReturnExpr: bodyIsReturnExpr, + ValidForCallStmt: validForCallStmt, + NumResults: sig.Results().Len(), + }}, nil +} + +// parseCompact parses a Go source file of the form "package _\n func f() { ... }" +// and returns the sole function declaration. +func parseCompact(fset *token.FileSet, content []byte) (*ast.FuncDecl, error) { + const mode = parser.ParseComments | parser.SkipObjectResolution | parser.AllErrors + f, err := parser.ParseFile(fset, "callee.go", content, mode) + if err != nil { + return nil, fmt.Errorf("internal error: cannot compact file: %v", err) + } + return f.Decls[0].(*ast.FuncDecl), nil +} + +// deref removes a pointer type constructor from the core type of t. +func deref(t types.Type) types.Type { + if ptr, ok := typeparams.CoreType(t).(*types.Pointer); ok { + return ptr.Elem() + } + return t +} + +func isField(obj types.Object) bool { + if v, ok := obj.(*types.Var); ok && v.IsField() { + return true + } + return false +} + +func isMethod(obj types.Object) bool { + if f, ok := obj.(*types.Func); ok && f.Type().(*types.Signature).Recv() != nil { + return true + } + return false +} + +// -- serialization -- + +var ( + _ gob.GobEncoder = (*Callee)(nil) + _ gob.GobDecoder = (*Callee)(nil) +) + +func (callee *Callee) GobEncode() ([]byte, error) { + var out bytes.Buffer + if err := gob.NewEncoder(&out).Encode(callee.impl); err != nil { + return nil, err + } + return out.Bytes(), nil +} + +func (callee *Callee) GobDecode(data []byte) error { + if err := gob.NewDecoder(bytes.NewReader(data)).Decode(&callee.impl); err != nil { + return err + } + fset := token.NewFileSet() + decl, err := parseCompact(fset, callee.impl.Content) + if err != nil { + return err + } + callee.impl.fset = fset + callee.impl.decl = decl + return nil +} diff --git a/internal/refactor/inline/inline.go b/internal/refactor/inline/inline.go new file mode 100644 index 00000000000..9167498fc35 --- /dev/null +++ b/internal/refactor/inline/inline.go @@ -0,0 +1,688 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package inline implements inlining of Go function calls. +// +// The client provides information about the caller and callee, +// including the source text, syntax tree, and type information, and +// the inliner returns the modified source file for the caller, or an +// error if the inlining operation is invalid (for example because the +// function body refers to names that are inaccessible to the caller). +// +// Although this interface demands more information from the client +// than might seem necessary, it enables smoother integration with +// existing batch and interactive tools that have their own ways of +// managing the processes of reading, parsing, and type-checking +// packages. In particular, this package does not assume that the +// caller and callee belong to the same token.FileSet or +// types.Importer realms. +// +// In general, inlining consists of modifying a function or method +// call expression f(a1, ..., an) so that the name of the function f +// is replaced ("literalized") by a literal copy of the function +// declaration, with free identifiers suitably modified to use the +// locally appropriate identifiers or perhaps constant argument +// values. +// +// Inlining must not change the semantics of the call. Semantics +// preservation is crucial for clients such as codebase maintenance +// tools that automatically inline all calls to designated functions +// on a large scale. Such tools must not introduce subtle behavior +// changes. (Fully inlining a call is dynamically observable using +// reflection over the call stack, but this exception to the rule is +// explicitly allowed.) +// +// In some special cases it is possible to entirely replace ("reduce") +// the call by a copy of the function's body in which parameters have +// been replaced by arguments, but this is surprisingly tricky for a +// number of reasons, some of which are listed here for illustration: +// +// - Any effects of the call argument expressions must be preserved, +// even if the corresponding parameters are never referenced, or are +// referenced multiple times, or are referenced in a different order +// from the arguments. +// +// - Even an argument expression as simple as ptr.x may not be +// referentially transparent, because another argument may have the +// effect of changing the value of ptr. +// +// - Although constants are referentially transparent, as a matter of +// style we do not wish to duplicate literals that are referenced +// multiple times in the body because this undoes proper factoring. +// Also, string literals may be arbitrarily large. +// +// - If the function body consists of statements other than just +// "return expr", in some contexts it may be syntactically +// impossible to replace the call expression by the body statements. +// Consider "} else if x := f(); cond { ... }". +// (Go has no equivalent to Lisp's progn or Rust's blocks.) +// +// - Similarly, without the equivalent of Rust-style blocks and +// first-class tuples, there is no general way to reduce a call +// to a function such as +// > func(params)(args)(results) { stmts; return body } +// to an expression such as +// > { var params = args; stmts; body } +// or even a statement such as +// > results = { var params = args; stmts; body } +// Consequently the declaration and scope of the result variables, +// and the assignment and control-flow implications of the return +// statement, must be dealt with by cases. +// +// - A standalone call statement that calls a function whose body is +// "return expr" cannot be simply replaced by the body expression +// if it is not itself a call or channel receive expression; it is +// necessary to explicitly discard the result using "_ = expr". +// +// Similarly, if the body is a call expression, only calls to some +// built-in functions with no result (such as copy or panic) are +// permitted as statements, whereas others (such as append) return +// a result that must be used, even if just by discarding. +// +// - If a parameter or result variable is updated by an assignment +// within the function body, it cannot always be safely replaced +// by a variable in the caller. For example, given +// > func f(a int) int { a++; return a } +// The call y = f(x) cannot be replaced by { x++; y = x } because +// this would change the value of the caller's variable x. +// Only if the caller is finished with x is this safe. +// +// A similar argument applies to parameter or result variables +// that escape: by eliminating a variable, inlining would change +// the identity of the variable that escapes. +// +// - If the function body uses 'defer' and the inlined call is not a +// tail-call, inlining may delay the deferred effects. +// +// - Each control label that is used by both caller and callee must +// be α-renamed. +// +// - Given +// > func f() uint8 { return 0 } +// > var x any = f() +// reducing the call to var x any = 0 is unsound because it +// discards the implicit conversion. We may need to make each +// argument->parameter and return->result assignment conversion +// implicit if the types differ. Assignments to variadic +// parameters may need to explicitly construct a slice. +// +// More complex callee functions are inlinable with more elaborate and +// invasive changes to the statements surrounding the call expression. +// +// TODO(adonovan): future work: +// +// - Handle more of the above special cases by careful analysis, +// thoughtful factoring of the large design space, and thorough +// test coverage. +// +// - Write a fuzz-like test that selects function calls at +// random in the corpus, inlines them, and checks that the +// result is either a sensible error or a valid transformation. +// +// - Eliminate parameters that are unreferenced in the callee +// and whose argument expression is side-effect free. +// +// - Afford the client more control such as a limit on the total +// increase in line count, or a refusal to inline using the +// general approach (replacing name by function literal). This +// could be achieved by returning metadata alongside the result +// and having the client conditionally discard the change. +// +// - Is it acceptable to skip effects that are limited to runtime +// panics? Can we avoid evaluating an argument x.f +// or a[i] when the corresponding parameter is unused? +// +// - When caller syntax permits a block, replace argument-to-parameter +// assignment by a set of local var decls, e.g. f(1, 2) would +// become { var x, y = 1, 2; body... }. +// +// But even this is complicated: a single var decl initializer +// cannot declare all the parameters and initialize them to their +// arguments in one go if they have varied types. Instead, +// one must use multiple specs such as: +// > { var x int = 1; var y int32 = 2; body ...} +// but this means that the initializer expression for y is +// within the scope of x, so it may require α-renaming. +// +// It is tempting to use a short var decl { x, y := 1, 2; body ...} +// as it permits simultaneous declaration and initialization +// of many variables of varied type. However, one must take care +// to convert each argument expression to the correct parameter +// variable type, perhaps explicitly. (Consider "x := 1 << 64".) +// +// Also, as a matter of style, having all parameter declarations +// and argument expressions in a single statement is potentially +// unwieldy. +// +// - Support inlining of generic functions, replacing type parameters +// by their instantiations. +// +// - Support inlining of calls to function literals such as: +// > f := func(...) { ...} +// > f() +// including recursive ones: +// > var f func(...) +// > f = func(...) { ...f...} +// > f() +// But note that the existing algorithm makes widespread assumptions +// that the callee is a package-level function or method. +// +// - Eliminate parens inserted conservatively when they are redundant. +// +// - Allow non-'go' build systems such as Bazel/Blaze a chance to +// decide whether an import is accessible using logic other than +// "/internal/" path segments. This could be achieved by returning +// the list of added import paths. +// +// - Inlining a function from another module may change the +// effective version of the Go language spec that governs it. We +// should probably make the client responsible for rejecting +// attempts to inline from newer callees to older callers, since +// there's no way for this package to access module versions. +// +// - Use an alternative implementation of the import-organizing +// operation that doesn't require operating on a complete file +// (and reformatting). Then return the results in a higher-level +// form as a set of import additions and deletions plus a single +// diff that encloses the call expression. This interface could +// perhaps be implemented atop imports.Process by post-processing +// its result to obtain the abstract import changes and discarding +// its formatted output. +package inline + +import ( + "bytes" + "fmt" + "go/ast" + "go/token" + "go/types" + "log" + pathpkg "path" + "reflect" + "sort" + "strings" + + "golang.org/x/tools/go/ast/astutil" + "golang.org/x/tools/go/types/typeutil" + "golang.org/x/tools/imports" + "golang.org/x/tools/internal/typeparams" +) + +// A Caller describes the function call and its enclosing context. +// +// The client is responsible for populating this struct and passing it to Inline. +type Caller struct { + Fset *token.FileSet + Types *types.Package + Info *types.Info + File *ast.File + Call *ast.CallExpr + Content []byte +} + +func (caller *Caller) offset(pos token.Pos) int { return offsetOf(caller.Fset, pos) } + +// Inline inlines the called function (callee) into the function call (caller) +// and returns the updated, formatted content of the caller source file. +func Inline(caller *Caller, callee_ *Callee) ([]byte, error) { + callee := &callee_.impl + + // -- check caller -- + + // Inlining of dynamic calls is not currently supported, + // even for local closure calls. + if typeutil.StaticCallee(caller.Info, caller.Call) == nil { + // e.g. interface method + return nil, fmt.Errorf("cannot inline: not a static function call") + } + + // Reject cross-package inlining if callee has + // free references to unexported symbols. + samePkg := caller.Types.Path() == callee.PkgPath + if !samePkg && len(callee.Unexported) > 0 { + return nil, fmt.Errorf("cannot inline call to %s because body refers to non-exported %s", + callee.Name, callee.Unexported[0]) + } + + // -- analyze callee's free references in caller context -- + + // syntax path enclosing Call, innermost first (Path[0]=Call) + callerPath, _ := astutil.PathEnclosingInterval(caller.File, caller.Call.Pos(), caller.Call.End()) + callerLookup := func(name string, pos token.Pos) types.Object { + for _, n := range callerPath { + // The function body scope (containing not just params) + // is associated with FuncDecl.Type, not FuncDecl.Body. + if decl, ok := n.(*ast.FuncDecl); ok { + n = decl.Type + } + if scope := caller.Info.Scopes[n]; scope != nil { + if _, obj := scope.LookupParent(name, pos); obj != nil { + return obj + } + } + } + return nil + } + + // Import map, initially populated with caller imports. + // + // For simplicity we ignore existing dot imports, so that a + // qualified identifier (QI) in the callee is always + // represented by a QI in the caller, allowing us to treat a + // QI like a selection on a package name. + importMap := make(map[string]string) // maps package path to local name + for _, imp := range caller.File.Imports { + if pkgname, ok := importedPkgName(caller.Info, imp); ok && pkgname.Name() != "." { + importMap[pkgname.Imported().Path()] = pkgname.Name() + } + } + + // localImportName returns the local name for a given imported package path. + var newImports []string + localImportName := func(path string) string { + name, ok := importMap[path] + if !ok { + // import added by callee + // + // Choose local PkgName based on last segment of + // package path plus, if needed, a numeric suffix to + // ensure uniqueness. + // + // TODO(adonovan): preserve the PkgName used + // in the original source, or, for a dot import, + // use the package's declared name. + base := pathpkg.Base(path) + name = base + for n := 0; callerLookup(name, caller.Call.Pos()) != nil; n++ { + name = fmt.Sprintf("%s%d", base, n) + } + + // TODO(adonovan): don't use a renaming import + // unless the local name differs from either + // the package name or the last segment of path. + // This requires that we tabulate (path, declared name, local name) + // triples for each package referenced by the callee. + newImports = append(newImports, fmt.Sprintf("%s %q", name, path)) + importMap[path] = name + } + return name + } + + // Compute the renaming of the callee's free identifiers. + objRenames := make([]string, len(callee.FreeObjs)) // "" => no rename + for i, obj := range callee.FreeObjs { + // obj is a free object of the callee. + // + // Possible cases are: + // - nil or a builtin + // => check not shadowed in caller. + // - package-level var/func/const/types + // => same package: check not shadowed in caller. + // => otherwise: import other package form a qualified identifier. + // (Unexported cross-package references were rejected already.) + // - type parameter + // => not yet supported + // - pkgname + // => import other package and use its local name. + // + // There can be no free references to labels, fields, or methods. + + var newName string + if obj.Kind == "pkgname" { + // Use locally appropriate import, creating as needed. + newName = localImportName(obj.PkgPath) // imported package + + } else if !obj.ValidPos { + // Built-in function, type, or nil: check not shadowed at caller. + found := callerLookup(obj.Name, caller.Call.Pos()) // can't fail + if found.Pos().IsValid() { + return nil, fmt.Errorf("cannot inline because built-in %q is shadowed in caller by a %s (line %d)", + obj.Name, objectKind(found), + caller.Fset.Position(found.Pos()).Line) + } + + newName = obj.Name + + } else { + // Must be reference to package-level var/func/const/type, + // since type parameters are not yet supported. + newName = obj.Name + qualify := false + if obj.PkgPath == callee.PkgPath { + // reference within callee package + if samePkg { + // Caller and callee are in same package. + // Check caller has not shadowed the decl. + found := callerLookup(obj.Name, caller.Call.Pos()) // can't fail + if !isPkgLevel(found) { + return nil, fmt.Errorf("cannot inline because %q is shadowed in caller by a %s (line %d)", + obj.Name, objectKind(found), + caller.Fset.Position(found.Pos()).Line) + } + } else { + // Cross-package reference. + qualify = true + } + } else { + // Reference to a package-level declaration + // in another package, without a qualified identifier: + // it must be a dot import. + qualify = true + } + + // Form a qualified identifier, pkg.Name. + if qualify { + pkgName := localImportName(obj.PkgPath) + newName = pkgName + "." + newName + } + } + objRenames[i] = newName + } + + // Compute edits to inlined callee. + type edit struct { + start, end int // byte offsets wrt callee.content + new string + } + var edits []edit + + // Give explicit blank "_" names to all method parameters + // (including receiver) since we will make the receiver a regular + // parameter and one cannot mix named and unnamed parameters. + // e.g. func (T) f(int, string) -> (_ T, _ int, _ string) + if callee.decl.Recv != nil { + ensureNamed := func(params *ast.FieldList) { + for _, param := range params.List { + if param.Names == nil { + offset := callee.offset(param.Type.Pos()) + edits = append(edits, edit{ + start: offset, + end: offset, + new: "_ ", + }) + } + } + } + ensureNamed(callee.decl.Recv) + ensureNamed(callee.decl.Type.Params) + } + + // Generate replacements for each free identifier. + for _, ref := range callee.FreeRefs { + if repl := objRenames[ref.Object]; repl != "" { + edits = append(edits, edit{ + start: ref.Start, + end: ref.End, + new: repl, + }) + } + } + + // Edits are non-overlapping but insertions and edits may be coincident. + // Preserve original order. + sort.SliceStable(edits, func(i, j int) bool { + return edits[i].start < edits[j].start + }) + + // Check that all imports (in particular, the new ones) are accessible. + // TODO(adonovan): allow customization of the accessibility relation (e.g. for Bazel). + for path := range importMap { + // TODO(adonovan): better segment hygiene. + if i := strings.Index(path, "/internal/"); i >= 0 { + if !strings.HasPrefix(caller.Types.Path(), path[:i]) { + return nil, fmt.Errorf("can't inline function %v as its body refers to inaccessible package %q", callee.Name, path) + } + } + } + + // The transformation is expressed by splicing substrings of + // the two source files, because syntax trees don't preserve + // comments faithfully (see #20744). + var out bytes.Buffer + + // 'replace' emits to out the specified range of the callee, + // applying all edits that fall completely within it. + replace := func(start, end int) { + off := start + for _, edit := range edits { + if start <= edit.start && edit.end <= end { + out.Write(callee.Content[off:edit.start]) + out.WriteString(edit.new) + off = edit.end + } + } + out.Write(callee.Content[off:end]) + } + + // Insert new imports after last existing import, + // to avoid migration of pre-import comments. + // The imports will be organized later. + { + offset := caller.offset(caller.File.Name.End()) // after package decl + if len(caller.File.Imports) > 0 { + // It's tempting to insert the new import after the last ImportSpec, + // but that may not be at the end of the import decl. + // Consider: import ( "a"; "b" ‸ ) + for _, decl := range caller.File.Decls { + if decl, ok := decl.(*ast.GenDecl); ok && decl.Tok == token.IMPORT { + offset = caller.offset(decl.End()) // after import decl + } + } + } + out.Write(caller.Content[:offset]) + out.WriteString("\n") + for _, imp := range newImports { + fmt.Fprintf(&out, "import %s\n", imp) + } + out.Write(caller.Content[offset:caller.offset(caller.Call.Pos())]) + } + + // Special case: a call to a function whose body consists only + // of "return expr" may be replaced by the expression, so long as: + // + // (a) There are no receiver or parameter argument expressions + // whose side effects must be considered. + // (b) There are no named parameter or named result variables + // that could potentially escape. + // + // TODO(adonovan): expand this special case to cover more scenarios. + // Consider each parameter in turn. If: + // - the parameter does not escape and is never assigned; + // - its argument is pure (no effects or panics--basically just idents and literals) + // and referentially transparent (not new(T) or &T{...}) or referenced at most once; and + // - the argument and parameter have the same type + // then the parameter can be eliminated and each reference + // to it replaced by the argument. + // If: + // - all parameters can be so replaced; + // - and the body is just "return expr"; + // - and the result vars are unnamed or never referenced (and thus cannot escape); + // then the call expression can be replaced by its body expression. + if callee.BodyIsReturnExpr && + callee.decl.Recv == nil && // no receiver arg effects to consider + len(caller.Call.Args) == 0 && // no argument effects to consider + !hasNamedVars(callee.decl.Type.Params) && // no param vars escape + !hasNamedVars(callee.decl.Type.Results) { // no result vars escape + + // A single return operand inlined to an expression + // context may need parens. Otherwise: + // func two() int { return 1+1 } + // print(-two()) => print(-1+1) // oops! + parens := callee.NumResults == 1 + + // If the call is a standalone statement, but the + // callee body is not suitable as a standalone statement + // (f() or <-ch), explicitly discard the results: + // _, _ = expr + if isCallStmt(callerPath) { + parens = false + + if !callee.ValidForCallStmt { + for i := 0; i < callee.NumResults; i++ { + if i > 0 { + out.WriteString(", ") + } + out.WriteString("_") + } + out.WriteString(" = ") + } + } + + // Emit the body expression(s). + for i, res := range callee.decl.Body.List[0].(*ast.ReturnStmt).Results { + if i > 0 { + out.WriteString(", ") + } + if parens { + out.WriteString("(") + } + replace(callee.offset(res.Pos()), callee.offset(res.End())) + if parens { + out.WriteString(")") + } + } + goto rest + } + + // Emit a function literal in place of the callee name, + // with appropriate replacements. + out.WriteString("func (") + if recv := callee.decl.Recv; recv != nil { + // Move method receiver to head of ordinary parameters. + replace(callee.offset(recv.Opening+1), callee.offset(recv.Closing)) + if len(callee.decl.Type.Params.List) > 0 { + out.WriteString(", ") + } + } + replace(callee.offset(callee.decl.Type.Params.Opening+1), + callee.offset(callee.decl.End())) + + // Emit call arguments. + out.WriteString("(") + if callee.decl.Recv != nil { + // Move receiver argument x.f(...) to argument list f(x, ...). + recv := astutil.Unparen(caller.Call.Fun).(*ast.SelectorExpr).X + + // If the receiver argument and parameter have + // different pointerness, make the "&" or "*" explicit. + argPtr := is[*types.Pointer](typeparams.CoreType(caller.Info.TypeOf(recv))) + paramPtr := is[*ast.StarExpr](callee.decl.Recv.List[0].Type) + if !argPtr && paramPtr { + out.WriteString("&") + } else if argPtr && !paramPtr { + out.WriteString("*") + } + + out.Write(caller.Content[caller.offset(recv.Pos()):caller.offset(recv.End())]) + + if len(caller.Call.Args) > 0 { + out.WriteString(", ") + } + } + // Append ordinary args, sans initial "(". + out.Write(caller.Content[caller.offset(caller.Call.Lparen+1):caller.offset(caller.Call.End())]) + + // Append rest of caller file. +rest: + out.Write(caller.Content[caller.offset(caller.Call.End()):]) + + // Reformat, and organize imports. + // + // TODO(adonovan): this looks at the user's cache state. + // Replace with a simpler implementation since + // all the necessary imports are present but merely untidy. + // That will be faster, and also less prone to nondeterminism + // if there are bugs in our logic for import maintenance. + // + // However, golang.org/x/tools/internal/imports.ApplyFixes is + // too simple as it requires the caller to have figured out + // all the logical edits. In our case, we know all the new + // imports that are needed (see newImports), each of which can + // be specified as: + // + // &imports.ImportFix{ + // StmtInfo: imports.ImportInfo{path, name, + // IdentName: name, + // FixType: imports.AddImport, + // } + // + // but we don't know which imports are made redundant by the + // inlining itself. For example, inlining a call to + // fmt.Println may make the "fmt" import redundant. + // + // Also, both imports.Process and internal/imports.ApplyFixes + // reformat the entire file, which is not ideal for clients + // such as gopls. (That said, the point of a canonical format + // is arguably that any tool can reformat as needed without + // this being inconvenient.) + res, err := imports.Process("output", out.Bytes(), nil) + if err != nil { + if false { // debugging + log.Printf("cannot reformat: %v <<%s>>", err, &out) + } + return nil, err // cannot reformat (a bug?) + } + return res, nil +} + +// -- helpers -- + +func is[T any](x any) bool { + _, ok := x.(T) + return ok +} + +func within(pos token.Pos, n ast.Node) bool { + return n.Pos() <= pos && pos <= n.End() +} + +func offsetOf(fset *token.FileSet, pos token.Pos) int { + return fset.PositionFor(pos, false).Offset +} + +// importedPkgName returns the PkgName object declared by an ImportSpec. +// TODO(adonovan): make this a method of types.Info (#62037). +func importedPkgName(info *types.Info, imp *ast.ImportSpec) (*types.PkgName, bool) { + var obj types.Object + if imp.Name != nil { + obj = info.Defs[imp.Name] + } else { + obj = info.Implicits[imp] + } + pkgname, ok := obj.(*types.PkgName) + return pkgname, ok +} + +func isPkgLevel(obj types.Object) bool { + return obj.Pkg().Scope().Lookup(obj.Name()) == obj +} + +// objectKind returns an object's kind (e.g. var, func, const, typename). +func objectKind(obj types.Object) string { + return strings.TrimPrefix(strings.ToLower(reflect.TypeOf(obj).String()), "*types.") +} + +// isCallStmt reports whether the function call (specified +// as a PathEnclosingInterval) appears within an ExprStmt. +func isCallStmt(callPath []ast.Node) bool { + _ = callPath[0].(*ast.CallExpr) + for _, n := range callPath[1:] { + switch n.(type) { + case *ast.ParenExpr: + continue + case *ast.ExprStmt: + return true + } + break + } + return false +} + +// hasNamedVars reports whether a function parameter tuple uses named variables. +// +// TODO(adonovan): this is a placeholder for a more complex analysis to detect +// whether inlining might cause named param/result variables to escape. +func hasNamedVars(tuple *ast.FieldList) bool { + return tuple != nil && len(tuple.List) > 0 && tuple.List[0].Names != nil +} diff --git a/internal/refactor/inline/inline_test.go b/internal/refactor/inline/inline_test.go new file mode 100644 index 00000000000..042784082cc --- /dev/null +++ b/internal/refactor/inline/inline_test.go @@ -0,0 +1,322 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package inline_test + +import ( + "bytes" + "encoding/gob" + "fmt" + "go/ast" + "go/token" + "os" + "path/filepath" + "regexp" + "testing" + + "golang.org/x/tools/go/ast/astutil" + "golang.org/x/tools/go/expect" + "golang.org/x/tools/go/packages" + "golang.org/x/tools/go/types/typeutil" + "golang.org/x/tools/internal/diff" + "golang.org/x/tools/internal/refactor/inline" + "golang.org/x/tools/txtar" +) + +// Test executes test scenarios specified by files in testdata/*.txtar. +func Test(t *testing.T) { + files, err := filepath.Glob("testdata/*.txtar") + if err != nil { + t.Fatal(err) + } + for _, file := range files { + file := file + t.Run(filepath.Base(file), func(t *testing.T) { + t.Parallel() + + // Extract archive to temporary tree. + ar, err := txtar.ParseFile(file) + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + if err := extractTxtar(ar, dir); err != nil { + t.Fatal(err) + } + + // Load packages. + cfg := &packages.Config{ + Dir: dir, + Mode: packages.LoadAllSyntax, + Env: append(os.Environ(), + "GO111MODULES=on", + "GOPATH=", + "GOWORK=off", + "GOPROXY=off"), + } + pkgs, err := packages.Load(cfg, "./...") + if err != nil { + t.Errorf("Load: %v", err) + } + // Report parse/type errors; they may be benign. + packages.Visit(pkgs, nil, func(pkg *packages.Package) { + for _, err := range pkg.Errors { + t.Log(err) + } + }) + + // Process @inline notes in comments in initial packages. + for _, pkg := range pkgs { + for _, file := range pkg.Syntax { + // Read file content (for @inline regexp, and inliner). + content, err := os.ReadFile(pkg.Fset.File(file.Pos()).Name()) + if err != nil { + t.Error(err) + continue + } + + // Read and process @inline notes. + notes, err := expect.ExtractGo(pkg.Fset, file) + if err != nil { + t.Errorf("parsing notes in %q: %v", pkg.Fset.File(file.Pos()).Name(), err) + continue + } + for _, note := range notes { + posn := pkg.Fset.Position(note.Pos) + if note.Name != "inline" { + t.Errorf("%s: invalid marker @%s", posn, note.Name) + continue + } + if nargs := len(note.Args); nargs != 2 { + t.Errorf("@inline: want 2 args, got %d", nargs) + continue + } + pattern, ok := note.Args[0].(*regexp.Regexp) + if !ok { + t.Errorf("%s: @inline(rx, want): want regular expression rx", posn) + continue + } + + // want is a []byte (success) or *Regexp (failure) + var want any + switch x := note.Args[1].(type) { + case string, expect.Identifier: + for _, file := range ar.Files { + if file.Name == fmt.Sprint(x) { + want = file.Data + break + } + } + if want == nil { + t.Errorf("%s: @inline(rx, want): archive entry %q not found", posn, x) + continue + } + case *regexp.Regexp: + want = x + default: + t.Errorf("%s: @inline(rx, want): want file name (to assert success) or error message regexp (to assert failure)", posn) + continue + } + t.Log("doInlineNote", posn) + if err := doInlineNote(pkg, file, content, pattern, posn, want); err != nil { + t.Errorf("%s: @inline(%v, %v): %v", posn, note.Args[0], note.Args[1], err) + continue + } + } + } + } + }) + } +} + +// doInlineNote executes an assertion specified by a single +// @inline(re"pattern", want) note in a comment. It finds the first +// match of regular expression 'pattern' on the same line, finds the +// innermost enclosing CallExpr, and inlines it. +// +// Finally it checks that, on success, the transformed file is equal +// to want (a []byte), or on failure that the error message matches +// want (a *Regexp). +func doInlineNote(pkg *packages.Package, file *ast.File, content []byte, pattern *regexp.Regexp, posn token.Position, want any) error { + // Find extent of pattern match within commented line. + var startPos, endPos token.Pos + { + tokFile := pkg.Fset.File(file.Pos()) + lineStartOffset := int(tokFile.LineStart(posn.Line)) - tokFile.Base() + line := content[lineStartOffset:] + if i := bytes.IndexByte(line, '\n'); i >= 0 { + line = line[:i] + } + matches := pattern.FindSubmatchIndex(line) + var start, end int // offsets + switch len(matches) { + case 2: + // no subgroups: return the range of the regexp expression + start, end = matches[0], matches[1] + case 4: + // one subgroup: return its range + start, end = matches[2], matches[3] + default: + return fmt.Errorf("invalid location regexp %q: expect either 0 or 1 subgroups, got %d", + pattern, len(matches)/2-1) + } + startPos = tokFile.Pos(lineStartOffset + start) + endPos = tokFile.Pos(lineStartOffset + end) + } + + // Find innermost call enclosing the pattern match. + var caller *inline.Caller + { + path, _ := astutil.PathEnclosingInterval(file, startPos, endPos) + for _, n := range path { + if call, ok := n.(*ast.CallExpr); ok { + caller = &inline.Caller{ + Fset: pkg.Fset, + Types: pkg.Types, + Info: pkg.TypesInfo, + File: file, + Call: call, + Content: content, + } + break + } + } + if caller == nil { + return fmt.Errorf("no enclosing call") + } + } + + // Is it a static function call? + fn := typeutil.StaticCallee(caller.Info, caller.Call) + if fn == nil { + return fmt.Errorf("cannot inline: not a static call") + } + + // Find callee function. + var ( + calleePkg *packages.Package + calleeDecl *ast.FuncDecl + ) + { + var same func(*ast.FuncDecl) bool + // Is the call within the package? + if fn.Pkg() == caller.Types { + calleePkg = pkg // same as caller + same = func(decl *ast.FuncDecl) bool { + return decl.Name.Pos() == fn.Pos() + } + } else { + // Different package. Load it now. + // (The primary load loaded all dependencies, + // but we choose to load it again, with + // a distinct token.FileSet and types.Importer, + // to keep the implementation honest.) + cfg := &packages.Config{ + // TODO(adonovan): get the original module root more cleanly + Dir: filepath.Dir(filepath.Dir(pkg.GoFiles[0])), + Fset: token.NewFileSet(), + Mode: packages.LoadSyntax, + } + roots, err := packages.Load(cfg, fn.Pkg().Path()) + if err != nil { + return fmt.Errorf("loading callee package: %v", err) + } + if packages.PrintErrors(roots) > 0 { + return fmt.Errorf("callee package had errors") // (see log) + } + calleePkg = roots[0] + posn := caller.Fset.Position(fn.Pos()) // callee posn wrt caller package + same = func(decl *ast.FuncDecl) bool { + // We can't rely on columns in export data: + // some variants replace it with 1. + // We can't expect file names to have the same prefix. + // export data for go1.20 std packages have $GOROOT written in + // them, so how are we supposed to find the source? Yuck! + // Ugh. need to samefile? Nope $GOROOT just won't work + // This is highly client specific anyway. + posn2 := calleePkg.Fset.Position(decl.Name.Pos()) + return posn.Filename == posn2.Filename && + posn.Line == posn2.Line + } + } + + for _, file := range calleePkg.Syntax { + for _, decl := range file.Decls { + if decl, ok := decl.(*ast.FuncDecl); ok && same(decl) { + calleeDecl = decl + goto found + } + } + } + return fmt.Errorf("can't find FuncDecl for callee") // can't happen? + found: + } + + // Do the inlining. For the purposes of the test, + // AnalyzeCallee and Inline are a single operation. + got, err := func() ([]byte, error) { + filename := calleePkg.Fset.File(calleeDecl.Pos()).Name() + content, err := os.ReadFile(filename) + if err != nil { + return nil, err + } + callee, err := inline.AnalyzeCallee( + calleePkg.Fset, + calleePkg.Types, + calleePkg.TypesInfo, + calleeDecl, + content) + if err != nil { + return nil, err + } + + // Perform Gob transcoding so that it is exercised by the test. + var enc bytes.Buffer + if err := gob.NewEncoder(&enc).Encode(callee); err != nil { + return nil, fmt.Errorf("internal error: gob encoding failed: %v", err) + } + *callee = inline.Callee{} + if err := gob.NewDecoder(&enc).Decode(callee); err != nil { + return nil, fmt.Errorf("internal error: gob decoding failed: %v", err) + } + + return inline.Inline(caller, callee) + }() + if err != nil { + if wantRE, ok := want.(*regexp.Regexp); ok { + if !wantRE.MatchString(err.Error()) { + return fmt.Errorf("Inline failed with wrong error: %v (want error matching %q)", err, want) + } + return nil // expected error + } + return fmt.Errorf("Inline failed: %v", err) // success was expected + } + + // Inline succeeded. + if want, ok := want.([]byte); ok { + got = append(bytes.TrimSpace(got), '\n') + want = append(bytes.TrimSpace(want), '\n') + if diff := diff.Unified("want", "got", string(want), string(got)); diff != "" { + return fmt.Errorf("Inline returned wrong output:\n%s\nWant:\n%s\nDiff:\n%s", + got, want, diff) + } + return nil + } + return fmt.Errorf("Inline succeeded unexpectedly: want error matching %q, got <<%s>>", want, got) + +} + +// TODO(adonovan): publish this a helper (#61386). +func extractTxtar(ar *txtar.Archive, dir string) error { + for _, file := range ar.Files { + name := filepath.Join(dir, file.Name) + if err := os.MkdirAll(filepath.Dir(name), 0777); err != nil { + return err + } + if err := os.WriteFile(name, file.Data, 0666); err != nil { + return err + } + } + return nil +} diff --git a/internal/refactor/inline/testdata/basic-err.txtar b/internal/refactor/inline/testdata/basic-err.txtar new file mode 100644 index 00000000000..18e0eb7adb3 --- /dev/null +++ b/internal/refactor/inline/testdata/basic-err.txtar @@ -0,0 +1,24 @@ +Test of inlining a function that references err.Error, +which is often a special case because it has no position. + +-- go.mod -- +module testdata +go 1.12 + +-- a/a.go -- +package a + +import "io" + +var _ = getError(io.EOF) //@ inline(re"getError", getError) + +func getError(err error) string { return err.Error() } + +-- getError -- +package a + +import "io" + +var _ = func(err error) string { return err.Error() }(io.EOF) //@ inline(re"getError", getError) + +func getError(err error) string { return err.Error() } diff --git a/internal/refactor/inline/testdata/basic-literal.txtar b/internal/refactor/inline/testdata/basic-literal.txtar new file mode 100644 index 00000000000..50bac33456a --- /dev/null +++ b/internal/refactor/inline/testdata/basic-literal.txtar @@ -0,0 +1,19 @@ +Most basic test of inlining by literalization. + +-- go.mod -- +module testdata +go 1.12 + +-- a/a.go -- +package a + +var _ = add(1, 2) //@ inline(re"add", add) + +func add(x, y int) int { return x + y } + +-- add -- +package a + +var _ = func(x, y int) int { return x + y }(1, 2) //@ inline(re"add", add) + +func add(x, y int) int { return x + y } diff --git a/internal/refactor/inline/testdata/basic-reduce.txtar b/internal/refactor/inline/testdata/basic-reduce.txtar new file mode 100644 index 00000000000..9eedbc05f1e --- /dev/null +++ b/internal/refactor/inline/testdata/basic-reduce.txtar @@ -0,0 +1,19 @@ +Most basic test of inlining by reduction. + +-- go.mod -- +module testdata +go 1.12 + +-- a/a.go -- +package a + +var _ = zero() //@ inline(re"zero", zero) + +func zero() int { return 0 } + +-- zero -- +package a + +var _ = (0) //@ inline(re"zero", zero) + +func zero() int { return 0 } diff --git a/internal/refactor/inline/testdata/comments.txtar b/internal/refactor/inline/testdata/comments.txtar new file mode 100644 index 00000000000..0482e919a48 --- /dev/null +++ b/internal/refactor/inline/testdata/comments.txtar @@ -0,0 +1,56 @@ +Inlining, whether by literalization or reduction, +preserves comments in the callee. + +-- go.mod -- +module testdata +go 1.12 + +-- a/f.go -- +package a + +func _() { + f() //@ inline(re"f", f) +} + +func f() { + // a + /* b */ g() /* c */ + // d +} + +-- f -- +package a + +func _() { + func() { + // a + /* b */ + g() /* c */ + // d + }() //@ inline(re"f", f) +} + +func f() { + // a + /* b */ + g() /* c */ + // d +} + +-- a/g.go -- +package a + +func _() { + println(g()) //@ inline(re"g", g) +} + +func g() int { return 1 /*hello*/ + /*there*/ 1 } + +-- g -- +package a + +func _() { + println((1 /*hello*/ + /*there*/ 1)) //@ inline(re"g", g) +} + +func g() int { return 1 /*hello*/ + /*there*/ 1 } diff --git a/internal/refactor/inline/testdata/crosspkg.txtar b/internal/refactor/inline/testdata/crosspkg.txtar new file mode 100644 index 00000000000..43dc63f32ea --- /dev/null +++ b/internal/refactor/inline/testdata/crosspkg.txtar @@ -0,0 +1,77 @@ +Test of cross-package inlining. +The first case creates a new import, +the second reuses an existing one. + +-- go.mod -- +module testdata +go 1.12 + +-- a/a.go -- +package a + +// This comment does not migrate. + +import ( + "fmt" + "testdata/b" +) + +// Nor this one. + +func A() { + fmt.Println() + b.B1() //@ inline(re"B1", b1result) + b.B2() //@ inline(re"B2", b2result) +} + +-- b/b.go -- +package b + +import "testdata/c" +import "fmt" + +func B1() { c.C() } +func B2() { fmt.Println() } + +-- c/c.go -- +package c + +func C() {} + +-- b1result -- +package a + +// This comment does not migrate. + +import ( + "fmt" + "testdata/b" + + c "testdata/c" +) + +// Nor this one. + +func A() { + fmt.Println() + func() { c.C() }() //@ inline(re"B1", b1result) + b.B2() //@ inline(re"B2", b2result) +} + +-- b2result -- +package a + +// This comment does not migrate. + +import ( + "fmt" + "testdata/b" +) + +// Nor this one. + +func A() { + fmt.Println() + b.B1() //@ inline(re"B1", b1result) + func() { fmt.Println() }() //@ inline(re"B2", b2result) +} diff --git a/internal/refactor/inline/testdata/dotimport.txtar b/internal/refactor/inline/testdata/dotimport.txtar new file mode 100644 index 00000000000..7e886afdb94 --- /dev/null +++ b/internal/refactor/inline/testdata/dotimport.txtar @@ -0,0 +1,35 @@ +Test of inlining a function that uses a dot import. + +-- go.mod -- +module testdata +go 1.12 + +-- a/a.go -- +package a + +func A() {} + +-- b/b.go -- +package b + +import . "testdata/a" + +func B() { A() } + +-- c/c.go -- +package c + +import "testdata/b" + +func _() { + b.B() //@ inline(re"B", result) +} + +-- result -- +package c + +import a "testdata/a" + +func _() { + func() { a.A() }() //@ inline(re"B", result) +} diff --git a/internal/refactor/inline/testdata/err-basic.txtar b/internal/refactor/inline/testdata/err-basic.txtar new file mode 100644 index 00000000000..54377c70c4b --- /dev/null +++ b/internal/refactor/inline/testdata/err-basic.txtar @@ -0,0 +1,30 @@ +Basic errors: +- Inlining of generic functions is not yet supported. + +We can't express tests for the error resulting from inlining a +conversion T(x), a call to a literal func(){}(), a call to a +func-typed var, or a call to an interface method, since all of these +cause the test driver to fail to locate the callee, so +it doesn't even reach the Indent function. + +-- go.mod -- +module testdata +go 1.12 + +-- a/generic.go -- +package a + +func _() { + f[int]() //@ inline(re"f", re"type parameters are not yet supported") +} + +func f[T any]() {} + +-- a/nobody.go -- +package a + +func _() { + g() //@ inline(re"g", re"has no body") +} + +func g() diff --git a/internal/refactor/inline/testdata/err-shadow-builtin.txtar b/internal/refactor/inline/testdata/err-shadow-builtin.txtar new file mode 100644 index 00000000000..543d38fe540 --- /dev/null +++ b/internal/refactor/inline/testdata/err-shadow-builtin.txtar @@ -0,0 +1,36 @@ +Failures to inline because callee references a builtin that +is shadowed by caller. + +-- go.mod -- +module testdata +go 1.12 + +-- a/nil.go -- +package a + +func _() { + const nil = 1 + _ = f() //@ inline(re"f", re"nil.*shadowed.*by.*const .line 4") +} + +func f() *int { return nil } + +-- a/append.go -- +package a + +func _() { + type append int + g(nil) //@ inline(re"g", re"append.*shadowed.*by.*typename .line 4") +} + +func g(x []int) { _ = append(x, x...) } + +-- a/type.go -- +package a + +func _() { + type int uint8 + _ = h(0) //@ inline(re"h", re"int.*shadowed.*by.*typename .line 4") +} + +func h(x int) int { return x + 1 } diff --git a/internal/refactor/inline/testdata/err-shadow-pkg.txtar b/internal/refactor/inline/testdata/err-shadow-pkg.txtar new file mode 100644 index 00000000000..4338b8b31cd --- /dev/null +++ b/internal/refactor/inline/testdata/err-shadow-pkg.txtar @@ -0,0 +1,36 @@ +Test of failure to inline because callee references a +package-level decl that is shadowed by caller. + +Observe that the first call to f can be inlined because +the shadowing has not yet occurred; but the second call +to f is within the scope of the local constant v. + +-- go.mod -- +module testdata +go 1.12 + +-- a/a.go -- +package a + +func _() { + f() //@ inline(re"f", result) + const v = 1 + f() //@ inline(re"f", re"v.*shadowed.*by.*const .line 5") +} + +func f() int { return v } + +var v int + +-- result -- +package a + +func _() { + _ = v //@ inline(re"f", result) + const v = 1 + f() //@ inline(re"f", re"v.*shadowed.*by.*const .line 5") +} + +func f() int { return v } + +var v int diff --git a/internal/refactor/inline/testdata/err-unexported.txtar b/internal/refactor/inline/testdata/err-unexported.txtar new file mode 100644 index 00000000000..9ba91e5195d --- /dev/null +++ b/internal/refactor/inline/testdata/err-unexported.txtar @@ -0,0 +1,31 @@ +Errors from attempting to import a function from another +package whose body refers to unexported declarations. + +-- go.mod -- +module testdata +go 1.12 + +-- a/a.go -- +package a + +func A1() { b() } +func b() {} + +func A2() { var x T; print(x.f) } +type T struct { f int } + +func A3() { _ = &T{f: 0} } + +func A4() { _ = &T{0} } + +-- b/b.go -- +package b + +import "testdata/a" + +func _() { + a.A1() //@ inline(re"A1", re`body refers to non-exported b`) + a.A2() //@ inline(re"A2", re`body refers to non-exported \(testdata/a.T\).f`) + a.A3() //@ inline(re"A3", re`body refers to non-exported \(testdata/a.T\).f`) + a.A4() //@ inline(re"A4", re`body refers to non-exported \(testdata/a.T\).f`) +} diff --git a/internal/refactor/inline/testdata/exprstmt.txtar b/internal/refactor/inline/testdata/exprstmt.txtar new file mode 100644 index 00000000000..449ce35c454 --- /dev/null +++ b/internal/refactor/inline/testdata/exprstmt.txtar @@ -0,0 +1,99 @@ +Inlining an expression into an ExprStmt. +Call and receive expressions can be inlined directly +(though calls to only some builtins can be reduced). +All other expressions are inlined as "_ = expr". + +-- go.mod -- +module testdata +go 1.12 + +-- a/call.go -- +package a + +func _() { + call() //@ inline(re"call", call) +} + +func call() int { return recv() } + +-- call -- +package a + +func _() { + recv() //@ inline(re"call", call) +} + +func call() int { return recv() } + +-- a/recv.go -- +package a + +func _() { + recv() //@ inline(re"recv", recv) +} + +func recv() int { return <-(chan int)(nil) } + +-- recv -- +package a + +func _() { + <-(chan int)(nil) //@ inline(re"recv", recv) +} + +func recv() int { return <-(chan int)(nil) } + +-- a/constant.go -- +package a + +func _() { + constant() //@ inline(re"constant", constant) +} + +func constant() int { return 0 } + +-- constant -- +package a + +func _() { + _ = 0 //@ inline(re"constant", constant) +} + +func constant() int { return 0 } + +-- a/builtin.go -- +package a + +func _() { + builtin() //@ inline(re"builtin", builtin) +} + +func builtin() int { return len("") } + +-- builtin -- +package a + +func _() { + _ = len("") //@ inline(re"builtin", builtin) +} + +func builtin() int { return len("") } + +-- a/copy.go -- +package a + +func _() { + _copy() //@ inline(re"copy", copy) +} + +func _copy() int { return copy([]int(nil), []int(nil)) } + +-- copy -- +package a + +func _() { + copy([]int(nil), []int(nil)) //@ inline(re"copy", copy) +} + +func _copy() int { return copy([]int(nil), []int(nil)) } + diff --git a/internal/refactor/inline/testdata/import-shadow.txtar b/internal/refactor/inline/testdata/import-shadow.txtar new file mode 100644 index 00000000000..913c9cbe01a --- /dev/null +++ b/internal/refactor/inline/testdata/import-shadow.txtar @@ -0,0 +1,41 @@ +Test of heuristic for generating a fresh import PkgName. +The names c and c0 are taken, so it uses c1. + +-- go.mod -- +module testdata +go 1.12 + +-- a/a.go -- +package a + +import "testdata/b" + +func A() { + const c = 1 + type c0 int + b.B() //@ inline(re"B", result) +} + +-- b/b.go -- +package b + +import "testdata/c" + +func B() { c.C() } + +-- c/c.go -- +package c + +func C() {} + +-- result -- +package a + +import c1 "testdata/c" + +func A() { + const c = 1 + type c0 int + func() { c1.C() }() //@ inline(re"B", result) +} + diff --git a/internal/refactor/inline/testdata/internal.txtar b/internal/refactor/inline/testdata/internal.txtar new file mode 100644 index 00000000000..92a0fef4c0a --- /dev/null +++ b/internal/refactor/inline/testdata/internal.txtar @@ -0,0 +1,29 @@ +Test of inlining a function that references an +internal package that is not accessible to the caller. + +(c -> b -> b/internal/a) + +-- go.mod -- +module testdata +go 1.12 + +-- b/internal/a/a.go -- +package a + +func A() {} + +-- b/b.go -- +package b + +import "testdata/b/internal/a" + +func B() { a.A() } + +-- c/c.go -- +package c + +import "testdata/b" + +func _() { + b.B() //@ inline(re"B", re`body refers to inaccessible package "testdata/b/internal/a"`) +} diff --git a/internal/refactor/inline/testdata/method.txtar b/internal/refactor/inline/testdata/method.txtar new file mode 100644 index 00000000000..a4e02d575ca --- /dev/null +++ b/internal/refactor/inline/testdata/method.txtar @@ -0,0 +1,104 @@ +Test of inlining a method call. + +The call to (*T).g0 implicitly takes the address &x. + +The f1/g1 methods have parameters, exercising the +splicing of the receiver into the parameter list. +Notice that the unnamed parameters become named. + +-- go.mod -- +module testdata +go 1.12 + +-- a/f0.go -- +package a + +type T int +func (T) f0() {} + +func _(x T) { + x.f0() //@ inline(re"f0", f0) +} + +-- f0 -- +package a + +type T int + +func (T) f0() {} + +func _(x T) { + func(_ T) {}(x) //@ inline(re"f0", f0) +} + +-- a/g0.go -- +package a + +func (recv *T) g0() {} + +func _(x T) { + x.g0() //@ inline(re"g0", g0) +} + +-- g0 -- +package a + +func (recv *T) g0() {} + +func _(x T) { + func(recv *T) {}(&x) //@ inline(re"g0", g0) +} + +-- a/f1.go -- +package a + +func (T) f1(int, int) {} + +func _(x T) { + x.f1(1, 2) //@ inline(re"f1", f1) +} + +-- f1 -- +package a + +func (T) f1(int, int) {} + +func _(x T) { + func(_ T, _ int, _ int) {}(x, 1, 2) //@ inline(re"f1", f1) +} + +-- a/g1.go -- +package a + +func (recv *T) g1(int, int) {} + +func _(x T) { + x.g1(1, 2) //@ inline(re"g1", g1) +} + +-- g1 -- +package a + +func (recv *T) g1(int, int) {} + +func _(x T) { + func(recv *T, _ int, _ int) {}(&x, 1, 2) //@ inline(re"g1", g1) +} + +-- a/h.go -- +package a + +func (T) h() int { return 1 } + +func _() { + new(T).h() //@ inline(re"h", h) +} + +-- h -- +package a + +func (T) h() int { return 1 } + +func _() { + func(_ T) int { return 1 }(*new(T)) //@ inline(re"h", h) +} diff --git a/internal/refactor/inline/testdata/n-ary.txtar b/internal/refactor/inline/testdata/n-ary.txtar new file mode 100644 index 00000000000..2de97358aed --- /dev/null +++ b/internal/refactor/inline/testdata/n-ary.txtar @@ -0,0 +1,79 @@ +Tests of various n-ary result function cases. + +-- go.mod -- +module testdata +go 1.12 + +-- a/a.go -- +package a + +func _() { + println(f1()) //@ inline(re"f1", f1) +} + +func f1() (int, int) { return 1, 1 } + +-- f1 -- +package a + +func _() { + println(1, 1) //@ inline(re"f1", f1) +} + +func f1() (int, int) { return 1, 1 } + +-- b/b.go -- +package b + +func _() { + f2() //@ inline(re"f2", f2) +} + +func f2() (int, int) { return 2, 2 } + +-- f2 -- +package b + +func _() { + _, _ = 2, 2 //@ inline(re"f2", f2) +} + +func f2() (int, int) { return 2, 2 } + +-- c/c.go -- +package c + +func _() { + _, _ = f3() //@ inline(re"f3", f3) +} + +func f3() (int, int) { return f3A() } +func f3A() (x, y int) + +-- f3 -- +package c + +func _() { + _, _ = f3A() //@ inline(re"f3", f3) +} + +func f3() (int, int) { return f3A() } +func f3A() (x, y int) + +-- d/d.go -- +package d + +func _() { + println(-f4()) //@ inline(re"f4", f4) +} + +func f4() int { return 2 + 2 } + +-- f4 -- +package d + +func _() { + println(-(2 + 2)) //@ inline(re"f4", f4) +} + +func f4() int { return 2 + 2 } diff --git a/internal/refactor/inline/testdata/revdotimport.txtar b/internal/refactor/inline/testdata/revdotimport.txtar new file mode 100644 index 00000000000..f8b895e9218 --- /dev/null +++ b/internal/refactor/inline/testdata/revdotimport.txtar @@ -0,0 +1,43 @@ +Test of inlining a function into a context that already +dot-imports the necessary additional import. + +-- go.mod -- +module testdata +go 1.12 + +-- a/a.go -- +package a + +func A() {} + +-- b/b.go -- +package b + +import "testdata/a" + +func B() { a.A() } + +-- c/c.go -- +package c + +import . "testdata/a" +import "testdata/b" + +func _() { + A() + b.B() //@ inline(re"B", result) +} + +-- result -- +package c + +import ( + . "testdata/a" + + a "testdata/a" +) + +func _() { + A() + func() { a.A() }() //@ inline(re"B", result) +} From 5e8d21aa5354ad8e14124765ee07380f43de5e23 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Mon, 28 Aug 2023 14:56:34 -0400 Subject: [PATCH 040/178] gopls/internal/lsp/source: implement refactor.inline code action This change uses the new inlining package to implement the refactor.inline code action. Fixes golang/go#59243 Change-Id: I9455e1989c6e077849c72924e336c0c738c5cebc Reviewed-on: https://go-review.googlesource.com/c/tools/+/523696 Reviewed-by: Robert Findley gopls-CI: kokoro Run-TryBot: Alan Donovan TryBot-Result: Gopher Robot --- gopls/internal/lsp/cmd/cmd.go | 5 +- gopls/internal/lsp/cmd/suggested_fix.go | 24 +++- gopls/internal/lsp/code_action.go | 42 ++++++- gopls/internal/lsp/source/fix.go | 4 + gopls/internal/lsp/source/inline.go | 113 ++++++++++++++++++ gopls/internal/lsp/source/options.go | 1 + gopls/internal/lsp/source/stub.go | 17 ++- gopls/internal/lsp/tests/tests.go | 1 + .../marker/testdata/codeaction/inline.txt | 23 ++++ 9 files changed, 219 insertions(+), 11 deletions(-) create mode 100644 gopls/internal/lsp/source/inline.go create mode 100644 gopls/internal/regtest/marker/testdata/codeaction/inline.txt diff --git a/gopls/internal/lsp/cmd/cmd.go b/gopls/internal/lsp/cmd/cmd.go index 3474ed73352..08e7a64e00a 100644 --- a/gopls/internal/lsp/cmd/cmd.go +++ b/gopls/internal/lsp/cmd/cmd.go @@ -524,7 +524,10 @@ func (c *cmdClient) Configuration(ctx context.Context, p *protocol.ParamConfigur } func (c *cmdClient) ApplyEdit(ctx context.Context, p *protocol.ApplyWorkspaceEditParams) (*protocol.ApplyWorkspaceEditResult, error) { - return &protocol.ApplyWorkspaceEditResult{Applied: false, FailureReason: "not implemented"}, nil + return &protocol.ApplyWorkspaceEditResult{ + Applied: false, + FailureReason: "the gopls command-line client does not apply edits", + }, nil } func (c *cmdClient) PublishDiagnostics(ctx context.Context, p *protocol.PublishDiagnosticsParams) error { diff --git a/gopls/internal/lsp/cmd/suggested_fix.go b/gopls/internal/lsp/cmd/suggested_fix.go index c0770bc489e..5e267e71bfd 100644 --- a/gopls/internal/lsp/cmd/suggested_fix.go +++ b/gopls/internal/lsp/cmd/suggested_fix.go @@ -101,12 +101,30 @@ func (s *suggestedFix) Run(ctx context.Context, args ...string) error { // Gather edits from matching code actions. var edits []protocol.TextEdit for _, a := range actions { - if a.Command != nil { - return fmt.Errorf("ExecuteCommand is not yet supported on the command line (action: %v)", a.Title) - } + // Without -all, apply only "preferred" fixes. if !a.IsPreferred && !s.All { continue } + + // Execute any command. + // This may cause the server to make + // an ApplyEdit downcall to the client. + if a.Command != nil { + if _, err := conn.ExecuteCommand(ctx, &protocol.ExecuteCommandParams{ + Command: a.Command.Command, + Arguments: a.Command.Arguments, + }); err != nil { + return err + } + // The specification says that commands should + // be executed _after_ edits are applied, not + // instead of them, but we don't want to + // duplicate edits. + continue + } + + // Partially apply CodeAction.Edit, a WorkspaceEdit. + // (See also conn.Client.applyWorkspaceEdit(a.Edit)). if !from.HasPosition() { for _, c := range a.Edit.DocumentChanges { if c.TextDocumentEdit != nil { diff --git a/gopls/internal/lsp/code_action.go b/gopls/internal/lsp/code_action.go index 69df978f0fc..938edcf0eeb 100644 --- a/gopls/internal/lsp/code_action.go +++ b/gopls/internal/lsp/code_action.go @@ -194,7 +194,10 @@ func (s *Server) codeAction(ctx context.Context, params *protocol.CodeActionPara } // Code actions requiring type information. - if len(stubMethodsDiagnostics) > 0 || want[protocol.RefactorRewrite] || want[protocol.GoTest] { + if len(stubMethodsDiagnostics) > 0 || + want[protocol.RefactorRewrite] || + want[protocol.RefactorInline] || + want[protocol.GoTest] { pkg, pgf, err := source.NarrowestPackageForFile(ctx, snapshot, fh.URI()) if err != nil { return nil, err @@ -250,6 +253,14 @@ func (s *Server) codeAction(ctx context.Context, params *protocol.CodeActionPara actions = append(actions, rewrites...) } + if want[protocol.RefactorInline] { + rewrites, err := refactorInline(ctx, snapshot, pkg, pgf, fh, params.Range) + if err != nil { + return nil, err + } + actions = append(actions, rewrites...) + } + if want[protocol.GoTest] { fixes, err := goTest(ctx, snapshot, pkg, pgf, params.Range) if err != nil { @@ -499,6 +510,35 @@ func refactorRewrite(ctx context.Context, snapshot source.Snapshot, pkg source.P return actions, nil } +// refactorInline returns inline actions available at the specified range. +func refactorInline(ctx context.Context, snapshot source.Snapshot, pkg source.Package, pgf *source.ParsedGoFile, fh source.FileHandle, rng protocol.Range) ([]protocol.CodeAction, error) { + var commands []protocol.Command + + // If range is within call expression, offer inline action. + if _, fn, err := source.EnclosingStaticCall(pkg, pgf, rng); err == nil { + cmd, err := command.NewApplyFixCommand(fmt.Sprintf("Inline call to %s", fn.Name()), command.ApplyFixArgs{ + URI: protocol.URIFromSpanURI(pgf.URI), + Fix: source.InlineCall, + Range: rng, + }) + if err != nil { + return nil, err + } + commands = append(commands, cmd) + } + + // Convert commands to actions. + var actions []protocol.CodeAction + for i := range commands { + actions = append(actions, protocol.CodeAction{ + Title: commands[i].Title, + Kind: protocol.RefactorInline, + Command: &commands[i], + }) + } + return actions, nil +} + func documentChanges(fh source.FileHandle, edits []protocol.TextEdit) []protocol.DocumentChanges { return []protocol.DocumentChanges{ { diff --git a/gopls/internal/lsp/source/fix.go b/gopls/internal/lsp/source/fix.go index f9d901c196c..7a715a8ff5a 100644 --- a/gopls/internal/lsp/source/fix.go +++ b/gopls/internal/lsp/source/fix.go @@ -35,6 +35,8 @@ type ( singleFileFixFunc func(fset *token.FileSet, start, end token.Pos, src []byte, file *ast.File, pkg *types.Package, info *types.Info) (*analysis.SuggestedFix, error) ) +// These strings identify kinds of suggested fix, both in Analyzer.Fix +// and in the ApplyFix subcommand (see ExecuteCommand and ApplyFixArgs.Fix). const ( FillStruct = "fill_struct" StubMethods = "stub_methods" @@ -42,6 +44,7 @@ const ( ExtractVariable = "extract_variable" ExtractFunction = "extract_function" ExtractMethod = "extract_method" + InlineCall = "inline_call" InvertIfCondition = "invert_if_condition" AddEmbedImport = "add_embed_import" ) @@ -51,6 +54,7 @@ var suggestedFixes = map[string]SuggestedFixFunc{ FillStruct: singleFile(fillstruct.SuggestedFix), UndeclaredName: singleFile(undeclaredname.SuggestedFix), ExtractVariable: singleFile(extractVariable), + InlineCall: inlineCall, ExtractFunction: singleFile(extractFunction), ExtractMethod: singleFile(extractMethod), InvertIfCondition: singleFile(invertIfCondition), diff --git a/gopls/internal/lsp/source/inline.go b/gopls/internal/lsp/source/inline.go new file mode 100644 index 00000000000..6a8d57d412a --- /dev/null +++ b/gopls/internal/lsp/source/inline.go @@ -0,0 +1,113 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package source + +// This file defines the refactor.inline code action. + +import ( + "context" + "fmt" + "go/ast" + "go/token" + "go/types" + + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/ast/astutil" + "golang.org/x/tools/go/types/typeutil" + "golang.org/x/tools/gopls/internal/lsp/protocol" + "golang.org/x/tools/gopls/internal/lsp/safetoken" + "golang.org/x/tools/gopls/internal/span" + "golang.org/x/tools/internal/diff" + "golang.org/x/tools/internal/refactor/inline" +) + +// EnclosingStaticCall returns the innermost function call enclosing +// the selected range, along with the callee. +func EnclosingStaticCall(pkg Package, pgf *ParsedGoFile, rng protocol.Range) (*ast.CallExpr, *types.Func, error) { + start, end, err := pgf.RangePos(rng) + if err != nil { + return nil, nil, err + } + path, _ := astutil.PathEnclosingInterval(pgf.File, start, end) + + var call *ast.CallExpr +loop: + for _, n := range path { + switch n := n.(type) { + case *ast.FuncLit: + break loop + case *ast.CallExpr: + call = n + break loop + } + } + if call == nil { + return nil, nil, fmt.Errorf("no enclosing call") + } + if safetoken.Line(pgf.Tok, call.Lparen) != safetoken.Line(pgf.Tok, start) { + return nil, nil, fmt.Errorf("enclosing call is not on this line") + } + fn := typeutil.StaticCallee(pkg.GetTypesInfo(), call) + if fn == nil { + return nil, nil, fmt.Errorf("not a static call to a Go function") + } + return call, fn, nil +} + +func inlineCall(ctx context.Context, snapshot Snapshot, fh FileHandle, rng protocol.Range) (*token.FileSet, *analysis.SuggestedFix, error) { + // Find enclosing static call. + callerPkg, callerPGF, err := NarrowestPackageForFile(ctx, snapshot, fh.URI()) + if err != nil { + return nil, nil, err + } + call, fn, err := EnclosingStaticCall(callerPkg, callerPGF, rng) + if err != nil { + return nil, nil, err + } + + // Locate callee by file/line and analyze it. + calleePosn := safetoken.StartPosition(callerPkg.FileSet(), fn.Pos()) + calleePkg, calleePGF, err := NarrowestPackageForFile(ctx, snapshot, span.URIFromPath(calleePosn.Filename)) + if err != nil { + return nil, nil, err + } + var calleeDecl *ast.FuncDecl + for _, decl := range calleePGF.File.Decls { + if decl, ok := decl.(*ast.FuncDecl); ok { + posn := safetoken.StartPosition(calleePkg.FileSet(), decl.Name.Pos()) + if posn.Line == calleePosn.Line && posn.Column == calleePosn.Column { + calleeDecl = decl + break + } + } + } + if calleeDecl == nil { + return nil, nil, fmt.Errorf("can't find callee") + } + callee, err := inline.AnalyzeCallee(calleePkg.FileSet(), calleePkg.GetTypes(), calleePkg.GetTypesInfo(), calleeDecl, calleePGF.Src) + if err != nil { + return nil, nil, err + } + + // Inline the call. + caller := &inline.Caller{ + Fset: callerPkg.FileSet(), + Types: callerPkg.GetTypes(), + Info: callerPkg.GetTypesInfo(), + File: callerPGF.File, + Call: call, + Content: callerPGF.Src, + } + got, err := inline.Inline(caller, callee) + if err != nil { + return nil, nil, err + } + + // Suggest the fix. + return callerPkg.FileSet(), &analysis.SuggestedFix{ + Message: fmt.Sprintf("inline call of %v", callee), + TextEdits: diffToTextEdits(callerPGF.Tok, diff.Bytes(callerPGF.Src, got)), + }, nil +} diff --git a/gopls/internal/lsp/source/options.go b/gopls/internal/lsp/source/options.go index 45899e62969..334cc9dc798 100644 --- a/gopls/internal/lsp/source/options.go +++ b/gopls/internal/lsp/source/options.go @@ -104,6 +104,7 @@ func DefaultOptions() *Options { protocol.SourceOrganizeImports: true, protocol.QuickFix: true, protocol.RefactorRewrite: true, + protocol.RefactorInline: true, protocol.RefactorExtract: true, }, Mod: { diff --git a/gopls/internal/lsp/source/stub.go b/gopls/internal/lsp/source/stub.go index 1f886796d92..482c18a1bfa 100644 --- a/gopls/internal/lsp/source/stub.go +++ b/gopls/internal/lsp/source/stub.go @@ -22,6 +22,7 @@ import ( "golang.org/x/tools/gopls/internal/lsp/analysis/stubmethods" "golang.org/x/tools/gopls/internal/lsp/protocol" "golang.org/x/tools/gopls/internal/lsp/safetoken" + "golang.org/x/tools/internal/diff" "golang.org/x/tools/internal/tokeninternal" "golang.org/x/tools/internal/typeparams" ) @@ -231,15 +232,19 @@ func (%s%s%s) %s%s { // Report the diff. diffs := snapshot.View().Options().ComputeEdits(string(input), output.String()) - var edits []analysis.TextEdit + return tokeninternal.FileSetFor(declPGF.Tok), // edits use declPGF.Tok + &analysis.SuggestedFix{TextEdits: diffToTextEdits(declPGF.Tok, diffs)}, + nil +} + +func diffToTextEdits(tok *token.File, diffs []diff.Edit) []analysis.TextEdit { + edits := make([]analysis.TextEdit, 0, len(diffs)) for _, edit := range diffs { edits = append(edits, analysis.TextEdit{ - Pos: declPGF.Tok.Pos(edit.Start), - End: declPGF.Tok.Pos(edit.End), + Pos: tok.Pos(edit.Start), + End: tok.Pos(edit.End), NewText: []byte(edit.New), }) } - return tokeninternal.FileSetFor(declPGF.Tok), // edits use declPGF.Tok - &analysis.SuggestedFix{TextEdits: edits}, - nil + return edits } diff --git a/gopls/internal/lsp/tests/tests.go b/gopls/internal/lsp/tests/tests.go index 65e9c0e9532..150bec9bc7b 100644 --- a/gopls/internal/lsp/tests/tests.go +++ b/gopls/internal/lsp/tests/tests.go @@ -230,6 +230,7 @@ func DefaultOptions(o *source.Options) { protocol.SourceOrganizeImports: true, protocol.QuickFix: true, protocol.RefactorRewrite: true, + protocol.RefactorInline: true, protocol.RefactorExtract: true, protocol.SourceFixAll: true, }, diff --git a/gopls/internal/regtest/marker/testdata/codeaction/inline.txt b/gopls/internal/regtest/marker/testdata/codeaction/inline.txt new file mode 100644 index 00000000000..8f1ea924864 --- /dev/null +++ b/gopls/internal/regtest/marker/testdata/codeaction/inline.txt @@ -0,0 +1,23 @@ +This is a minimal test of the refactor.inline code action. + +-- go.mod -- +module testdata/codeaction +go 1.18 + +-- a/a.go -- +package a + +func _() { + println(add(1, 2)) //@codeaction("refactor.inline", "add", ")", inline) +} + +func add(x, y int) int { return x + y } + +-- @inline/a/a.go -- +package a + +func _() { + println(func(x, y int) int { return x + y }(1, 2)) //@codeaction("refactor.inline", "add", ")", inline) +} + +func add(x, y int) int { return x + y } From 9658d2e94ba2a446437e1fb10302f167524b2029 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Tue, 29 Aug 2023 15:40:52 -0400 Subject: [PATCH 041/178] internal/refactor/inline: NeedGoPackages in test Fixes golang/go#62351 Change-Id: I867444c516c113f7e74f29cd5e488e23b6f6b2cc Reviewed-on: https://go-review.googlesource.com/c/tools/+/524057 Run-TryBot: Alan Donovan Auto-Submit: Alan Donovan Reviewed-by: Hyang-Ah Hana Kim TryBot-Result: Gopher Robot --- internal/refactor/inline/inline_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/refactor/inline/inline_test.go b/internal/refactor/inline/inline_test.go index 042784082cc..f77d2851f17 100644 --- a/internal/refactor/inline/inline_test.go +++ b/internal/refactor/inline/inline_test.go @@ -21,11 +21,14 @@ import ( "golang.org/x/tools/go/types/typeutil" "golang.org/x/tools/internal/diff" "golang.org/x/tools/internal/refactor/inline" + "golang.org/x/tools/internal/testenv" "golang.org/x/tools/txtar" ) // Test executes test scenarios specified by files in testdata/*.txtar. func Test(t *testing.T) { + testenv.NeedsGoPackages(t) + files, err := filepath.Glob("testdata/*.txtar") if err != nil { t.Fatal(err) From e3671fc6117f572c77c83068b82fcecb61af30cf Mon Sep 17 00:00:00 2001 From: "Hana (Hyang-Ah) Kim" Date: Wed, 30 Aug 2023 14:24:37 -0400 Subject: [PATCH 042/178] internal/telemetry: unconditionally trigger upload logic Change-Id: Ib035753d3c292ba0dd360447884bb46ae8ec409c Reviewed-on: https://go-review.googlesource.com/c/tools/+/524535 Run-TryBot: Hyang-Ah Hana Kim TryBot-Result: Gopher Robot Reviewed-by: Robert Findley --- gopls/internal/telemetry/telemetry.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/gopls/internal/telemetry/telemetry.go b/gopls/internal/telemetry/telemetry.go index 067e4f8e6aa..a20a40f5057 100644 --- a/gopls/internal/telemetry/telemetry.go +++ b/gopls/internal/telemetry/telemetry.go @@ -9,7 +9,6 @@ package telemetry import ( "fmt" - "os" "time" "golang.org/x/telemetry/counter" @@ -20,9 +19,7 @@ import ( // Start starts telemetry instrumentation. func Start() { counter.Open() - if os.Getenv("GOPLS_TELEMETRY_EXP") != "" { - go packAndUpload() - } + go packAndUpload() } func packAndUpload() { From 8234134986237e66077e84b6916430f907a922c4 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Wed, 30 Aug 2023 17:54:51 -0400 Subject: [PATCH 043/178] gopls/internal/lsp/frob: add defensive header I can't explain the crash in the bug report: the frob logic looks sound, which leaves these possibilities: (a) the provided data is garbage or is being trampled (but the caller logic looks sound); (b) the file contents are corrupted (but the filecache SHA256 checksum was fine); (c) there's a RAM problem (but that always feels like a cop-out explanation). I've added a magic number to the file header so that there's a chance we'll detect some variants of a and b. Updates golang/go#62383 Change-Id: Icd32a2dc6ab019f3deee1b332428e0313c93a6ff Reviewed-on: https://go-review.googlesource.com/c/tools/+/524655 Run-TryBot: Alan Donovan TryBot-Result: Gopher Robot Reviewed-by: Robert Findley --- gopls/internal/lsp/filecache/filecache.go | 7 ++++--- gopls/internal/lsp/frob/frob.go | 12 ++++++++---- gopls/internal/lsp/frob/frob_test.go | 4 ++++ 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/gopls/internal/lsp/filecache/filecache.go b/gopls/internal/lsp/filecache/filecache.go index 7d4b8b1b424..6877780c29c 100644 --- a/gopls/internal/lsp/filecache/filecache.go +++ b/gopls/internal/lsp/filecache/filecache.go @@ -60,10 +60,11 @@ type memKey struct { key [32]byte } -// Get retrieves from the cache and returns a newly allocated -// copy of the value most recently supplied to Set(kind, key), -// possibly by another process. +// Get retrieves from the cache and returns the value most recently +// supplied to Set(kind, key), possibly by another process. // Get returns ErrNotFound if the value was not found. +// +// Callers should not modify the returned array. func Get(kind string, key [32]byte) ([]byte, error) { // First consult the read-through memory cache. // Note that memory cache hits do not update the times diff --git a/gopls/internal/lsp/frob/frob.go b/gopls/internal/lsp/frob/frob.go index 57f1ef5014c..7d037328424 100644 --- a/gopls/internal/lsp/frob/frob.go +++ b/gopls/internal/lsp/frob/frob.go @@ -5,8 +5,8 @@ // Package frob is a fast restricted object encoder/decoder in the // spirit of encoding/gob. // -// As with gob, types that recursively contain functions, -// channels, and unsafe.Pointers cannot encoded, but frob has these +// As with gob, types that recursively contain functions, channels, +// and unsafe.Pointers cannot be encoded, but frob has these // additional restrictions: // // - Interface values are not supported; this avoids the need for @@ -24,8 +24,6 @@ // // - There is no error handling. All errors are reported by panicking. // -// - Types that (recursively) contain private struct fields are not permitted. -// // - Values are serialized as trees, not graphs, so shared subgraphs // are encoded repeatedly. // @@ -123,12 +121,15 @@ func (fr *frob) addElem(t reflect.Type) { fr.elems = append(fr.elems, frobFor(t)) } +const magic = "frob" + func (fr *frob) Encode(v any) []byte { rv := reflect.ValueOf(v) if rv.Type() != fr.t { panic(fmt.Sprintf("got %v, want %v", rv.Type(), fr.t)) } w := &writer{} + w.bytes([]byte(magic)) fr.encode(w, rv) if uint64(len(w.data))>>32 != 0 { panic("too large") // includes all cases where len doesn't fit in 32 bits @@ -244,6 +245,9 @@ func (fr *frob) Decode(data []byte, ptr any) { panic(fmt.Sprintf("got %v, want %v", rv.Type(), fr.t)) } rd := &reader{data} + if string(rd.bytes(4)) != magic { + panic("not a frob-encoded message") + } fr.decode(rd, rv) if len(rd.data) > 0 { panic("surplus bytes") diff --git a/gopls/internal/lsp/frob/frob_test.go b/gopls/internal/lsp/frob/frob_test.go index 892f18b0eec..6a0f6e729db 100644 --- a/gopls/internal/lsp/frob/frob_test.go +++ b/gopls/internal/lsp/frob/frob_test.go @@ -18,6 +18,8 @@ func TestBasics(t *testing.T) { B [2]int C *Basics D map[string]int + E []byte + F []string } codec := frob.CodecFor[Basics]() @@ -29,6 +31,8 @@ func TestBasics(t *testing.T) { B: [...]int{3, 4}, D: map[string]int{"one": 1}, }, + E: []byte("hello"), + F: []string{s1, s2}, } var y Basics codec.Decode(codec.Encode(x), &y) From 914b218fc34ea54ba61ea1ba53cb42d9890142f0 Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Thu, 31 Aug 2023 12:19:05 -0400 Subject: [PATCH 044/178] gopls/internal/lsp/analysis/unusedparams: document the blank identifier Improve documentation around the blank identifier in the unusedparams analyzer. Referring to "underscored" names is confusing, as that may reasonably be interpreted as referring to names like "_foo". Fixes golang/go#60682 Change-Id: I90e4bc47fad230c5843e69395c055ebe77ad498a Reviewed-on: https://go-review.googlesource.com/c/tools/+/524835 TryBot-Result: Gopher Robot Run-TryBot: Robert Findley Reviewed-by: Alan Donovan --- gopls/doc/analyzers.md | 2 +- gopls/internal/lsp/analysis/unusedparams/unusedparams.go | 2 +- gopls/internal/lsp/source/api_json.go | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/gopls/doc/analyzers.md b/gopls/doc/analyzers.md index 48c98e0cb39..9a592d4b890 100644 --- a/gopls/doc/analyzers.md +++ b/gopls/doc/analyzers.md @@ -632,7 +632,7 @@ any parameters that are not being used. To reduce false positives it ignores: - methods -- parameters that do not have a name or are underscored +- parameters that do not have a name or have the name '_' (the blank identifier) - functions in test files - functions with empty bodies or those with just a return stmt diff --git a/gopls/internal/lsp/analysis/unusedparams/unusedparams.go b/gopls/internal/lsp/analysis/unusedparams/unusedparams.go index 4c933c8fb86..e0ef5ef8dfb 100644 --- a/gopls/internal/lsp/analysis/unusedparams/unusedparams.go +++ b/gopls/internal/lsp/analysis/unusedparams/unusedparams.go @@ -24,7 +24,7 @@ any parameters that are not being used. To reduce false positives it ignores: - methods -- parameters that do not have a name or are underscored +- parameters that do not have a name or have the name '_' (the blank identifier) - functions in test files - functions with empty bodies or those with just a return stmt` diff --git a/gopls/internal/lsp/source/api_json.go b/gopls/internal/lsp/source/api_json.go index 97f6384ab82..7635edba2e6 100644 --- a/gopls/internal/lsp/source/api_json.go +++ b/gopls/internal/lsp/source/api_json.go @@ -410,7 +410,7 @@ var GeneratedAPIJSON = &APIJSON{ }, { Name: "\"unusedparams\"", - Doc: "check for unused parameters of functions\n\nThe unusedparams analyzer checks functions to see if there are\nany parameters that are not being used.\n\nTo reduce false positives it ignores:\n- methods\n- parameters that do not have a name or are underscored\n- functions in test files\n- functions with empty bodies or those with just a return stmt", + Doc: "check for unused parameters of functions\n\nThe unusedparams analyzer checks functions to see if there are\nany parameters that are not being used.\n\nTo reduce false positives it ignores:\n- methods\n- parameters that do not have a name or have the name '_' (the blank identifier)\n- functions in test files\n- functions with empty bodies or those with just a return stmt", Default: "false", }, { @@ -1132,7 +1132,7 @@ var GeneratedAPIJSON = &APIJSON{ }, { Name: "unusedparams", - Doc: "check for unused parameters of functions\n\nThe unusedparams analyzer checks functions to see if there are\nany parameters that are not being used.\n\nTo reduce false positives it ignores:\n- methods\n- parameters that do not have a name or are underscored\n- functions in test files\n- functions with empty bodies or those with just a return stmt", + Doc: "check for unused parameters of functions\n\nThe unusedparams analyzer checks functions to see if there are\nany parameters that are not being used.\n\nTo reduce false positives it ignores:\n- methods\n- parameters that do not have a name or have the name '_' (the blank identifier)\n- functions in test files\n- functions with empty bodies or those with just a return stmt", }, { Name: "unusedresult", From 1bfa8e3224560a65ead5ffea49dd707a10d1c52a Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Thu, 31 Aug 2023 16:32:04 -0400 Subject: [PATCH 045/178] gopls: update uses of deprecated ioutil APIs Change-Id: I35acdc8a1e8e558f905fd6879b3ac8332e7c2325 Reviewed-on: https://go-review.googlesource.com/c/tools/+/524836 Reviewed-by: Alan Donovan TryBot-Result: Gopher Robot Reviewed-by: qiulaidongfeng <2645477756@qq.com> Run-TryBot: Robert Findley --- gopls/internal/lsp/cache/mod_tidy.go | 3 +-- gopls/internal/lsp/cache/snapshot.go | 5 ++--- gopls/internal/lsp/cache/view.go | 5 ++--- gopls/internal/lsp/cache/view_test.go | 8 ++------ gopls/internal/lsp/command.go | 3 +-- gopls/internal/lsp/source/gc_annotations.go | 5 ++--- 6 files changed, 10 insertions(+), 19 deletions(-) diff --git a/gopls/internal/lsp/cache/mod_tidy.go b/gopls/internal/lsp/cache/mod_tidy.go index a96793bdbc2..202310dd497 100644 --- a/gopls/internal/lsp/cache/mod_tidy.go +++ b/gopls/internal/lsp/cache/mod_tidy.go @@ -9,7 +9,6 @@ import ( "fmt" "go/ast" "go/token" - "io/ioutil" "os" "path/filepath" "strconv" @@ -118,7 +117,7 @@ func modTidyImpl(ctx context.Context, snapshot *snapshot, filename string, pm *s // Go directly to disk to get the temporary mod file, // since it is always on disk. - tempContents, err := ioutil.ReadFile(tmpURI.Filename()) + tempContents, err := os.ReadFile(tmpURI.Filename()) if err != nil { return nil, err } diff --git a/gopls/internal/lsp/cache/snapshot.go b/gopls/internal/lsp/cache/snapshot.go index 2c2f79bf25d..ed32b9175a0 100644 --- a/gopls/internal/lsp/cache/snapshot.go +++ b/gopls/internal/lsp/cache/snapshot.go @@ -14,7 +14,6 @@ import ( "go/token" "go/types" "io" - "io/ioutil" "log" "os" "path/filepath" @@ -456,11 +455,11 @@ func (s *snapshot) RunGoCommands(ctx context.Context, allowNetwork bool, wd stri return false, nil, nil, nil } var modBytes, sumBytes []byte - modBytes, err = ioutil.ReadFile(tmpURI.Filename()) + modBytes, err = os.ReadFile(tmpURI.Filename()) if err != nil && !os.IsNotExist(err) { return false, nil, nil, err } - sumBytes, err = ioutil.ReadFile(strings.TrimSuffix(tmpURI.Filename(), ".mod") + ".sum") + sumBytes, err = os.ReadFile(strings.TrimSuffix(tmpURI.Filename(), ".mod") + ".sum") if err != nil && !os.IsNotExist(err) { return false, nil, nil, err } diff --git a/gopls/internal/lsp/cache/view.go b/gopls/internal/lsp/cache/view.go index 70395d1a259..fbdb6047a78 100644 --- a/gopls/internal/lsp/cache/view.go +++ b/gopls/internal/lsp/cache/view.go @@ -11,7 +11,6 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" "os" "path" "path/filepath" @@ -351,7 +350,7 @@ func (v *View) ID() string { return v.id } // longer needed. func tempModFile(modFh source.FileHandle, gosum []byte) (tmpURI span.URI, cleanup func(), err error) { filenameHash := source.Hashf("%s", modFh.URI().Filename()) - tmpMod, err := ioutil.TempFile("", fmt.Sprintf("go.%s.*.mod", filenameHash)) + tmpMod, err := os.CreateTemp("", fmt.Sprintf("go.%s.*.mod", filenameHash)) if err != nil { return "", nil, err } @@ -386,7 +385,7 @@ func tempModFile(modFh source.FileHandle, gosum []byte) (tmpURI span.URI, cleanu // Create an analogous go.sum, if one exists. if gosum != nil { - if err := ioutil.WriteFile(tmpSumName, gosum, 0655); err != nil { + if err := os.WriteFile(tmpSumName, gosum, 0655); err != nil { return "", nil, err } } diff --git a/gopls/internal/lsp/cache/view_test.go b/gopls/internal/lsp/cache/view_test.go index 90471ed4401..21b10b6a982 100644 --- a/gopls/internal/lsp/cache/view_test.go +++ b/gopls/internal/lsp/cache/view_test.go @@ -6,7 +6,6 @@ package cache import ( "context" "encoding/json" - "io/ioutil" "os" "path/filepath" "testing" @@ -20,17 +19,14 @@ import ( ) func TestCaseInsensitiveFilesystem(t *testing.T) { - base, err := ioutil.TempDir("", t.Name()) - if err != nil { - t.Fatal(err) - } + base := t.TempDir() inner := filepath.Join(base, "a/B/c/DEFgh") if err := os.MkdirAll(inner, 0777); err != nil { t.Fatal(err) } file := filepath.Join(inner, "f.go") - if err := ioutil.WriteFile(file, []byte("hi"), 0777); err != nil { + if err := os.WriteFile(file, []byte("hi"), 0777); err != nil { t.Fatal(err) } if _, err := os.Stat(filepath.Join(inner, "F.go")); err != nil { diff --git a/gopls/internal/lsp/command.go b/gopls/internal/lsp/command.go index ff646709d63..32bb00c9366 100644 --- a/gopls/internal/lsp/command.go +++ b/gopls/internal/lsp/command.go @@ -11,7 +11,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "os" "os/exec" "path/filepath" @@ -629,7 +628,7 @@ func collectFileEdits(ctx context.Context, snapshot source.Snapshot, uri span.UR // file and leave it unsaved. We would rather apply the changes directly, // especially to go.sum, which should be mostly invisible to the user. if !snapshot.IsOpen(uri) { - err := ioutil.WriteFile(uri.Filename(), newContent, 0666) + err := os.WriteFile(uri.Filename(), newContent, 0666) return nil, err } diff --git a/gopls/internal/lsp/source/gc_annotations.go b/gopls/internal/lsp/source/gc_annotations.go index b1299b8d891..842548c9515 100644 --- a/gopls/internal/lsp/source/gc_annotations.go +++ b/gopls/internal/lsp/source/gc_annotations.go @@ -9,7 +9,6 @@ import ( "context" "encoding/json" "fmt" - "io/ioutil" "os" "path/filepath" "strings" @@ -45,7 +44,7 @@ func GCOptimizationDetails(ctx context.Context, snapshot Snapshot, m *Metadata) if err := os.MkdirAll(outDir, 0700); err != nil { return nil, err } - tmpFile, err := ioutil.TempFile(os.TempDir(), "gopls-x") + tmpFile, err := os.CreateTemp(os.TempDir(), "gopls-x") if err != nil { return nil, err } @@ -99,7 +98,7 @@ func GCOptimizationDetails(ctx context.Context, snapshot Snapshot, m *Metadata) } func parseDetailsFile(filename string, options *Options) (span.URI, []*Diagnostic, error) { - buf, err := ioutil.ReadFile(filename) + buf, err := os.ReadFile(filename) if err != nil { return "", nil, err } From 5fc00b44cd9b4227deece2f4611a2e3d4ce47475 Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Thu, 31 Aug 2023 16:29:19 -0400 Subject: [PATCH 046/178] gopls/internal: move Options and FileKind from View to Snapshot In preparation for making snapshots truly idempotent, move certain methods that depend on settings into the snapshot. Nothing should access settings through the View. For golang/go#42814 Change-Id: Ib3ced985dc89515f5a6e6049c7be316342706e54 Reviewed-on: https://go-review.googlesource.com/c/tools/+/524837 Run-TryBot: Robert Findley TryBot-Result: Gopher Robot Reviewed-by: Alan Donovan --- gopls/internal/lsp/cache/load.go | 2 +- gopls/internal/lsp/cache/mod.go | 2 +- gopls/internal/lsp/cache/mod_tidy.go | 2 +- gopls/internal/lsp/cache/snapshot.go | 10 +++++++- gopls/internal/lsp/code_action.go | 10 ++++---- gopls/internal/lsp/code_lens.go | 4 ++-- gopls/internal/lsp/command.go | 9 ++++--- gopls/internal/lsp/completion.go | 4 ++-- gopls/internal/lsp/definition.go | 4 ++-- gopls/internal/lsp/diagnostics.go | 2 +- gopls/internal/lsp/folding_range.go | 2 +- gopls/internal/lsp/format.go | 2 +- gopls/internal/lsp/highlight.go | 2 +- gopls/internal/lsp/hover.go | 2 +- gopls/internal/lsp/inlay_hint.go | 2 +- gopls/internal/lsp/link.go | 17 +++++++------ gopls/internal/lsp/mod/diagnostics.go | 2 +- gopls/internal/lsp/mod/format.go | 2 +- gopls/internal/lsp/mod/hover.go | 8 +++---- gopls/internal/lsp/references.go | 2 +- gopls/internal/lsp/semantic.go | 9 ++++--- .../lsp/source/completion/completion.go | 2 +- .../internal/lsp/source/completion/format.go | 4 ++-- gopls/internal/lsp/source/diagnostics.go | 2 +- gopls/internal/lsp/source/format.go | 8 +++---- gopls/internal/lsp/source/gc_annotations.go | 2 +- gopls/internal/lsp/source/hover.go | 4 ++-- gopls/internal/lsp/source/inlay_hint.go | 2 +- gopls/internal/lsp/source/rename.go | 4 ++-- gopls/internal/lsp/source/signature_help.go | 4 ++-- gopls/internal/lsp/source/stub.go | 2 +- gopls/internal/lsp/source/types_format.go | 4 ++-- gopls/internal/lsp/source/view.go | 24 +++++++++---------- gopls/internal/lsp/source/workspace_symbol.go | 4 ++-- gopls/internal/lsp/symbols.go | 4 ++-- gopls/internal/lsp/text_synchronization.go | 2 +- gopls/internal/lsp/work/format.go | 2 +- gopls/internal/lsp/work/hover.go | 2 +- gopls/internal/vulncheck/command.go | 4 ++-- 39 files changed, 92 insertions(+), 87 deletions(-) diff --git a/gopls/internal/lsp/cache/load.go b/gopls/internal/lsp/cache/load.go index eb302b7b332..05d44329c20 100644 --- a/gopls/internal/lsp/cache/load.go +++ b/gopls/internal/lsp/cache/load.go @@ -67,7 +67,7 @@ func (s *snapshot) load(ctx context.Context, allowNetwork bool, scopes ...loadSc panic(fmt.Sprintf("internal error: load called with multiple scopes when a file scope is present (file: %s)", uri)) } fh := s.FindFile(uri) - if fh == nil || s.View().FileKind(fh) != source.Go { + if fh == nil || s.FileKind(fh) != source.Go { // Don't try to load a file that doesn't exist, or isn't a go file. continue } diff --git a/gopls/internal/lsp/cache/mod.go b/gopls/internal/lsp/cache/mod.go index 8b6331d056d..db0ab0a64b8 100644 --- a/gopls/internal/lsp/cache/mod.go +++ b/gopls/internal/lsp/cache/mod.go @@ -213,7 +213,7 @@ func sumFilename(modURI span.URI) string { func (s *snapshot) ModWhy(ctx context.Context, fh source.FileHandle) (map[string]string, error) { uri := fh.URI() - if s.View().FileKind(fh) != source.Mod { + if s.FileKind(fh) != source.Mod { return nil, fmt.Errorf("%s is not a go.mod file", uri) } diff --git a/gopls/internal/lsp/cache/mod_tidy.go b/gopls/internal/lsp/cache/mod_tidy.go index 202310dd497..64e02d1c01e 100644 --- a/gopls/internal/lsp/cache/mod_tidy.go +++ b/gopls/internal/lsp/cache/mod_tidy.go @@ -168,7 +168,7 @@ func modTidyDiagnostics(ctx context.Context, snapshot *snapshot, pm *source.Pars for _, req := range wrongDirectness { // Handle dependencies that are incorrectly labeled indirect and // vice versa. - srcDiag, err := directnessDiagnostic(pm.Mapper, req, snapshot.View().Options().ComputeEdits) + srcDiag, err := directnessDiagnostic(pm.Mapper, req, snapshot.Options().ComputeEdits) if err != nil { // We're probably in a bad state if we can't compute a // directnessDiagnostic, but try to keep going so as to not suppress diff --git a/gopls/internal/lsp/cache/snapshot.go b/gopls/internal/lsp/cache/snapshot.go index ed32b9175a0..cbc9c6e7e16 100644 --- a/gopls/internal/lsp/cache/snapshot.go +++ b/gopls/internal/lsp/cache/snapshot.go @@ -283,6 +283,14 @@ func (s *snapshot) View() source.View { return s.view } +func (s *snapshot) FileKind(h source.FileHandle) source.FileKind { + return s.view.FileKind(h) +} + +func (s *snapshot) Options() *source.Options { + return s.view.Options() // temporarily return view options. +} + func (s *snapshot) BackgroundContext() context.Context { return s.backgroundCtx } @@ -894,7 +902,7 @@ const fileExtensions = "go,mod,sum,work" func (s *snapshot) fileWatchingGlobPatterns(ctx context.Context) map[string]struct{} { extensions := fileExtensions - for _, ext := range s.View().Options().TemplateExtensions { + for _, ext := range s.Options().TemplateExtensions { extensions += "," + ext } // Work-around microsoft/vscode#100870 by making sure that we are, diff --git a/gopls/internal/lsp/code_action.go b/gopls/internal/lsp/code_action.go index 938edcf0eeb..bef4e34d68f 100644 --- a/gopls/internal/lsp/code_action.go +++ b/gopls/internal/lsp/code_action.go @@ -38,8 +38,8 @@ func (s *Server) codeAction(ctx context.Context, params *protocol.CodeActionPara uri := fh.URI() // Determine the supported actions for this file kind. - kind := snapshot.View().FileKind(fh) - supportedCodeActions, ok := snapshot.View().Options().SupportedCodeActions[kind] + kind := snapshot.FileKind(fh) + supportedCodeActions, ok := snapshot.Options().SupportedCodeActions[kind] if !ok { return nil, fmt.Errorf("no supported code actions for %v file kind", kind) } @@ -185,7 +185,7 @@ func (s *Server) codeAction(ctx context.Context, params *protocol.CodeActionPara } var stubMethodsDiagnostics []protocol.Diagnostic - if wantQuickFixes && snapshot.View().Options().IsAnalyzerEnabled(stubmethods.Analyzer.Name) { + if wantQuickFixes && snapshot.Options().IsAnalyzerEnabled(stubmethods.Analyzer.Name) { for _, pd := range diagnostics { if stubmethods.MatchesMessage(pd.Message) { stubMethodsDiagnostics = append(stubMethodsDiagnostics, pd) @@ -453,7 +453,7 @@ func refactorRewrite(ctx context.Context, snapshot source.Snapshot, pkg source.P // // TODO: Consider removing the inspection after convenienceAnalyzers are removed. inspect := inspector.New([]*ast.File{pgf.File}) - if snapshot.View().Options().IsAnalyzerEnabled(fillstruct.Analyzer.Name) { + if snapshot.Options().IsAnalyzerEnabled(fillstruct.Analyzer.Name) { for _, d := range fillstruct.DiagnoseFillableStructs(inspect, start, end, pkg.GetTypes(), pkg.GetTypesInfo()) { rng, err := pgf.Mapper.PosRange(pgf.Tok, d.Pos, d.End) if err != nil { @@ -480,7 +480,7 @@ func refactorRewrite(ctx context.Context, snapshot source.Snapshot, pkg source.P }) } - if snapshot.View().Options().IsAnalyzerEnabled(infertypeargs.Analyzer.Name) { + if snapshot.Options().IsAnalyzerEnabled(infertypeargs.Analyzer.Name) { for _, d := range infertypeargs.DiagnoseInferableTypeArgs(pkg.FileSet(), inspect, start, end, pkg.GetTypes(), pkg.GetTypesInfo()) { if len(d.SuggestedFixes) != 1 { panic(fmt.Sprintf("unexpected number of suggested fixes from infertypeargs: %v", len(d.SuggestedFixes))) diff --git a/gopls/internal/lsp/code_lens.go b/gopls/internal/lsp/code_lens.go index 0167a78dc30..da7598604b0 100644 --- a/gopls/internal/lsp/code_lens.go +++ b/gopls/internal/lsp/code_lens.go @@ -27,7 +27,7 @@ func (s *Server) codeLens(ctx context.Context, params *protocol.CodeLensParams) return nil, err } var lenses map[command.Command]source.LensFunc - switch snapshot.View().FileKind(fh) { + switch snapshot.FileKind(fh) { case source.Mod: lenses = mod.LensFuncs() case source.Go: @@ -38,7 +38,7 @@ func (s *Server) codeLens(ctx context.Context, params *protocol.CodeLensParams) } var result []protocol.CodeLens for cmd, lf := range lenses { - if !snapshot.View().Options().Codelenses[string(cmd)] { + if !snapshot.Options().Codelenses[string(cmd)] { continue } added, err := lf(ctx, snapshot, fh) diff --git a/gopls/internal/lsp/command.go b/gopls/internal/lsp/command.go index 32bb00c9366..388030e4bcc 100644 --- a/gopls/internal/lsp/command.go +++ b/gopls/internal/lsp/command.go @@ -426,7 +426,7 @@ func dropDependency(snapshot source.Snapshot, pm *source.ParsedModule, modulePat return nil, err } // Calculate the edits to be made due to the change. - diff := snapshot.View().Options().ComputeEdits(string(pm.Mapper.Content), string(newContent)) + diff := snapshot.Options().ComputeEdits(string(pm.Mapper.Content), string(newContent)) return source.ToProtocolEdits(pm.Mapper, diff) } @@ -633,7 +633,7 @@ func collectFileEdits(ctx context.Context, snapshot source.Snapshot, uri span.UR } m := protocol.NewMapper(fh.URI(), oldContent) - diff := snapshot.View().Options().ComputeEdits(string(oldContent), string(newContent)) + diff := snapshot.Options().ComputeEdits(string(oldContent), string(newContent)) edits, err := source.ToProtocolEdits(m, diff) if err != nil { return nil, err @@ -899,7 +899,7 @@ type pkgLoadConfig struct { func (c *commandHandler) FetchVulncheckResult(ctx context.Context, arg command.URIArg) (map[protocol.DocumentURI]*govulncheck.Result, error) { ret := map[protocol.DocumentURI]*govulncheck.Result{} err := c.run(ctx, commandConfig{forURI: arg.URI}, func(ctx context.Context, deps commandDeps) error { - if deps.snapshot.View().Options().Vulncheck == source.ModeVulncheckImports { + if deps.snapshot.Options().Vulncheck == source.ModeVulncheckImports { for _, modfile := range deps.snapshot.ModFiles() { res, err := deps.snapshot.ModVuln(ctx, modfile) if err != nil { @@ -936,8 +936,7 @@ func (c *commandHandler) RunGovulncheck(ctx context.Context, args command.Vulnch }, func(ctx context.Context, deps commandDeps) error { tokenChan <- deps.work.Token() - view := deps.snapshot.View() - opts := view.Options() + opts := deps.snapshot.Options() // quickly test if gopls is compiled to support govulncheck // by checking vulncheck.Main. Alternatively, we can continue and // let the `gopls vulncheck` command fail. This is lighter-weight. diff --git a/gopls/internal/lsp/completion.go b/gopls/internal/lsp/completion.go index f464bb2d6bf..209f26be3cb 100644 --- a/gopls/internal/lsp/completion.go +++ b/gopls/internal/lsp/completion.go @@ -29,7 +29,7 @@ func (s *Server) completion(ctx context.Context, params *protocol.CompletionPara } var candidates []completion.CompletionItem var surrounding *completion.Selection - switch snapshot.View().FileKind(fh) { + switch snapshot.FileKind(fh) { case source.Go: candidates, surrounding, err = completion.Completion(ctx, snapshot, fh, params.Position, params.Context) case source.Mod: @@ -65,7 +65,7 @@ func (s *Server) completion(ctx context.Context, params *protocol.CompletionPara // When using deep completions/fuzzy matching, report results as incomplete so // client fetches updated completions after every key stroke. - options := snapshot.View().Options() + options := snapshot.Options() incompleteResults := options.DeepCompletion || options.Matcher == source.Fuzzy items := toProtocolCompletionItems(candidates, rng, options) diff --git a/gopls/internal/lsp/definition.go b/gopls/internal/lsp/definition.go index 89cf86efc05..fb691ef9d16 100644 --- a/gopls/internal/lsp/definition.go +++ b/gopls/internal/lsp/definition.go @@ -26,7 +26,7 @@ func (s *Server) definition(ctx context.Context, params *protocol.DefinitionPara if !ok { return nil, err } - switch kind := snapshot.View().FileKind(fh); kind { + switch kind := snapshot.FileKind(fh); kind { case source.Tmpl: return template.Definition(snapshot, fh, params.Position) case source.Go: @@ -51,7 +51,7 @@ func (s *Server) typeDefinition(ctx context.Context, params *protocol.TypeDefini if !ok { return nil, err } - switch kind := snapshot.View().FileKind(fh); kind { + switch kind := snapshot.FileKind(fh); kind { case source.Go: return source.TypeDefinition(ctx, snapshot, fh, params.Position) default: diff --git a/gopls/internal/lsp/diagnostics.go b/gopls/internal/lsp/diagnostics.go index dbc163aa529..2ae50586416 100644 --- a/gopls/internal/lsp/diagnostics.go +++ b/gopls/internal/lsp/diagnostics.go @@ -161,7 +161,7 @@ func (s *Server) diagnoseSnapshots(snapshots map[source.Snapshot][]span.URI, onD diagnosticWG.Add(1) go func(snapshot source.Snapshot, uris []span.URI) { defer diagnosticWG.Done() - s.diagnoseSnapshot(snapshot, uris, onDisk, snapshot.View().Options().DiagnosticsDelay) + s.diagnoseSnapshot(snapshot, uris, onDisk, snapshot.Options().DiagnosticsDelay) }(snapshot, uris) } diagnosticWG.Wait() diff --git a/gopls/internal/lsp/folding_range.go b/gopls/internal/lsp/folding_range.go index ecbe93f1d8d..e3b4987d391 100644 --- a/gopls/internal/lsp/folding_range.go +++ b/gopls/internal/lsp/folding_range.go @@ -23,7 +23,7 @@ func (s *Server) foldingRange(ctx context.Context, params *protocol.FoldingRange return nil, err } - ranges, err := source.FoldingRange(ctx, snapshot, fh, snapshot.View().Options().LineFoldingOnly) + ranges, err := source.FoldingRange(ctx, snapshot, fh, snapshot.Options().LineFoldingOnly) if err != nil { return nil, err } diff --git a/gopls/internal/lsp/format.go b/gopls/internal/lsp/format.go index 47659ba94a5..a6197a68e59 100644 --- a/gopls/internal/lsp/format.go +++ b/gopls/internal/lsp/format.go @@ -24,7 +24,7 @@ func (s *Server) formatting(ctx context.Context, params *protocol.DocumentFormat if !ok { return nil, err } - switch snapshot.View().FileKind(fh) { + switch snapshot.FileKind(fh) { case source.Mod: return mod.Format(ctx, snapshot, fh) case source.Go: diff --git a/gopls/internal/lsp/highlight.go b/gopls/internal/lsp/highlight.go index a3c898a0a77..c0c2502e5f1 100644 --- a/gopls/internal/lsp/highlight.go +++ b/gopls/internal/lsp/highlight.go @@ -24,7 +24,7 @@ func (s *Server) documentHighlight(ctx context.Context, params *protocol.Documen return nil, err } - if snapshot.View().FileKind(fh) == source.Tmpl { + if snapshot.FileKind(fh) == source.Tmpl { return template.Highlight(ctx, snapshot, fh, params.Position) } diff --git a/gopls/internal/lsp/hover.go b/gopls/internal/lsp/hover.go index 9c5bc0d308d..eef59920ae4 100644 --- a/gopls/internal/lsp/hover.go +++ b/gopls/internal/lsp/hover.go @@ -25,7 +25,7 @@ func (s *Server) hover(ctx context.Context, params *protocol.HoverParams) (*prot if !ok { return nil, err } - switch snapshot.View().FileKind(fh) { + switch snapshot.FileKind(fh) { case source.Mod: return mod.Hover(ctx, snapshot, fh, params.Position) case source.Go: diff --git a/gopls/internal/lsp/inlay_hint.go b/gopls/internal/lsp/inlay_hint.go index 67a6625c0e1..39b51abcbc6 100644 --- a/gopls/internal/lsp/inlay_hint.go +++ b/gopls/internal/lsp/inlay_hint.go @@ -23,7 +23,7 @@ func (s *Server) inlayHint(ctx context.Context, params *protocol.InlayHintParams if !ok { return nil, err } - switch snapshot.View().FileKind(fh) { + switch snapshot.FileKind(fh) { case source.Mod: return mod.InlayHint(ctx, snapshot, fh, params.Range) case source.Go: diff --git a/gopls/internal/lsp/link.go b/gopls/internal/lsp/link.go index 4ad745fc1f2..f04e265a08b 100644 --- a/gopls/internal/lsp/link.go +++ b/gopls/internal/lsp/link.go @@ -32,7 +32,7 @@ func (s *Server) documentLink(ctx context.Context, params *protocol.DocumentLink if !ok { return nil, err } - switch snapshot.View().FileKind(fh) { + switch snapshot.FileKind(fh) { case source.Mod: links, err = modLinks(ctx, snapshot, fh) case source.Go: @@ -69,7 +69,7 @@ func modLinks(ctx context.Context, snapshot source.Snapshot, fh source.FileHandl } // Shift the start position to the location of the // dependency within the require statement. - target := source.BuildLink(snapshot.View().Options().LinkTarget, "mod/"+req.Mod.String(), "") + target := source.BuildLink(snapshot.Options().LinkTarget, "mod/"+req.Mod.String(), "") l, err := toProtocolLink(pm.Mapper, target, start+i, start+i+len(dep)) if err != nil { return nil, err @@ -82,7 +82,7 @@ func modLinks(ctx context.Context, snapshot source.Snapshot, fh source.FileHandl } // Get all the links that are contained in the comments of the file. - urlRegexp := snapshot.View().Options().URLRegexp + urlRegexp := snapshot.Options().URLRegexp for _, expr := range pm.File.Syntax.Stmt { comments := expr.Comment() if comments == nil { @@ -103,7 +103,6 @@ func modLinks(ctx context.Context, snapshot source.Snapshot, fh source.FileHandl // goLinks returns the set of hyperlink annotations for the specified Go file. func goLinks(ctx context.Context, snapshot source.Snapshot, fh source.FileHandle) ([]protocol.DocumentLink, error) { - view := snapshot.View() pgf, err := snapshot.ParseGo(ctx, fh, source.ParseFull) if err != nil { @@ -113,12 +112,12 @@ func goLinks(ctx context.Context, snapshot source.Snapshot, fh source.FileHandle var links []protocol.DocumentLink // Create links for import specs. - if view.Options().ImportShortcut.ShowLinks() { + if snapshot.Options().ImportShortcut.ShowLinks() { // If links are to pkg.go.dev, append module version suffixes. // This requires the import map from the package metadata. Ignore errors. var depsByImpPath map[source.ImportPath]source.PackageID - if strings.ToLower(view.Options().LinkTarget) == "pkg.go.dev" { + if strings.ToLower(snapshot.Options().LinkTarget) == "pkg.go.dev" { if meta, err := source.NarrowestMetadataForFile(ctx, snapshot, fh.URI()); err == nil { depsByImpPath = meta.DepsByImpPath } @@ -130,7 +129,7 @@ func goLinks(ctx context.Context, snapshot source.Snapshot, fh source.FileHandle continue // bad import } // See golang/go#36998: don't link to modules matching GOPRIVATE. - if view.IsGoPrivatePath(string(importPath)) { + if snapshot.View().IsGoPrivatePath(string(importPath)) { continue } @@ -145,7 +144,7 @@ func goLinks(ctx context.Context, snapshot source.Snapshot, fh source.FileHandle if err != nil { return nil, err } - targetURL := source.BuildLink(view.Options().LinkTarget, urlPath, "") + targetURL := source.BuildLink(snapshot.Options().LinkTarget, urlPath, "") // Account for the quotation marks in the positions. l, err := toProtocolLink(pgf.Mapper, targetURL, start+len(`"`), end-len(`"`)) if err != nil { @@ -155,7 +154,7 @@ func goLinks(ctx context.Context, snapshot source.Snapshot, fh source.FileHandle } } - urlRegexp := snapshot.View().Options().URLRegexp + urlRegexp := snapshot.Options().URLRegexp // Gather links found in string literals. var str []*ast.BasicLit diff --git a/gopls/internal/lsp/mod/diagnostics.go b/gopls/internal/lsp/mod/diagnostics.go index bb0346e6034..43fc0a24481 100644 --- a/gopls/internal/lsp/mod/diagnostics.go +++ b/gopls/internal/lsp/mod/diagnostics.go @@ -192,7 +192,7 @@ func ModVulnerabilityDiagnostics(ctx context.Context, snapshot source.Snapshot, diagSource := source.Govulncheck vs := snapshot.View().Vulnerabilities(fh.URI())[fh.URI()] - if vs == nil && snapshot.View().Options().Vulncheck == source.ModeVulncheckImports { + if vs == nil && snapshot.Options().Vulncheck == source.ModeVulncheckImports { vs, err = snapshot.ModVuln(ctx, fh.URI()) if err != nil { return nil, err diff --git a/gopls/internal/lsp/mod/format.go b/gopls/internal/lsp/mod/format.go index 9c3942ee06d..daa12dac9a4 100644 --- a/gopls/internal/lsp/mod/format.go +++ b/gopls/internal/lsp/mod/format.go @@ -25,6 +25,6 @@ func Format(ctx context.Context, snapshot source.Snapshot, fh source.FileHandle) return nil, err } // Calculate the edits to be made due to the change. - diffs := snapshot.View().Options().ComputeEdits(string(pm.Mapper.Content), string(formatted)) + diffs := snapshot.Options().ComputeEdits(string(pm.Mapper.Content), string(formatted)) return source.ToProtocolEdits(pm.Mapper, diffs) } diff --git a/gopls/internal/lsp/mod/hover.go b/gopls/internal/lsp/mod/hover.go index fbd3c000013..bc754dcb911 100644 --- a/gopls/internal/lsp/mod/hover.go +++ b/gopls/internal/lsp/mod/hover.go @@ -82,7 +82,7 @@ func hoverOnRequireStatement(ctx context.Context, pm *source.ParsedModule, offse // Get the vulnerability info. fromGovulncheck := true vs := snapshot.View().Vulnerabilities(fh.URI())[fh.URI()] - if vs == nil && snapshot.View().Options().Vulncheck == source.ModeVulncheckImports { + if vs == nil && snapshot.Options().Vulncheck == source.ModeVulncheckImports { var err error vs, err = snapshot.ModVuln(ctx, fh.URI()) if err != nil { @@ -109,7 +109,7 @@ func hoverOnRequireStatement(ctx context.Context, pm *source.ParsedModule, offse if err != nil { return nil, err } - options := snapshot.View().Options() + options := snapshot.Options() isPrivate := snapshot.View().IsGoPrivatePath(req.Mod.Path) header := formatHeader(req.Mod.Path, options) explanation = formatExplanation(explanation, req, options, isPrivate) @@ -140,7 +140,7 @@ func hoverOnModuleStatement(ctx context.Context, pm *source.ParsedModule, offset fromGovulncheck := true vs := snapshot.View().Vulnerabilities(fh.URI())[fh.URI()] - if vs == nil && snapshot.View().Options().Vulncheck == source.ModeVulncheckImports { + if vs == nil && snapshot.Options().Vulncheck == source.ModeVulncheckImports { vs, err = snapshot.ModVuln(ctx, fh.URI()) if err != nil { return nil, false @@ -150,7 +150,7 @@ func hoverOnModuleStatement(ctx context.Context, pm *source.ParsedModule, offset modpath := "stdlib" goVersion := snapshot.View().GoVersionString() affecting, nonaffecting := lookupVulns(vs, modpath, goVersion) - options := snapshot.View().Options() + options := snapshot.Options() vulns := formatVulnerabilities(modpath, affecting, nonaffecting, options, fromGovulncheck) return &protocol.Hover{ diff --git a/gopls/internal/lsp/references.go b/gopls/internal/lsp/references.go index 09e1e6349a1..cc89b381088 100644 --- a/gopls/internal/lsp/references.go +++ b/gopls/internal/lsp/references.go @@ -23,7 +23,7 @@ func (s *Server) references(ctx context.Context, params *protocol.ReferenceParam if !ok { return nil, err } - if snapshot.View().FileKind(fh) == source.Tmpl { + if snapshot.FileKind(fh) == source.Tmpl { return template.References(ctx, snapshot, fh, params) } return source.References(ctx, snapshot, fh, params.Position, params.Context.IncludeDeclaration) diff --git a/gopls/internal/lsp/semantic.go b/gopls/internal/lsp/semantic.go index 021fa5efabb..12ee8dae903 100644 --- a/gopls/internal/lsp/semantic.go +++ b/gopls/internal/lsp/semantic.go @@ -63,13 +63,12 @@ func (s *Server) computeSemanticTokens(ctx context.Context, td protocol.TextDocu if !ok { return nil, err } - vv := snapshot.View() - if !vv.Options().SemanticTokens { + if !snapshot.Options().SemanticTokens { // return an error, so if the option changes // the client won't remember the wrong answer return nil, fmt.Errorf("semantictokens are disabled") } - kind := snapshot.View().FileKind(fh) + kind := snapshot.FileKind(fh) if kind == source.Tmpl { // this is a little cumbersome to avoid both exporting 'encoded' and its methods // and to avoid import cycles @@ -111,8 +110,8 @@ func (s *Server) computeSemanticTokens(ctx context.Context, td protocol.TextDocu fset: pkg.FileSet(), tokTypes: s.session.Options().SemanticTypes, tokMods: s.session.Options().SemanticMods, - noStrings: vv.Options().NoSemanticString, - noNumbers: vv.Options().NoSemanticNumber, + noStrings: snapshot.Options().NoSemanticString, + noNumbers: snapshot.Options().NoSemanticNumber, } if err := e.init(); err != nil { // e.init should never return an error, unless there's some diff --git a/gopls/internal/lsp/source/completion/completion.go b/gopls/internal/lsp/source/completion/completion.go index a4095f37832..e0a221f6017 100644 --- a/gopls/internal/lsp/source/completion/completion.go +++ b/gopls/internal/lsp/source/completion/completion.go @@ -508,7 +508,7 @@ func Completion(ctx context.Context, snapshot source.Snapshot, fh source.FileHan scopes := source.CollectScopes(pkg.GetTypesInfo(), path, pos) scopes = append(scopes, pkg.GetTypes().Scope(), types.Universe) - opts := snapshot.View().Options() + opts := snapshot.Options() c := &completer{ pkg: pkg, snapshot: snapshot, diff --git a/gopls/internal/lsp/source/completion/format.go b/gopls/internal/lsp/source/completion/format.go index 848f52d3132..89c5cb4ae97 100644 --- a/gopls/internal/lsp/source/completion/format.go +++ b/gopls/internal/lsp/source/completion/format.go @@ -263,9 +263,9 @@ Suffixes: // TODO(rfindley): It doesn't look like this does the right thing for // multi-line comments. if strings.HasPrefix(comment.Text(), "Deprecated") { - if c.snapshot.View().Options().CompletionTags { + if c.snapshot.Options().CompletionTags { item.Tags = []protocol.CompletionItemTag{protocol.ComplDeprecated} - } else if c.snapshot.View().Options().CompletionDeprecated { + } else if c.snapshot.Options().CompletionDeprecated { item.Deprecated = true } } diff --git a/gopls/internal/lsp/source/diagnostics.go b/gopls/internal/lsp/source/diagnostics.go index ad56253a5a9..ff41c570ddd 100644 --- a/gopls/internal/lsp/source/diagnostics.go +++ b/gopls/internal/lsp/source/diagnostics.go @@ -32,7 +32,7 @@ func Analyze(ctx context.Context, snapshot Snapshot, pkgIDs map[PackageID]unit, return nil, ctx.Err() } - options := snapshot.View().Options() + options := snapshot.Options() categories := []map[string]*Analyzer{ options.DefaultAnalyzers, options.StaticcheckAnalyzers, diff --git a/gopls/internal/lsp/source/format.go b/gopls/internal/lsp/source/format.go index 047edfc4839..6eed4cb9d0b 100644 --- a/gopls/internal/lsp/source/format.go +++ b/gopls/internal/lsp/source/format.go @@ -62,7 +62,7 @@ func Format(ctx context.Context, snapshot Snapshot, fh FileHandle) ([]protocol.T // Apply additional formatting, if any is supported. Currently, the only // supported additional formatter is gofumpt. - if format := snapshot.View().Options().GofumptFormat; snapshot.View().Options().Gofumpt && format != nil { + if format := snapshot.Options().GofumptFormat; snapshot.Options().Gofumpt && format != nil { // gofumpt can customize formatting based on language version and module // path, if available. // @@ -155,7 +155,7 @@ func computeImportEdits(ctx context.Context, snapshot Snapshot, pgf *ParsedGoFil // ComputeOneImportFixEdits returns text edits for a single import fix. func ComputeOneImportFixEdits(snapshot Snapshot, pgf *ParsedGoFile, fix *imports.ImportFix) ([]protocol.TextEdit, error) { options := &imports.Options{ - LocalPrefix: snapshot.View().Options().Local, + LocalPrefix: snapshot.Options().Local, // Defaults. AllErrors: true, Comments: true, @@ -194,7 +194,7 @@ func computeFixEdits(snapshot Snapshot, pgf *ParsedGoFile, options *imports.Opti if fixedData == nil || fixedData[len(fixedData)-1] != '\n' { fixedData = append(fixedData, '\n') // ApplyFixes may miss the newline, go figure. } - edits := snapshot.View().Options().ComputeEdits(left, string(fixedData)) + edits := snapshot.Options().ComputeEdits(left, string(fixedData)) return protocolEditsFromSource([]byte(left), edits) } @@ -304,7 +304,7 @@ func computeTextEdits(ctx context.Context, snapshot Snapshot, pgf *ParsedGoFile, _, done := event.Start(ctx, "source.computeTextEdits") defer done() - edits := snapshot.View().Options().ComputeEdits(string(pgf.Src), formatted) + edits := snapshot.Options().ComputeEdits(string(pgf.Src), formatted) return ToProtocolEdits(pgf.Mapper, edits) } diff --git a/gopls/internal/lsp/source/gc_annotations.go b/gopls/internal/lsp/source/gc_annotations.go index 842548c9515..2a21473aaf2 100644 --- a/gopls/internal/lsp/source/gc_annotations.go +++ b/gopls/internal/lsp/source/gc_annotations.go @@ -74,7 +74,7 @@ func GCOptimizationDetails(ctx context.Context, snapshot Snapshot, m *Metadata) return nil, err } reports := make(map[span.URI][]*Diagnostic) - opts := snapshot.View().Options() + opts := snapshot.Options() var parseError error for _, fn := range files { uri, diagnostics, err := parseDetailsFile(fn, opts) diff --git a/gopls/internal/lsp/source/hover.go b/gopls/internal/lsp/source/hover.go index 6fc4d792875..a6830751a91 100644 --- a/gopls/internal/lsp/source/hover.go +++ b/gopls/internal/lsp/source/hover.go @@ -71,13 +71,13 @@ func Hover(ctx context.Context, snapshot Snapshot, fh FileHandle, position proto if h == nil { return nil, nil } - hover, err := formatHover(h, snapshot.View().Options()) + hover, err := formatHover(h, snapshot.Options()) if err != nil { return nil, err } return &protocol.Hover{ Contents: protocol.MarkupContent{ - Kind: snapshot.View().Options().PreferredContentFormat, + Kind: snapshot.Options().PreferredContentFormat, Value: hover, }, Range: rng, diff --git a/gopls/internal/lsp/source/inlay_hint.go b/gopls/internal/lsp/source/inlay_hint.go index f323d56cb2c..f75cd3621e3 100644 --- a/gopls/internal/lsp/source/inlay_hint.go +++ b/gopls/internal/lsp/source/inlay_hint.go @@ -88,7 +88,7 @@ func InlayHint(ctx context.Context, snapshot Snapshot, fh FileHandle, pRng proto } // Collect a list of the inlay hints that are enabled. - inlayHintOptions := snapshot.View().Options().InlayHintOptions + inlayHintOptions := snapshot.Options().InlayHintOptions var enabledHints []InlayHintFunc for hint, enabled := range inlayHintOptions.Hints { if !enabled { diff --git a/gopls/internal/lsp/source/rename.go b/gopls/internal/lsp/source/rename.go index eb1fdce622f..ad6184966f4 100644 --- a/gopls/internal/lsp/source/rename.go +++ b/gopls/internal/lsp/source/rename.go @@ -152,7 +152,7 @@ func PrepareRename(ctx context.Context, snapshot Snapshot, f FileHandle, pp prot func prepareRenamePackageName(ctx context.Context, snapshot Snapshot, pgf *ParsedGoFile) (*PrepareItem, error) { // Does the client support file renaming? fileRenameSupported := false - for _, op := range snapshot.View().Options().SupportedResourceOperations { + for _, op := range snapshot.Options().SupportedResourceOperations { if op == protocol.Rename { fileRenameSupported = true break @@ -727,7 +727,7 @@ func renamePackageName(ctx context.Context, s Snapshot, f FileHandle, newName Pa } // Calculate the edits to be made due to the change. - edits := s.View().Options().ComputeEdits(string(pm.Mapper.Content), string(newContent)) + edits := s.Options().ComputeEdits(string(pm.Mapper.Content), string(newContent)) renamingEdits[pm.URI] = append(renamingEdits[pm.URI], edits...) } diff --git a/gopls/internal/lsp/source/signature_help.go b/gopls/internal/lsp/source/signature_help.go index ce9b2678e46..1420fc3ee15 100644 --- a/gopls/internal/lsp/source/signature_help.go +++ b/gopls/internal/lsp/source/signature_help.go @@ -117,7 +117,7 @@ FindCall: } return &protocol.SignatureInformation{ Label: name + s.Format(), - Documentation: stringToSigInfoDocumentation(s.doc, snapshot.View().Options()), + Documentation: stringToSigInfoDocumentation(s.doc, snapshot.Options()), Parameters: paramInfo, }, activeParam, nil } @@ -134,7 +134,7 @@ func builtinSignature(ctx context.Context, snapshot Snapshot, callExpr *ast.Call activeParam := activeParameter(callExpr, len(sig.params), sig.variadic, pos) return &protocol.SignatureInformation{ Label: sig.name + sig.Format(), - Documentation: stringToSigInfoDocumentation(sig.doc, snapshot.View().Options()), + Documentation: stringToSigInfoDocumentation(sig.doc, snapshot.Options()), Parameters: paramInfo, }, activeParam, nil diff --git a/gopls/internal/lsp/source/stub.go b/gopls/internal/lsp/source/stub.go index 482c18a1bfa..fd2b357032c 100644 --- a/gopls/internal/lsp/source/stub.go +++ b/gopls/internal/lsp/source/stub.go @@ -231,7 +231,7 @@ func (%s%s%s) %s%s { } // Report the diff. - diffs := snapshot.View().Options().ComputeEdits(string(input), output.String()) + diffs := snapshot.Options().ComputeEdits(string(input), output.String()) return tokeninternal.FileSetFor(declPGF.Tok), // edits use declPGF.Tok &analysis.SuggestedFix{TextEdits: diffToTextEdits(declPGF.Tok, diffs)}, nil diff --git a/gopls/internal/lsp/source/types_format.go b/gopls/internal/lsp/source/types_format.go index 3c371711967..1fcad26bb11 100644 --- a/gopls/internal/lsp/source/types_format.go +++ b/gopls/internal/lsp/source/types_format.go @@ -115,7 +115,7 @@ func NewBuiltinSignature(ctx context.Context, s Snapshot, name string) (*signatu params, _ := formatFieldList(ctx, fset, decl.Type.Params, variadic) results, needResultParens := formatFieldList(ctx, fset, decl.Type.Results, false) d := decl.Doc.Text() - switch s.View().Options().HoverKind { + switch s.Options().HoverKind { case SynopsisDocumentation: d = doc.Synopsis(d) case NoDocumentation: @@ -245,7 +245,7 @@ func NewSignature(ctx context.Context, s Snapshot, pkg Package, sig *types.Signa if comment != nil { d = comment.Text() } - switch s.View().Options().HoverKind { + switch s.Options().HoverKind { case SynopsisDocumentation: d = doc.Synopsis(d) case NoDocumentation: diff --git a/gopls/internal/lsp/source/view.go b/gopls/internal/lsp/source/view.go index b47e5b800ce..fe51cf0e5b6 100644 --- a/gopls/internal/lsp/source/view.go +++ b/gopls/internal/lsp/source/view.go @@ -58,6 +58,18 @@ type Snapshot interface { // subsequent snapshots in a view may not have adjacent global IDs. GlobalID() GlobalSnapshotID + // FileKind returns the type of a file. + // + // We can't reliably deduce the kind from the file name alone, + // as some editors can be told to interpret a buffer as + // language different from the file name heuristic, e.g. that + // an .html file actually contains Go "html/template" syntax, + // or even that a .go file contains Python. + FileKind(FileHandle) FileKind + + // Options returns the options associated with this snapshot. + Options() *Options + // View returns the View associated with this snapshot. View() View @@ -353,9 +365,6 @@ type View interface { // Folder returns the folder with which this view was created. Folder() span.URI - // Options returns a copy of the Options for this view. - Options() *Options - // Snapshot returns the current snapshot for the view, and a // release function that must be called when the Snapshot is // no longer needed. @@ -388,15 +397,6 @@ type View interface { // required by modfile. SetVulnerabilities(modfile span.URI, vulncheckResult *govulncheck.Result) - // FileKind returns the type of a file. - // - // We can't reliably deduce the kind from the file name alone, - // as some editors can be told to interpret a buffer as - // language different from the file name heuristic, e.g. that - // an .html file actually contains Go "html/template" syntax, - // or even that a .go file contains Python. - FileKind(FileHandle) FileKind - // GoVersion returns the configured Go version for this view. GoVersion() int diff --git a/gopls/internal/lsp/source/workspace_symbol.go b/gopls/internal/lsp/source/workspace_symbol.go index eb774a5df53..c656889fdb6 100644 --- a/gopls/internal/lsp/source/workspace_symbol.go +++ b/gopls/internal/lsp/source/workspace_symbol.go @@ -313,12 +313,12 @@ func collectSymbols(ctx context.Context, views []View, matcherType SymbolMatcher // whether a URI is in any open workspace. roots = append(roots, strings.TrimRight(string(v.Folder()), "/")) - filters := v.Options().DirectoryFilters + filters := snapshot.Options().DirectoryFilters filterer := NewFilterer(filters) folder := filepath.ToSlash(v.Folder().Filename()) workspaceOnly := true - if v.Options().SymbolScope == AllSymbolScope { + if snapshot.Options().SymbolScope == AllSymbolScope { workspaceOnly = false } symbols, err := snapshot.Symbols(ctx, workspaceOnly) diff --git a/gopls/internal/lsp/symbols.go b/gopls/internal/lsp/symbols.go index b31d980484c..18bae059e79 100644 --- a/gopls/internal/lsp/symbols.go +++ b/gopls/internal/lsp/symbols.go @@ -24,7 +24,7 @@ func (s *Server) documentSymbol(ctx context.Context, params *protocol.DocumentSy return []interface{}{}, err } var docSymbols []protocol.DocumentSymbol - switch snapshot.View().FileKind(fh) { + switch snapshot.FileKind(fh) { case source.Tmpl: docSymbols, err = template.DocumentSymbols(snapshot, fh) case source.Go: @@ -40,7 +40,7 @@ func (s *Server) documentSymbol(ctx context.Context, params *protocol.DocumentSy // TODO: Remove this once the lsp deprecates SymbolInformation. symbols := make([]interface{}, len(docSymbols)) for i, s := range docSymbols { - if snapshot.View().Options().HierarchicalDocumentSymbolSupport { + if snapshot.Options().HierarchicalDocumentSymbolSupport { symbols[i] = s continue } diff --git a/gopls/internal/lsp/text_synchronization.go b/gopls/internal/lsp/text_synchronization.go index 5c6ebcc086b..5584287a78c 100644 --- a/gopls/internal/lsp/text_synchronization.go +++ b/gopls/internal/lsp/text_synchronization.go @@ -280,7 +280,7 @@ func (s *Server) didModifyFiles(ctx context.Context, modifications []source.File for snapshot, uris := range snapshots { for _, uri := range uris { mod := modMap[uri] - if snapshot.View().Options().ChattyDiagnostics || mod.Action == source.Open || mod.Action == source.Close { + if snapshot.Options().ChattyDiagnostics || mod.Action == source.Open || mod.Action == source.Close { s.mustPublishDiagnostics(uri) } } diff --git a/gopls/internal/lsp/work/format.go b/gopls/internal/lsp/work/format.go index e852eb4d27e..70cbe59d174 100644 --- a/gopls/internal/lsp/work/format.go +++ b/gopls/internal/lsp/work/format.go @@ -23,6 +23,6 @@ func Format(ctx context.Context, snapshot source.Snapshot, fh source.FileHandle) } formatted := modfile.Format(pw.File.Syntax) // Calculate the edits to be made due to the change. - diffs := snapshot.View().Options().ComputeEdits(string(pw.Mapper.Content), string(formatted)) + diffs := snapshot.Options().ComputeEdits(string(pw.Mapper.Content), string(formatted)) return source.ToProtocolEdits(pw.Mapper, diffs) } diff --git a/gopls/internal/lsp/work/hover.go b/gopls/internal/lsp/work/hover.go index 558eebc824b..d777acdf3b4 100644 --- a/gopls/internal/lsp/work/hover.go +++ b/gopls/internal/lsp/work/hover.go @@ -62,7 +62,7 @@ func Hover(ctx context.Context, snapshot source.Snapshot, fh source.FileHandle, if err != nil { return nil, err } - options := snapshot.View().Options() + options := snapshot.Options() return &protocol.Hover{ Contents: protocol.MarkupContent{ Kind: options.PreferredContentFormat, diff --git a/gopls/internal/vulncheck/command.go b/gopls/internal/vulncheck/command.go index 82711188583..4a3d3d2dcc0 100644 --- a/gopls/internal/vulncheck/command.go +++ b/gopls/internal/vulncheck/command.go @@ -197,7 +197,7 @@ func vulnerablePackages(ctx context.Context, snapshot source.Snapshot, modfile s return nil, err } cli, err := client.NewClient( - findGOVULNDB(snapshot.View().Options().EnvSlice()), + findGOVULNDB(snapshot.Options().EnvSlice()), client.Options{HTTPCache: govulncheck.NewInMemoryCache(fsCache)}) if err != nil { return nil, err @@ -209,7 +209,7 @@ func vulnerablePackages(ctx context.Context, snapshot source.Snapshot, modfile s mu sync.Mutex ) - goVersion := snapshot.View().Options().Env[GoVersionForVulnTest] + goVersion := snapshot.Options().Env[GoVersionForVulnTest] if goVersion == "" { goVersion = snapshot.View().GoVersionString() } From 63c7da0095b2d69cce50d5c706c1e282e0a4a80a Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Thu, 31 Aug 2023 16:49:21 -0400 Subject: [PATCH 047/178] gopls: more 1.18 cleanup; use strings.Cut and os functions Change-Id: Id2be028f3af23877d7707d3d883d5e05ee2b59a4 Reviewed-on: https://go-review.googlesource.com/c/tools/+/524839 Run-TryBot: Robert Findley Reviewed-by: Alan Donovan gopls-CI: kokoro TryBot-Result: Gopher Robot LUCI-TryBot-Result: Go LUCI Reviewed-by: qiulaidongfeng <2645477756@qq.com> --- gopls/internal/lsp/cache/cycle_test.go | 2 +- gopls/internal/lsp/cache/snapshot.go | 10 +--------- gopls/internal/lsp/lru/lru.go | 2 -- gopls/internal/lsp/lru/lru_test.go | 2 -- gopls/internal/lsp/regtest/marker.go | 15 ++------------- gopls/internal/lsp/regtest/regtest.go | 3 +-- gopls/internal/lsp/regtest/runner.go | 3 +-- 7 files changed, 6 insertions(+), 31 deletions(-) diff --git a/gopls/internal/lsp/cache/cycle_test.go b/gopls/internal/lsp/cache/cycle_test.go index d08e8e0b73f..25edbbfe338 100644 --- a/gopls/internal/lsp/cache/cycle_test.go +++ b/gopls/internal/lsp/cache/cycle_test.go @@ -76,7 +76,7 @@ func TestBreakImportCycles(t *testing.T) { } if s != "" { for _, item := range strings.Split(s, ";") { - nodeID, succIDs, ok := cut(item, "->") + nodeID, succIDs, ok := strings.Cut(item, "->") node := makeNode(nodeID) if ok { for _, succID := range strings.Split(succIDs, ",") { diff --git a/gopls/internal/lsp/cache/snapshot.go b/gopls/internal/lsp/cache/snapshot.go index cbc9c6e7e16..a1fe4753e2f 100644 --- a/gopls/internal/lsp/cache/snapshot.go +++ b/gopls/internal/lsp/cache/snapshot.go @@ -1874,20 +1874,12 @@ https://github.com/golang/tools/blob/master/gopls/doc/settings.md#buildflags-str // Most likely, each call site of inVendor needs to be reconsidered to // understand and correctly implement the desired behavior. func inVendor(uri span.URI) bool { - _, after, found := cut(string(uri), "/vendor/") + _, after, found := strings.Cut(string(uri), "/vendor/") // Only subdirectories of /vendor/ are considered vendored // (/vendor/a/foo.go is vendored, /vendor/foo.go is not). return found && strings.Contains(after, "/") } -// TODO(adonovan): replace with strings.Cut when we can assume go1.18. -func cut(s, sep string) (before, after string, found bool) { - if i := strings.Index(s, sep); i >= 0 { - return s[:i], s[i+len(sep):], true - } - return s, "", false -} - // unappliedChanges is a file source that handles an uncloned snapshot. type unappliedChanges struct { originalSnapshot *snapshot diff --git a/gopls/internal/lsp/lru/lru.go b/gopls/internal/lsp/lru/lru.go index 5750f412bb0..b75fc852d2d 100644 --- a/gopls/internal/lsp/lru/lru.go +++ b/gopls/internal/lsp/lru/lru.go @@ -11,8 +11,6 @@ import ( "sync" ) -type any = interface{} // TODO: remove once gopls only builds at go1.18+ - // A Cache is a fixed-size in-memory LRU cache. type Cache struct { capacity int diff --git a/gopls/internal/lsp/lru/lru_test.go b/gopls/internal/lsp/lru/lru_test.go index 165a64780cb..a9e6407a7c6 100644 --- a/gopls/internal/lsp/lru/lru_test.go +++ b/gopls/internal/lsp/lru/lru_test.go @@ -17,8 +17,6 @@ import ( "golang.org/x/tools/gopls/internal/lsp/lru" ) -type any = interface{} // TODO: remove once gopls only builds at go1.18+ - func TestCache(t *testing.T) { type get struct { key string diff --git a/gopls/internal/lsp/regtest/marker.go b/gopls/internal/lsp/regtest/marker.go index 1c364bdc952..36dcda39647 100644 --- a/gopls/internal/lsp/regtest/marker.go +++ b/gopls/internal/lsp/regtest/marker.go @@ -746,8 +746,7 @@ func loadMarkerTest(name string, content []byte) (*markerTest, error) { test.env = make(map[string]string) fields := strings.Fields(string(file.Data)) for _, field := range fields { - // TODO: use strings.Cut once we are on 1.18+. - key, value, ok := cut(field, "=") + key, value, ok := strings.Cut(field, "=") if !ok { return nil, fmt.Errorf("env vars must be formatted as var=value, got %q", field) } @@ -755,7 +754,7 @@ func loadMarkerTest(name string, content []byte) (*markerTest, error) { } case strings.HasPrefix(file.Name, "@"): // golden content - id, name, _ := cut(file.Name[len("@"):], "/") + id, name, _ := strings.Cut(file.Name[len("@"):], "/") // Note that a file.Name of just "@id" gives (id, name) = ("id", ""). if _, ok := test.golden[id]; !ok { test.golden[id] = &Golden{ @@ -800,16 +799,6 @@ func loadMarkerTest(name string, content []byte) (*markerTest, error) { return test, nil } -// cut is a copy of strings.Cut. -// -// TODO: once we only support Go 1.18+, just use strings.Cut. -func cut(s, sep string) (before, after string, found bool) { - if i := strings.Index(s, sep); i >= 0 { - return s[:i], s[i+len(sep):], true - } - return s, "", false -} - // formatTest formats the test as a txtar archive. func formatTest(test *markerTest) ([]byte, error) { arch := &txtar.Archive{ diff --git a/gopls/internal/lsp/regtest/regtest.go b/gopls/internal/lsp/regtest/regtest.go index 02c0ad06bb9..7def1d77da7 100644 --- a/gopls/internal/lsp/regtest/regtest.go +++ b/gopls/internal/lsp/regtest/regtest.go @@ -8,7 +8,6 @@ import ( "context" "flag" "fmt" - "io/ioutil" "os" "runtime" "testing" @@ -133,7 +132,7 @@ func Main(m *testing.M, hook func(*source.Options)) { } } - dir, err := ioutil.TempDir("", "gopls-regtest-") + dir, err := os.MkdirTemp("", "gopls-regtest-") if err != nil { panic(fmt.Errorf("creating regtest temp directory: %v", err)) } diff --git a/gopls/internal/lsp/regtest/runner.go b/gopls/internal/lsp/regtest/runner.go index 4f085e720fc..e4aa2f312fa 100644 --- a/gopls/internal/lsp/regtest/runner.go +++ b/gopls/internal/lsp/regtest/runner.go @@ -9,7 +9,6 @@ import ( "context" "fmt" "io" - "io/ioutil" "net" "os" "path/filepath" @@ -370,7 +369,7 @@ func (r *Runner) separateProcessServer(optsHook func(*source.Options)) jsonrpc2. } r.startRemoteOnce.Do(func() { - socketDir, err := ioutil.TempDir(r.tempDir, "gopls-regtest-socket") + socketDir, err := os.MkdirTemp(r.tempDir, "gopls-regtest-socket") if err != nil { r.remoteErr = err return From cb85f8f57e5bc9f26662a5a8a1c45b7538b8d212 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Tue, 23 May 2023 13:00:02 -0400 Subject: [PATCH 048/178] gopls/internal/lsp/cmd: consolidate editing flags Four operations (rename, fix, imports, format) apply edits to the client's files. This change causes them to behave the same w.r.t. the -write, -diff, -preserve, and -list flags. Specifically, the default behavior prints the edited file contents (sans filename or newline). The -write, -diff, and -list flags suppress the default behavior but are otherwise orthogonal. The -preserve flag causes -write to save backups. (These semantics were based on the previous behavior of 'format', which I suspect is the most important of the subcommands by usage.) Also: - clarify the documentation of the fix operation, including examples of cases that now work, such as InvalidIfaceAssign (see the attached issue). - support CodeAction edit commands, which may cause the server to make an ApplyEdit downcall. ApplyEdit is handled using the same logic and flags as the client-initiated edits. - handle various discarded errors properly. - remove unnecessary Contexts. - add tests. Note: this CL changes the behavior of the rename and imports commands when multiple flags are provided. Before, flags had precedence -w > -d > default. Now, the -w -d -l flags are orthogonal and only if none are provided is the changed output printed. Also, the rename operation no longer prints the name of each changed file. Updates golang/go#60290 Change-Id: If986267ca4365e95f379d8243de6f302f7e756c0 Reviewed-on: https://go-review.googlesource.com/c/tools/+/497756 Reviewed-by: Robert Findley Run-TryBot: Alan Donovan TryBot-Result: Gopher Robot --- gopls/internal/lsp/cmd/cmd.go | 116 ++++++++++++++++-- gopls/internal/lsp/cmd/format.go | 42 +------ gopls/internal/lsp/cmd/imports.go | 30 +---- gopls/internal/lsp/cmd/rename.go | 65 +--------- gopls/internal/lsp/cmd/suggested_fix.go | 66 +++++----- .../internal/lsp/cmd/test/integration_test.go | 54 +++++--- gopls/internal/lsp/cmd/usage/fix.hlp | 37 +++++- gopls/internal/lsp/cmd/usage/format.hlp | 8 +- gopls/internal/lsp/cmd/usage/imports.hlp | 8 +- gopls/internal/lsp/cmd/usage/rename.hlp | 8 +- internal/tool/tool.go | 3 + 11 files changed, 245 insertions(+), 192 deletions(-) diff --git a/gopls/internal/lsp/cmd/cmd.go b/gopls/internal/lsp/cmd/cmd.go index 08e7a64e00a..2d637f29a70 100644 --- a/gopls/internal/lsp/cmd/cmd.go +++ b/gopls/internal/lsp/cmd/cmd.go @@ -21,6 +21,7 @@ import ( "text/tabwriter" "time" + "golang.org/x/exp/slices" "golang.org/x/tools/gopls/internal/lsp" "golang.org/x/tools/gopls/internal/lsp/browser" "golang.org/x/tools/gopls/internal/lsp/cache" @@ -30,6 +31,7 @@ import ( "golang.org/x/tools/gopls/internal/lsp/protocol" "golang.org/x/tools/gopls/internal/lsp/source" "golang.org/x/tools/gopls/internal/span" + "golang.org/x/tools/internal/diff" "golang.org/x/tools/internal/jsonrpc2" "golang.org/x/tools/internal/tool" "golang.org/x/tools/internal/xcontext" @@ -75,6 +77,26 @@ type Application struct { // PrepareOptions is called to update the options when a new view is built. // It is primarily to allow the behavior of gopls to be modified by hooks. PrepareOptions func(*source.Options) + + // editFlags holds flags that control how file edit operations + // are applied, in particular when the server makes an ApplyEdits + // downcall to the client. Present only for commands that apply edits. + editFlags *EditFlags +} + +// EditFlags defines flags common to {fix,format,imports,rename} +// that control how edits are applied to the client's files. +// +// The type is exported for flag reflection. +// +// The -write, -diff, and -list flags are orthogonal but any +// of them suppresses the default behavior, which is to print +// the edited file contents. +type EditFlags struct { + Write bool `flag:"w,write" help:"write edited content to source files"` + Preserve bool `flag:"preserve" help:"with -write, make copies of original files"` + Diff bool `flag:"d,diff" help:"display diffs instead of edited file content"` + List bool `flag:"l,list" help:"display names of edited files"` } func (app *Application) verbose() bool { @@ -524,10 +546,86 @@ func (c *cmdClient) Configuration(ctx context.Context, p *protocol.ParamConfigur } func (c *cmdClient) ApplyEdit(ctx context.Context, p *protocol.ApplyWorkspaceEditParams) (*protocol.ApplyWorkspaceEditResult, error) { - return &protocol.ApplyWorkspaceEditResult{ - Applied: false, - FailureReason: "the gopls command-line client does not apply edits", - }, nil + if err := c.applyWorkspaceEdit(&p.Edit); err != nil { + return &protocol.ApplyWorkspaceEditResult{FailureReason: err.Error()}, nil + } + return &protocol.ApplyWorkspaceEditResult{Applied: true}, nil +} + +// applyWorkspaceEdit applies a complete WorkspaceEdit to the client's +// files, honoring the preferred edit mode specified by cli.app.editMode. +// (Used by rename and by ApplyEdit downcalls.) +func (cli *cmdClient) applyWorkspaceEdit(edit *protocol.WorkspaceEdit) error { + var orderedURIs []span.URI + edits := map[span.URI][]protocol.TextEdit{} + for _, c := range edit.DocumentChanges { + if c.TextDocumentEdit != nil { + uri := fileURI(c.TextDocumentEdit.TextDocument.URI) + edits[uri] = append(edits[uri], c.TextDocumentEdit.Edits...) + orderedURIs = append(orderedURIs, uri) + } + if c.RenameFile != nil { + return fmt.Errorf("client does not support file renaming (%s -> %s)", + c.RenameFile.OldURI, + c.RenameFile.NewURI) + } + } + slices.Sort(orderedURIs) + for _, uri := range orderedURIs { + f := cli.openFile(uri) + if f.err != nil { + return f.err + } + if err := applyTextEdits(f.mapper, edits[uri], cli.app.editFlags); err != nil { + return err + } + } + return nil +} + +// applyTextEdits applies a list of edits to the mapper file content, +// using the preferred edit mode. It is a no-op if there are no edits. +func applyTextEdits(mapper *protocol.Mapper, edits []protocol.TextEdit, flags *EditFlags) error { + if len(edits) == 0 { + return nil + } + newContent, renameEdits, err := source.ApplyProtocolEdits(mapper, edits) + if err != nil { + return err + } + + filename := mapper.URI.Filename() + + if flags.List { + fmt.Println(filename) + } + + if flags.Write { + if flags.Preserve { + if err := os.Rename(filename, filename+".orig"); err != nil { + return err + } + } + if err := os.WriteFile(filename, newContent, 0644); err != nil { + return err + } + } + + if flags.Diff { + unified, err := diff.ToUnified(filename+".orig", filename, string(mapper.Content), renameEdits) + if err != nil { + return err + } + fmt.Print(unified) + } + + // No flags: just print edited file content. + // TODO(adonovan): how is this ever useful with multiple files? + if !(flags.List || flags.Write || flags.Diff) { + os.Stdout.Write(newContent) + } + + return nil } func (c *cmdClient) PublishDiagnostics(ctx context.Context, p *protocol.PublishDiagnosticsParams) error { @@ -542,7 +640,7 @@ func (c *cmdClient) PublishDiagnostics(ctx context.Context, p *protocol.PublishD c.filesMu.Lock() defer c.filesMu.Unlock() - file := c.getFile(ctx, fileURI(p.URI)) + file := c.getFile(fileURI(p.URI)) file.diagnostics = append(file.diagnostics, p.Diagnostics...) // Perform a crude in-place deduplication. @@ -610,7 +708,7 @@ func (c *cmdClient) InlineValueRefresh(context.Context) error { return nil } -func (c *cmdClient) getFile(ctx context.Context, uri span.URI) *cmdFile { +func (c *cmdClient) getFile(uri span.URI) *cmdFile { file, found := c.files[uri] if !found || file.err != nil { file = &cmdFile{ @@ -629,17 +727,17 @@ func (c *cmdClient) getFile(ctx context.Context, uri span.URI) *cmdFile { return file } -func (c *cmdClient) openFile(ctx context.Context, uri span.URI) *cmdFile { +func (c *cmdClient) openFile(uri span.URI) *cmdFile { c.filesMu.Lock() defer c.filesMu.Unlock() - return c.getFile(ctx, uri) + return c.getFile(uri) } // TODO(adonovan): provide convenience helpers to: // - map a (URI, protocol.Range) to a MappedRange; // - parse a command-line argument to a MappedRange. func (c *connection) openFile(ctx context.Context, uri span.URI) (*cmdFile, error) { - file := c.client.openFile(ctx, uri) + file := c.client.openFile(uri) if file.err != nil { return nil, file.err } diff --git a/gopls/internal/lsp/cmd/format.go b/gopls/internal/lsp/cmd/format.go index 517a4d33d9f..73a8d7f582b 100644 --- a/gopls/internal/lsp/cmd/format.go +++ b/gopls/internal/lsp/cmd/format.go @@ -8,21 +8,14 @@ import ( "context" "flag" "fmt" - "io/ioutil" - "os" "golang.org/x/tools/gopls/internal/lsp/protocol" - "golang.org/x/tools/gopls/internal/lsp/source" "golang.org/x/tools/gopls/internal/span" - "golang.org/x/tools/internal/diff" ) // format implements the format verb for gopls. type format struct { - Diff bool `flag:"d,diff" help:"display diffs instead of rewriting files"` - Write bool `flag:"w,write" help:"write result to (source) file instead of stdout"` - List bool `flag:"l,list" help:"list files whose formatting differs from gofmt's"` - + EditFlags app *Application } @@ -47,10 +40,9 @@ format-flags: // results to stdout. func (c *format) Run(ctx context.Context, args ...string) error { if len(args) == 0 { - // no files, so no results return nil } - // now we ready to kick things off + c.app.editFlags = &c.EditFlags conn, err := c.app.connect(ctx, nil) if err != nil { return err @@ -62,7 +54,6 @@ func (c *format) Run(ctx context.Context, args ...string) error { if err != nil { return err } - filename := spn.URI().Filename() loc, err := file.mapper.SpanLocation(spn) if err != nil { return err @@ -77,33 +68,8 @@ func (c *format) Run(ctx context.Context, args ...string) error { if err != nil { return fmt.Errorf("%v: %v", spn, err) } - formatted, sedits, err := source.ApplyProtocolEdits(file.mapper, edits) - if err != nil { - return fmt.Errorf("%v: %v", spn, err) - } - printIt := true - if c.List { - printIt = false - if len(edits) > 0 { - fmt.Println(filename) - } - } - if c.Write { - printIt = false - if len(edits) > 0 { - ioutil.WriteFile(filename, formatted, 0644) - } - } - if c.Diff { - printIt = false - unified, err := diff.ToUnified(filename+".orig", filename, string(file.mapper.Content), sedits) - if err != nil { - return err - } - fmt.Print(unified) - } - if printIt { - os.Stdout.Write(formatted) + if err := applyTextEdits(file.mapper, edits, c.app.editFlags); err != nil { + return err } } return nil diff --git a/gopls/internal/lsp/cmd/imports.go b/gopls/internal/lsp/cmd/imports.go index 537c8f164f1..d014d03881e 100644 --- a/gopls/internal/lsp/cmd/imports.go +++ b/gopls/internal/lsp/cmd/imports.go @@ -8,21 +8,15 @@ import ( "context" "flag" "fmt" - "io/ioutil" - "os" "golang.org/x/tools/gopls/internal/lsp/protocol" - "golang.org/x/tools/gopls/internal/lsp/source" "golang.org/x/tools/gopls/internal/span" - "golang.org/x/tools/internal/diff" "golang.org/x/tools/internal/tool" ) // imports implements the import verb for gopls. type imports struct { - Diff bool `flag:"d,diff" help:"display diffs instead of rewriting files"` - Write bool `flag:"w,write" help:"write result to (source) file instead of stdout"` - + EditFlags app *Application } @@ -49,6 +43,7 @@ func (t *imports) Run(ctx context.Context, args ...string) error { if len(args) != 1 { return tool.CommandLineErrorf("imports expects 1 argument") } + t.app.editFlags = &t.EditFlags conn, err := t.app.connect(ctx, nil) if err != nil { return err @@ -82,24 +77,5 @@ func (t *imports) Run(ctx context.Context, args ...string) error { } } } - newContent, sedits, err := source.ApplyProtocolEdits(file.mapper, edits) - if err != nil { - return fmt.Errorf("%v: %v", edits, err) - } - filename := file.uri.Filename() - switch { - case t.Write: - if len(edits) > 0 { - ioutil.WriteFile(filename, newContent, 0644) - } - case t.Diff: - unified, err := diff.ToUnified(filename+".orig", filename, string(file.mapper.Content), sedits) - if err != nil { - return err - } - fmt.Print(unified) - default: - os.Stdout.Write(newContent) - } - return nil + return applyTextEdits(file.mapper, edits, t.app.editFlags) } diff --git a/gopls/internal/lsp/cmd/rename.go b/gopls/internal/lsp/cmd/rename.go index 8a1ae36d7e7..5ad7aa44494 100644 --- a/gopls/internal/lsp/cmd/rename.go +++ b/gopls/internal/lsp/cmd/rename.go @@ -8,24 +8,15 @@ import ( "context" "flag" "fmt" - "io/ioutil" - "os" - "path/filepath" - "sort" "golang.org/x/tools/gopls/internal/lsp/protocol" - "golang.org/x/tools/gopls/internal/lsp/source" "golang.org/x/tools/gopls/internal/span" - "golang.org/x/tools/internal/diff" "golang.org/x/tools/internal/tool" ) // rename implements the rename verb for gopls. type rename struct { - Diff bool `flag:"d,diff" help:"display diffs instead of rewriting files"` - Write bool `flag:"w,write" help:"write result to (source) file instead of stdout"` - Preserve bool `flag:"preserve" help:"preserve original files"` - + EditFlags app *Application } @@ -54,6 +45,7 @@ func (r *rename) Run(ctx context.Context, args ...string) error { if len(args) != 2 { return tool.CommandLineErrorf("definition expects 2 arguments (position, new name)") } + r.app.editFlags = &r.EditFlags conn, err := r.app.connect(ctx, nil) if err != nil { return err @@ -78,56 +70,5 @@ func (r *rename) Run(ctx context.Context, args ...string) error { if err != nil { return err } - var orderedURIs []string - edits := map[span.URI][]protocol.TextEdit{} - for _, c := range edit.DocumentChanges { - if c.TextDocumentEdit != nil { - uri := fileURI(c.TextDocumentEdit.TextDocument.URI) - edits[uri] = append(edits[uri], c.TextDocumentEdit.Edits...) - orderedURIs = append(orderedURIs, string(uri)) - } - } - sort.Strings(orderedURIs) - changeCount := len(orderedURIs) - - for _, u := range orderedURIs { - uri := span.URIFromURI(u) - cmdFile, err := conn.openFile(ctx, uri) - if err != nil { - return err - } - filename := cmdFile.uri.Filename() - - newContent, renameEdits, err := source.ApplyProtocolEdits(cmdFile.mapper, edits[uri]) - if err != nil { - return fmt.Errorf("%v: %v", edits, err) - } - - switch { - case r.Write: - fmt.Fprintln(os.Stderr, filename) - if r.Preserve { - if err := os.Rename(filename, filename+".orig"); err != nil { - return fmt.Errorf("%v: %v", edits, err) - } - } - ioutil.WriteFile(filename, newContent, 0644) - case r.Diff: - unified, err := diff.ToUnified(filename+".orig", filename, string(cmdFile.mapper.Content), renameEdits) - if err != nil { - return err - } - fmt.Print(unified) - default: - if len(orderedURIs) > 1 { - fmt.Printf("%s:\n", filepath.Base(filename)) - } - os.Stdout.Write(newContent) - if changeCount > 1 { // if this wasn't last change, print newline - fmt.Println() - } - changeCount -= 1 - } - } - return nil + return conn.client.applyWorkspaceEdit(edit) } diff --git a/gopls/internal/lsp/cmd/suggested_fix.go b/gopls/internal/lsp/cmd/suggested_fix.go index 5e267e71bfd..a3e6093912a 100644 --- a/gopls/internal/lsp/cmd/suggested_fix.go +++ b/gopls/internal/lsp/cmd/suggested_fix.go @@ -8,21 +8,21 @@ import ( "context" "flag" "fmt" - "io/ioutil" - "os" "golang.org/x/tools/gopls/internal/lsp/protocol" - "golang.org/x/tools/gopls/internal/lsp/source" "golang.org/x/tools/gopls/internal/span" - "golang.org/x/tools/internal/diff" "golang.org/x/tools/internal/tool" ) +// TODO(adonovan): this command has a very poor user interface. It +// should have a way to query the available fixes for a file (without +// a span), enumerate the valid fix kinds, enable all fixes, and not +// require the pointless -all flag. See issue #60290. + // suggestedFix implements the fix verb for gopls. type suggestedFix struct { - Diff bool `flag:"d,diff" help:"display diffs instead of rewriting files"` - Write bool `flag:"w,write" help:"write result to (source) file instead of stdout"` - All bool `flag:"a,all" help:"apply all fixes, not just preferred fixes"` + EditFlags + All bool `flag:"a,all" help:"apply all fixes, not just preferred fixes"` app *Application } @@ -33,8 +33,33 @@ func (s *suggestedFix) Usage() string { return "[fix-flags] " } func (s *suggestedFix) ShortHelp() string { return "apply suggested fixes" } func (s *suggestedFix) DetailedHelp(f *flag.FlagSet) { fmt.Fprintf(f.Output(), ` -Example: apply suggested fixes for this file - $ gopls fix -w internal/lsp/cmd/check.go +Example: apply fixes to this file, rewriting it: + + $ gopls fix -a -w internal/lsp/cmd/check.go + +The -a (-all) flag causes all fixes, not just preferred ones, to be +applied, but since no fixes are currently preferred, this flag is +essentially mandatory. + +Arguments after the filename are interpreted as LSP CodeAction kinds +to be applied; the default set is {"quickfix"}, but valid kinds include: + + quickfix + refactor + refactor.extract + refactor.inline + refactor.rewrite + source.organizeImports + source.fixAll + +CodeAction kinds are hierarchical, so "refactor" includes +"refactor.inline". There is currently no way to enable or even +enumerate all kinds. + +Example: apply any "refactor.rewrite" fixes at the specific byte +offset within this file: + + $ gopls fix -a internal/lsp/cmd/check.go:#43 refactor.rewrite fix-flags: `) @@ -49,6 +74,7 @@ func (s *suggestedFix) Run(ctx context.Context, args ...string) error { if len(args) < 1 { return tool.CommandLineErrorf("fix expects at least 1 argument") } + s.app.editFlags = &s.EditFlags conn, err := s.app.connect(ctx, nil) if err != nil { return err @@ -163,25 +189,5 @@ func (s *suggestedFix) Run(ctx context.Context, args ...string) error { } } - newContent, sedits, err := source.ApplyProtocolEdits(file.mapper, edits) - if err != nil { - return fmt.Errorf("%v: %v", edits, err) - } - - filename := file.uri.Filename() - switch { - case s.Write: - if len(edits) > 0 { - ioutil.WriteFile(filename, newContent, 0644) - } - case s.Diff: - diffs, err := diff.ToUnified(filename+".orig", filename, string(file.mapper.Content), sedits) - if err != nil { - return err - } - fmt.Print(diffs) - default: - os.Stdout.Write(newContent) - } - return nil + return applyTextEdits(file.mapper, edits, s.app.editFlags) } diff --git a/gopls/internal/lsp/cmd/test/integration_test.go b/gopls/internal/lsp/cmd/test/integration_test.go index b7b6d9bd86c..4ee9e3eb7c5 100644 --- a/gopls/internal/lsp/cmd/test/integration_test.go +++ b/gopls/internal/lsp/cmd/test/integration_test.go @@ -20,8 +20,8 @@ package cmdtest // TODO(adonovan): // - Use markers to represent positions in the input and in assertions. // - Coverage of cross-cutting things like cwd, environ, span parsing, etc. -// - Subcommands that accept -write and -diff flags should implement -// them consistently wrt the default behavior; factor their tests. +// - Subcommands that accept -write and -diff flags implement them +// consistently; factor their tests. // - Add missing test for 'vulncheck' subcommand. // - Add tests for client-only commands: serve, bug, help, api-json, licenses. @@ -396,7 +396,7 @@ func _() { res := gopls(t, tree, "imports", "a.go") res.checkExit(true) if res.stdout != want { - t.Errorf("format: got <<%s>>, want <<%s>>", res.stdout, want) + t.Errorf("imports: got <<%s>>, want <<%s>>", res.stdout, want) } } // -diff: show a unified diff @@ -783,12 +783,13 @@ go 1.18 package a type T int func f() (int, string) { return } -`) - want := ` + +-- b.go -- package a -type T int -func f() (int, string) { return 0, "" } -`[1:] +import "io" +var _ io.Reader = C{} +type C struct{} +`) // no arguments { @@ -796,20 +797,45 @@ func f() (int, string) { return 0, "" } res.checkExit(false) res.checkStderr("expects at least 1 argument") } - // success (-a enables fillreturns) + // success with default kinds, {quickfix}. + // -a is always required because no fix is currently "preferred" (!) { res := gopls(t, tree, "fix", "-a", "a.go") res.checkExit(true) got := res.stdout + want := ` +package a +type T int +func f() (int, string) { return 0, "" } + +`[1:] + if got != want { + t.Errorf("fix: got <<%s>>, want <<%s>>\nstderr:\n%s", got, want, res.stderr) + } + } + // success, with explicit CodeAction kind and diagnostics span. + { + res := gopls(t, tree, "fix", "-a", "b.go:#40", "quickfix") + res.checkExit(true) + got := res.stdout + want := ` +package a + +import "io" + +var _ io.Reader = C{} + +type C struct{} + +// Read implements io.Reader. +func (C) Read(p []byte) (n int, err error) { + panic("unimplemented") +} +`[1:] if got != want { t.Errorf("fix: got <<%s>>, want <<%s>>\nstderr:\n%s", got, want, res.stderr) } } - // TODO(adonovan): more tests: - // - -write, -diff: factor with imports, format, rename. - // - without -all flag - // - args[2:] is an optional list of protocol.CodeActionKind enum values. - // - a span argument with a range causes filtering. } // TestWorkspaceSymbol tests the 'workspace_symbol' subcommand (../workspace_symbol.go). diff --git a/gopls/internal/lsp/cmd/usage/fix.hlp b/gopls/internal/lsp/cmd/usage/fix.hlp index 4789a6c5b37..39e464da59d 100644 --- a/gopls/internal/lsp/cmd/usage/fix.hlp +++ b/gopls/internal/lsp/cmd/usage/fix.hlp @@ -3,13 +3,42 @@ apply suggested fixes Usage: gopls [flags] fix [fix-flags] -Example: apply suggested fixes for this file - $ gopls fix -w internal/lsp/cmd/check.go +Example: apply fixes to this file, rewriting it: + + $ gopls fix -a -w internal/lsp/cmd/check.go + +The -a (-all) flag causes all fixes, not just preferred ones, to be +applied, but since no fixes are currently preferred, this flag is +essentially mandatory. + +Arguments after the filename are interpreted as LSP CodeAction kinds +to be applied; the default set is {"quickfix"}, but valid kinds include: + + quickfix + refactor + refactor.extract + refactor.inline + refactor.rewrite + source.organizeImports + source.fixAll + +CodeAction kinds are hierarchical, so "refactor" includes +"refactor.inline". There is currently no way to enable or even +enumerate all kinds. + +Example: apply any "refactor.rewrite" fixes at the specific byte +offset within this file: + + $ gopls fix -a internal/lsp/cmd/check.go:#43 refactor.rewrite fix-flags: -a,-all apply all fixes, not just preferred fixes -d,-diff - display diffs instead of rewriting files + display diffs instead of edited file content + -l,-list + display names of edited files + -preserve + with -write, make copies of original files -w,-write - write result to (source) file instead of stdout + write edited content to source files diff --git a/gopls/internal/lsp/cmd/usage/format.hlp b/gopls/internal/lsp/cmd/usage/format.hlp index 7ef0bbe4314..fedb5895282 100644 --- a/gopls/internal/lsp/cmd/usage/format.hlp +++ b/gopls/internal/lsp/cmd/usage/format.hlp @@ -11,8 +11,10 @@ Example: reformat this file: format-flags: -d,-diff - display diffs instead of rewriting files + display diffs instead of edited file content -l,-list - list files whose formatting differs from gofmt's + display names of edited files + -preserve + with -write, make copies of original files -w,-write - write result to (source) file instead of stdout + write edited content to source files diff --git a/gopls/internal/lsp/cmd/usage/imports.hlp b/gopls/internal/lsp/cmd/usage/imports.hlp index 295f4daa2d4..6e0517296ec 100644 --- a/gopls/internal/lsp/cmd/usage/imports.hlp +++ b/gopls/internal/lsp/cmd/usage/imports.hlp @@ -9,6 +9,10 @@ Example: update imports statements in a file: imports-flags: -d,-diff - display diffs instead of rewriting files + display diffs instead of edited file content + -l,-list + display names of edited files + -preserve + with -write, make copies of original files -w,-write - write result to (source) file instead of stdout + write edited content to source files diff --git a/gopls/internal/lsp/cmd/usage/rename.hlp b/gopls/internal/lsp/cmd/usage/rename.hlp index ae58cbf60a7..7b6d7f96b55 100644 --- a/gopls/internal/lsp/cmd/usage/rename.hlp +++ b/gopls/internal/lsp/cmd/usage/rename.hlp @@ -11,8 +11,10 @@ Example: rename-flags: -d,-diff - display diffs instead of rewriting files + display diffs instead of edited file content + -l,-list + display names of edited files -preserve - preserve original files + with -write, make copies of original files -w,-write - write result to (source) file instead of stdout + write edited content to source files diff --git a/internal/tool/tool.go b/internal/tool/tool.go index f4dd8d1c562..36ba55bea39 100644 --- a/internal/tool/tool.go +++ b/internal/tool/tool.go @@ -220,6 +220,9 @@ func addFlags(f *flag.FlagSet, field reflect.StructField, value reflect.Value) * if value.Kind() != reflect.Struct { return nil } + + // TODO(adonovan): there's no need for this special treatment of Profile: + // The caller can use f.Lookup("profile.cpu") etc instead. p, _ := value.Addr().Interface().(*Profile) // go through all the fields of the struct for i := 0; i < value.Type().NumField(); i++ { From ea5e7c6cd496574f847276deb47b01b9520d6411 Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Thu, 31 Aug 2023 17:00:21 -0400 Subject: [PATCH 049/178] gopls/internal/lsp/cache: delete unused mustEncode Change-Id: Id487772ae3d2b252d1b946f42af9520b2c990613 Reviewed-on: https://go-review.googlesource.com/c/tools/+/524840 TryBot-Result: Gopher Robot gopls-CI: kokoro Run-TryBot: Robert Findley Auto-Submit: Robert Findley LUCI-TryBot-Result: Go LUCI Reviewed-by: Alan Donovan --- gopls/internal/lsp/cache/analysis.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/gopls/internal/lsp/cache/analysis.go b/gopls/internal/lsp/cache/analysis.go index 1cfa84dba59..5676a7814a2 100644 --- a/gopls/internal/lsp/cache/analysis.go +++ b/gopls/internal/lsp/cache/analysis.go @@ -1395,14 +1395,6 @@ func requiredAnalyzers(analyzers []*analysis.Analyzer) []*analysis.Analyzer { return result } -func mustEncode(x interface{}) []byte { - var buf bytes.Buffer - if err := gob.NewEncoder(&buf).Encode(x); err != nil { - log.Fatalf("internal error encoding %T: %v", x, err) - } - return buf.Bytes() -} - var analyzeSummaryCodec = frob.CodecFor[*analyzeSummary]() // -- data types for serialization of analysis.Diagnostic and source.Diagnostic -- From 7663a409cd096befd0afbd20646f4b6f8f90b05e Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Fri, 1 Sep 2023 10:37:56 -0400 Subject: [PATCH 050/178] internal/cmd/deadcode: add -generated flag -generated=false will suppress output of unreachable functions in generated Go source files (as determined by ast.IsGenerated). Change-Id: Iab5aa9fbc497a9bcb6a10124e2fe7ab892ad1936 Reviewed-on: https://go-review.googlesource.com/c/tools/+/524946 Reviewed-by: Robert Findley TryBot-Result: Gopher Robot Auto-Submit: Alan Donovan Run-TryBot: Alan Donovan --- internal/cmd/deadcode/deadcode.go | 71 ++++++++++++++++++++++++++++--- 1 file changed, 64 insertions(+), 7 deletions(-) diff --git a/internal/cmd/deadcode/deadcode.go b/internal/cmd/deadcode/deadcode.go index 60e22cb5552..f3388aa6161 100644 --- a/internal/cmd/deadcode/deadcode.go +++ b/internal/cmd/deadcode/deadcode.go @@ -8,6 +8,7 @@ import ( _ "embed" "flag" "fmt" + "go/ast" "go/token" "io" "log" @@ -32,10 +33,11 @@ var ( testFlag = flag.Bool("test", false, "include implicit test packages and executables") tagsFlag = flag.String("tags", "", "comma-separated list of extra build tags (see: go help buildconstraint)") - filterFlag = flag.String("filter", "", "report only packages matching this regular expression (default: module of first package)") - lineFlag = flag.Bool("line", false, "show output in a line-oriented format") - cpuProfile = flag.String("cpuprofile", "", "write CPU profile to this file") - memProfile = flag.String("memprofile", "", "write memory profile to this file") + filterFlag = flag.String("filter", "", "report only packages matching this regular expression (default: module of first package)") + generatedFlag = flag.Bool("generated", true, "report dead functions in generated Go files") + lineFlag = flag.Bool("line", false, "show output in a line-oriented format") + cpuProfile = flag.String("cpuprofile", "", "write CPU profile to this file") + memProfile = flag.String("memprofile", "", "write memory profile to this file") ) func usage() { @@ -104,6 +106,18 @@ func main() { log.Fatalf("packages contain errors") } + // (Optionally) gather names of generated files. + generated := make(map[string]bool) + if !*generatedFlag { + packages.Visit(initial, nil, func(p *packages.Package) { + for _, file := range p.Syntax { + if isGenerated(file) { + generated[p.Fset.File(file.Pos()).Name()] = true + } + } + }) + } + // If -filter is unset, use first module (if available). if *filterFlag == "" { if mod := initial[0].Module; mod != nil && mod.Path != "" { @@ -176,6 +190,13 @@ func main() { } posn := prog.Fset.Position(fn.Pos()) + + // If -generated=false, skip functions declared in generated Go files. + // (Functions called by them may still be reported as dead.) + if generated[posn.Filename] { + continue + } + if !reachablePosn[posn] { reachablePosn[posn] = true // suppress dups with same pos @@ -220,9 +241,6 @@ func main() { return xposn.Line < yposn.Line }) - // TODO(adonovan): add an option to skip (or indicate) - // dead functions in generated files (see ast.IsGenerated). - if *lineFlag { // line-oriented output for _, fn := range fns { @@ -238,3 +256,42 @@ func main() { } } } + +// TODO(adonovan): use go1.21's ast.IsGenerated. + +// isGenerated reports whether the file was generated by a program, +// not handwritten, by detecting the special comment described +// at https://go.dev/s/generatedcode. +// +// The syntax tree must have been parsed with the ParseComments flag. +// Example: +// +// f, err := parser.ParseFile(fset, filename, src, parser.ParseComments|parser.PackageClauseOnly) +// if err != nil { ... } +// gen := ast.IsGenerated(f) +func isGenerated(file *ast.File) bool { + _, ok := generator(file) + return ok +} + +func generator(file *ast.File) (string, bool) { + for _, group := range file.Comments { + for _, comment := range group.List { + if comment.Pos() > file.Package { + break // after package declaration + } + // opt: check Contains first to avoid unnecessary array allocation in Split. + const prefix = "// Code generated " + if strings.Contains(comment.Text, prefix) { + for _, line := range strings.Split(comment.Text, "\n") { + if rest, ok := strings.CutPrefix(line, prefix); ok { + if gen, ok := strings.CutSuffix(rest, " DO NOT EDIT."); ok { + return gen, true + } + } + } + } + } + } + return "", false +} From a1a928ddbedd8082c24d6223aa9ce1f00f922286 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Fri, 1 Sep 2023 10:42:17 -0400 Subject: [PATCH 051/178] gopls: remove dead code This change removes functions reported as unreachable by internal/cmd/deadcode. Also, move the "generated" file marker to the correct positions. Change-Id: I56f3c056cd010914ac5d4e410f1ee4a44289cc77 Reviewed-on: https://go-review.googlesource.com/c/tools/+/524760 Run-TryBot: Alan Donovan Reviewed-by: Robert Findley Auto-Submit: Alan Donovan TryBot-Result: Gopher Robot --- gopls/internal/lsp/cmd/cmd.go | 9 ------- gopls/internal/lsp/command/command_gen.go | 4 +-- gopls/internal/lsp/command/gen/gen.go | 4 +-- gopls/internal/lsp/source/options.go | 8 ------ gopls/internal/lsp/tests/tests.go | 32 ----------------------- 5 files changed, 4 insertions(+), 53 deletions(-) diff --git a/gopls/internal/lsp/cmd/cmd.go b/gopls/internal/lsp/cmd/cmd.go index 2d637f29a70..267456bc4be 100644 --- a/gopls/internal/lsp/cmd/cmd.go +++ b/gopls/internal/lsp/cmd/cmd.go @@ -348,15 +348,6 @@ func (app *Application) connect(ctx context.Context, onProgress func(*protocol.P } } -// CloseTestConnections terminates shared connections used in command tests. It -// should only be called from tests. -func CloseTestConnections(ctx context.Context) { - for _, c := range internalConnections { - c.Shutdown(ctx) - c.Exit(ctx) - } -} - func (app *Application) connectRemote(ctx context.Context, remote string) (*connection, error) { conn, err := lsprpc.ConnectToRemote(ctx, remote) if err != nil { diff --git a/gopls/internal/lsp/command/command_gen.go b/gopls/internal/lsp/command/command_gen.go index 25a101cb36e..00b76579601 100644 --- a/gopls/internal/lsp/command/command_gen.go +++ b/gopls/internal/lsp/command/command_gen.go @@ -7,10 +7,10 @@ //go:build !generate // +build !generate -package command - // Code generated by generate.go. DO NOT EDIT. +package command + import ( "context" "fmt" diff --git a/gopls/internal/lsp/command/gen/gen.go b/gopls/internal/lsp/command/gen/gen.go index b3f89c8d773..9f0453c62cc 100644 --- a/gopls/internal/lsp/command/gen/gen.go +++ b/gopls/internal/lsp/command/gen/gen.go @@ -25,10 +25,10 @@ const src = `// Copyright 2021 The Go Authors. All rights reserved. //go:build !generate // +build !generate -package command - // Code generated by generate.go. DO NOT EDIT. +package command + import ( {{range $k, $v := .Imports -}} "{{$k}}" diff --git a/gopls/internal/lsp/source/options.go b/gopls/internal/lsp/source/options.go index 334cc9dc798..67818fad34a 100644 --- a/gopls/internal/lsp/source/options.go +++ b/gopls/internal/lsp/source/options.go @@ -1311,14 +1311,6 @@ func (e *SoftError) Error() string { return e.msg } -// softErrorf reports an error that does not affect the functionality of gopls -// (a warning in the UI). -// The formatted message will be shown to the user unmodified. -func (r *OptionResult) softErrorf(format string, values ...interface{}) { - msg := fmt.Sprintf(format, values...) - r.Error = &SoftError{msg} -} - // deprecated reports the current setting as deprecated. If 'replacement' is // non-nil, it is suggested to the user. func (r *OptionResult) deprecated(replacement string) { diff --git a/gopls/internal/lsp/tests/tests.go b/gopls/internal/lsp/tests/tests.go index 150bec9bc7b..4f5fc3c5080 100644 --- a/gopls/internal/lsp/tests/tests.go +++ b/gopls/internal/lsp/tests/tests.go @@ -1159,38 +1159,6 @@ func SpanName(spn span.Span) string { return fmt.Sprintf("%v_%v_%v", uriName(spn.URI()), spn.Start().Line(), spn.Start().Column()) } -func CopyFolderToTempDir(folder string) (string, error) { - if _, err := os.Stat(folder); err != nil { - return "", err - } - dst, err := ioutil.TempDir("", "modfile_test") - if err != nil { - return "", err - } - fds, err := ioutil.ReadDir(folder) - if err != nil { - return "", err - } - for _, fd := range fds { - srcfp := filepath.Join(folder, fd.Name()) - stat, err := os.Stat(srcfp) - if err != nil { - return "", err - } - if !stat.Mode().IsRegular() { - return "", fmt.Errorf("cannot copy non regular file %s", srcfp) - } - contents, err := ioutil.ReadFile(srcfp) - if err != nil { - return "", err - } - if err := ioutil.WriteFile(filepath.Join(dst, fd.Name()), contents, stat.Mode()); err != nil { - return "", err - } - } - return dst, nil -} - func shouldSkip(data *Data, uri span.URI) bool { if data.ModfileFlagAvailable { return false From 010e045c4eb965ad42d8feccd193abb1b80ee42f Mon Sep 17 00:00:00 2001 From: Robert Findley Date: Fri, 1 Sep 2023 10:05:02 -0400 Subject: [PATCH 052/178] internal/persistent: use generics Now that we're on 1.18+, make internal/persistent.Map generic. Change-Id: I3403241fe22e28f969d7feb09a752b52f0d2ee4d Reviewed-on: https://go-review.googlesource.com/c/tools/+/524759 gopls-CI: kokoro LUCI-TryBot-Result: Go LUCI Reviewed-by: Alan Donovan --- gopls/internal/lsp/cache/check.go | 9 ++-- gopls/internal/lsp/cache/load.go | 3 +- gopls/internal/lsp/cache/maps.go | 31 +++-------- gopls/internal/lsp/cache/mod.go | 6 +-- gopls/internal/lsp/cache/mod_tidy.go | 2 +- gopls/internal/lsp/cache/mod_vuln.go | 2 +- gopls/internal/lsp/cache/session.go | 17 +++--- gopls/internal/lsp/cache/snapshot.go | 27 +++++----- gopls/internal/lsp/cache/symbols.go | 3 +- internal/constraints/constraint.go | 52 ++++++++++++++++++ internal/persistent/map.go | 81 ++++++++++++++-------------- internal/persistent/map_test.go | 36 ++++++------- 12 files changed, 148 insertions(+), 121 deletions(-) create mode 100644 internal/constraints/constraint.go diff --git a/gopls/internal/lsp/cache/check.go b/gopls/internal/lsp/cache/check.go index 74404af98c1..b7267e983ec 100644 --- a/gopls/internal/lsp/cache/check.go +++ b/gopls/internal/lsp/cache/check.go @@ -849,7 +849,7 @@ func (s *snapshot) getPackageHandles(ctx context.Context, ids []PackageID) (map[ unfinishedSuccs: int32(len(m.DepsByPkgPath)), } if entry, hit := b.s.packages.Get(m.ID); hit { - n.ph = entry.(*packageHandle) + n.ph = entry } if n.unfinishedSuccs == 0 { leaves = append(leaves, n) @@ -1118,12 +1118,11 @@ func (b *packageHandleBuilder) buildPackageHandle(ctx context.Context, n *handle } // Check the packages map again in case another goroutine got there first. - if alt, ok := b.s.packages.Get(n.m.ID); ok && alt.(*packageHandle).validated { - altPH := alt.(*packageHandle) - if altPH.m != n.m { + if alt, ok := b.s.packages.Get(n.m.ID); ok && alt.validated { + if alt.m != n.m { bug.Reportf("existing package handle does not match for %s", n.m.ID) } - n.ph = altPH + n.ph = alt } else { b.s.packages.Set(n.m.ID, n.ph, nil) } diff --git a/gopls/internal/lsp/cache/load.go b/gopls/internal/lsp/cache/load.go index 05d44329c20..03db2a35d0d 100644 --- a/gopls/internal/lsp/cache/load.go +++ b/gopls/internal/lsp/cache/load.go @@ -217,8 +217,7 @@ func (s *snapshot) load(ctx context.Context, allowNetwork bool, scopes ...loadSc s.mu.Lock() // Assert the invariant s.packages.Get(id).m == s.meta.metadata[id]. - s.packages.Range(func(k, v interface{}) { - id, ph := k.(PackageID), v.(*packageHandle) + s.packages.Range(func(id PackageID, ph *packageHandle) { if s.meta.metadata[id] != ph.m { panic("inconsistent metadata") } diff --git a/gopls/internal/lsp/cache/maps.go b/gopls/internal/lsp/cache/maps.go index de6187da255..3fa866cb840 100644 --- a/gopls/internal/lsp/cache/maps.go +++ b/gopls/internal/lsp/cache/maps.go @@ -10,21 +10,14 @@ import ( "golang.org/x/tools/internal/persistent" ) -// TODO(euroelessar): Use generics once support for go1.17 is dropped. - type filesMap struct { - impl *persistent.Map + impl *persistent.Map[span.URI, source.FileHandle] overlayMap map[span.URI]*Overlay // the subset that are overlays } -// uriLessInterface is the < relation for "any" values containing span.URIs. -func uriLessInterface(a, b interface{}) bool { - return a.(span.URI) < b.(span.URI) -} - func newFilesMap() filesMap { return filesMap{ - impl: persistent.NewMap(uriLessInterface), + impl: new(persistent.Map[span.URI, source.FileHandle]), overlayMap: make(map[span.URI]*Overlay), } } @@ -53,9 +46,7 @@ func (m filesMap) Get(key span.URI) (source.FileHandle, bool) { } func (m filesMap) Range(do func(key span.URI, value source.FileHandle)) { - m.impl.Range(func(key, value interface{}) { - do(key.(span.URI), value.(source.FileHandle)) - }) + m.impl.Range(do) } func (m filesMap) Set(key span.URI, value source.FileHandle) { @@ -86,19 +77,13 @@ func (m filesMap) overlays() []*Overlay { return overlays } -func packageIDLessInterface(x, y interface{}) bool { - return x.(PackageID) < y.(PackageID) -} - type knownDirsSet struct { - impl *persistent.Map + impl *persistent.Map[span.URI, struct{}] } func newKnownDirsSet() knownDirsSet { return knownDirsSet{ - impl: persistent.NewMap(func(a, b interface{}) bool { - return a.(span.URI) < b.(span.URI) - }), + impl: new(persistent.Map[span.URI, struct{}]), } } @@ -118,8 +103,8 @@ func (s knownDirsSet) Contains(key span.URI) bool { } func (s knownDirsSet) Range(do func(key span.URI)) { - s.impl.Range(func(key, value interface{}) { - do(key.(span.URI)) + s.impl.Range(func(key span.URI, value struct{}) { + do(key) }) } @@ -128,7 +113,7 @@ func (s knownDirsSet) SetAll(other knownDirsSet) { } func (s knownDirsSet) Insert(key span.URI) { - s.impl.Set(key, nil, nil) + s.impl.Set(key, struct{}{}, nil) } func (s knownDirsSet) Remove(key span.URI) { diff --git a/gopls/internal/lsp/cache/mod.go b/gopls/internal/lsp/cache/mod.go index db0ab0a64b8..8a452ab086d 100644 --- a/gopls/internal/lsp/cache/mod.go +++ b/gopls/internal/lsp/cache/mod.go @@ -52,7 +52,7 @@ func (s *snapshot) ParseMod(ctx context.Context, fh source.FileHandle) (*source. } // Await result. - v, err := s.awaitPromise(ctx, entry.(*memoize.Promise)) + v, err := s.awaitPromise(ctx, entry) if err != nil { return nil, err } @@ -130,7 +130,7 @@ func (s *snapshot) ParseWork(ctx context.Context, fh source.FileHandle) (*source } // Await result. - v, err := s.awaitPromise(ctx, entry.(*memoize.Promise)) + v, err := s.awaitPromise(ctx, entry) if err != nil { return nil, err } @@ -240,7 +240,7 @@ func (s *snapshot) ModWhy(ctx context.Context, fh source.FileHandle) (map[string } // Await result. - v, err := s.awaitPromise(ctx, entry.(*memoize.Promise)) + v, err := s.awaitPromise(ctx, entry) if err != nil { return nil, err } diff --git a/gopls/internal/lsp/cache/mod_tidy.go b/gopls/internal/lsp/cache/mod_tidy.go index 64e02d1c01e..b806edb7499 100644 --- a/gopls/internal/lsp/cache/mod_tidy.go +++ b/gopls/internal/lsp/cache/mod_tidy.go @@ -85,7 +85,7 @@ func (s *snapshot) ModTidy(ctx context.Context, pm *source.ParsedModule) (*sourc } // Await result. - v, err := s.awaitPromise(ctx, entry.(*memoize.Promise)) + v, err := s.awaitPromise(ctx, entry) if err != nil { return nil, err } diff --git a/gopls/internal/lsp/cache/mod_vuln.go b/gopls/internal/lsp/cache/mod_vuln.go index 942ca52525c..dcd58bfa94a 100644 --- a/gopls/internal/lsp/cache/mod_vuln.go +++ b/gopls/internal/lsp/cache/mod_vuln.go @@ -55,7 +55,7 @@ func (s *snapshot) ModVuln(ctx context.Context, modURI span.URI) (*govulncheck.R } // Await result. - v, err := s.awaitPromise(ctx, entry.(*memoize.Promise)) + v, err := s.awaitPromise(ctx, entry) if err != nil { return nil, err } diff --git a/gopls/internal/lsp/cache/session.go b/gopls/internal/lsp/cache/session.go index 6b75f10b36f..cd51e6d498a 100644 --- a/gopls/internal/lsp/cache/session.go +++ b/gopls/internal/lsp/cache/session.go @@ -20,6 +20,7 @@ import ( "golang.org/x/tools/internal/event" "golang.org/x/tools/internal/gocommand" "golang.org/x/tools/internal/imports" + "golang.org/x/tools/internal/memoize" "golang.org/x/tools/internal/persistent" "golang.org/x/tools/internal/xcontext" ) @@ -169,18 +170,18 @@ func (s *Session) createView(ctx context.Context, name string, folder span.URI, backgroundCtx: backgroundCtx, cancel: cancel, store: s.cache.store, - packages: persistent.NewMap(packageIDLessInterface), + packages: new(persistent.Map[PackageID, *packageHandle]), meta: new(metadataGraph), files: newFilesMap(), - activePackages: persistent.NewMap(packageIDLessInterface), - symbolizeHandles: persistent.NewMap(uriLessInterface), + activePackages: new(persistent.Map[PackageID, *Package]), + symbolizeHandles: new(persistent.Map[span.URI, *memoize.Promise]), workspacePackages: make(map[PackageID]PackagePath), unloadableFiles: make(map[span.URI]struct{}), - parseModHandles: persistent.NewMap(uriLessInterface), - parseWorkHandles: persistent.NewMap(uriLessInterface), - modTidyHandles: persistent.NewMap(uriLessInterface), - modVulnHandles: persistent.NewMap(uriLessInterface), - modWhyHandles: persistent.NewMap(uriLessInterface), + parseModHandles: new(persistent.Map[span.URI, *memoize.Promise]), + parseWorkHandles: new(persistent.Map[span.URI, *memoize.Promise]), + modTidyHandles: new(persistent.Map[span.URI, *memoize.Promise]), + modVulnHandles: new(persistent.Map[span.URI, *memoize.Promise]), + modWhyHandles: new(persistent.Map[span.URI, *memoize.Promise]), knownSubdirs: newKnownDirsSet(), workspaceModFiles: wsModFiles, workspaceModFilesErr: wsModFilesErr, diff --git a/gopls/internal/lsp/cache/snapshot.go b/gopls/internal/lsp/cache/snapshot.go index a1fe4753e2f..a914880a4e3 100644 --- a/gopls/internal/lsp/cache/snapshot.go +++ b/gopls/internal/lsp/cache/snapshot.go @@ -101,7 +101,7 @@ type snapshot struct { // symbolizeHandles maps each file URI to a handle for the future // result of computing the symbols declared in that file. - symbolizeHandles *persistent.Map // from span.URI to *memoize.Promise[symbolizeResult] + symbolizeHandles *persistent.Map[span.URI, *memoize.Promise] // *memoize.Promise[symbolizeResult] // packages maps a packageKey to a *packageHandle. // It may be invalidated when a file's content changes. @@ -110,13 +110,13 @@ type snapshot struct { // - packages.Get(id).meta == meta.metadata[id] for all ids // - if a package is in packages, then all of its dependencies should also // be in packages, unless there is a missing import - packages *persistent.Map // from packageID to *packageHandle + packages *persistent.Map[PackageID, *packageHandle] // activePackages maps a package ID to a memoized active package, or nil if // the package is known not to be open. // // IDs not contained in the map are not known to be open or not open. - activePackages *persistent.Map // from packageID to *Package + activePackages *persistent.Map[PackageID, *Package] // workspacePackages contains the workspace's packages, which are loaded // when the view is created. It contains no intermediate test variants. @@ -137,18 +137,18 @@ type snapshot struct { // parseModHandles keeps track of any parseModHandles for the snapshot. // The handles need not refer to only the view's go.mod file. - parseModHandles *persistent.Map // from span.URI to *memoize.Promise[parseModResult] + parseModHandles *persistent.Map[span.URI, *memoize.Promise] // *memoize.Promise[parseModResult] // parseWorkHandles keeps track of any parseWorkHandles for the snapshot. // The handles need not refer to only the view's go.work file. - parseWorkHandles *persistent.Map // from span.URI to *memoize.Promise[parseWorkResult] + parseWorkHandles *persistent.Map[span.URI, *memoize.Promise] // *memoize.Promise[parseWorkResult] // Preserve go.mod-related handles to avoid garbage-collecting the results // of various calls to the go command. The handles need not refer to only // the view's go.mod file. - modTidyHandles *persistent.Map // from span.URI to *memoize.Promise[modTidyResult] - modWhyHandles *persistent.Map // from span.URI to *memoize.Promise[modWhyResult] - modVulnHandles *persistent.Map // from span.URI to *memoize.Promise[modVulnResult] + modTidyHandles *persistent.Map[span.URI, *memoize.Promise] // *memoize.Promise[modTidyResult] + modWhyHandles *persistent.Map[span.URI, *memoize.Promise] // *memoize.Promise[modWhyResult] + modVulnHandles *persistent.Map[span.URI, *memoize.Promise] // *memoize.Promise[modVulnResult] // knownSubdirs is the set of subdirectory URIs in the workspace, // used to create glob patterns for file watching. @@ -871,7 +871,7 @@ func (s *snapshot) getActivePackage(id PackageID) *Package { defer s.mu.Unlock() if value, ok := s.activePackages.Get(id); ok { - return value.(*Package) // possibly nil, if we have already checked this id. + return value } return nil } @@ -895,7 +895,7 @@ func (s *snapshot) setActivePackage(id PackageID, pkg *Package) { func (s *snapshot) resetActivePackagesLocked() { s.activePackages.Destroy() - s.activePackages = persistent.NewMap(packageIDLessInterface) + s.activePackages = new(persistent.Map[PackageID, *Package]) } const fileExtensions = "go,mod,sum,work" @@ -2189,7 +2189,7 @@ func (s *snapshot) clone(ctx, bgCtx context.Context, changes map[span.URI]*fileC result.packages.Delete(id) } else { if entry, hit := result.packages.Get(id); hit { - ph := entry.(*packageHandle).clone(false) + ph := entry.clone(false) result.packages.Set(id, ph, nil) } } @@ -2291,12 +2291,11 @@ func (s *snapshot) clone(ctx, bgCtx context.Context, changes map[span.URI]*fileC // changed that happens not to be present in the map, but that's OK: the goal // of this function is to guarantee that IF the nearest mod file is present in // the map, it is invalidated. -func deleteMostRelevantModFile(m *persistent.Map, changed span.URI) { +func deleteMostRelevantModFile(m *persistent.Map[span.URI, *memoize.Promise], changed span.URI) { var mostRelevant span.URI changedFile := changed.Filename() - m.Range(func(key, value interface{}) { - modURI := key.(span.URI) + m.Range(func(modURI span.URI, _ *memoize.Promise) { if len(modURI) > len(mostRelevant) { if source.InDir(filepath.Dir(modURI.Filename()), changedFile) { mostRelevant = modURI diff --git a/gopls/internal/lsp/cache/symbols.go b/gopls/internal/lsp/cache/symbols.go index 466d9dc71a6..3ecd794303b 100644 --- a/gopls/internal/lsp/cache/symbols.go +++ b/gopls/internal/lsp/cache/symbols.go @@ -15,7 +15,6 @@ import ( "golang.org/x/tools/gopls/internal/lsp/protocol" "golang.org/x/tools/gopls/internal/lsp/source" "golang.org/x/tools/gopls/internal/span" - "golang.org/x/tools/internal/memoize" ) // symbolize returns the result of symbolizing the file identified by uri, using a cache. @@ -51,7 +50,7 @@ func (s *snapshot) symbolize(ctx context.Context, uri span.URI) ([]source.Symbol } // Await result. - v, err := s.awaitPromise(ctx, entry.(*memoize.Promise)) + v, err := s.awaitPromise(ctx, entry) if err != nil { return nil, err } diff --git a/internal/constraints/constraint.go b/internal/constraints/constraint.go new file mode 100644 index 00000000000..4e6ab61ea34 --- /dev/null +++ b/internal/constraints/constraint.go @@ -0,0 +1,52 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package constraints defines a set of useful constraints to be used +// with type parameters. +package constraints + +// Copied from x/exp/constraints. + +// Signed is a constraint that permits any signed integer type. +// If future releases of Go add new predeclared signed integer types, +// this constraint will be modified to include them. +type Signed interface { + ~int | ~int8 | ~int16 | ~int32 | ~int64 +} + +// Unsigned is a constraint that permits any unsigned integer type. +// If future releases of Go add new predeclared unsigned integer types, +// this constraint will be modified to include them. +type Unsigned interface { + ~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64 | ~uintptr +} + +// Integer is a constraint that permits any integer type. +// If future releases of Go add new predeclared integer types, +// this constraint will be modified to include them. +type Integer interface { + Signed | Unsigned +} + +// Float is a constraint that permits any floating-point type. +// If future releases of Go add new predeclared floating-point types, +// this constraint will be modified to include them. +type Float interface { + ~float32 | ~float64 +} + +// Complex is a constraint that permits any complex numeric type. +// If future releases of Go add new predeclared complex numeric types, +// this constraint will be modified to include them. +type Complex interface { + ~complex64 | ~complex128 +} + +// Ordered is a constraint that permits any ordered type: any type +// that supports the operators < <= >= >. +// If future releases of Go add new ordered types, +// this constraint will be modified to include them. +type Ordered interface { + Integer | Float | ~string +} diff --git a/internal/persistent/map.go b/internal/persistent/map.go index a9d878f4146..02389f89dc5 100644 --- a/internal/persistent/map.go +++ b/internal/persistent/map.go @@ -12,6 +12,8 @@ import ( "math/rand" "strings" "sync/atomic" + + "golang.org/x/tools/internal/constraints" ) // Implementation details: @@ -25,9 +27,7 @@ import ( // Each argument is followed by a delta change to its reference counter. // In case if no change is expected, the delta will be `-0`. -// Map is an associative mapping from keys to values, both represented as -// interface{}. Key comparison and iteration order is defined by a -// client-provided function that implements a strict weak order. +// Map is an associative mapping from keys to values. // // Maps can be Cloned in constant time. // Get, Store, and Delete operations are done on average in logarithmic time. @@ -38,16 +38,23 @@ import ( // // Internally the implementation is based on a randomized persistent treap: // https://en.wikipedia.org/wiki/Treap. -type Map struct { - less func(a, b interface{}) bool +// +// The zero value is ready to use. +type Map[K constraints.Ordered, V any] struct { + // Map is a generic wrapper around a non-generic implementation to avoid a + // significant increase in the size of the executable. root *mapNode } -func (m *Map) String() string { +func (*Map[K, V]) less(l, r any) bool { + return l.(K) < r.(K) +} + +func (m *Map[K, V]) String() string { var buf strings.Builder buf.WriteByte('{') var sep string - m.Range(func(k, v interface{}) { + m.Range(func(k K, v V) { fmt.Fprintf(&buf, "%s%v: %v", sep, k, v) sep = ", " }) @@ -56,7 +63,7 @@ func (m *Map) String() string { } type mapNode struct { - key interface{} + key any value *refValue weight uint64 refCount int32 @@ -65,11 +72,11 @@ type mapNode struct { type refValue struct { refCount int32 - value interface{} - release func(key, value interface{}) + value any + release func(key, value any) } -func newNodeWithRef(key, value interface{}, release func(key, value interface{})) *mapNode { +func newNodeWithRef[K constraints.Ordered, V any](key K, value V, release func(key, value any)) *mapNode { return &mapNode{ key: key, value: &refValue{ @@ -116,20 +123,10 @@ func (node *mapNode) decref() { } } -// NewMap returns a new map whose keys are ordered by the given comparison -// function (a strict weak order). It is the responsibility of the caller to -// Destroy it at later time. -func NewMap(less func(a, b interface{}) bool) *Map { - return &Map{ - less: less, - } -} - // Clone returns a copy of the given map. It is a responsibility of the caller // to Destroy it at later time. -func (pm *Map) Clone() *Map { - return &Map{ - less: pm.less, +func (pm *Map[K, V]) Clone() *Map[K, V] { + return &Map[K, V]{ root: pm.root.incref(), } } @@ -137,24 +134,26 @@ func (pm *Map) Clone() *Map { // Destroy destroys the map. // // After Destroy, the Map should not be used again. -func (pm *Map) Destroy() { +func (pm *Map[K, V]) Destroy() { // The implementation of these two functions is the same, // but their intent is different. pm.Clear() } // Clear removes all entries from the map. -func (pm *Map) Clear() { +func (pm *Map[K, V]) Clear() { pm.root.decref() pm.root = nil } // Range calls f sequentially in ascending key order for all entries in the map. -func (pm *Map) Range(f func(key, value interface{})) { - pm.root.forEach(f) +func (pm *Map[K, V]) Range(f func(key K, value V)) { + pm.root.forEach(func(k, v any) { + f(k.(K), v.(V)) + }) } -func (node *mapNode) forEach(f func(key, value interface{})) { +func (node *mapNode) forEach(f func(key, value any)) { if node == nil { return } @@ -163,26 +162,26 @@ func (node *mapNode) forEach(f func(key, value interface{})) { node.right.forEach(f) } -// Get returns the map value associated with the specified key, or nil if no entry -// is present. The ok result indicates whether an entry was found in the map. -func (pm *Map) Get(key interface{}) (interface{}, bool) { +// Get returns the map value associated with the specified key. +// The ok result indicates whether an entry was found in the map. +func (pm *Map[K, V]) Get(key K) (V, bool) { node := pm.root for node != nil { - if pm.less(key, node.key) { + if key < node.key.(K) { node = node.left - } else if pm.less(node.key, key) { + } else if node.key.(K) < key { node = node.right } else { - return node.value.value, true + return node.value.value.(V), true } } - return nil, false + var zero V + return zero, false } // SetAll updates the map with key/value pairs from the other map, overwriting existing keys. // It is equivalent to calling Set for each entry in the other map but is more efficient. -// Both maps must have the same comparison function, otherwise behavior is undefined. -func (pm *Map) SetAll(other *Map) { +func (pm *Map[K, V]) SetAll(other *Map[K, V]) { root := pm.root pm.root = union(root, other.root, pm.less, true) root.decref() @@ -191,7 +190,7 @@ func (pm *Map) SetAll(other *Map) { // Set updates the value associated with the specified key. // If release is non-nil, it will be called with entry's key and value once the // key is no longer contained in the map or any clone. -func (pm *Map) Set(key, value interface{}, release func(key, value interface{})) { +func (pm *Map[K, V]) Set(key K, value V, release func(key, value any)) { first := pm.root second := newNodeWithRef(key, value, release) pm.root = union(first, second, pm.less, true) @@ -205,7 +204,7 @@ func (pm *Map) Set(key, value interface{}, release func(key, value interface{})) // union(first:-0, second:-0) (result:+1) // Union borrows both subtrees without affecting their refcount and returns a // new reference that the caller is expected to call decref. -func union(first, second *mapNode, less func(a, b interface{}) bool, overwrite bool) *mapNode { +func union(first, second *mapNode, less func(any, any) bool, overwrite bool) *mapNode { if first == nil { return second.incref() } @@ -243,7 +242,7 @@ func union(first, second *mapNode, less func(a, b interface{}) bool, overwrite b // split(n:-0) (left:+1, mid:+1, right:+1) // Split borrows n without affecting its refcount, and returns three // new references that the caller is expected to call decref. -func split(n *mapNode, key interface{}, less func(a, b interface{}) bool, requireMid bool) (left, mid, right *mapNode) { +func split(n *mapNode, key any, less func(any, any) bool, requireMid bool) (left, mid, right *mapNode) { if n == nil { return nil, nil, nil } @@ -272,7 +271,7 @@ func split(n *mapNode, key interface{}, less func(a, b interface{}) bool, requir } // Delete deletes the value for a key. -func (pm *Map) Delete(key interface{}) { +func (pm *Map[K, V]) Delete(key K) { root := pm.root left, mid, right := split(root, key, pm.less, true) if mid == nil { diff --git a/internal/persistent/map_test.go b/internal/persistent/map_test.go index 9f89a1d300c..c73e5662d90 100644 --- a/internal/persistent/map_test.go +++ b/internal/persistent/map_test.go @@ -18,7 +18,7 @@ type mapEntry struct { } type validatedMap struct { - impl *Map + impl *Map[int, int] expected map[int]int // current key-value mapping. deleted map[mapEntry]int // maps deleted entries to their clock time of last deletion seen map[mapEntry]int // maps seen entries to their clock time of last insertion @@ -30,9 +30,7 @@ func TestSimpleMap(t *testing.T) { seenEntries := make(map[mapEntry]int) m1 := &validatedMap{ - impl: NewMap(func(a, b interface{}) bool { - return a.(int) < b.(int) - }), + impl: new(Map[int, int]), expected: make(map[int]int), deleted: deletedEntries, seen: seenEntries, @@ -123,9 +121,7 @@ func TestRandomMap(t *testing.T) { seenEntries := make(map[mapEntry]int) m := &validatedMap{ - impl: NewMap(func(a, b interface{}) bool { - return a.(int) < b.(int) - }), + impl: new(Map[int, int]), expected: make(map[int]int), deleted: deletedEntries, seen: seenEntries, @@ -165,9 +161,7 @@ func TestUpdate(t *testing.T) { seenEntries := make(map[mapEntry]int) m1 := &validatedMap{ - impl: NewMap(func(a, b interface{}) bool { - return a.(int) < b.(int) - }), + impl: new(Map[int, int]), expected: make(map[int]int), deleted: deletedEntries, seen: seenEntries, @@ -233,7 +227,7 @@ func dumpMap(t *testing.T, prefix string, n *mapNode) { func (vm *validatedMap) validate(t *testing.T) { t.Helper() - validateNode(t, vm.impl.root, vm.impl.less) + validateNode(t, vm.impl.root) // Note: this validation may not make sense if maps were constructed using // SetAll operations. If this proves to be problematic, remove the clock, @@ -246,23 +240,23 @@ func (vm *validatedMap) validate(t *testing.T) { } actualMap := make(map[int]int, len(vm.expected)) - vm.impl.Range(func(key, value interface{}) { - if other, ok := actualMap[key.(int)]; ok { + vm.impl.Range(func(key, value int) { + if other, ok := actualMap[key]; ok { t.Fatalf("key is present twice, key: %d, first value: %d, second value: %d", key, value, other) } - actualMap[key.(int)] = value.(int) + actualMap[key] = value }) assertSameMap(t, actualMap, vm.expected) } -func validateNode(t *testing.T, node *mapNode, less func(a, b interface{}) bool) { +func validateNode(t *testing.T, node *mapNode) { if node == nil { return } if node.left != nil { - if less(node.key, node.left.key) { + if node.key.(int) < node.left.key.(int) { t.Fatalf("left child has larger key: %v vs %v", node.left.key, node.key) } if node.left.weight > node.weight { @@ -271,7 +265,7 @@ func validateNode(t *testing.T, node *mapNode, less func(a, b interface{}) bool) } if node.right != nil { - if less(node.right.key, node.key) { + if node.right.key.(int) < node.key.(int) { t.Fatalf("right child has smaller key: %v vs %v", node.right.key, node.key) } if node.right.weight > node.weight { @@ -279,8 +273,8 @@ func validateNode(t *testing.T, node *mapNode, less func(a, b interface{}) bool) } } - validateNode(t, node.left, less) - validateNode(t, node.right, less) + validateNode(t, node.left) + validateNode(t, node.right) } func (vm *validatedMap) setAll(t *testing.T, other *validatedMap) { @@ -300,7 +294,7 @@ func (vm *validatedMap) set(t *testing.T, key, value int) { vm.clock++ vm.seen[entry] = vm.clock - vm.impl.Set(key, value, func(deletedKey, deletedValue interface{}) { + vm.impl.Set(key, value, func(deletedKey, deletedValue any) { if deletedKey != key || deletedValue != value { t.Fatalf("unexpected passed in deleted entry: %v/%v, expected: %v/%v", deletedKey, deletedValue, key, value) } @@ -346,7 +340,7 @@ func (vm *validatedMap) destroy() { vm.impl.Destroy() } -func assertSameMap(t *testing.T, map1, map2 interface{}) { +func assertSameMap(t *testing.T, map1, map2 any) { t.Helper() if !reflect.DeepEqual(map1, map2) { From 77c6ac601f791bef45ef25c870e2d07fdaef6250 Mon Sep 17 00:00:00 2001 From: "Hana (Hyang-Ah) Kim" Date: Fri, 1 Sep 2023 15:07:52 -0400 Subject: [PATCH 053/178] gopls/internal/telemetry: don't schedule the next upload The telemetry upload package implementation queries UploadConfig only once and reuses it throughout the process lifetime. So, periodic upload doesn't work. Assume users frequently restart gopls or eventually we will move the upload logic to go which is frequently invoked, and run upload only once at the start up. It's still better to clean up the cached upload config object after upload is complete. Fixes golang/go#62405 Change-Id: I39395cf876d9f0e570a71da5284420f570dcdc6b Reviewed-on: https://go-review.googlesource.com/c/tools/+/524819 TryBot-Result: Gopher Robot Run-TryBot: Hyang-Ah Hana Kim Reviewed-by: Peter Weinberger --- gopls/internal/telemetry/telemetry.go | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/gopls/internal/telemetry/telemetry.go b/gopls/internal/telemetry/telemetry.go index a20a40f5057..db75e1a7fbf 100644 --- a/gopls/internal/telemetry/telemetry.go +++ b/gopls/internal/telemetry/telemetry.go @@ -9,7 +9,6 @@ package telemetry import ( "fmt" - "time" "golang.org/x/telemetry/counter" "golang.org/x/telemetry/upload" @@ -19,14 +18,8 @@ import ( // Start starts telemetry instrumentation. func Start() { counter.Open() - go packAndUpload() -} - -func packAndUpload() { - start := time.Now() - upload.Run(nil) - elapsed := time.Since(start) - time.AfterFunc(24*time.Hour-elapsed, packAndUpload) + // upload only once at startup, hoping that users restart gopls often. + go upload.Run(nil) } // RecordClientInfo records gopls client info. From 5a9656936d83c03440e5b437421cb0fb92e62e31 Mon Sep 17 00:00:00 2001 From: Robert Findley Date: Fri, 1 Sep 2023 14:50:05 -0400 Subject: [PATCH 054/178] gopls/internal/lsp/cmd: don't use x/exp/slices A direct dependency on x/exp/slices was added in CL 497756. As a matter of policy, we don't depend on x/exp from gopls. Also, this left the module in an untidy state. I suspect that the slices package was added by a bad goimports operation, which is another issue entirely. In any case, the standard library slices package is not available in 1.18. Change-Id: I76b78313537ef9918317ecec25c4b12ed526c62f Reviewed-on: https://go-review.googlesource.com/c/tools/+/524817 gopls-CI: kokoro LUCI-TryBot-Result: Go LUCI Auto-Submit: Robert Findley Reviewed-by: Alan Donovan --- gopls/internal/lsp/cmd/cmd.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gopls/internal/lsp/cmd/cmd.go b/gopls/internal/lsp/cmd/cmd.go index 267456bc4be..340073d8a5a 100644 --- a/gopls/internal/lsp/cmd/cmd.go +++ b/gopls/internal/lsp/cmd/cmd.go @@ -21,7 +21,6 @@ import ( "text/tabwriter" "time" - "golang.org/x/exp/slices" "golang.org/x/tools/gopls/internal/lsp" "golang.org/x/tools/gopls/internal/lsp/browser" "golang.org/x/tools/gopls/internal/lsp/cache" @@ -547,13 +546,13 @@ func (c *cmdClient) ApplyEdit(ctx context.Context, p *protocol.ApplyWorkspaceEdi // files, honoring the preferred edit mode specified by cli.app.editMode. // (Used by rename and by ApplyEdit downcalls.) func (cli *cmdClient) applyWorkspaceEdit(edit *protocol.WorkspaceEdit) error { - var orderedURIs []span.URI + var orderedURIs []string edits := map[span.URI][]protocol.TextEdit{} for _, c := range edit.DocumentChanges { if c.TextDocumentEdit != nil { uri := fileURI(c.TextDocumentEdit.TextDocument.URI) edits[uri] = append(edits[uri], c.TextDocumentEdit.Edits...) - orderedURIs = append(orderedURIs, uri) + orderedURIs = append(orderedURIs, string(uri)) } if c.RenameFile != nil { return fmt.Errorf("client does not support file renaming (%s -> %s)", @@ -561,8 +560,9 @@ func (cli *cmdClient) applyWorkspaceEdit(edit *protocol.WorkspaceEdit) error { c.RenameFile.NewURI) } } - slices.Sort(orderedURIs) - for _, uri := range orderedURIs { + sort.Strings(orderedURIs) + for _, u := range orderedURIs { + uri := span.URIFromURI(u) f := cli.openFile(uri) if f.err != nil { return f.err From 2c6ba93996da7073f913c9918ac8dd99411f58b3 Mon Sep 17 00:00:00 2001 From: Robert Findley Date: Fri, 1 Sep 2023 14:52:18 -0400 Subject: [PATCH 055/178] gopls: tidy for 1.17+ Gopls no longer needs to be tidy with -compat=1.16. Change-Id: I5fd6d786f11d6ea346967663aa0af0f686c12c79 Reviewed-on: https://go-review.googlesource.com/c/tools/+/524818 LUCI-TryBot-Result: Go LUCI Reviewed-by: Alan Donovan Auto-Submit: Robert Findley gopls-CI: kokoro --- gopls/go.sum | 34 ---------------------------------- 1 file changed, 34 deletions(-) diff --git a/gopls/go.sum b/gopls/go.sum index 4085d484b6c..da346fed138 100644 --- a/gopls/go.sum +++ b/gopls/go.sum @@ -1,21 +1,12 @@ -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= -github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= -github.com/google/go-cmdtest v0.4.1-0.20220921163831-55ab3332a786/go.mod h1:apVn/GCasLZUVpAJ6oWAuyP7Ne7CEsQbTnc0plM3m+o= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/safehtml v0.0.2/go.mod h1:L4KWwDsUJdECRAEpZoBn3O64bQaywRscowZjJAzjHnU= github.com/google/safehtml v0.1.0 h1:EwLKo8qawTKfsi0orxcQAZzu07cICaBeFMegAU9eaT8= github.com/google/safehtml v0.1.0/go.mod h1:L4KWwDsUJdECRAEpZoBn3O64bQaywRscowZjJAzjHnU= @@ -25,59 +16,39 @@ github.com/jba/templatecheck v0.6.0 h1:SwM8C4hlK/YNLsdcXStfnHWE2HKkuTVwy5FKQHt5r github.com/jba/templatecheck v0.6.0/go.mod h1:/1k7EajoSErFI9GLHAsiIJEaNLt3ALKNw2TV7z2SYv4= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= -golang.org/x/exp/typeparams v0.0.0-20221208152030-732eee02a75a/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/exp/typeparams v0.0.0-20221212164502-fae10dda9338 h1:2O2DON6y3XMJiQRAS1UWU+54aec2uopH3x7MAiqGW6Y= golang.org/x/exp/typeparams v0.0.0-20221212164502-fae10dda9338/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= -golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211213223007-03aa0b5f6827/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/telemetry v0.0.0-20230822160736-17171dbf1d76 h1:Lv25uIMpljmSMN0+GCC+xgiC/4ikIdKMkQfw/EVq2Nk= golang.org/x/telemetry v0.0.0-20230822160736-17171dbf1d76/go.mod h1:kO7uNSGGmqCHII6C0TYfaLwSBIfcyhj53//nu0+Fy4A= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -86,9 +57,6 @@ golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/vuln v0.0.0-20230110180137-6ad3e3d07815 h1:A9kONVi4+AnuOr1dopsibH6hLi1Huy54cbeJxnq4vmU= golang.org/x/vuln v0.0.0-20230110180137-6ad3e3d07815/go.mod h1:XJiVExZgoZfrrxoTeVsFYrSSk1snhfpOEC95JL+A4T0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= @@ -99,12 +67,10 @@ gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.2.2/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY= honnef.co/go/tools v0.4.2 h1:6qXr+R5w+ktL5UkwEbPp+fEvfyoMPche6GkOpGHZcLc= honnef.co/go/tools v0.4.2/go.mod h1:36ZgoUOrqOk1GxwHhyryEkq8FQWkUO2xGuSMhUCcdvA= mvdan.cc/gofumpt v0.4.0 h1:JVf4NN1mIpHogBj7ABpgOyZc65/UUOkKQFkoURsz4MM= mvdan.cc/gofumpt v0.4.0/go.mod h1:PljLOHDeZqgS8opHRKLzp2It2VBuSdteAgqUfzMTxlQ= mvdan.cc/unparam v0.0.0-20211214103731-d0ef000c54e5 h1:Jh3LAeMt1eGpxomyu3jVkmVZWW2MxZ1qIIV2TZ/nRio= -mvdan.cc/unparam v0.0.0-20211214103731-d0ef000c54e5/go.mod h1:b8RRCBm0eeiWR8cfN88xeq2G5SG3VKGO+5UPWi5FSOY= mvdan.cc/xurls/v2 v2.4.0 h1:tzxjVAj+wSBmDcF6zBB7/myTy3gX9xvi8Tyr28AuQgc= mvdan.cc/xurls/v2 v2.4.0/go.mod h1:+GEjq9uNjqs8LQfM9nVnM8rff0OQ5Iash5rzX+N1CSg= From 44f7796438e567ee014586c24a7afbab9a1ebed1 Mon Sep 17 00:00:00 2001 From: Robert Findley Date: Fri, 1 Sep 2023 14:04:38 -0400 Subject: [PATCH 056/178] gopls: add and enable the slog analyzer This analyzer is included in go/vet, and so gopls should have it as well. Change-Id: Ib5cbee44a1f38c4aa45d75bcaa7a345d88099d8e Reviewed-on: https://go-review.googlesource.com/c/tools/+/524764 Reviewed-by: Alan Donovan Auto-Submit: Robert Findley gopls-CI: kokoro LUCI-TryBot-Result: Go LUCI --- gopls/doc/analyzers.md | 18 +++++++++++++++ gopls/internal/lsp/source/api_json.go | 11 +++++++++ gopls/internal/lsp/source/options.go | 2 ++ .../marker/testdata/diagnostics/analyzers.txt | 23 +++++++++++++++++-- 4 files changed, 52 insertions(+), 2 deletions(-) diff --git a/gopls/doc/analyzers.md b/gopls/doc/analyzers.md index 9a592d4b890..2ff9434d0b6 100644 --- a/gopls/doc/analyzers.md +++ b/gopls/doc/analyzers.md @@ -494,6 +494,24 @@ This is one of the simplifications that "gofmt -s" applies. **Enabled by default.** +## **slog** + +check for invalid structured logging calls + +The slog checker looks for calls to functions from the log/slog +package that take alternating key-value pairs. It reports calls +where an argument in a key position is neither a string nor a +slog.Attr, and where a final key is missing its value. +For example,it would report + + slog.Warn("message", 11, "k") // slog.Warn arg "11" should be a string or a slog.Attr + +and + + slog.Info("message", "k1", v1, "k2") // call to slog.Info missing a final value + +**Enabled by default.** + ## **sortslice** check the argument type of sort.Slice diff --git a/gopls/internal/lsp/source/api_json.go b/gopls/internal/lsp/source/api_json.go index 7635edba2e6..60425db2c5c 100644 --- a/gopls/internal/lsp/source/api_json.go +++ b/gopls/internal/lsp/source/api_json.go @@ -358,6 +358,11 @@ var GeneratedAPIJSON = &APIJSON{ Doc: "check for slice simplifications\n\nA slice expression of the form:\n\ts[a:len(s)]\nwill be simplified to:\n\ts[a:]\n\nThis is one of the simplifications that \"gofmt -s\" applies.", Default: "true", }, + { + Name: "\"slog\"", + Doc: "check for invalid structured logging calls\n\nThe slog checker looks for calls to functions from the log/slog\npackage that take alternating key-value pairs. It reports calls\nwhere an argument in a key position is neither a string nor a\nslog.Attr, and where a final key is missing its value.\nFor example,it would report\n\n\tslog.Warn(\"message\", 11, \"k\") // slog.Warn arg \"11\" should be a string or a slog.Attr\n\nand\n\n\tslog.Info(\"message\", \"k1\", v1, \"k2\") // call to slog.Info missing a final value", + Default: "true", + }, { Name: "\"sortslice\"", Doc: "check the argument type of sort.Slice\n\nsort.Slice requires an argument of a slice type. Check that\nthe interface{} value passed to sort.Slice is actually a slice.", @@ -1070,6 +1075,12 @@ var GeneratedAPIJSON = &APIJSON{ Doc: "check for slice simplifications\n\nA slice expression of the form:\n\ts[a:len(s)]\nwill be simplified to:\n\ts[a:]\n\nThis is one of the simplifications that \"gofmt -s\" applies.", Default: true, }, + { + Name: "slog", + Doc: "check for invalid structured logging calls\n\nThe slog checker looks for calls to functions from the log/slog\npackage that take alternating key-value pairs. It reports calls\nwhere an argument in a key position is neither a string nor a\nslog.Attr, and where a final key is missing its value.\nFor example,it would report\n\n\tslog.Warn(\"message\", 11, \"k\") // slog.Warn arg \"11\" should be a string or a slog.Attr\n\nand\n\n\tslog.Info(\"message\", \"k1\", v1, \"k2\") // call to slog.Info missing a final value", + URL: "https://pkg.go.dev/golang.org/x/tools/go/analysis/passes/slog", + Default: true, + }, { Name: "sortslice", Doc: "check the argument type of sort.Slice\n\nsort.Slice requires an argument of a slice type. Check that\nthe interface{} value passed to sort.Slice is actually a slice.", diff --git a/gopls/internal/lsp/source/options.go b/gopls/internal/lsp/source/options.go index 67818fad34a..2b91f834d6a 100644 --- a/gopls/internal/lsp/source/options.go +++ b/gopls/internal/lsp/source/options.go @@ -39,6 +39,7 @@ import ( "golang.org/x/tools/go/analysis/passes/printf" "golang.org/x/tools/go/analysis/passes/shadow" "golang.org/x/tools/go/analysis/passes/shift" + "golang.org/x/tools/go/analysis/passes/slog" "golang.org/x/tools/go/analysis/passes/sortslice" "golang.org/x/tools/go/analysis/passes/stdmethods" "golang.org/x/tools/go/analysis/passes/stringintconv" @@ -1549,6 +1550,7 @@ func defaultAnalyzers() map[string]*Analyzer { nilfunc.Analyzer.Name: {Analyzer: nilfunc.Analyzer, Enabled: true}, printf.Analyzer.Name: {Analyzer: printf.Analyzer, Enabled: true}, shift.Analyzer.Name: {Analyzer: shift.Analyzer, Enabled: true}, + slog.Analyzer.Name: {Analyzer: slog.Analyzer, Enabled: true}, stdmethods.Analyzer.Name: {Analyzer: stdmethods.Analyzer, Enabled: true}, stringintconv.Analyzer.Name: {Analyzer: stringintconv.Analyzer, Enabled: true}, structtag.Analyzer.Name: {Analyzer: structtag.Analyzer, Enabled: true}, diff --git a/gopls/internal/regtest/marker/testdata/diagnostics/analyzers.txt b/gopls/internal/regtest/marker/testdata/diagnostics/analyzers.txt index 6e7e4650578..e98674b94f4 100644 --- a/gopls/internal/regtest/marker/testdata/diagnostics/analyzers.txt +++ b/gopls/internal/regtest/marker/testdata/diagnostics/analyzers.txt @@ -1,24 +1,32 @@ Test of warning diagnostics from various analyzers: -tests, copylocks, printf, and timeformat. +copylocks, printf, slog, tests, and timeformat. -- go.mod -- module example.com go 1.12 +-- flags -- +-min_go=go1.21 + -- bad_test.go -- package analyzer import ( "fmt" + "log/slog" "sync" "testing" "time" ) -func Testbad(t *testing.T) { //@diag("", re"Testbad has malformed name: first letter after 'Test' must not be lowercase") +// copylocks +func _() { var x sync.Mutex _ = x //@diag("x", re"assignment copies lock value to _: sync.Mutex") +} +// printf +func _() { printfWrapper("%s") //@diag(re`printfWrapper\(.*\)`, re"example.com.printfWrapper format %s reads arg #1, but call has 0 args") } @@ -26,7 +34,18 @@ func printfWrapper(format string, args ...interface{}) { fmt.Printf(format, args...) } +// slog +func _() { + slog.Info("msg", 1) //@diag("1", re`slog.Info arg "1" should be a string or a slog.Attr`) +} + +// tests +func Testbad(t *testing.T) { //@diag("", re"Testbad has malformed name: first letter after 'Test' must not be lowercase") +} + +// timeformat func _() { now := time.Now() fmt.Println(now.Format("2006-02-01")) //@diag("2006-02-01", re"2006-02-01 should be 2006-01-02") } + From 38b898b246a939ee85545de1da16fd710a72a9c4 Mon Sep 17 00:00:00 2001 From: Robert Findley Date: Fri, 1 Sep 2023 12:54:24 -0400 Subject: [PATCH 057/178] internal/persistent: add Set Add a simple Set wrapper around persistent.Map, with a new test. Follow-up CLs will replace ad-hoc sets in gopls with a persistent.Set. Change-Id: Idd5fc5389719d3f59d658d8d9cb8fc0206e35797 Reviewed-on: https://go-review.googlesource.com/c/tools/+/524761 LUCI-TryBot-Result: Go LUCI Reviewed-by: Alan Donovan Auto-Submit: Robert Findley gopls-CI: kokoro --- internal/persistent/set.go | 78 +++++++++++++++++++ internal/persistent/set_test.go | 132 ++++++++++++++++++++++++++++++++ 2 files changed, 210 insertions(+) create mode 100644 internal/persistent/set.go create mode 100644 internal/persistent/set_test.go diff --git a/internal/persistent/set.go b/internal/persistent/set.go new file mode 100644 index 00000000000..348de5a71d2 --- /dev/null +++ b/internal/persistent/set.go @@ -0,0 +1,78 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package persistent + +import "golang.org/x/tools/internal/constraints" + +// Set is a collection of elements of type K. +// +// It uses immutable data structures internally, so that sets can be cloned in +// constant time. +// +// The zero value is a valid empty set. +type Set[K constraints.Ordered] struct { + impl *Map[K, struct{}] +} + +// Clone creates a copy of the receiver. +func (s *Set[K]) Clone() *Set[K] { + clone := new(Set[K]) + if s.impl != nil { + clone.impl = s.impl.Clone() + } + return clone +} + +// Destroy destroys the set. +// +// After Destroy, the Set should not be used again. +func (s *Set[K]) Destroy() { + if s.impl != nil { + s.impl.Destroy() + } +} + +// Contains reports whether s contains the given key. +func (s *Set[K]) Contains(key K) bool { + if s.impl == nil { + return false + } + _, ok := s.impl.Get(key) + return ok +} + +// Range calls f sequentially in ascending key order for all entries in the set. +func (s *Set[K]) Range(f func(key K)) { + if s.impl != nil { + s.impl.Range(func(key K, _ struct{}) { + f(key) + }) + } +} + +// AddAll adds all elements from other to the receiver set. +func (s *Set[K]) AddAll(other *Set[K]) { + if other.impl != nil { + if s.impl == nil { + s.impl = new(Map[K, struct{}]) + } + s.impl.SetAll(other.impl) + } +} + +// Add adds an element to the set. +func (s *Set[K]) Add(key K) { + if s.impl == nil { + s.impl = new(Map[K, struct{}]) + } + s.impl.Set(key, struct{}{}, nil) +} + +// Remove removes an element from the set. +func (s *Set[K]) Remove(key K) { + if s.impl != nil { + s.impl.Delete(key) + } +} diff --git a/internal/persistent/set_test.go b/internal/persistent/set_test.go new file mode 100644 index 00000000000..59025140bce --- /dev/null +++ b/internal/persistent/set_test.go @@ -0,0 +1,132 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package persistent_test + +import ( + "fmt" + "strings" + "testing" + + "golang.org/x/tools/internal/constraints" + "golang.org/x/tools/internal/persistent" +) + +func TestSet(t *testing.T) { + const ( + add = iota + remove + ) + type op struct { + op int + v int + } + + tests := []struct { + label string + ops []op + want []int + }{ + {"empty", nil, nil}, + {"singleton", []op{{add, 1}}, []int{1}}, + {"add and remove", []op{ + {add, 1}, + {remove, 1}, + }, nil}, + {"interleaved and remove", []op{ + {add, 1}, + {add, 2}, + {remove, 1}, + {add, 3}, + }, []int{2, 3}}, + } + + for _, test := range tests { + t.Run(test.label, func(t *testing.T) { + var s persistent.Set[int] + for _, op := range test.ops { + switch op.op { + case add: + s.Add(op.v) + case remove: + s.Remove(op.v) + } + } + + if d := diff(&s, test.want); d != "" { + t.Errorf("unexpected diff:\n%s", d) + } + }) + } +} + +func TestSet_Clone(t *testing.T) { + s1 := new(persistent.Set[int]) + s1.Add(1) + s1.Add(2) + s2 := s1.Clone() + s1.Add(3) + s2.Add(4) + if d := diff(s1, []int{1, 2, 3}); d != "" { + t.Errorf("s1: unexpected diff:\n%s", d) + } + if d := diff(s2, []int{1, 2, 4}); d != "" { + t.Errorf("s2: unexpected diff:\n%s", d) + } +} + +func TestSet_AddAll(t *testing.T) { + s1 := new(persistent.Set[int]) + s1.Add(1) + s1.Add(2) + s2 := new(persistent.Set[int]) + s2.Add(2) + s2.Add(3) + s2.Add(4) + s3 := new(persistent.Set[int]) + + s := new(persistent.Set[int]) + s.AddAll(s1) + s.AddAll(s2) + s.AddAll(s3) + + if d := diff(s1, []int{1, 2}); d != "" { + t.Errorf("s1: unexpected diff:\n%s", d) + } + if d := diff(s2, []int{2, 3, 4}); d != "" { + t.Errorf("s2: unexpected diff:\n%s", d) + } + if d := diff(s3, nil); d != "" { + t.Errorf("s3: unexpected diff:\n%s", d) + } + if d := diff(s, []int{1, 2, 3, 4}); d != "" { + t.Errorf("s: unexpected diff:\n%s", d) + } +} + +func diff[K constraints.Ordered](got *persistent.Set[K], want []K) string { + wantSet := make(map[K]struct{}) + for _, w := range want { + wantSet[w] = struct{}{} + } + var diff []string + got.Range(func(key K) { + if _, ok := wantSet[key]; !ok { + diff = append(diff, fmt.Sprintf("+%v", key)) + } + }) + for key := range wantSet { + if !got.Contains(key) { + diff = append(diff, fmt.Sprintf("-%v", key)) + } + } + if len(diff) > 0 { + d := new(strings.Builder) + for _, l := range diff { + fmt.Fprintln(d, l) + } + return d.String() + } + return "" +} From 21090a2aa8d3719d5a5d4264e41696529522f0bd Mon Sep 17 00:00:00 2001 From: Robert Findley Date: Fri, 1 Sep 2023 13:00:44 -0400 Subject: [PATCH 058/178] gopls/internal/lsp/cache: use persistent.Set in a couple places Use the new persistent.Set type to track known dirs and unloadable files. Change-Id: I3e0d4bdc846f4c37a0046a01bf67d83bc06b9598 Reviewed-on: https://go-review.googlesource.com/c/tools/+/524762 LUCI-TryBot-Result: Go LUCI Reviewed-by: Alan Donovan Auto-Submit: Robert Findley --- gopls/internal/lsp/cache/maps.go | 43 ---------------------------- gopls/internal/lsp/cache/session.go | 12 ++++---- gopls/internal/lsp/cache/snapshot.go | 30 +++++++++---------- 3 files changed, 19 insertions(+), 66 deletions(-) diff --git a/gopls/internal/lsp/cache/maps.go b/gopls/internal/lsp/cache/maps.go index 3fa866cb840..edb72d5c123 100644 --- a/gopls/internal/lsp/cache/maps.go +++ b/gopls/internal/lsp/cache/maps.go @@ -76,46 +76,3 @@ func (m filesMap) overlays() []*Overlay { } return overlays } - -type knownDirsSet struct { - impl *persistent.Map[span.URI, struct{}] -} - -func newKnownDirsSet() knownDirsSet { - return knownDirsSet{ - impl: new(persistent.Map[span.URI, struct{}]), - } -} - -func (s knownDirsSet) Clone() knownDirsSet { - return knownDirsSet{ - impl: s.impl.Clone(), - } -} - -func (s knownDirsSet) Destroy() { - s.impl.Destroy() -} - -func (s knownDirsSet) Contains(key span.URI) bool { - _, ok := s.impl.Get(key) - return ok -} - -func (s knownDirsSet) Range(do func(key span.URI)) { - s.impl.Range(func(key span.URI, value struct{}) { - do(key) - }) -} - -func (s knownDirsSet) SetAll(other knownDirsSet) { - s.impl.SetAll(other.impl) -} - -func (s knownDirsSet) Insert(key span.URI) { - s.impl.Set(key, struct{}{}, nil) -} - -func (s knownDirsSet) Remove(key span.URI) { - s.impl.Delete(key) -} diff --git a/gopls/internal/lsp/cache/session.go b/gopls/internal/lsp/cache/session.go index cd51e6d498a..1e463fa3f4f 100644 --- a/gopls/internal/lsp/cache/session.go +++ b/gopls/internal/lsp/cache/session.go @@ -176,13 +176,13 @@ func (s *Session) createView(ctx context.Context, name string, folder span.URI, activePackages: new(persistent.Map[PackageID, *Package]), symbolizeHandles: new(persistent.Map[span.URI, *memoize.Promise]), workspacePackages: make(map[PackageID]PackagePath), - unloadableFiles: make(map[span.URI]struct{}), + unloadableFiles: new(persistent.Set[span.URI]), parseModHandles: new(persistent.Map[span.URI, *memoize.Promise]), parseWorkHandles: new(persistent.Map[span.URI, *memoize.Promise]), modTidyHandles: new(persistent.Map[span.URI, *memoize.Promise]), modVulnHandles: new(persistent.Map[span.URI, *memoize.Promise]), modWhyHandles: new(persistent.Map[span.URI, *memoize.Promise]), - knownSubdirs: newKnownDirsSet(), + knownSubdirs: new(persistent.Set[span.URI]), workspaceModFiles: wsModFiles, workspaceModFilesErr: wsModFilesErr, pkgIndex: typerefs.NewPackageIndex(), @@ -619,15 +619,15 @@ func (s *Session) ExpandModificationsToDirectories(ctx context.Context, changes // knownDirectories returns all of the directories known to the given // snapshots, including workspace directories and their subdirectories. // It is responsibility of the caller to destroy the returned set. -func knownDirectories(ctx context.Context, snapshots []*snapshot) knownDirsSet { - result := newKnownDirsSet() +func knownDirectories(ctx context.Context, snapshots []*snapshot) *persistent.Set[span.URI] { + result := new(persistent.Set[span.URI]) for _, snapshot := range snapshots { dirs := snapshot.dirs(ctx) for _, dir := range dirs { - result.Insert(dir) + result.Add(dir) } knownSubdirs := snapshot.getKnownSubdirs(dirs) - result.SetAll(knownSubdirs) + result.AddAll(knownSubdirs) knownSubdirs.Destroy() } return result diff --git a/gopls/internal/lsp/cache/snapshot.go b/gopls/internal/lsp/cache/snapshot.go index a914880a4e3..94eceed869b 100644 --- a/gopls/internal/lsp/cache/snapshot.go +++ b/gopls/internal/lsp/cache/snapshot.go @@ -130,7 +130,7 @@ type snapshot struct { shouldLoad map[PackageID][]PackagePath // unloadableFiles keeps track of files that we've failed to load. - unloadableFiles map[span.URI]struct{} + unloadableFiles *persistent.Set[span.URI] // TODO(rfindley): rename the handles below to "promises". A promise is // different from a handle (we mutate the package handle.) @@ -152,7 +152,7 @@ type snapshot struct { // knownSubdirs is the set of subdirectory URIs in the workspace, // used to create glob patterns for file watching. - knownSubdirs knownDirsSet + knownSubdirs *persistent.Set[span.URI] knownSubdirsCache map[string]struct{} // memo of knownSubdirs as a set of filenames // unprocessedSubdirChanges are any changes that might affect the set of // subdirectories in the workspace. They are not reflected to knownSubdirs @@ -269,6 +269,7 @@ func (s *snapshot) destroy(destroyedBy string) { s.modTidyHandles.Destroy() s.modVulnHandles.Destroy() s.modWhyHandles.Destroy() + s.unloadableFiles.Destroy() } func (s *snapshot) SequenceID() uint64 { @@ -750,7 +751,7 @@ func (s *snapshot) MetadataForFile(ctx context.Context, uri span.URI) ([]*source } // Check if uri is known to be unloadable. - _, unloadable := s.unloadableFiles[uri] + unloadable := s.unloadableFiles.Contains(uri) s.mu.Unlock() @@ -803,7 +804,7 @@ func (s *snapshot) MetadataForFile(ctx context.Context, uri span.URI) ([]*source // so if we get here and still have // no IDs, uri is unloadable. if !unloadable && len(ids) == 0 { - s.unloadableFiles[uri] = struct{}{} + s.unloadableFiles.Add(uri) } // Sort packages "narrowest" to "widest" (in practice: @@ -1017,14 +1018,14 @@ func (s *snapshot) collectAllKnownSubdirs(ctx context.Context) { defer s.mu.Unlock() s.knownSubdirs.Destroy() - s.knownSubdirs = newKnownDirsSet() + s.knownSubdirs = new(persistent.Set[span.URI]) s.knownSubdirsCache = nil s.files.Range(func(uri span.URI, fh source.FileHandle) { s.addKnownSubdirLocked(uri, dirs) }) } -func (s *snapshot) getKnownSubdirs(wsDirs []span.URI) knownDirsSet { +func (s *snapshot) getKnownSubdirs(wsDirs []span.URI) *persistent.Set[span.URI] { s.mu.Lock() defer s.mu.Unlock() @@ -1075,7 +1076,7 @@ func (s *snapshot) addKnownSubdirLocked(uri span.URI, dirs []span.URI) { if s.knownSubdirs.Contains(uri) { break } - s.knownSubdirs.Insert(uri) + s.knownSubdirs.Add(uri) dir = filepath.Dir(dir) s.knownSubdirsCache = nil } @@ -1592,7 +1593,7 @@ func (s *snapshot) reloadOrphanedOpenFiles(ctx context.Context) error { s.mu.Lock() loadable := files[:0] for _, file := range files { - if _, unloadable := s.unloadableFiles[file.URI()]; !unloadable { + if !s.unloadableFiles.Contains(file.URI()) { loadable = append(loadable, file) } } @@ -1655,7 +1656,7 @@ func (s *snapshot) reloadOrphanedOpenFiles(ctx context.Context) error { // metadata graph that resulted from loading. uri := file.URI() if len(s.meta.ids[uri]) == 0 { - s.unloadableFiles[uri] = struct{}{} + s.unloadableFiles.Add(uri) } } @@ -1969,7 +1970,7 @@ func (s *snapshot) clone(ctx, bgCtx context.Context, changes map[span.URI]*fileC files: s.files.Clone(), symbolizeHandles: s.symbolizeHandles.Clone(), workspacePackages: make(map[PackageID]PackagePath, len(s.workspacePackages)), - unloadableFiles: make(map[span.URI]struct{}, len(s.unloadableFiles)), + unloadableFiles: s.unloadableFiles.Clone(), // see the TODO for unloadableFiles below parseModHandles: s.parseModHandles.Clone(), parseWorkHandles: s.parseWorkHandles.Clone(), modTidyHandles: s.modTidyHandles.Clone(), @@ -1993,16 +1994,11 @@ func (s *snapshot) clone(ctx, bgCtx context.Context, changes map[span.URI]*fileC // incref/decref operation that might destroy it prematurely.) release := result.Acquire() - // Copy the set of unloadable files. - // - // TODO(rfindley): this looks wrong. Shouldn't we clear unloadableFiles on + // TODO(rfindley): this looks wrong. Should we clear unloadableFiles on // changes to environment or workspace layout, or more generally on any // metadata change? // // Maybe not, as major configuration changes cause a new view. - for k, v := range s.unloadableFiles { - result.unloadableFiles[k] = v - } // Add all of the known subdirectories, but don't update them for the // changed files. We need to rebuild the workspace module to know the @@ -2119,7 +2115,7 @@ func (s *snapshot) clone(ctx, bgCtx context.Context, changes map[span.URI]*fileC // TODO(rfindley): this also looks wrong, as typing in an unloadable file // will result in repeated reloads. We should only delete if metadata // changed. - delete(result.unloadableFiles, uri) + result.unloadableFiles.Remove(uri) } // Deleting an import can cause list errors due to import cycles to be From a807ccf39a240aea24e1fd02ff1ada1a94e87fba Mon Sep 17 00:00:00 2001 From: Gopher Robot Date: Tue, 5 Sep 2023 16:33:55 +0000 Subject: [PATCH 059/178] go.mod: update golang.org/x dependencies Update golang.org/x dependencies to their latest tagged versions. Change-Id: Ibc57cfb67f76bdcacf21e01be5a97b64c2ec4733 Reviewed-on: https://go-review.googlesource.com/c/tools/+/525636 Run-TryBot: Gopher Robot Reviewed-by: Dmitri Shuralyov TryBot-Result: Gopher Robot Reviewed-by: Dmitri Shuralyov Auto-Submit: Gopher Robot Reviewed-by: Heschi Kreinick --- go.mod | 4 ++-- go.sum | 14 +++++++------- gopls/go.mod | 4 ++-- gopls/go.sum | 48 +++++++++++++++++++++++++++++++++++++++++------- 4 files changed, 52 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 26c7cdc9aec..24cca8bec0f 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,8 @@ go 1.18 // tagx:compat 1.16 require ( github.com/yuin/goldmark v1.4.13 golang.org/x/mod v0.12.0 - golang.org/x/net v0.14.0 - golang.org/x/sys v0.11.0 + golang.org/x/net v0.15.0 + golang.org/x/sys v0.12.0 ) require golang.org/x/sync v0.3.0 diff --git a/go.sum b/go.sum index d2f1de7aba3..2c884cc6e39 100644 --- a/go.sum +++ b/go.sum @@ -2,7 +2,7 @@ github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= @@ -12,8 +12,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= -golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -26,19 +26,19 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= diff --git a/gopls/go.mod b/gopls/go.mod index f1c3aa130ae..3198e0f81bb 100644 --- a/gopls/go.mod +++ b/gopls/go.mod @@ -9,9 +9,9 @@ require ( github.com/sergi/go-diff v1.1.0 golang.org/x/mod v0.12.0 golang.org/x/sync v0.3.0 - golang.org/x/sys v0.11.0 + golang.org/x/sys v0.12.0 golang.org/x/telemetry v0.0.0-20230822160736-17171dbf1d76 - golang.org/x/text v0.12.0 + golang.org/x/text v0.13.0 golang.org/x/tools v0.6.0 golang.org/x/vuln v0.0.0-20230110180137-6ad3e3d07815 gopkg.in/yaml.v3 v3.0.1 diff --git a/gopls/go.sum b/gopls/go.sum index da346fed138..78b66349fe3 100644 --- a/gopls/go.sum +++ b/gopls/go.sum @@ -1,12 +1,21 @@ +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= +github.com/google/go-cmdtest v0.4.1-0.20220921163831-55ab3332a786/go.mod h1:apVn/GCasLZUVpAJ6oWAuyP7Ne7CEsQbTnc0plM3m+o= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/safehtml v0.0.2/go.mod h1:L4KWwDsUJdECRAEpZoBn3O64bQaywRscowZjJAzjHnU= github.com/google/safehtml v0.1.0 h1:EwLKo8qawTKfsi0orxcQAZzu07cICaBeFMegAU9eaT8= github.com/google/safehtml v0.1.0/go.mod h1:L4KWwDsUJdECRAEpZoBn3O64bQaywRscowZjJAzjHnU= @@ -16,47 +25,70 @@ github.com/jba/templatecheck v0.6.0 h1:SwM8C4hlK/YNLsdcXStfnHWE2HKkuTVwy5FKQHt5r github.com/jba/templatecheck v0.6.0/go.mod h1:/1k7EajoSErFI9GLHAsiIJEaNLt3ALKNw2TV7z2SYv4= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= +golang.org/x/exp/typeparams v0.0.0-20221208152030-732eee02a75a/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/exp/typeparams v0.0.0-20221212164502-fae10dda9338 h1:2O2DON6y3XMJiQRAS1UWU+54aec2uopH3x7MAiqGW6Y= golang.org/x/exp/typeparams v0.0.0-20221212164502-fae10dda9338/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211213223007-03aa0b5f6827/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/telemetry v0.0.0-20230822160736-17171dbf1d76 h1:Lv25uIMpljmSMN0+GCC+xgiC/4ikIdKMkQfw/EVq2Nk= golang.org/x/telemetry v0.0.0-20230822160736-17171dbf1d76/go.mod h1:kO7uNSGGmqCHII6C0TYfaLwSBIfcyhj53//nu0+Fy4A= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= -golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/vuln v0.0.0-20230110180137-6ad3e3d07815 h1:A9kONVi4+AnuOr1dopsibH6hLi1Huy54cbeJxnq4vmU= golang.org/x/vuln v0.0.0-20230110180137-6ad3e3d07815/go.mod h1:XJiVExZgoZfrrxoTeVsFYrSSk1snhfpOEC95JL+A4T0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= @@ -67,10 +99,12 @@ gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.2.2/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY= honnef.co/go/tools v0.4.2 h1:6qXr+R5w+ktL5UkwEbPp+fEvfyoMPche6GkOpGHZcLc= honnef.co/go/tools v0.4.2/go.mod h1:36ZgoUOrqOk1GxwHhyryEkq8FQWkUO2xGuSMhUCcdvA= mvdan.cc/gofumpt v0.4.0 h1:JVf4NN1mIpHogBj7ABpgOyZc65/UUOkKQFkoURsz4MM= mvdan.cc/gofumpt v0.4.0/go.mod h1:PljLOHDeZqgS8opHRKLzp2It2VBuSdteAgqUfzMTxlQ= mvdan.cc/unparam v0.0.0-20211214103731-d0ef000c54e5 h1:Jh3LAeMt1eGpxomyu3jVkmVZWW2MxZ1qIIV2TZ/nRio= +mvdan.cc/unparam v0.0.0-20211214103731-d0ef000c54e5/go.mod h1:b8RRCBm0eeiWR8cfN88xeq2G5SG3VKGO+5UPWi5FSOY= mvdan.cc/xurls/v2 v2.4.0 h1:tzxjVAj+wSBmDcF6zBB7/myTy3gX9xvi8Tyr28AuQgc= mvdan.cc/xurls/v2 v2.4.0/go.mod h1:+GEjq9uNjqs8LQfM9nVnM8rff0OQ5Iash5rzX+N1CSg= From b5e55d198461206bca9558e65cdd518f8e4f2735 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Fri, 1 Sep 2023 17:18:31 -0400 Subject: [PATCH 060/178] go/analysis/analysistest: give better hint in SuggestedFix assertion Also, better documentation on the underlying cause. Change-Id: I0ee93b6e9f2ada52d9a32a322b77fda783ddf076 Reviewed-on: https://go-review.googlesource.com/c/tools/+/525215 TryBot-Result: Gopher Robot Run-TryBot: Alan Donovan Reviewed-by: Robert Findley --- go/analysis/analysistest/analysistest.go | 34 +++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/go/analysis/analysistest/analysistest.go b/go/analysis/analysistest/analysistest.go index 6a27edb1064..63ca6e9eb2e 100644 --- a/go/analysis/analysistest/analysistest.go +++ b/go/analysis/analysistest/analysistest.go @@ -98,6 +98,34 @@ type Testing interface { // println() // } // } +// +// # Conflicts +// +// A single analysis pass may offer two or more suggested fixes that +// (1) conflict but are nonetheless logically composable, (e.g. +// because both update the import declaration), or (2) are +// fundamentally incompatible (e.g. alternative fixes to the same +// statement). +// +// It is up to the driver to decide how to apply such fixes. A +// sophisticated driver could attempt to resolve conflicts of the +// first kind, but this test driver simply reports the fact of the +// conflict with the expectation that the user will split their tests +// into nonconflicting parts. +// +// Conflicts of the second kind can be avoided by giving the +// alternative fixes different names (SuggestedFix.Message) and using +// a multi-section .txtar file with a named section for each +// alternative fix. +// +// Analyzers that compute fixes from a textual diff of the +// before/after file contents (instead of directly from syntax tree +// positions) may produce fixes that, although logically +// non-conflicting, nonetheless conflict due to the particulars of the +// diff algorithm. In such cases it may suffice to introduce +// sufficient separation of the statements in the test input so that +// the computed diffs do not overlap. If that fails, break the test +// into smaller parts. func RunWithSuggestedFixes(t Testing, dir string, a *analysis.Analyzer, patterns ...string) []*Result { r := Run(t, dir, a, patterns...) @@ -135,7 +163,7 @@ func RunWithSuggestedFixes(t Testing, dir string, a *analysis.Analyzer, patterns continue } if _, ok := fileContents[file]; !ok { - contents, err := ioutil.ReadFile(file.Name()) + contents, err := os.ReadFile(file.Name()) if err != nil { t.Errorf("error reading %s: %v", file.Name(), err) } @@ -186,7 +214,7 @@ func RunWithSuggestedFixes(t Testing, dir string, a *analysis.Analyzer, patterns found = true out, err := diff.ApplyBytes(orig, edits) if err != nil { - t.Errorf("%s: error applying fixes: %v", file.Name(), err) + t.Errorf("%s: error applying fixes: %v (see possible explanations at RunWithSuggestedFixes)", file.Name(), err) continue } // the file may contain multiple trailing @@ -220,7 +248,7 @@ func RunWithSuggestedFixes(t Testing, dir string, a *analysis.Analyzer, patterns out, err := diff.ApplyBytes(orig, catchallEdits) if err != nil { - t.Errorf("%s: error applying fixes: %v", file.Name(), err) + t.Errorf("%s: error applying fixes: %v (see possible explanations at RunWithSuggestedFixes)", file.Name(), err) continue } want := string(ar.Comment) From fe324ac19e64c9fdec4d551d73e6eb264f80cf8a Mon Sep 17 00:00:00 2001 From: Dmitri Shuralyov Date: Tue, 5 Sep 2023 12:56:44 -0400 Subject: [PATCH 061/178] all: tidy with -compat=1.18 If the goal of the tidy compatibility target is to include the 4 most recent major Go releases, then it can be 1.18 now as Go 1.21.0 is out. Also drop the tagx:compat directive so that relui will know to to use the go.mod's default of 'go 1.18' when running 'go mod tidy'. This is for the next time it updates and tags x/tools. Change-Id: I67421d603afd1ddd76350515a8345035aa3e50df Reviewed-on: https://go-review.googlesource.com/c/tools/+/525617 gopls-CI: kokoro Reviewed-by: Dmitri Shuralyov LUCI-TryBot-Result: Go LUCI Reviewed-by: Robert Findley Auto-Submit: Dmitri Shuralyov --- go.mod | 2 +- go.sum | 36 ------------------------------------ gopls/go.sum | 34 ---------------------------------- 3 files changed, 1 insertion(+), 71 deletions(-) diff --git a/go.mod b/go.mod index 24cca8bec0f..4a01af35e72 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module golang.org/x/tools -go 1.18 // tagx:compat 1.16 +go 1.18 require ( github.com/yuin/goldmark v1.4.13 diff --git a/go.sum b/go.sum index 2c884cc6e39..3f717edf725 100644 --- a/go.sum +++ b/go.sum @@ -1,46 +1,10 @@ github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/gopls/go.sum b/gopls/go.sum index 78b66349fe3..45d1c2edb85 100644 --- a/gopls/go.sum +++ b/gopls/go.sum @@ -1,21 +1,12 @@ -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= -github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= -github.com/google/go-cmdtest v0.4.1-0.20220921163831-55ab3332a786/go.mod h1:apVn/GCasLZUVpAJ6oWAuyP7Ne7CEsQbTnc0plM3m+o= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/safehtml v0.0.2/go.mod h1:L4KWwDsUJdECRAEpZoBn3O64bQaywRscowZjJAzjHnU= github.com/google/safehtml v0.1.0 h1:EwLKo8qawTKfsi0orxcQAZzu07cICaBeFMegAU9eaT8= github.com/google/safehtml v0.1.0/go.mod h1:L4KWwDsUJdECRAEpZoBn3O64bQaywRscowZjJAzjHnU= @@ -25,59 +16,39 @@ github.com/jba/templatecheck v0.6.0 h1:SwM8C4hlK/YNLsdcXStfnHWE2HKkuTVwy5FKQHt5r github.com/jba/templatecheck v0.6.0/go.mod h1:/1k7EajoSErFI9GLHAsiIJEaNLt3ALKNw2TV7z2SYv4= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= -golang.org/x/exp/typeparams v0.0.0-20221208152030-732eee02a75a/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/exp/typeparams v0.0.0-20221212164502-fae10dda9338 h1:2O2DON6y3XMJiQRAS1UWU+54aec2uopH3x7MAiqGW6Y= golang.org/x/exp/typeparams v0.0.0-20221212164502-fae10dda9338/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= -golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211213223007-03aa0b5f6827/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/telemetry v0.0.0-20230822160736-17171dbf1d76 h1:Lv25uIMpljmSMN0+GCC+xgiC/4ikIdKMkQfw/EVq2Nk= golang.org/x/telemetry v0.0.0-20230822160736-17171dbf1d76/go.mod h1:kO7uNSGGmqCHII6C0TYfaLwSBIfcyhj53//nu0+Fy4A= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -86,9 +57,6 @@ golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/vuln v0.0.0-20230110180137-6ad3e3d07815 h1:A9kONVi4+AnuOr1dopsibH6hLi1Huy54cbeJxnq4vmU= golang.org/x/vuln v0.0.0-20230110180137-6ad3e3d07815/go.mod h1:XJiVExZgoZfrrxoTeVsFYrSSk1snhfpOEC95JL+A4T0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= @@ -99,12 +67,10 @@ gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.2.2/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY= honnef.co/go/tools v0.4.2 h1:6qXr+R5w+ktL5UkwEbPp+fEvfyoMPche6GkOpGHZcLc= honnef.co/go/tools v0.4.2/go.mod h1:36ZgoUOrqOk1GxwHhyryEkq8FQWkUO2xGuSMhUCcdvA= mvdan.cc/gofumpt v0.4.0 h1:JVf4NN1mIpHogBj7ABpgOyZc65/UUOkKQFkoURsz4MM= mvdan.cc/gofumpt v0.4.0/go.mod h1:PljLOHDeZqgS8opHRKLzp2It2VBuSdteAgqUfzMTxlQ= mvdan.cc/unparam v0.0.0-20211214103731-d0ef000c54e5 h1:Jh3LAeMt1eGpxomyu3jVkmVZWW2MxZ1qIIV2TZ/nRio= -mvdan.cc/unparam v0.0.0-20211214103731-d0ef000c54e5/go.mod h1:b8RRCBm0eeiWR8cfN88xeq2G5SG3VKGO+5UPWi5FSOY= mvdan.cc/xurls/v2 v2.4.0 h1:tzxjVAj+wSBmDcF6zBB7/myTy3gX9xvi8Tyr28AuQgc= mvdan.cc/xurls/v2 v2.4.0/go.mod h1:+GEjq9uNjqs8LQfM9nVnM8rff0OQ5Iash5rzX+N1CSg= From 36c4f987d2d9f02ffe121e06d2260f69880f9d50 Mon Sep 17 00:00:00 2001 From: Robert Findley Date: Fri, 1 Sep 2023 13:37:54 -0400 Subject: [PATCH 062/178] gopls/internal/lsp/cache: simplify tracking of snapshot directories Great care was taken to track known directories in the snapshot without blocking in snapshot.Clone, introducing significant complexity. This complexity can be avoided by instead keeping track of observed directories as files are set in the snapshot. These directories need only be reset when files are deleted from the snapshot, which is a relatively rare event. Also rename filesMap->fileMap, and move to filemap.go, with a new unit test. This reduces some path dependence on seen files, as the set of directories is well defined and depends only on the files in the snapshot. Previously, when a file was removed, gopls called Stat to check if the directory still existed, which leads to path dependence: an add+remove was not the same as nothing at all. Updates golang/go#57558 Change-Id: I5fd89ce870fa7d8afd19471d150396b1e4ea8875 Reviewed-on: https://go-review.googlesource.com/c/tools/+/525616 Reviewed-by: Alan Donovan LUCI-TryBot-Result: Go LUCI --- gopls/internal/lsp/cache/filemap.go | 134 ++++++++++++++ gopls/internal/lsp/cache/filemap_test.go | 108 ++++++++++++ gopls/internal/lsp/cache/maps.go | 78 -------- gopls/internal/lsp/cache/session.go | 51 ++---- gopls/internal/lsp/cache/snapshot.go | 216 +++++++---------------- gopls/internal/lsp/cache/view.go | 1 - gopls/internal/lsp/cache/workspace.go | 33 ---- internal/persistent/map.go | 4 +- 8 files changed, 324 insertions(+), 301 deletions(-) create mode 100644 gopls/internal/lsp/cache/filemap.go create mode 100644 gopls/internal/lsp/cache/filemap_test.go delete mode 100644 gopls/internal/lsp/cache/maps.go diff --git a/gopls/internal/lsp/cache/filemap.go b/gopls/internal/lsp/cache/filemap.go new file mode 100644 index 00000000000..8b74cf732d1 --- /dev/null +++ b/gopls/internal/lsp/cache/filemap.go @@ -0,0 +1,134 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cache + +import ( + "path/filepath" + + "golang.org/x/tools/gopls/internal/lsp/source" + "golang.org/x/tools/gopls/internal/span" + "golang.org/x/tools/internal/persistent" +) + +// A fileMap maps files in the snapshot, with some additional bookkeeping: +// It keeps track of overlays as well as directories containing any observed +// file. +type fileMap struct { + files *persistent.Map[span.URI, source.FileHandle] + overlays *persistent.Map[span.URI, *Overlay] // the subset of files that are overlays + dirs *persistent.Set[string] // all dirs containing files; if nil, dirs have not been initialized +} + +func newFileMap() *fileMap { + return &fileMap{ + files: new(persistent.Map[span.URI, source.FileHandle]), + overlays: new(persistent.Map[span.URI, *Overlay]), + dirs: new(persistent.Set[string]), + } +} + +func (m *fileMap) Clone() *fileMap { + m2 := &fileMap{ + files: m.files.Clone(), + overlays: m.overlays.Clone(), + } + if m.dirs != nil { + m2.dirs = m.dirs.Clone() + } + return m2 +} + +func (m *fileMap) Destroy() { + m.files.Destroy() + m.overlays.Destroy() + if m.dirs != nil { + m.dirs.Destroy() + } +} + +// Get returns the file handle mapped by the given key, or (nil, false) if the +// key is not present. +func (m *fileMap) Get(key span.URI) (source.FileHandle, bool) { + return m.files.Get(key) +} + +// Range calls f for each (uri, fh) in the map. +func (m *fileMap) Range(f func(uri span.URI, fh source.FileHandle)) { + m.files.Range(f) +} + +// Set stores the given file handle for key, updating overlays and directories +// accordingly. +func (m *fileMap) Set(key span.URI, fh source.FileHandle) { + m.files.Set(key, fh, nil) + + // update overlays + if o, ok := fh.(*Overlay); ok { + m.overlays.Set(key, o, nil) + } else { + // Setting a non-overlay must delete the corresponding overlay, to preserve + // the accuracy of the overlay set. + m.overlays.Delete(key) + } + + // update dirs + if m.dirs == nil { + m.initDirs() + } else { + m.addDirs(key) + } +} + +func (m *fileMap) initDirs() { + m.dirs = new(persistent.Set[string]) + m.files.Range(func(u span.URI, _ source.FileHandle) { + m.addDirs(u) + }) +} + +// addDirs adds all directories containing u to the dirs set. +func (m *fileMap) addDirs(u span.URI) { + dir := filepath.Dir(u.Filename()) + for dir != "" && !m.dirs.Contains(dir) { + m.dirs.Add(dir) + dir = filepath.Dir(dir) + } +} + +// Delete removes a file from the map, and updates overlays and dirs +// accordingly. +func (m *fileMap) Delete(key span.URI) { + m.files.Delete(key) + m.overlays.Delete(key) + + // Deleting a file may cause the set of dirs to shrink; therefore we must + // re-evaluate the dir set. + // + // Do this lazily, to avoid work if there are multiple deletions in a row. + if m.dirs != nil { + m.dirs.Destroy() + m.dirs = nil + } +} + +// Overlays returns a new unordered array of overlay files. +func (m *fileMap) Overlays() []*Overlay { + var overlays []*Overlay + m.overlays.Range(func(_ span.URI, o *Overlay) { + overlays = append(overlays, o) + }) + return overlays +} + +// Dirs reports returns the set of dirs observed by the fileMap. +// +// This operation mutates the fileMap. +// The result must not be mutated by the caller. +func (m *fileMap) Dirs() *persistent.Set[string] { + if m.dirs == nil { + m.initDirs() + } + return m.dirs +} diff --git a/gopls/internal/lsp/cache/filemap_test.go b/gopls/internal/lsp/cache/filemap_test.go new file mode 100644 index 00000000000..3d5bab41c67 --- /dev/null +++ b/gopls/internal/lsp/cache/filemap_test.go @@ -0,0 +1,108 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cache + +import ( + "path/filepath" + "sort" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "golang.org/x/tools/gopls/internal/lsp/source" + "golang.org/x/tools/gopls/internal/span" +) + +func TestFileMap(t *testing.T) { + const ( + set = iota + del + ) + type op struct { + op int // set or remove + path string + overlay bool + } + tests := []struct { + label string + ops []op + wantFiles []string + wantOverlays []string + wantDirs []string + }{ + {"empty", nil, nil, nil, nil}, + {"singleton", []op{ + {set, "/a/b", false}, + }, []string{"/a/b"}, nil, []string{"/", "/a"}}, + {"overlay", []op{ + {set, "/a/b", true}, + }, []string{"/a/b"}, []string{"/a/b"}, []string{"/", "/a"}}, + {"replace overlay", []op{ + {set, "/a/b", true}, + {set, "/a/b", false}, + }, []string{"/a/b"}, nil, []string{"/", "/a"}}, + {"multi dir", []op{ + {set, "/a/b", false}, + {set, "/c/d", false}, + }, []string{"/a/b", "/c/d"}, nil, []string{"/", "/a", "/c"}}, + {"empty dir", []op{ + {set, "/a/b", false}, + {set, "/c/d", false}, + {del, "/a/b", false}, + }, []string{"/c/d"}, nil, []string{"/", "/c"}}, + } + + // Normalize paths for windows compatibility. + normalize := func(path string) string { + return strings.TrimPrefix(filepath.ToSlash(path), "C:") // the span packages adds 'C:' + } + + for _, test := range tests { + t.Run(test.label, func(t *testing.T) { + m := newFileMap() + for _, op := range test.ops { + uri := span.URIFromPath(filepath.FromSlash(op.path)) + switch op.op { + case set: + var fh source.FileHandle + if op.overlay { + fh = &Overlay{uri: uri} + } else { + fh = &DiskFile{uri: uri} + } + m.Set(uri, fh) + case del: + m.Delete(uri) + } + } + + var gotFiles []string + m.Range(func(uri span.URI, _ source.FileHandle) { + gotFiles = append(gotFiles, normalize(uri.Filename())) + }) + sort.Strings(gotFiles) + if diff := cmp.Diff(test.wantFiles, gotFiles); diff != "" { + t.Errorf("Files mismatch (-want +got):\n%s", diff) + } + + var gotOverlays []string + for _, o := range m.Overlays() { + gotOverlays = append(gotOverlays, normalize(o.URI().Filename())) + } + if diff := cmp.Diff(test.wantOverlays, gotOverlays); diff != "" { + t.Errorf("Overlays mismatch (-want +got):\n%s", diff) + } + + var gotDirs []string + m.Dirs().Range(func(dir string) { + gotDirs = append(gotDirs, normalize(dir)) + }) + sort.Strings(gotDirs) + if diff := cmp.Diff(test.wantDirs, gotDirs); diff != "" { + t.Errorf("Dirs mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/gopls/internal/lsp/cache/maps.go b/gopls/internal/lsp/cache/maps.go deleted file mode 100644 index edb72d5c123..00000000000 --- a/gopls/internal/lsp/cache/maps.go +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package cache - -import ( - "golang.org/x/tools/gopls/internal/lsp/source" - "golang.org/x/tools/gopls/internal/span" - "golang.org/x/tools/internal/persistent" -) - -type filesMap struct { - impl *persistent.Map[span.URI, source.FileHandle] - overlayMap map[span.URI]*Overlay // the subset that are overlays -} - -func newFilesMap() filesMap { - return filesMap{ - impl: new(persistent.Map[span.URI, source.FileHandle]), - overlayMap: make(map[span.URI]*Overlay), - } -} - -func (m filesMap) Clone() filesMap { - overlays := make(map[span.URI]*Overlay, len(m.overlayMap)) - for k, v := range m.overlayMap { - overlays[k] = v - } - return filesMap{ - impl: m.impl.Clone(), - overlayMap: overlays, - } -} - -func (m filesMap) Destroy() { - m.impl.Destroy() -} - -func (m filesMap) Get(key span.URI) (source.FileHandle, bool) { - value, ok := m.impl.Get(key) - if !ok { - return nil, false - } - return value.(source.FileHandle), true -} - -func (m filesMap) Range(do func(key span.URI, value source.FileHandle)) { - m.impl.Range(do) -} - -func (m filesMap) Set(key span.URI, value source.FileHandle) { - m.impl.Set(key, value, nil) - - if o, ok := value.(*Overlay); ok { - m.overlayMap[key] = o - } else { - // Setting a non-overlay must delete the corresponding overlay, to preserve - // the accuracy of the overlay set. - delete(m.overlayMap, key) - } -} - -func (m *filesMap) Delete(key span.URI) { - m.impl.Delete(key) - delete(m.overlayMap, key) -} - -// overlays returns a new unordered array of overlay files. -func (m filesMap) overlays() []*Overlay { - // In practice we will always have at least one overlay, so there is no need - // to optimize for the len=0 case by returning a nil slice. - overlays := make([]*Overlay, 0, len(m.overlayMap)) - for _, o := range m.overlayMap { - overlays = append(overlays, o) - } - return overlays -} diff --git a/gopls/internal/lsp/cache/session.go b/gopls/internal/lsp/cache/session.go index 1e463fa3f4f..e6f48118636 100644 --- a/gopls/internal/lsp/cache/session.go +++ b/gopls/internal/lsp/cache/session.go @@ -172,7 +172,7 @@ func (s *Session) createView(ctx context.Context, name string, folder span.URI, store: s.cache.store, packages: new(persistent.Map[PackageID, *packageHandle]), meta: new(metadataGraph), - files: newFilesMap(), + files: newFileMap(), activePackages: new(persistent.Map[PackageID, *Package]), symbolizeHandles: new(persistent.Map[span.URI, *memoize.Promise]), workspacePackages: make(map[PackageID]PackagePath), @@ -182,7 +182,6 @@ func (s *Session) createView(ctx context.Context, name string, folder span.URI, modTidyHandles: new(persistent.Map[span.URI, *memoize.Promise]), modVulnHandles: new(persistent.Map[span.URI, *memoize.Promise]), modWhyHandles: new(persistent.Map[span.URI, *memoize.Promise]), - knownSubdirs: new(persistent.Set[span.URI]), workspaceModFiles: wsModFiles, workspaceModFilesErr: wsModFilesErr, pkgIndex: typerefs.NewPackageIndex(), @@ -594,15 +593,23 @@ func (s *Session) ExpandModificationsToDirectories(ctx context.Context, changes } s.viewMu.Unlock() - knownDirs := knownDirectories(ctx, snapshots) - defer knownDirs.Destroy() - + // Expand the modification to any file we could care about, which we define + // to be any file observed by any of the snapshots. + // + // There may be other files in the directory, but if we haven't read them yet + // we don't need to invalidate them. var result []source.FileModification for _, c := range changes { - if !knownDirs.Contains(c.URI) { + expanded := make(map[span.URI]bool) + for _, snapshot := range snapshots { + for _, uri := range snapshot.filesInDir(c.URI) { + expanded[uri] = true + } + } + if len(expanded) == 0 { result = append(result, c) } else { - for uri := range knownFilesInDir(ctx, snapshots, c.URI) { + for uri := range expanded { result = append(result, source.FileModification{ URI: uri, Action: c.Action, @@ -616,36 +623,6 @@ func (s *Session) ExpandModificationsToDirectories(ctx context.Context, changes return result } -// knownDirectories returns all of the directories known to the given -// snapshots, including workspace directories and their subdirectories. -// It is responsibility of the caller to destroy the returned set. -func knownDirectories(ctx context.Context, snapshots []*snapshot) *persistent.Set[span.URI] { - result := new(persistent.Set[span.URI]) - for _, snapshot := range snapshots { - dirs := snapshot.dirs(ctx) - for _, dir := range dirs { - result.Add(dir) - } - knownSubdirs := snapshot.getKnownSubdirs(dirs) - result.AddAll(knownSubdirs) - knownSubdirs.Destroy() - } - return result -} - -// knownFilesInDir returns the files known to the snapshots in the session. -// It does not respect symlinks. -func knownFilesInDir(ctx context.Context, snapshots []*snapshot, dir span.URI) map[span.URI]struct{} { - files := map[span.URI]struct{}{} - - for _, snapshot := range snapshots { - for _, uri := range snapshot.knownFilesInDir(ctx, dir) { - files[uri] = struct{}{} - } - } - return files -} - // Precondition: caller holds s.viewMu lock. // TODO(rfindley): move this to fs_overlay.go. func (fs *overlayFS) updateOverlays(ctx context.Context, changes []source.FileModification) error { diff --git a/gopls/internal/lsp/cache/snapshot.go b/gopls/internal/lsp/cache/snapshot.go index 94eceed869b..afadd5c7cf1 100644 --- a/gopls/internal/lsp/cache/snapshot.go +++ b/gopls/internal/lsp/cache/snapshot.go @@ -97,7 +97,7 @@ type snapshot struct { // files maps file URIs to their corresponding FileHandles. // It may invalidated when a file's content changes. - files filesMap + files *fileMap // symbolizeHandles maps each file URI to a handle for the future // result of computing the symbols declared in that file. @@ -150,15 +150,6 @@ type snapshot struct { modWhyHandles *persistent.Map[span.URI, *memoize.Promise] // *memoize.Promise[modWhyResult] modVulnHandles *persistent.Map[span.URI, *memoize.Promise] // *memoize.Promise[modVulnResult] - // knownSubdirs is the set of subdirectory URIs in the workspace, - // used to create glob patterns for file watching. - knownSubdirs *persistent.Set[span.URI] - knownSubdirsCache map[string]struct{} // memo of knownSubdirs as a set of filenames - // unprocessedSubdirChanges are any changes that might affect the set of - // subdirectories in the workspace. They are not reflected to knownSubdirs - // during the snapshot cloning step as it can slow down cloning. - unprocessedSubdirChanges []*fileChange - // workspaceModFiles holds the set of mod files active in this snapshot. // // This is either empty, a single entry for the workspace go.mod file, or the @@ -262,7 +253,6 @@ func (s *snapshot) destroy(destroyedBy string) { s.packages.Destroy() s.activePackages.Destroy() s.files.Destroy() - s.knownSubdirs.Destroy() s.symbolizeHandles.Destroy() s.parseModHandles.Destroy() s.parseWorkHandles.Destroy() @@ -629,7 +619,7 @@ func (s *snapshot) overlays() []*Overlay { s.mu.Lock() defer s.mu.Unlock() - return s.files.overlays() + return s.files.Overlays() } // Package data kinds, identifying various package data that may be stored in @@ -899,10 +889,8 @@ func (s *snapshot) resetActivePackagesLocked() { s.activePackages = new(persistent.Map[PackageID, *Package]) } -const fileExtensions = "go,mod,sum,work" - func (s *snapshot) fileWatchingGlobPatterns(ctx context.Context) map[string]struct{} { - extensions := fileExtensions + extensions := "go,mod,sum,work" for _, ext := range s.Options().TemplateExtensions { extensions += "," + ext } @@ -920,19 +908,17 @@ func (s *snapshot) fileWatchingGlobPatterns(ctx context.Context) map[string]stru } // Add a pattern for each Go module in the workspace that is not within the view. - dirs := s.dirs(ctx) + dirs := s.workspaceDirs(ctx) for _, dir := range dirs { - dirName := dir.Filename() - // If the directory is within the view's folder, we're already watching // it with the first pattern above. - if source.InDir(s.view.folder.Filename(), dirName) { + if source.InDir(s.view.folder.Filename(), dir) { continue } // TODO(rstambler): If microsoft/vscode#3025 is resolved before // microsoft/vscode#101042, we will need a work-around for Windows // drive letter casing. - patterns[fmt.Sprintf("%s/**/*.{%s}", dirName, extensions)] = struct{}{} + patterns[fmt.Sprintf("%s/**/*.{%s}", dir, extensions)] = struct{}{} } if s.watchSubdirs() { @@ -943,6 +929,11 @@ func (s *snapshot) fileWatchingGlobPatterns(ctx context.Context) map[string]stru // directories. There may be thousands of patterns, each a single // directory. // + // We compute this set by looking at files that we've previously observed. + // This may miss changed to directories that we haven't observed, but that + // shouldn't matter as there is nothing to invalidate (if a directory falls + // in forest, etc). + // // (A previous iteration created a single glob pattern holding a union of // all the directories, but this was found to cause VS Code to get stuck // for several minutes after a buffer was saved twice in a workspace that @@ -956,6 +947,46 @@ func (s *snapshot) fileWatchingGlobPatterns(ctx context.Context) map[string]stru return patterns } +func (s *snapshot) addKnownSubdirs(patterns map[string]unit, wsDirs []string) { + s.mu.Lock() + defer s.mu.Unlock() + + s.files.Dirs().Range(func(dir string) { + for _, wsDir := range wsDirs { + if source.InDir(wsDir, dir) { + patterns[dir] = unit{} + } + } + }) +} + +// workspaceDirs returns the workspace directories for the loaded modules. +// +// A workspace directory is, roughly speaking, a directory for which we care +// about file changes. +func (s *snapshot) workspaceDirs(ctx context.Context) []string { + dirSet := make(map[string]unit) + + // Dirs should, at the very least, contain the working directory and folder. + dirSet[s.view.workingDir().Filename()] = unit{} + dirSet[s.view.folder.Filename()] = unit{} + + // Additionally, if e.g. go.work indicates other workspace modules, we should + // include their directories too. + if s.workspaceModFilesErr == nil { + for modFile := range s.workspaceModFiles { + dir := filepath.Dir(modFile.Filename()) + dirSet[dir] = unit{} + } + } + var dirs []string + for d := range dirSet { + dirs = append(dirs, d) + } + sort.Strings(dirs) + return dirs +} + // watchSubdirs reports whether gopls should request separate file watchers for // each relevant subdirectory. This is necessary only for clients (namely VS // Code) that do not send notifications for individual files in a directory @@ -985,127 +1016,19 @@ func (s *snapshot) watchSubdirs() bool { } } -func (s *snapshot) addKnownSubdirs(patterns map[string]struct{}, wsDirs []span.URI) { +// filesInDir returns all files observed by the snapshot that are contained in +// a directory with the provided URI. +func (s *snapshot) filesInDir(uri span.URI) []span.URI { s.mu.Lock() defer s.mu.Unlock() - // First, process any pending changes and update the set of known - // subdirectories. - // It may change list of known subdirs and therefore invalidate the cache. - s.applyKnownSubdirsChangesLocked(wsDirs) - - // TODO(adonovan): is it still necessary to memoize the Range - // and URI.Filename operations? - if s.knownSubdirsCache == nil { - s.knownSubdirsCache = make(map[string]struct{}) - s.knownSubdirs.Range(func(uri span.URI) { - s.knownSubdirsCache[uri.Filename()] = struct{}{} - }) - } - - for pattern := range s.knownSubdirsCache { - patterns[pattern] = struct{}{} - } -} - -// collectAllKnownSubdirs collects all of the subdirectories within the -// snapshot's workspace directories. None of the workspace directories are -// included. -func (s *snapshot) collectAllKnownSubdirs(ctx context.Context) { - dirs := s.dirs(ctx) - - s.mu.Lock() - defer s.mu.Unlock() - - s.knownSubdirs.Destroy() - s.knownSubdirs = new(persistent.Set[span.URI]) - s.knownSubdirsCache = nil - s.files.Range(func(uri span.URI, fh source.FileHandle) { - s.addKnownSubdirLocked(uri, dirs) - }) -} - -func (s *snapshot) getKnownSubdirs(wsDirs []span.URI) *persistent.Set[span.URI] { - s.mu.Lock() - defer s.mu.Unlock() - - // First, process any pending changes and update the set of known - // subdirectories. - s.applyKnownSubdirsChangesLocked(wsDirs) - - return s.knownSubdirs.Clone() -} - -func (s *snapshot) applyKnownSubdirsChangesLocked(wsDirs []span.URI) { - for _, c := range s.unprocessedSubdirChanges { - if c.isUnchanged { - continue - } - if !c.exists { - s.removeKnownSubdirLocked(c.fileHandle.URI()) - } else { - s.addKnownSubdirLocked(c.fileHandle.URI(), wsDirs) - } - } - s.unprocessedSubdirChanges = nil -} - -func (s *snapshot) addKnownSubdirLocked(uri span.URI, dirs []span.URI) { - dir := filepath.Dir(uri.Filename()) - // First check if the directory is already known, because then we can - // return early. - if s.knownSubdirs.Contains(span.URIFromPath(dir)) { - return - } - var matched span.URI - for _, wsDir := range dirs { - if source.InDir(wsDir.Filename(), dir) { - matched = wsDir - break - } - } - // Don't watch any directory outside of the workspace directories. - if matched == "" { - return - } - for { - if dir == "" || dir == matched.Filename() { - break - } - uri := span.URIFromPath(dir) - if s.knownSubdirs.Contains(uri) { - break - } - s.knownSubdirs.Add(uri) - dir = filepath.Dir(dir) - s.knownSubdirsCache = nil - } -} - -func (s *snapshot) removeKnownSubdirLocked(uri span.URI) { - dir := filepath.Dir(uri.Filename()) - for dir != "" { - uri := span.URIFromPath(dir) - if !s.knownSubdirs.Contains(uri) { - break - } - if info, _ := os.Stat(dir); info == nil { - s.knownSubdirs.Remove(uri) - s.knownSubdirsCache = nil - } - dir = filepath.Dir(dir) + dir := uri.Filename() + if !s.files.Dirs().Contains(dir) { + return nil } -} - -// knownFilesInDir returns the files known to the given snapshot that are in -// the given directory. It does not respect symlinks. -func (s *snapshot) knownFilesInDir(ctx context.Context, dir span.URI) []span.URI { var files []span.URI - s.mu.Lock() - defer s.mu.Unlock() - - s.files.Range(func(uri span.URI, fh source.FileHandle) { - if source.InDir(dir.Filename(), uri.Filename()) { + s.files.Range(func(uri span.URI, _ source.FileHandle) { + if source.InDir(dir, uri.Filename()) { files = append(files, uri) } }) @@ -1976,7 +1899,6 @@ func (s *snapshot) clone(ctx, bgCtx context.Context, changes map[span.URI]*fileC modTidyHandles: s.modTidyHandles.Clone(), modWhyHandles: s.modWhyHandles.Clone(), modVulnHandles: s.modVulnHandles.Clone(), - knownSubdirs: s.knownSubdirs.Clone(), workspaceModFiles: wsModFiles, workspaceModFilesErr: wsModFilesErr, importGraph: s.importGraph, @@ -2000,15 +1922,6 @@ func (s *snapshot) clone(ctx, bgCtx context.Context, changes map[span.URI]*fileC // // Maybe not, as major configuration changes cause a new view. - // Add all of the known subdirectories, but don't update them for the - // changed files. We need to rebuild the workspace module to know the - // true set of known subdirectories, but we don't want to do that in clone. - result.knownSubdirs = s.knownSubdirs.Clone() - result.knownSubdirsCache = s.knownSubdirsCache - for _, c := range changes { - result.unprocessedSubdirChanges = append(result.unprocessedSubdirChanges, c) - } - // directIDs keeps track of package IDs that have directly changed. // Note: this is not a set, it's a map from id to invalidateMetadata. directIDs := map[PackageID]bool{} @@ -2099,14 +2012,17 @@ func (s *snapshot) clone(ctx, bgCtx context.Context, changes map[span.URI]*fileC result.parseModHandles.Delete(uri) result.parseWorkHandles.Delete(uri) + // Handle the invalidated file; it may have new contents or not exist. + // + // Note, we can't simply delete the file unconditionally and let it be + // re-read, as (1) the snapshot must observe all overlays, and (2) deleting + // a file forces directories to be reevaluated, as it may be the last file + // in a directory. We want to avoid that work in the common case where a + // file has simply changed. if !change.exists { result.files.Delete(uri) } else { - // TODO(golang/go#57558): the line below is strictly necessary to ensure - // that snapshots have each overlay, but it is problematic that we must - // set any content in snapshot.clone: if the file has changed, let it be - // re-read. result.files.Set(uri, change.fileHandle) } diff --git a/gopls/internal/lsp/cache/view.go b/gopls/internal/lsp/cache/view.go index fbdb6047a78..d7ddc31fe0e 100644 --- a/gopls/internal/lsp/cache/view.go +++ b/gopls/internal/lsp/cache/view.go @@ -778,7 +778,6 @@ func (s *snapshot) initialize(ctx context.Context, firstAttempt bool) { } s.loadWorkspace(ctx, firstAttempt) - s.collectAllKnownSubdirs(ctx) } func (s *snapshot) loadWorkspace(ctx context.Context, firstAttempt bool) (loadErr error) { diff --git a/gopls/internal/lsp/cache/workspace.go b/gopls/internal/lsp/cache/workspace.go index 28179f5a0b9..e241d02b49a 100644 --- a/gopls/internal/lsp/cache/workspace.go +++ b/gopls/internal/lsp/cache/workspace.go @@ -11,7 +11,6 @@ import ( "io/fs" "os" "path/filepath" - "sort" "strings" "golang.org/x/mod/modfile" @@ -60,38 +59,6 @@ func computeWorkspaceModFiles(ctx context.Context, gomod, gowork span.URI, go111 return nil, nil } -// dirs returns the workspace directories for the loaded modules. -// -// A workspace directory is, roughly speaking, a directory for which we care -// about file changes. This is used for the purpose of registering file -// watching patterns, and expanding directory modifications to their adjacent -// files. -// -// TODO(rfindley): move this to snapshot.go. -// TODO(rfindley): can we make this abstraction simpler and/or more accurate? -func (s *snapshot) dirs(ctx context.Context) []span.URI { - dirSet := make(map[span.URI]struct{}) - - // Dirs should, at the very least, contain the working directory and folder. - dirSet[s.view.workingDir()] = struct{}{} - dirSet[s.view.folder] = struct{}{} - - // Additionally, if e.g. go.work indicates other workspace modules, we should - // include their directories too. - if s.workspaceModFilesErr == nil { - for modFile := range s.workspaceModFiles { - dir := filepath.Dir(modFile.Filename()) - dirSet[span.URIFromPath(dir)] = struct{}{} - } - } - var dirs []span.URI - for d := range dirSet { - dirs = append(dirs, d) - } - sort.Slice(dirs, func(i, j int) bool { return dirs[i] < dirs[j] }) - return dirs -} - // isGoMod reports if uri is a go.mod file. func isGoMod(uri span.URI) bool { return filepath.Base(uri.Filename()) == "go.mod" diff --git a/internal/persistent/map.go b/internal/persistent/map.go index 02389f89dc5..64cd500c65a 100644 --- a/internal/persistent/map.go +++ b/internal/persistent/map.go @@ -30,8 +30,8 @@ import ( // Map is an associative mapping from keys to values. // // Maps can be Cloned in constant time. -// Get, Store, and Delete operations are done on average in logarithmic time. -// Maps can be Updated in O(m log(n/m)) time for maps of size n and m, where m < n. +// Get, Set, and Delete operations are done on average in logarithmic time. +// Maps can be merged (via SetAll) in O(m log(n/m)) time for maps of size n and m, where m < n. // // Values are reference counted, and a client-supplied release function // is called when a value is no longer referenced by a map or any clone. From 56a186636b8f3f0b74f8b0857399ce28e2569ed1 Mon Sep 17 00:00:00 2001 From: "Hana (Hyang-Ah) Kim" Date: Tue, 5 Sep 2023 14:57:44 -0400 Subject: [PATCH 063/178] gopls: update staticcheck (v0.4.5) Includes some crash fixes and support for go1.21. See: https: //github.com/dominikh/go-tools/releases/tag/2023.1.5 https: //github.com/dominikh/go-tools/releases/tag/2023.1.4 Change-Id: I01f2a740f6d7e5a1622c7ff796637dbd7a2d2bf3 Reviewed-on: https://go-review.googlesource.com/c/tools/+/525775 Commit-Queue: Hyang-Ah Hana Kim TryBot-Result: Gopher Robot Auto-Submit: Hyang-Ah Hana Kim Run-TryBot: Hyang-Ah Hana Kim Reviewed-by: Robert Findley --- gopls/go.mod | 4 ++-- gopls/go.sum | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/gopls/go.mod b/gopls/go.mod index 3198e0f81bb..d04addfe084 100644 --- a/gopls/go.mod +++ b/gopls/go.mod @@ -12,10 +12,10 @@ require ( golang.org/x/sys v0.12.0 golang.org/x/telemetry v0.0.0-20230822160736-17171dbf1d76 golang.org/x/text v0.13.0 - golang.org/x/tools v0.6.0 + golang.org/x/tools v0.9.4-0.20230601214343-86c93e8732cc golang.org/x/vuln v0.0.0-20230110180137-6ad3e3d07815 gopkg.in/yaml.v3 v3.0.1 - honnef.co/go/tools v0.4.2 + honnef.co/go/tools v0.4.5 mvdan.cc/gofumpt v0.4.0 mvdan.cc/xurls/v2 v2.4.0 ) diff --git a/gopls/go.sum b/gopls/go.sum index 45d1c2edb85..2782d7303f0 100644 --- a/gopls/go.sum +++ b/gopls/go.sum @@ -67,8 +67,8 @@ gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.4.2 h1:6qXr+R5w+ktL5UkwEbPp+fEvfyoMPche6GkOpGHZcLc= -honnef.co/go/tools v0.4.2/go.mod h1:36ZgoUOrqOk1GxwHhyryEkq8FQWkUO2xGuSMhUCcdvA= +honnef.co/go/tools v0.4.5 h1:YGD4H+SuIOOqsyoLOpZDWcieM28W47/zRO7f+9V3nvo= +honnef.co/go/tools v0.4.5/go.mod h1:GUV+uIBCLpdf0/v6UhHHG/yzI/z6qPskBeQCjcNB96k= mvdan.cc/gofumpt v0.4.0 h1:JVf4NN1mIpHogBj7ABpgOyZc65/UUOkKQFkoURsz4MM= mvdan.cc/gofumpt v0.4.0/go.mod h1:PljLOHDeZqgS8opHRKLzp2It2VBuSdteAgqUfzMTxlQ= mvdan.cc/unparam v0.0.0-20211214103731-d0ef000c54e5 h1:Jh3LAeMt1eGpxomyu3jVkmVZWW2MxZ1qIIV2TZ/nRio= From 15a23a914c953ce280d89d1d9d20004e7fd7afdd Mon Sep 17 00:00:00 2001 From: Robert Findley Date: Fri, 1 Sep 2023 13:01:23 -0400 Subject: [PATCH 064/178] gopls/internal/lsp/cache: consolidate logic for updating maps in clone Various snapshot maps are updated in a similar way during cloning. Though it's not obvious, the location of these updates should not matter. Factor them out into a helper to reduce some of the complexity of the clone method. Along the way, make one semantic change: only invalidate unloadable files if a change has affected their metadata. Add some TODOs for further cleanup. Change-Id: Iac04e5502519f1e34ae8b70bc21e3670857d1053 Reviewed-on: https://go-review.googlesource.com/c/tools/+/524763 LUCI-TryBot-Result: Go LUCI Reviewed-by: Alan Donovan Auto-Submit: Robert Findley gopls-CI: kokoro --- gopls/internal/lsp/cache/filemap.go | 26 ++++++++++- gopls/internal/lsp/cache/session.go | 2 + gopls/internal/lsp/cache/snapshot.go | 69 ++++++++++------------------ 3 files changed, 51 insertions(+), 46 deletions(-) diff --git a/gopls/internal/lsp/cache/filemap.go b/gopls/internal/lsp/cache/filemap.go index 8b74cf732d1..faf35197697 100644 --- a/gopls/internal/lsp/cache/filemap.go +++ b/gopls/internal/lsp/cache/filemap.go @@ -29,7 +29,9 @@ func newFileMap() *fileMap { } } -func (m *fileMap) Clone() *fileMap { +// Clone creates a copy of the fileMap, incorporating the changes specified by +// the changes map. +func (m *fileMap) Clone(changes map[span.URI]*fileChange) *fileMap { m2 := &fileMap{ files: m.files.Clone(), overlays: m.overlays.Clone(), @@ -37,6 +39,28 @@ func (m *fileMap) Clone() *fileMap { if m.dirs != nil { m2.dirs = m.dirs.Clone() } + + // Handle file changes. + // + // Note, we can't simply delete the file unconditionally and let it be + // re-read by the snapshot, as (1) the snapshot must always observe all + // overlays, and (2) deleting a file forces directories to be reevaluated, as + // it may be the last file in a directory. We want to avoid that work in the + // common case where a file has simply changed. + // + // For that reason, we also do this in two passes, processing deletions + // first, as interleaved deletions and sets would result in the dirs map + // being recreated multiple times. + for uri, change := range changes { + if !change.exists { + m2.Delete(uri) + } + } + for uri, change := range changes { + if change.exists { + m2.Set(uri, change.fileHandle) + } + } return m2 } diff --git a/gopls/internal/lsp/cache/session.go b/gopls/internal/lsp/cache/session.go index e6f48118636..796e61be581 100644 --- a/gopls/internal/lsp/cache/session.go +++ b/gopls/internal/lsp/cache/session.go @@ -435,6 +435,8 @@ func (s *Session) DidModifyFiles(ctx context.Context, changes []source.FileModif checkViews := false for _, c := range changes { + // TODO(rfindley): go.work files need not be named "go.work" -- we need to + // check each view's source. if isGoMod(c.URI) || isGoWork(c.URI) { // Change, InvalidateMetadata, and UnknownFileAction actions do not cause // us to re-evaluate views. diff --git a/gopls/internal/lsp/cache/snapshot.go b/gopls/internal/lsp/cache/snapshot.go index afadd5c7cf1..1ed4fa74041 100644 --- a/gopls/internal/lsp/cache/snapshot.go +++ b/gopls/internal/lsp/cache/snapshot.go @@ -120,6 +120,7 @@ type snapshot struct { // workspacePackages contains the workspace's packages, which are loaded // when the view is created. It contains no intermediate test variants. + // TODO(rfindley): use a persistent.Map. workspacePackages map[PackageID]PackagePath // shouldLoad tracks packages that need to be reloaded, mapping a PackageID @@ -1869,6 +1870,8 @@ func (s *snapshot) clone(ctx, bgCtx context.Context, changes map[span.URI]*fileC // (e.g. "inconsistent vendoring detected"), or because // one or more modules may have moved into or out of the // vendor tree after 'go mod vendor' or 'rm -fr vendor/'. + // + // TODO(rfindley): revisit the location of this check. for uri := range changes { if inVendor(uri) && s.initializedErr != nil || strings.HasSuffix(string(uri), "/vendor/modules.txt") { @@ -1890,15 +1893,15 @@ func (s *snapshot) clone(ctx, bgCtx context.Context, changes map[span.URI]*fileC initializedErr: s.initializedErr, packages: s.packages.Clone(), activePackages: s.activePackages.Clone(), - files: s.files.Clone(), - symbolizeHandles: s.symbolizeHandles.Clone(), + files: s.files.Clone(changes), + symbolizeHandles: cloneWithout(s.symbolizeHandles, changes), workspacePackages: make(map[PackageID]PackagePath, len(s.workspacePackages)), - unloadableFiles: s.unloadableFiles.Clone(), // see the TODO for unloadableFiles below - parseModHandles: s.parseModHandles.Clone(), - parseWorkHandles: s.parseWorkHandles.Clone(), - modTidyHandles: s.modTidyHandles.Clone(), - modWhyHandles: s.modWhyHandles.Clone(), - modVulnHandles: s.modVulnHandles.Clone(), + unloadableFiles: s.unloadableFiles.Clone(), // not cloneWithout: typing in a file doesn't necessarily make it loadable + parseModHandles: cloneWithout(s.parseModHandles, changes), + parseWorkHandles: cloneWithout(s.parseWorkHandles, changes), + modTidyHandles: cloneWithout(s.modTidyHandles, changes), + modWhyHandles: cloneWithout(s.modWhyHandles, changes), + modVulnHandles: cloneWithout(s.modVulnHandles, changes), workspaceModFiles: wsModFiles, workspaceModFilesErr: wsModFilesErr, importGraph: s.importGraph, @@ -1916,12 +1919,6 @@ func (s *snapshot) clone(ctx, bgCtx context.Context, changes map[span.URI]*fileC // incref/decref operation that might destroy it prematurely.) release := result.Acquire() - // TODO(rfindley): this looks wrong. Should we clear unloadableFiles on - // changes to environment or workspace layout, or more generally on any - // metadata change? - // - // Maybe not, as major configuration changes cause a new view. - // directIDs keeps track of package IDs that have directly changed. // Note: this is not a set, it's a map from id to invalidateMetadata. directIDs := map[PackageID]bool{} @@ -1929,6 +1926,7 @@ func (s *snapshot) clone(ctx, bgCtx context.Context, changes map[span.URI]*fileC // Invalidate all package metadata if the workspace module has changed. if reinit { for k := range s.meta.metadata { + // TODO(rfindley): this seems brittle; can we just start over? directIDs[k] = true } } @@ -1939,14 +1937,6 @@ func (s *snapshot) clone(ctx, bgCtx context.Context, changes map[span.URI]*fileC anyFileAdded := false // adding a file can resolve missing dependencies for uri, change := range changes { - // Invalidate go.mod-related handles. - result.modTidyHandles.Delete(uri) - result.modWhyHandles.Delete(uri) - result.modVulnHandles.Delete(uri) - - // Invalidate handles for cached symbols. - result.symbolizeHandles.Delete(uri) - // The original FileHandle for this URI is cached on the snapshot. originalFH, _ := s.files.Get(uri) var originalOpen, newOpen bool @@ -1963,6 +1953,10 @@ func (s *snapshot) clone(ctx, bgCtx context.Context, changes map[span.URI]*fileC if strings.HasSuffix(uri.Filename(), ".go") { invalidateMetadata, pkgFileChanged, importDeleted = metadataChanges(ctx, s, originalFH, change.fileHandle) } + if invalidateMetadata { + // If this is a metadata-affecting change, perhaps a reload will succeed. + result.unloadableFiles.Remove(uri) + } invalidateMetadata = invalidateMetadata || forceReloadMetadata || reinit anyImportDeleted = anyImportDeleted || importDeleted @@ -2009,29 +2003,6 @@ func (s *snapshot) clone(ctx, bgCtx context.Context, changes map[span.URI]*fileC result.modWhyHandles.Clear() result.modVulnHandles.Clear() } - - result.parseModHandles.Delete(uri) - result.parseWorkHandles.Delete(uri) - - // Handle the invalidated file; it may have new contents or not exist. - // - // Note, we can't simply delete the file unconditionally and let it be - // re-read, as (1) the snapshot must observe all overlays, and (2) deleting - // a file forces directories to be reevaluated, as it may be the last file - // in a directory. We want to avoid that work in the common case where a - // file has simply changed. - if !change.exists { - result.files.Delete(uri) - } else { - result.files.Set(uri, change.fileHandle) - } - - // Make sure to remove the changed file from the unloadable set. - // - // TODO(rfindley): this also looks wrong, as typing in an unloadable file - // will result in repeated reloads. We should only delete if metadata - // changed. - result.unloadableFiles.Remove(uri) } // Deleting an import can cause list errors due to import cycles to be @@ -2195,6 +2166,14 @@ func (s *snapshot) clone(ctx, bgCtx context.Context, changes map[span.URI]*fileC return result, release } +func cloneWithout[V any](m *persistent.Map[span.URI, V], changes map[span.URI]*fileChange) *persistent.Map[span.URI, V] { + m2 := m.Clone() + for k := range changes { + m2.Delete(k) + } + return m2 +} + // deleteMostRelevantModFile deletes the mod file most likely to be the mod // file for the changed URI, if it exists. // From 01224cd51bc860d15015df147c57deba6c960f59 Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Wed, 6 Sep 2023 10:03:20 -0400 Subject: [PATCH 065/178] gopls/internal/lsp/cache: only evaluate fileMap.dirs when necessary There is no need to eagerly evaluate dirs during a Set. Change-Id: If225fb9e5b276848d5108c2be8a8a5f97c21a31e Reviewed-on: https://go-review.googlesource.com/c/tools/+/525996 Auto-Submit: Robert Findley Reviewed-by: Alan Donovan LUCI-TryBot-Result: Go LUCI --- gopls/internal/lsp/cache/filemap.go | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/gopls/internal/lsp/cache/filemap.go b/gopls/internal/lsp/cache/filemap.go index faf35197697..697a702a873 100644 --- a/gopls/internal/lsp/cache/filemap.go +++ b/gopls/internal/lsp/cache/filemap.go @@ -49,8 +49,7 @@ func (m *fileMap) Clone(changes map[span.URI]*fileChange) *fileMap { // common case where a file has simply changed. // // For that reason, we also do this in two passes, processing deletions - // first, as interleaved deletions and sets would result in the dirs map - // being recreated multiple times. + // first, as a set before a deletion would result in pointless work. for uri, change := range changes { if !change.exists { m2.Delete(uri) @@ -97,21 +96,12 @@ func (m *fileMap) Set(key span.URI, fh source.FileHandle) { m.overlays.Delete(key) } - // update dirs - if m.dirs == nil { - m.initDirs() - } else { + // update dirs, if they have been computed + if m.dirs != nil { m.addDirs(key) } } -func (m *fileMap) initDirs() { - m.dirs = new(persistent.Set[string]) - m.files.Range(func(u span.URI, _ source.FileHandle) { - m.addDirs(u) - }) -} - // addDirs adds all directories containing u to the dirs set. func (m *fileMap) addDirs(u span.URI) { dir := filepath.Dir(u.Filename()) @@ -152,7 +142,10 @@ func (m *fileMap) Overlays() []*Overlay { // The result must not be mutated by the caller. func (m *fileMap) Dirs() *persistent.Set[string] { if m.dirs == nil { - m.initDirs() + m.dirs = new(persistent.Set[string]) + m.files.Range(func(u span.URI, _ source.FileHandle) { + m.addDirs(u) + }) } return m.dirs } From 1889c0e11eedec4c2752282889ee039197777f75 Mon Sep 17 00:00:00 2001 From: Robert Findley Date: Wed, 6 Sep 2023 09:33:06 -0400 Subject: [PATCH 066/178] gopls/internal/lsp/cache: simplify file change propagation The fileChange type contained redundant information in its content, exists, and isUnchanged field: - content and exists may be derived from the file handle - isUnchanged implements an inaccurate heuristic, better derived from the actual delta of the change Eliminate the fileChange type in favor of just passing around FileHandles. Additionally, simplify snapshot.clone somewhat: - eliminate the unappliedChanges file source; the cloned in-progress snapshot is a valid file source - eliminate the skipID map, which appears to be redundant with pkgFileChanged detection in the main loop Change-Id: I2b22445bb4ef58fe1473315ceb47b9dceb856299 Reviewed-on: https://go-review.googlesource.com/c/tools/+/525979 Auto-Submit: Robert Findley gopls-CI: kokoro Reviewed-by: Alan Donovan LUCI-TryBot-Result: Go LUCI --- gopls/internal/lsp/cache/filemap.go | 12 +- gopls/internal/lsp/cache/fs_memoized.go | 2 +- gopls/internal/lsp/cache/fs_overlay.go | 2 +- gopls/internal/lsp/cache/session.go | 64 +++---- gopls/internal/lsp/cache/snapshot.go | 225 ++++++++++++------------ gopls/internal/lsp/cache/view.go | 10 +- gopls/internal/lsp/cache/workspace.go | 23 +-- gopls/internal/lsp/command.go | 4 +- gopls/internal/lsp/diagnostics.go | 2 +- gopls/internal/lsp/source/view.go | 7 +- 10 files changed, 161 insertions(+), 190 deletions(-) diff --git a/gopls/internal/lsp/cache/filemap.go b/gopls/internal/lsp/cache/filemap.go index 697a702a873..52b7a13ba95 100644 --- a/gopls/internal/lsp/cache/filemap.go +++ b/gopls/internal/lsp/cache/filemap.go @@ -31,7 +31,7 @@ func newFileMap() *fileMap { // Clone creates a copy of the fileMap, incorporating the changes specified by // the changes map. -func (m *fileMap) Clone(changes map[span.URI]*fileChange) *fileMap { +func (m *fileMap) Clone(changes map[span.URI]source.FileHandle) *fileMap { m2 := &fileMap{ files: m.files.Clone(), overlays: m.overlays.Clone(), @@ -50,14 +50,14 @@ func (m *fileMap) Clone(changes map[span.URI]*fileChange) *fileMap { // // For that reason, we also do this in two passes, processing deletions // first, as a set before a deletion would result in pointless work. - for uri, change := range changes { - if !change.exists { + for uri, fh := range changes { + if !fileExists(fh) { m2.Delete(uri) } } - for uri, change := range changes { - if change.exists { - m2.Set(uri, change.fileHandle) + for uri, fh := range changes { + if fileExists(fh) { + m2.Set(uri, fh) } } return m2 diff --git a/gopls/internal/lsp/cache/fs_memoized.go b/gopls/internal/lsp/cache/fs_memoized.go index 37f59e4ef26..bfc71205765 100644 --- a/gopls/internal/lsp/cache/fs_memoized.go +++ b/gopls/internal/lsp/cache/fs_memoized.go @@ -46,7 +46,7 @@ func (h *DiskFile) FileIdentity() source.FileIdentity { } } -func (h *DiskFile) Saved() bool { return true } +func (h *DiskFile) SameContentsOnDisk() bool { return true } func (h *DiskFile) Version() int32 { return 0 } func (h *DiskFile) Content() ([]byte, error) { return h.content, h.err } diff --git a/gopls/internal/lsp/cache/fs_overlay.go b/gopls/internal/lsp/cache/fs_overlay.go index 157eb8610f8..6764adda063 100644 --- a/gopls/internal/lsp/cache/fs_overlay.go +++ b/gopls/internal/lsp/cache/fs_overlay.go @@ -74,5 +74,5 @@ func (o *Overlay) FileIdentity() source.FileIdentity { func (o *Overlay) Content() ([]byte, error) { return o.content, nil } func (o *Overlay) Version() int32 { return o.version } -func (o *Overlay) Saved() bool { return o.saved } +func (o *Overlay) SameContentsOnDisk() bool { return o.saved } func (o *Overlay) Kind() source.FileKind { return o.kind } diff --git a/gopls/internal/lsp/cache/session.go b/gopls/internal/lsp/cache/session.go index 796e61be581..34644eac6bd 100644 --- a/gopls/internal/lsp/cache/session.go +++ b/gopls/internal/lsp/cache/session.go @@ -389,19 +389,6 @@ func (s *Session) ModifyFiles(ctx context.Context, changes []source.FileModifica return err } -// TODO(rfindley): fileChange seems redundant with source.FileModification. -// De-dupe into a common representation for changes. -type fileChange struct { - content []byte - exists bool - fileHandle source.FileHandle - - // isUnchanged indicates whether the file action is one that does not - // change the actual contents of the file. Opens and closes should not - // be treated like other changes, since the file content doesn't change. - isUnchanged bool -} - // DidModifyFiles reports a file modification to the session. It returns // the new snapshots after the modifications have been applied, paired with // the affected file URIs for those snapshots. @@ -482,7 +469,7 @@ func (s *Session) DidModifyFiles(ctx context.Context, changes []source.FileModif } // Collect information about views affected by these changes. - views := make(map[*View]map[span.URI]*fileChange) + views := make(map[*View]map[span.URI]source.FileHandle) affectedViews := map[span.URI][]*View{} // forceReloadMetadata records whether any change is the magic // source.InvalidateMetadata action. @@ -515,30 +502,15 @@ func (s *Session) DidModifyFiles(ctx context.Context, changes []source.FileModif } affectedViews[c.URI] = changedViews - isUnchanged := c.Action == source.Open || c.Action == source.Close - // Apply the changes to all affected views. + fh := mustReadFile(ctx, s, c.URI) for _, view := range changedViews { // Make sure that the file is added to the view's seenFiles set. view.markKnown(c.URI) if _, ok := views[view]; !ok { - views[view] = make(map[span.URI]*fileChange) - } - fh, err := s.ReadFile(ctx, c.URI) - if err != nil { - return nil, nil, err - } - content, err := fh.Content() - if err != nil { - // Ignore the error: the file may be deleted. - content = nil - } - views[view][c.URI] = &fileChange{ - content: content, - exists: err == nil, - fileHandle: fh, - isUnchanged: isUnchanged, + views[view] = make(map[span.URI]source.FileHandle) } + views[view][c.URI] = fh } } @@ -694,10 +666,7 @@ func (fs *overlayFS) updateOverlays(ctx context.Context, changes []source.FileMo } sameContentOnDisk = true default: - fh, err := fs.delegate.ReadFile(ctx, c.URI) - if err != nil { - return err - } + fh := mustReadFile(ctx, fs.delegate, c.URI) _, readErr := fh.Content() sameContentOnDisk = (readErr == nil && fh.FileIdentity().Hash == hash) } @@ -719,6 +688,29 @@ func (fs *overlayFS) updateOverlays(ctx context.Context, changes []source.FileMo return nil } +func mustReadFile(ctx context.Context, fs source.FileSource, uri span.URI) source.FileHandle { + ctx = xcontext.Detach(ctx) + fh, err := fs.ReadFile(ctx, uri) + if err != nil { + // ReadFile cannot fail with an uncancellable context. + bug.Reportf("reading file failed unexpectedly: %v", err) + return brokenFile{uri, err} + } + return fh +} + +// A brokenFile represents an unexpected failure to read a file. +type brokenFile struct { + uri span.URI + err error +} + +func (b brokenFile) URI() span.URI { return b.uri } +func (b brokenFile) FileIdentity() source.FileIdentity { return source.FileIdentity{URI: b.uri} } +func (b brokenFile) SameContentsOnDisk() bool { return false } +func (b brokenFile) Version() int32 { return 0 } +func (b brokenFile) Content() ([]byte, error) { return nil, b.err } + // FileWatchingGlobPatterns returns a new set of glob patterns to // watch every directory known by the view. For views within a module, // this is the module root, any directory in the module root, and any diff --git a/gopls/internal/lsp/cache/snapshot.go b/gopls/internal/lsp/cache/snapshot.go index 1ed4fa74041..908c426c9e0 100644 --- a/gopls/internal/lsp/cache/snapshot.go +++ b/gopls/internal/lsp/cache/snapshot.go @@ -1805,81 +1805,13 @@ func inVendor(uri span.URI) bool { return found && strings.Contains(after, "/") } -// unappliedChanges is a file source that handles an uncloned snapshot. -type unappliedChanges struct { - originalSnapshot *snapshot - changes map[span.URI]*fileChange -} - -func (ac *unappliedChanges) ReadFile(ctx context.Context, uri span.URI) (source.FileHandle, error) { - if c, ok := ac.changes[uri]; ok { - return c.fileHandle, nil - } - return ac.originalSnapshot.ReadFile(ctx, uri) -} - -func (s *snapshot) clone(ctx, bgCtx context.Context, changes map[span.URI]*fileChange, forceReloadMetadata bool) (*snapshot, func()) { +func (s *snapshot) clone(ctx, bgCtx context.Context, changes map[span.URI]source.FileHandle, forceReloadMetadata bool) (*snapshot, func()) { ctx, done := event.Start(ctx, "cache.snapshot.clone") defer done() - reinit := false - wsModFiles, wsModFilesErr := s.workspaceModFiles, s.workspaceModFilesErr - - if workURI, _ := s.view.GOWORK(); workURI != "" { - if change, ok := changes[workURI]; ok { - wsModFiles, wsModFilesErr = computeWorkspaceModFiles(ctx, s.view.gomod, workURI, s.view.effectiveGO111MODULE(), &unappliedChanges{ - originalSnapshot: s, - changes: changes, - }) - // TODO(rfindley): don't rely on 'isUnchanged' here. Use a content hash instead. - reinit = change.fileHandle.Saved() && !change.isUnchanged - } - } - - // Reinitialize if any workspace mod file has changed on disk. - for uri, change := range changes { - if _, ok := wsModFiles[uri]; ok && change.fileHandle.Saved() && !change.isUnchanged { - reinit = true - } - } - - // Finally, process sumfile changes that may affect loading. - for uri, change := range changes { - if !change.fileHandle.Saved() { - continue // like with go.mod files, we only reinit when things are saved - } - if filepath.Base(uri.Filename()) == "go.work.sum" && s.view.gowork != "" { - if filepath.Dir(uri.Filename()) == filepath.Dir(s.view.gowork) { - reinit = true - } - } - if filepath.Base(uri.Filename()) == "go.sum" { - dir := filepath.Dir(uri.Filename()) - modURI := span.URIFromPath(filepath.Join(dir, "go.mod")) - if _, active := wsModFiles[modURI]; active { - reinit = true - } - } - } - s.mu.Lock() defer s.mu.Unlock() - // Changes to vendor tree may require reinitialization, - // either because of an initialization error - // (e.g. "inconsistent vendoring detected"), or because - // one or more modules may have moved into or out of the - // vendor tree after 'go mod vendor' or 'rm -fr vendor/'. - // - // TODO(rfindley): revisit the location of this check. - for uri := range changes { - if inVendor(uri) && s.initializedErr != nil || - strings.HasSuffix(string(uri), "/vendor/modules.txt") { - reinit = true - break - } - } - bgCtx, cancel := context.WithCancel(bgCtx) result := &snapshot{ sequenceID: s.sequenceID + 1, @@ -1902,23 +1834,100 @@ func (s *snapshot) clone(ctx, bgCtx context.Context, changes map[span.URI]*fileC modTidyHandles: cloneWithout(s.modTidyHandles, changes), modWhyHandles: cloneWithout(s.modWhyHandles, changes), modVulnHandles: cloneWithout(s.modVulnHandles, changes), - workspaceModFiles: wsModFiles, - workspaceModFilesErr: wsModFilesErr, + workspaceModFiles: s.workspaceModFiles, + workspaceModFilesErr: s.workspaceModFilesErr, importGraph: s.importGraph, pkgIndex: s.pkgIndex, } + // Create a lease on the new snapshot. + // (Best to do this early in case the code below hides an + // incref/decref operation that might destroy it prematurely.) + release := result.Acquire() + + reinit := false + + // Changes to vendor tree may require reinitialization, + // either because of an initialization error + // (e.g. "inconsistent vendoring detected"), or because + // one or more modules may have moved into or out of the + // vendor tree after 'go mod vendor' or 'rm -fr vendor/'. + // + // TODO(rfindley): revisit the location of this check. + for uri := range changes { + if inVendor(uri) && s.initializedErr != nil || + strings.HasSuffix(string(uri), "/vendor/modules.txt") { + reinit = true + break + } + } + + // Collect observed file handles for changed URIs from the old snapshot, if + // they exist. Importantly, we don't call ReadFile here: consider the case + // where a file is added on disk; we don't want to read the newly added file + // into the old snapshot, as that will break our change detection below. + oldFiles := make(map[span.URI]source.FileHandle) + for uri := range changes { + if fh, ok := s.files.Get(uri); ok { + oldFiles[uri] = fh + } + } + // changedOnDisk determines if the new file handle may have changed on disk. + // It over-approximates, returning true if the new file is saved and either + // the old file wasn't saved, or the on-disk contents changed. + // + // oldFH may be nil. + changedOnDisk := func(oldFH, newFH source.FileHandle) bool { + if !newFH.SameContentsOnDisk() { + return false + } + if oe, ne := (oldFH != nil && fileExists(oldFH)), fileExists(newFH); !oe || !ne { + return oe != ne + } + return !oldFH.SameContentsOnDisk() || oldFH.FileIdentity() != newFH.FileIdentity() + } + + if workURI, _ := s.view.GOWORK(); workURI != "" { + if newFH, ok := changes[workURI]; ok { + result.workspaceModFiles, result.workspaceModFilesErr = computeWorkspaceModFiles(ctx, s.view.gomod, workURI, s.view.effectiveGO111MODULE(), result) + if changedOnDisk(oldFiles[workURI], newFH) { + reinit = true + } + } + } + + // Reinitialize if any workspace mod file has changed on disk. + for uri, newFH := range changes { + if _, ok := result.workspaceModFiles[uri]; ok && changedOnDisk(oldFiles[uri], newFH) { + reinit = true + } + } + + // Finally, process sumfile changes that may affect loading. + for uri, newFH := range changes { + if !changedOnDisk(oldFiles[uri], newFH) { + continue // like with go.mod files, we only reinit when things change on disk + } + dir, base := filepath.Split(uri.Filename()) + if base == "go.work.sum" && s.view.gowork != "" { + if dir == filepath.Dir(s.view.gowork) { + reinit = true + } + } + if base == "go.sum" { + modURI := span.URIFromPath(filepath.Join(dir, "go.mod")) + if _, active := result.workspaceModFiles[modURI]; active { + reinit = true + } + } + } + // The snapshot should be initialized if either s was uninitialized, or we've // detected a change that triggers reinitialization. if reinit { result.initialized = false } - // Create a lease on the new snapshot. - // (Best to do this early in case the code below hides an - // incref/decref operation that might destroy it prematurely.) - release := result.Acquire() - // directIDs keeps track of package IDs that have directly changed. // Note: this is not a set, it's a map from id to invalidateMetadata. directIDs := map[PackageID]bool{} @@ -1936,14 +1945,14 @@ func (s *snapshot) clone(ctx, bgCtx context.Context, changes map[span.URI]*fileC anyFileOpenedOrClosed := false // opened files affect workspace packages anyFileAdded := false // adding a file can resolve missing dependencies - for uri, change := range changes { + for uri, newFH := range changes { // The original FileHandle for this URI is cached on the snapshot. - originalFH, _ := s.files.Get(uri) - var originalOpen, newOpen bool - _, originalOpen = originalFH.(*Overlay) - _, newOpen = change.fileHandle.(*Overlay) - anyFileOpenedOrClosed = anyFileOpenedOrClosed || (originalOpen != newOpen) - anyFileAdded = anyFileAdded || (originalFH == nil && change.fileHandle != nil) + oldFH, _ := oldFiles[uri] // may be nil + _, oldOpen := oldFH.(*Overlay) + _, newOpen := newFH.(*Overlay) + + anyFileOpenedOrClosed = anyFileOpenedOrClosed || (oldOpen != newOpen) + anyFileAdded = anyFileAdded || (oldFH == nil || !fileExists(oldFH)) && fileExists(newFH) // If uri is a Go file, check if it has changed in a way that would // invalidate metadata. Note that we can't use s.view.FileKind here, @@ -1951,7 +1960,7 @@ func (s *snapshot) clone(ctx, bgCtx context.Context, changes map[span.URI]*fileC // but what the Go command sees. var invalidateMetadata, pkgFileChanged, importDeleted bool if strings.HasSuffix(uri.Filename(), ".go") { - invalidateMetadata, pkgFileChanged, importDeleted = metadataChanges(ctx, s, originalFH, change.fileHandle) + invalidateMetadata, pkgFileChanged, importDeleted = metadataChanges(ctx, s, oldFH, newFH) } if invalidateMetadata { // If this is a metadata-affecting change, perhaps a reload will succeed. @@ -1969,7 +1978,12 @@ func (s *snapshot) clone(ctx, bgCtx context.Context, changes map[span.URI]*fileC // Invalidate the previous modTidyHandle if any of the files have been // saved or if any of the metadata has been invalidated. - if invalidateMetadata || fileWasSaved(originalFH, change.fileHandle) { + // + // TODO(rfindley): this seems like too-aggressive invalidation of mod + // results. We should instead thread through overlays to the Go command + // invocation and only run this if invalidateMetadata (and perhaps then + // still do it less frequently). + if invalidateMetadata || fileWasSaved(oldFH, newFH) { // Only invalidate mod tidy results for the most relevant modfile in the // workspace. This is a potentially lossy optimization for workspaces // with many modules (such as google-cloud-go, which has 145 modules as @@ -1996,10 +2010,10 @@ func (s *snapshot) clone(ctx, bgCtx context.Context, changes map[span.URI]*fileC result.modTidyHandles.Clear() } - // TODO(rfindley): should we apply the above heuristic to mod vuln - // or mod handles as well? + // TODO(rfindley): should we apply the above heuristic to mod vuln or mod + // why handles as well? // - // TODO(rfindley): no tests fail if I delete the below line. + // TODO(rfindley): no tests fail if I delete the line below. result.modWhyHandles.Clear() result.modVulnHandles.Clear() } @@ -2079,25 +2093,6 @@ func (s *snapshot) clone(ctx, bgCtx context.Context, changes map[span.URI]*fileC result.activePackages.Delete(id) } - // If a file has been deleted, we must delete metadata for all packages - // containing that file. - // - // TODO(rfindley): why not keep invalid metadata in this case? If we - // otherwise allow operate on invalid metadata, why not continue to do so, - // skipping the missing file? - skipID := map[PackageID]bool{} - for _, c := range changes { - if c.exists { - continue - } - // The file has been deleted. - if ids, ok := s.meta.ids[c.fileHandle.URI()]; ok { - for _, id := range ids { - skipID[id] = true - } - } - } - // Any packages that need loading in s still need loading in the new // snapshot. for k, v := range s.shouldLoad { @@ -2134,10 +2129,11 @@ func (s *snapshot) clone(ctx, bgCtx context.Context, changes map[span.URI]*fileC } // Check whether the metadata should be deleted. - if skipID[k] || invalidateMetadata { + if invalidateMetadata { metadataUpdates[k] = nil continue } + } // Update metadata, if necessary. @@ -2166,7 +2162,7 @@ func (s *snapshot) clone(ctx, bgCtx context.Context, changes map[span.URI]*fileC return result, release } -func cloneWithout[V any](m *persistent.Map[span.URI, V], changes map[span.URI]*fileChange) *persistent.Map[span.URI, V] { +func cloneWithout[V any](m *persistent.Map[span.URI, V], changes map[span.URI]source.FileHandle) *persistent.Map[span.URI, V] { m2 := m.Clone() for k := range changes { m2.Delete(k) @@ -2291,9 +2287,9 @@ func fileWasSaved(originalFH, currentFH source.FileHandle) bool { // - importDeleted means that an import has been deleted, or we can't // determine if an import was deleted due to errors. func metadataChanges(ctx context.Context, lockedSnapshot *snapshot, oldFH, newFH source.FileHandle) (invalidate, pkgFileChanged, importDeleted bool) { - if oldFH == nil || newFH == nil { // existential changes - changed := (oldFH == nil) != (newFH == nil) - return changed, changed, (newFH == nil) // we don't know if an import was deleted + if oe, ne := oldFH != nil && fileExists(oldFH), fileExists(newFH); !oe || !ne { // existential changes + changed := oe != ne + return changed, changed, !ne // we don't know if an import was deleted } // If the file hasn't changed, there's no need to reload. @@ -2307,11 +2303,6 @@ func metadataChanges(ctx context.Context, lockedSnapshot *snapshot, oldFH, newFH newHeads, newErr := lockedSnapshot.view.parseCache.parseFiles(ctx, fset, source.ParseHeader, false, newFH) if oldErr != nil || newErr != nil { - // TODO(rfindley): we can get here if newFH does not exist. There is - // asymmetry, in that newFH may be non-nil even if the underlying file does - // not exist. - // - // We should not produce a non-nil filehandle for a file that does not exist. errChanged := (oldErr == nil) != (newErr == nil) return errChanged, errChanged, (newErr != nil) // we don't know if an import was deleted } diff --git a/gopls/internal/lsp/cache/view.go b/gopls/internal/lsp/cache/view.go index d7ddc31fe0e..6c3572a5e9d 100644 --- a/gopls/internal/lsp/cache/view.go +++ b/gopls/internal/lsp/cache/view.go @@ -72,7 +72,7 @@ type View struct { // fs is the file source used to populate this view. fs *overlayFS - // seenFiles tracks files that the view has accessed. + // knownFiles tracks files that the view has accessed. // TODO(golang/go#57558): this notion is fundamentally problematic, and // should be removed. knownFilesMu sync.Mutex @@ -910,7 +910,7 @@ func (s *snapshot) loadWorkspace(ctx context.Context, firstAttempt bool) (loadEr // // invalidateContent returns a non-nil snapshot for the new content, along with // a callback which the caller must invoke to release that snapshot. -func (v *View) invalidateContent(ctx context.Context, changes map[span.URI]*fileChange, forceReloadMetadata bool) (*snapshot, func()) { +func (v *View) invalidateContent(ctx context.Context, changes map[span.URI]source.FileHandle, forceReloadMetadata bool) (*snapshot, func()) { // Detach the context so that content invalidation cannot be canceled. ctx = xcontext.Detach(ctx) @@ -1052,11 +1052,11 @@ func (v *View) workingDir() span.URI { func findRootPattern(ctx context.Context, dir, basename string, fs source.FileSource) (string, error) { for dir != "" { target := filepath.Join(dir, basename) - exists, err := fileExists(ctx, span.URIFromPath(target), fs) + fh, err := fs.ReadFile(ctx, span.URIFromPath(target)) if err != nil { - return "", err // not readable or context cancelled + return "", err // context cancelled } - if exists { + if fileExists(fh) { return target, nil } // Trailing separators must be trimmed, otherwise filepath.Split is a noop. diff --git a/gopls/internal/lsp/cache/workspace.go b/gopls/internal/lsp/cache/workspace.go index e241d02b49a..e344f4950cc 100644 --- a/gopls/internal/lsp/cache/workspace.go +++ b/gopls/internal/lsp/cache/workspace.go @@ -9,7 +9,6 @@ import ( "errors" "fmt" "io/fs" - "os" "path/filepath" "strings" @@ -69,25 +68,11 @@ func isGoWork(uri span.URI) bool { return filepath.Base(uri.Filename()) == "go.work" } -// fileExists reports if the file uri exists within source. -func fileExists(ctx context.Context, uri span.URI, source source.FileSource) (bool, error) { - fh, err := source.ReadFile(ctx, uri) - if err != nil { - return false, err - } - return fileHandleExists(fh) -} - -// fileHandleExists reports if the file underlying fh actually exists. -func fileHandleExists(fh source.FileHandle) (bool, error) { +// fileExists reports whether the file has a Content (which may be empty). +// An overlay exists even if it is not reflected in the file system. +func fileExists(fh source.FileHandle) bool { _, err := fh.Content() - if err == nil { - return true, nil - } - if os.IsNotExist(err) { - return false, nil - } - return false, err + return err == nil } // errExhausted is returned by findModules if the file scan limit is reached. diff --git a/gopls/internal/lsp/command.go b/gopls/internal/lsp/command.go index 388030e4bcc..79d3a5c35d6 100644 --- a/gopls/internal/lsp/command.go +++ b/gopls/internal/lsp/command.go @@ -96,7 +96,7 @@ func (c *commandHandler) run(ctx context.Context, cfg commandConfig, run command if cfg.requireSave { var unsaved []string for _, overlay := range c.s.session.Overlays() { - if !overlay.Saved() { + if !overlay.SameContentsOnDisk() { unsaved = append(unsaved, overlay.URI().Filename()) } } @@ -1155,7 +1155,7 @@ func (c *commandHandler) RunGoWorkCommand(ctx context.Context, args command.RunG if err != nil { return fmt.Errorf("reading current go.work file: %v", err) } - if !fh.Saved() { + if !fh.SameContentsOnDisk() { return fmt.Errorf("must save workspace file %s before running go work commands", goworkURI) } } else { diff --git a/gopls/internal/lsp/diagnostics.go b/gopls/internal/lsp/diagnostics.go index 2ae50586416..4fbfd0acec3 100644 --- a/gopls/internal/lsp/diagnostics.go +++ b/gopls/internal/lsp/diagnostics.go @@ -573,7 +573,7 @@ func (s *Server) diagnosePkgs(ctx context.Context, snapshot source.Snapshot, toD fh := snapshot.FindFile(uri) // Don't publish gc details for unsaved buffers, since the underlying // logic operates on the file on disk. - if fh == nil || !fh.Saved() { + if fh == nil || !fh.SameContentsOnDisk() { continue } s.storeDiagnostics(snapshot, uri, gcDetailsSource, diags, true) diff --git a/gopls/internal/lsp/source/view.go b/gopls/internal/lsp/source/view.go index fe51cf0e5b6..8aea01f113b 100644 --- a/gopls/internal/lsp/source/view.go +++ b/gopls/internal/lsp/source/view.go @@ -409,6 +409,9 @@ type View interface { type FileSource interface { // ReadFile returns the FileHandle for a given URI, either by // reading the content of the file or by obtaining it from a cache. + // + // Invariant: ReadFile must only return an error in the case of context + // cancellation. If ctx.Err() is nil, the resulting error must also be nil. ReadFile(ctx context.Context, uri span.URI) (FileHandle, error) } @@ -768,9 +771,9 @@ type FileHandle interface { // FileIdentity returns a FileIdentity for the file, even if there was an // error reading it. FileIdentity() FileIdentity - // Saved reports whether the file has the same content on disk: + // SameContentsOnDisk reports whether the file has the same content on disk: // it is false for files open on an editor with unsaved edits. - Saved() bool + SameContentsOnDisk() bool // Version returns the file version, as defined by the LSP client. // For on-disk file handles, Version returns 0. Version() int32 From ab96daba6017a08171e3832bc677257303943eb1 Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Wed, 6 Sep 2023 14:17:20 -0400 Subject: [PATCH 067/178] gopls/internal/lsp/cache: move working dir to workspaceInformation Any change to the working dir must necessarily result in a new view (see minorOptionsChange). Therefore, it should be considered an immutable part of the view, and rename it to goCommandDir, which more closely matches its meaning (and there are various other things that could be considered "working" dirs). Also store the folder on the workspaceInformation, to move toward an immutable foundation that can be safely shared with the snapshot. Updates golang/go#57979 For golang/go#42814 Change-Id: I9a9e3aa18fa85bace827d9c8dd1607e851cfcfb8 Reviewed-on: https://go-review.googlesource.com/c/tools/+/526160 Auto-Submit: Robert Findley LUCI-TryBot-Result: Go LUCI Reviewed-by: Alan Donovan gopls-CI: kokoro --- gopls/internal/lsp/cache/imports.go | 4 +- gopls/internal/lsp/cache/load.go | 2 +- gopls/internal/lsp/cache/session.go | 1 - gopls/internal/lsp/cache/snapshot.go | 2 +- gopls/internal/lsp/cache/view.go | 62 +++++++++++++++++----------- 5 files changed, 41 insertions(+), 30 deletions(-) diff --git a/gopls/internal/lsp/cache/imports.go b/gopls/internal/lsp/cache/imports.go index 55085a2a1e0..f22d42c21a8 100644 --- a/gopls/internal/lsp/cache/imports.go +++ b/gopls/internal/lsp/cache/imports.go @@ -135,7 +135,7 @@ func populateProcessEnvFromSnapshot(ctx context.Context, pe *imports.ProcessEnv, // and has led to memory leaks in the past, when the snapshot was // unintentionally held past its lifetime. _, inv, cleanupInvocation, err := snapshot.goCommandInvocation(ctx, source.LoadWorkspace, &gocommand.Invocation{ - WorkingDir: snapshot.view.workingDir().Filename(), + WorkingDir: snapshot.view.goCommandDir.Filename(), }) if err != nil { return err @@ -154,7 +154,7 @@ func populateProcessEnvFromSnapshot(ctx context.Context, pe *imports.ProcessEnv, // We don't actually use the invocation, so clean it up now. cleanupInvocation() // TODO(rfindley): should this simply be inv.WorkingDir? - pe.WorkingDir = snapshot.view.workingDir().Filename() + pe.WorkingDir = snapshot.view.goCommandDir.Filename() return nil } diff --git a/gopls/internal/lsp/cache/load.go b/gopls/internal/lsp/cache/load.go index 03db2a35d0d..df68ba7429d 100644 --- a/gopls/internal/lsp/cache/load.go +++ b/gopls/internal/lsp/cache/load.go @@ -117,7 +117,7 @@ func (s *snapshot) load(ctx context.Context, allowNetwork bool, scopes ...loadSc flags |= source.AllowNetwork } _, inv, cleanup, err := s.goCommandInvocation(ctx, flags, &gocommand.Invocation{ - WorkingDir: s.view.workingDir().Filename(), + WorkingDir: s.view.goCommandDir.Filename(), }) if err != nil { return err diff --git a/gopls/internal/lsp/cache/session.go b/gopls/internal/lsp/cache/session.go index 34644eac6bd..af5190ba1f4 100644 --- a/gopls/internal/lsp/cache/session.go +++ b/gopls/internal/lsp/cache/session.go @@ -139,7 +139,6 @@ func (s *Session) createView(ctx context.Context, name string, folder span.URI, options: options, baseCtx: baseCtx, name: name, - folder: folder, moduleUpgrades: map[span.URI]map[string]string{}, vulns: map[span.URI]*govulncheck.Result{}, parseCache: s.parseCache, diff --git a/gopls/internal/lsp/cache/snapshot.go b/gopls/internal/lsp/cache/snapshot.go index 908c426c9e0..4010c985747 100644 --- a/gopls/internal/lsp/cache/snapshot.go +++ b/gopls/internal/lsp/cache/snapshot.go @@ -969,7 +969,7 @@ func (s *snapshot) workspaceDirs(ctx context.Context) []string { dirSet := make(map[string]unit) // Dirs should, at the very least, contain the working directory and folder. - dirSet[s.view.workingDir().Filename()] = unit{} + dirSet[s.view.goCommandDir.Filename()] = unit{} dirSet[s.view.folder.Filename()] = unit{} // Additionally, if e.g. go.work indicates other workspace modules, we should diff --git a/gopls/internal/lsp/cache/view.go b/gopls/internal/lsp/cache/view.go index 6c3572a5e9d..a1c7e6bab6a 100644 --- a/gopls/internal/lsp/cache/view.go +++ b/gopls/internal/lsp/cache/view.go @@ -52,8 +52,7 @@ type View struct { // Workspace information. The fields below are immutable, and together with // options define the build list. Any change to these fields results in a new // View. - folder span.URI // user-specified workspace folder - workspaceInformation // Go environment information + workspaceInformation // Go environment information importsState *importsState @@ -114,6 +113,9 @@ type View struct { // // This type is compared to see if the View needs to be reconstructed. type workspaceInformation struct { + // folder is the LSP workspace folder. + folder span.URI + // `go env` variables that need to be tracked by gopls. goEnv @@ -136,6 +138,13 @@ type workspaceInformation struct { // inGOPATH reports whether the workspace directory is contained in a GOPATH // directory. inGOPATH bool + + // goCommandDir is the dir to use for running go commands. + // + // The only case where this should matter is if we've narrowed the workspace to + // a single nested module. In that case, the go command won't be able to find + // the module unless we tell it the nested directory. + goCommandDir span.URI } // effectiveGO111MODULE reports the value of GO111MODULE effective in the go @@ -441,6 +450,11 @@ func (v *View) FileKind(fh source.FileHandle) source.FileKind { } func minorOptionsChange(a, b *source.Options) bool { + // TODO(rfindley): this function detects whether a view should be recreated, + // but this is also checked by the getWorkspaceInformation logic. + // + // We should eliminate this redundancy. + // Check if any of the settings that modify our understanding of files have // been changed. if !reflect.DeepEqual(a.Env, b.Env) { @@ -501,7 +515,7 @@ func viewEnv(v *View) string { (selected go env: %v) `, v.folder.Filename(), - v.workingDir().Filename(), + v.goCommandDir.Filename(), strings.TrimRight(v.workspaceInformation.goversionOutput, "\n"), v.snapshot.validBuildConfiguration(), buildFlags, @@ -586,8 +600,8 @@ func (v *View) contains(uri span.URI) bool { // user. It would be better to explicitly consider the set of active modules // wherever relevant. inGoDir := false - if source.InDir(v.workingDir().Filename(), v.folder.Filename()) { - inGoDir = source.InDir(v.workingDir().Filename(), uri.Filename()) + if source.InDir(v.goCommandDir.Filename(), v.folder.Filename()) { + inGoDir = source.InDir(v.goCommandDir.Filename(), uri.Filename()) } inFolder := source.InDir(v.folder.Filename(), uri.Filename()) @@ -946,7 +960,9 @@ func (s *Session) getWorkspaceInformation(ctx context.Context, folder span.URI, return workspaceInformation{}, fmt.Errorf("invalid workspace folder path: %w; check that the casing of the configured workspace folder path agrees with the casing reported by the operating system", err) } var err error - var info workspaceInformation + info := workspaceInformation{ + folder: folder, + } inv := gocommand.Invocation{ WorkingDir: folder.Filename(), Env: options.EnvSlice(), @@ -984,6 +1000,20 @@ func (s *Session) getWorkspaceInformation(ctx context.Context, folder span.URI, break } } + + // Compute the "working directory", which is where we run go commands. + // + // Note: if gowork is in use, this will default to the workspace folder. In + // the past, we would instead use the folder containing go.work. This should + // not make a difference, and in fact may improve go list error messages. + // + // TODO(golang/go#57514): eliminate the expandWorkspaceToModule setting + // entirely. + if options.ExpandWorkspaceToModule && info.gomod != "" { + info.goCommandDir = span.URIFromPath(filepath.Dir(info.gomod.Filename())) + } else { + info.goCommandDir = folder + } return info, nil } @@ -1025,24 +1055,6 @@ func findWorkspaceModFile(ctx context.Context, folderURI span.URI, fs source.Fil return "", nil } -// workingDir returns the directory from which to run Go commands. -// -// The only case where this should matter is if we've narrowed the workspace to -// a singular nested module. In that case, the go command won't be able to find -// the module unless we tell it the nested directory. -func (v *View) workingDir() span.URI { - // Note: if gowork is in use, this will default to the workspace folder. In - // the past, we would instead use the folder containing go.work. This should - // not make a difference, and in fact may improve go list error messages. - // - // TODO(golang/go#57514): eliminate the expandWorkspaceToModule setting - // entirely. - if v.Options().ExpandWorkspaceToModule && v.gomod != "" { - return span.URIFromPath(filepath.Dir(v.gomod.Filename())) - } - return v.folder -} - // findRootPattern looks for files with the given basename in dir or any parent // directory of dir, using the provided FileSource. It returns the first match, // starting from dir and search parents. @@ -1231,7 +1243,7 @@ func (s *snapshot) vendorEnabled(ctx context.Context, modURI span.URI, modConten // No vendor directory? // TODO(golang/go#57514): this is wrong if the working dir is not the module // root. - if fi, err := os.Stat(filepath.Join(s.view.workingDir().Filename(), "vendor")); err != nil || !fi.IsDir() { + if fi, err := os.Stat(filepath.Join(s.view.goCommandDir.Filename(), "vendor")); err != nil || !fi.IsDir() { return false, nil } From c6e442187054852685f5bccaf1a2e22d53a27bcd Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Wed, 6 Sep 2023 15:50:54 -0400 Subject: [PATCH 068/178] gopls/internal/lsp/tests: simplify options Inline EnableAllInlayHints into DefaultOptions and eliminate the apparently unnecessary EnableAllAnalyzers. Change-Id: I8a2587a2a27b275ad9b5d5e683e7baa7496938a5 Reviewed-on: https://go-review.googlesource.com/c/tools/+/526415 Reviewed-by: Peter Weinberger Reviewed-by: Alan Donovan Auto-Submit: Robert Findley LUCI-TryBot-Result: Go LUCI --- gopls/internal/lsp/lsp_test.go | 8 ------- gopls/internal/lsp/tests/tests.go | 8 +++++++ gopls/internal/lsp/tests/util.go | 35 ------------------------------- 3 files changed, 8 insertions(+), 43 deletions(-) diff --git a/gopls/internal/lsp/lsp_test.go b/gopls/internal/lsp/lsp_test.go index d7e1e33a8e0..0d99ebe0396 100644 --- a/gopls/internal/lsp/lsp_test.go +++ b/gopls/internal/lsp/lsp_test.go @@ -63,14 +63,6 @@ func testLSP(t *testing.T, datum *tests.Data) { defer session.RemoveView(view) - // Enable type error analyses for tests. - // TODO(golang/go#38212): Delete this once they are enabled by default. - tests.EnableAllAnalyzers(options) - session.SetViewOptions(ctx, view, options) - - // Enable all inlay hints for tests. - tests.EnableAllInlayHints(options) - // Only run the -modfile specific tests in module mode with Go 1.14 or above. datum.ModfileFlagAvailable = len(snapshot.ModFiles()) > 0 && testenv.Go1Point() >= 14 release() diff --git a/gopls/internal/lsp/tests/tests.go b/gopls/internal/lsp/tests/tests.go index 4f5fc3c5080..40ab900adca 100644 --- a/gopls/internal/lsp/tests/tests.go +++ b/gopls/internal/lsp/tests/tests.go @@ -248,6 +248,14 @@ func DefaultOptions(o *source.Options) { o.HierarchicalDocumentSymbolSupport = true o.SemanticTokens = true o.InternalOptions.NewDiff = "new" + + // Enable all inlay hints. + if o.Hints == nil { + o.Hints = make(map[string]bool) + } + for name := range source.AllInlayHints { + o.Hints[name] = true + } } func RunTests(t *testing.T, dataDir string, includeMultiModule bool, f func(*testing.T, *Data)) { diff --git a/gopls/internal/lsp/tests/util.go b/gopls/internal/lsp/tests/util.go index a4bfaa0152a..99a6393280e 100644 --- a/gopls/internal/lsp/tests/util.go +++ b/gopls/internal/lsp/tests/util.go @@ -441,38 +441,3 @@ func summarizeCompletionItems(i int, want, got []protocol.CompletionItem, reason } return msg.String() } - -func EnableAllAnalyzers(opts *source.Options) { - if opts.Analyses == nil { - opts.Analyses = make(map[string]bool) - } - for _, a := range opts.DefaultAnalyzers { - if !a.IsEnabled(opts) { - opts.Analyses[a.Analyzer.Name] = true - } - } - for _, a := range opts.TypeErrorAnalyzers { - if !a.IsEnabled(opts) { - opts.Analyses[a.Analyzer.Name] = true - } - } - for _, a := range opts.ConvenienceAnalyzers { - if !a.IsEnabled(opts) { - opts.Analyses[a.Analyzer.Name] = true - } - } - for _, a := range opts.StaticcheckAnalyzers { - if !a.IsEnabled(opts) { - opts.Analyses[a.Analyzer.Name] = true - } - } -} - -func EnableAllInlayHints(opts *source.Options) { - if opts.Hints == nil { - opts.Hints = make(map[string]bool) - } - for name := range source.AllInlayHints { - opts.Hints[name] = true - } -} From b0cdf010371e2581ac99bdfc23bd68e64b09d1a8 Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Wed, 6 Sep 2023 15:53:35 -0400 Subject: [PATCH 069/178] gopls/internal/lsp/tests: eliminate the go1.18 summary file We can now assume a Go version >= 1.18. Change-Id: I1faa4f9051f55a62fb2f4971f2006ca8f2dec494 Reviewed-on: https://go-review.googlesource.com/c/tools/+/526416 Reviewed-by: Alan Donovan LUCI-TryBot-Result: Go LUCI Auto-Submit: Robert Findley --- .../internal/lsp/testdata/summary.txt.golden | 12 ++++----- .../lsp/testdata/summary_go1.18.txt.golden | 25 ------------------- gopls/internal/lsp/tests/tests.go | 2 -- 3 files changed, 6 insertions(+), 33 deletions(-) delete mode 100644 gopls/internal/lsp/testdata/summary_go1.18.txt.golden diff --git a/gopls/internal/lsp/testdata/summary.txt.golden b/gopls/internal/lsp/testdata/summary.txt.golden index 4e6c3a08cdc..7375b821e69 100644 --- a/gopls/internal/lsp/testdata/summary.txt.golden +++ b/gopls/internal/lsp/testdata/summary.txt.golden @@ -1,23 +1,23 @@ -- summary -- CallHierarchyCount = 2 CodeLensCount = 5 -CompletionsCount = 263 -CompletionSnippetCount = 106 +CompletionsCount = 264 +CompletionSnippetCount = 115 UnimportedCompletionsCount = 5 DeepCompletionsCount = 5 FuzzyCompletionsCount = 8 -RankedCompletionsCount = 164 +RankedCompletionsCount = 174 CaseSensitiveCompletionsCount = 4 DiagnosticsCount = 23 FoldingRangesCount = 2 SemanticTokenCount = 3 -SuggestedFixCount = 74 +SuggestedFixCount = 80 MethodExtractionCount = 8 DefinitionsCount = 46 TypeDefinitionsCount = 18 HighlightsCount = 70 -InlayHintsCount = 4 -RenamesCount = 41 +InlayHintsCount = 5 +RenamesCount = 48 PrepareRenamesCount = 7 SignaturesCount = 33 LinksCount = 7 diff --git a/gopls/internal/lsp/testdata/summary_go1.18.txt.golden b/gopls/internal/lsp/testdata/summary_go1.18.txt.golden deleted file mode 100644 index 7375b821e69..00000000000 --- a/gopls/internal/lsp/testdata/summary_go1.18.txt.golden +++ /dev/null @@ -1,25 +0,0 @@ --- summary -- -CallHierarchyCount = 2 -CodeLensCount = 5 -CompletionsCount = 264 -CompletionSnippetCount = 115 -UnimportedCompletionsCount = 5 -DeepCompletionsCount = 5 -FuzzyCompletionsCount = 8 -RankedCompletionsCount = 174 -CaseSensitiveCompletionsCount = 4 -DiagnosticsCount = 23 -FoldingRangesCount = 2 -SemanticTokenCount = 3 -SuggestedFixCount = 80 -MethodExtractionCount = 8 -DefinitionsCount = 46 -TypeDefinitionsCount = 18 -HighlightsCount = 70 -InlayHintsCount = 5 -RenamesCount = 48 -PrepareRenamesCount = 7 -SignaturesCount = 33 -LinksCount = 7 -SelectionRangesCount = 3 - diff --git a/gopls/internal/lsp/tests/tests.go b/gopls/internal/lsp/tests/tests.go index 40ab900adca..41c9bad4aaf 100644 --- a/gopls/internal/lsp/tests/tests.go +++ b/gopls/internal/lsp/tests/tests.go @@ -56,8 +56,6 @@ var summaryFile = "summary.txt" func init() { if testenv.Go1Point() >= 21 { summaryFile = "summary_go1.21.txt" - } else if testenv.Go1Point() >= 18 { - summaryFile = "summary_go1.18.txt" } } From d06e891b039752aa256bd095cbd2ec323ed0e37a Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Wed, 6 Sep 2023 16:34:22 -0400 Subject: [PATCH 070/178] gopls/internal/regtest/marker: support folding ranges Port the foldingRange tests, using a different model for golden output. Also leave in a partial change to expose "unconsumed" notes to marker funcs. A follow-up change may use this for whole-file marker tests (if not, I'll delete the extraNotes field). For golang/go#54845 Change-Id: I5da1cc7eaeec44f4befb74445f1028394e255ba0 Reviewed-on: https://go-review.googlesource.com/c/tools/+/526795 Auto-Submit: Robert Findley TryBot-Result: Gopher Robot Commit-Queue: Robert Findley gopls-CI: kokoro Run-TryBot: Robert Findley Reviewed-by: Alan Donovan --- gopls/internal/lsp/fake/editor.go | 23 +- gopls/internal/lsp/lsp_test.go | 139 ---- gopls/internal/lsp/regtest/marker.go | 149 +++- gopls/internal/lsp/testdata/folding/a.go | 75 -- .../internal/lsp/testdata/folding/a.go.golden | 722 ------------------ .../lsp/testdata/folding/bad.go.golden | 81 -- gopls/internal/lsp/testdata/folding/bad.go.in | 18 - .../internal/lsp/testdata/summary.txt.golden | 1 - .../lsp/testdata/summary_go1.21.txt.golden | 1 - gopls/internal/lsp/tests/tests.go | 19 - .../marker/testdata/foldingrange/a.txt | 154 ++++ .../testdata/foldingrange/a_lineonly.txt | 163 ++++ .../marker/testdata/foldingrange/bad.txt | 41 + 13 files changed, 487 insertions(+), 1099 deletions(-) delete mode 100644 gopls/internal/lsp/testdata/folding/a.go delete mode 100644 gopls/internal/lsp/testdata/folding/a.go.golden delete mode 100644 gopls/internal/lsp/testdata/folding/bad.go.golden delete mode 100644 gopls/internal/lsp/testdata/folding/bad.go.in create mode 100644 gopls/internal/regtest/marker/testdata/foldingrange/a.txt create mode 100644 gopls/internal/regtest/marker/testdata/foldingrange/a_lineonly.txt create mode 100644 gopls/internal/regtest/marker/testdata/foldingrange/bad.txt diff --git a/gopls/internal/lsp/fake/editor.go b/gopls/internal/lsp/fake/editor.go index b6e507c291c..6fd46c3f133 100644 --- a/gopls/internal/lsp/fake/editor.go +++ b/gopls/internal/lsp/fake/editor.go @@ -7,6 +7,7 @@ package fake import ( "bytes" "context" + "encoding/json" "errors" "fmt" "os" @@ -109,6 +110,13 @@ type EditorConfig struct { // Settings holds user-provided configuration for the LSP server. Settings map[string]interface{} + + // CapabilitiesJSON holds JSON client capabilities to overlay over the + // editor's default client capabilities. + // + // Specifically, this JSON string will be unmarshalled into the editor's + // client capabilities struct, before sending to the server. + CapabilitiesJSON []byte } // NewEditor creates a new Editor. @@ -249,15 +257,13 @@ func (e *Editor) initialize(ctx context.Context) error { } params.InitializationOptions = makeSettings(e.sandbox, config) params.WorkspaceFolders = makeWorkspaceFolders(e.sandbox, config.WorkspaceFolders) + + // Set various client capabilities that are sought by gopls. params.Capabilities.Workspace.Configuration = true // support workspace/configuration params.Capabilities.Window.WorkDoneProgress = true // support window/workDoneProgress - - // TODO(rfindley): set client capabilities (note from the future: why?) - params.Capabilities.TextDocument.Completion.CompletionItem.TagSupport.ValueSet = []protocol.CompletionItemTag{protocol.ComplDeprecated} params.Capabilities.TextDocument.Completion.CompletionItem.SnippetSupport = true params.Capabilities.TextDocument.SemanticTokens.Requests.Full.Value = true - // copied from lsp/semantic.go to avoid import cycle in tests params.Capabilities.TextDocument.SemanticTokens.TokenTypes = []string{ "namespace", "type", "class", "enum", "interface", "struct", "typeParameter", "parameter", "variable", "property", "enumMember", @@ -268,14 +274,11 @@ func (e *Editor) initialize(ctx context.Context) error { "declaration", "definition", "readonly", "static", "deprecated", "abstract", "async", "modification", "documentation", "defaultLibrary", } - // The LSP tests have historically enabled this flag, // but really we should test both ways for older editors. params.Capabilities.TextDocument.DocumentSymbol.HierarchicalDocumentSymbolSupport = true - // Glob pattern watching is enabled. params.Capabilities.Workspace.DidChangeWatchedFiles.DynamicRegistration = true - // "rename" operations are used for package renaming. // // TODO(rfindley): add support for other resource operations (create, delete, ...) @@ -284,6 +287,12 @@ func (e *Editor) initialize(ctx context.Context) error { "rename", }, } + // Apply capabilities overlay. + if config.CapabilitiesJSON != nil { + if err := json.Unmarshal(config.CapabilitiesJSON, ¶ms.Capabilities); err != nil { + return fmt.Errorf("unmarshalling EditorConfig.CapabilitiesJSON: %v", err) + } + } trace := protocol.TraceValues("messages") params.Trace = &trace diff --git a/gopls/internal/lsp/lsp_test.go b/gopls/internal/lsp/lsp_test.go index 0d99ebe0396..40ebb9d636b 100644 --- a/gopls/internal/lsp/lsp_test.go +++ b/gopls/internal/lsp/lsp_test.go @@ -236,145 +236,6 @@ func (r *runner) Diagnostics(t *testing.T, uri span.URI, want []*source.Diagnost tests.CompareDiagnostics(t, uri, want, r.diagnostics[uri]) } -func (r *runner) FoldingRanges(t *testing.T, spn span.Span) { - uri := spn.URI() - view, err := r.server.session.ViewOf(uri) - if err != nil { - t.Fatal(err) - } - original := view.Options() - modified := original - defer r.server.session.SetViewOptions(r.ctx, view, original) - - for _, test := range []struct { - lineFoldingOnly bool - prefix string - }{ - {false, "foldingRange"}, - {true, "foldingRange-lineFolding"}, - } { - modified.LineFoldingOnly = test.lineFoldingOnly - view, err = r.server.session.SetViewOptions(r.ctx, view, modified) - if err != nil { - t.Error(err) - continue - } - ranges, err := r.server.FoldingRange(r.ctx, &protocol.FoldingRangeParams{ - TextDocument: protocol.TextDocumentIdentifier{ - URI: protocol.URIFromSpanURI(uri), - }, - }) - if err != nil { - t.Error(err) - continue - } - r.foldingRanges(t, test.prefix, uri, ranges) - } -} - -func (r *runner) foldingRanges(t *testing.T, prefix string, uri span.URI, ranges []protocol.FoldingRange) { - m, err := r.data.Mapper(uri) - if err != nil { - t.Fatal(err) - } - // Fold all ranges. - nonOverlapping := nonOverlappingRanges(ranges) - for i, rngs := range nonOverlapping { - got, err := foldRanges(m, string(m.Content), rngs) - if err != nil { - t.Error(err) - continue - } - tag := fmt.Sprintf("%s-%d", prefix, i) - want := string(r.data.Golden(t, tag, uri.Filename(), func() ([]byte, error) { - return []byte(got), nil - })) - - if want != got { - t.Errorf("%s: foldingRanges failed for %s, expected:\n%v\ngot:\n%v", tag, uri.Filename(), want, got) - } - } - - // Filter by kind. - kinds := []protocol.FoldingRangeKind{protocol.Imports, protocol.Comment} - for _, kind := range kinds { - var kindOnly []protocol.FoldingRange - for _, fRng := range ranges { - if fRng.Kind == string(kind) { - kindOnly = append(kindOnly, fRng) - } - } - - nonOverlapping := nonOverlappingRanges(kindOnly) - for i, rngs := range nonOverlapping { - got, err := foldRanges(m, string(m.Content), rngs) - if err != nil { - t.Error(err) - continue - } - tag := fmt.Sprintf("%s-%s-%d", prefix, kind, i) - want := string(r.data.Golden(t, tag, uri.Filename(), func() ([]byte, error) { - return []byte(got), nil - })) - - if want != got { - t.Errorf("%s: foldingRanges failed for %s, expected:\n%v\ngot:\n%v", tag, uri.Filename(), want, got) - } - } - - } -} - -func nonOverlappingRanges(ranges []protocol.FoldingRange) (res [][]protocol.FoldingRange) { - for _, fRng := range ranges { - setNum := len(res) - for i := 0; i < len(res); i++ { - canInsert := true - for _, rng := range res[i] { - if conflict(rng, fRng) { - canInsert = false - break - } - } - if canInsert { - setNum = i - break - } - } - if setNum == len(res) { - res = append(res, []protocol.FoldingRange{}) - } - res[setNum] = append(res[setNum], fRng) - } - return res -} - -func conflict(a, b protocol.FoldingRange) bool { - // a start position is <= b start positions - return (a.StartLine < b.StartLine || (a.StartLine == b.StartLine && a.StartCharacter <= b.StartCharacter)) && - (a.EndLine > b.StartLine || (a.EndLine == b.StartLine && a.EndCharacter > b.StartCharacter)) -} - -func foldRanges(m *protocol.Mapper, contents string, ranges []protocol.FoldingRange) (string, error) { - foldedText := "<>" - res := contents - // Apply the edits from the end of the file forward - // to preserve the offsets - // TODO(adonovan): factor to use diff.ApplyEdits, which validates the input. - for i := len(ranges) - 1; i >= 0; i-- { - r := ranges[i] - start, end, err := m.RangeOffsets(protocol.Range{ - Start: protocol.Position{Line: r.StartLine, Character: r.StartCharacter}, - End: protocol.Position{Line: r.EndLine, Character: r.EndCharacter}, - }) - if err != nil { - return "", err - } - res = res[:start] + foldedText + res[end:] - } - return res, nil -} - func (r *runner) SemanticTokens(t *testing.T, spn span.Span) { uri := spn.URI() filename := uri.Filename() diff --git a/gopls/internal/lsp/regtest/marker.go b/gopls/internal/lsp/regtest/marker.go index 36dcda39647..927eb072d5a 100644 --- a/gopls/internal/lsp/regtest/marker.go +++ b/gopls/internal/lsp/regtest/marker.go @@ -115,6 +115,10 @@ var update = flag.Bool("update", false, "if set, update test data during marker // TODO(rfindley): support flag values containing whitespace. // - "settings.json": this file is parsed as JSON, and used as the // session configuration (see gopls/doc/settings.md) +// - "capabilities.json": this file is parsed as JSON client capabilities, +// and applied as an overlay over the default editor client capabilities. +// see https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#clientCapabilities +// for more details. // - "env": this file is parsed as a list of VAR=VALUE fields specifying the // editor environment. // - Golden files: Within the archive, file names starting with '@' are @@ -161,6 +165,11 @@ var update = flag.Bool("update", false, "if set, update test data during marker // - def(src, dst location): perform a textDocument/definition request at // the src location, and check the result points to the dst location. // +// - foldingrange(golden): perform a textDocument/foldingRange for the +// current document, and compare with the golden content, which is the +// original source annotated with numbered tags delimiting the resulting +// ranges (e.g. <1 kind="..."> ... ). +// // - format(golden): perform a textDocument/format request for the enclosing // file, and compare against the named golden file. If the formatting // request succeeds, the golden file must contain the resulting formatted @@ -319,7 +328,6 @@ var update = flag.Bool("update", false, "if set, update test data during marker // - FuzzyCompletions // - CaseSensitiveCompletions // - RankCompletions -// - FoldingRanges // - Formats // - Imports // - SemanticTokens @@ -371,8 +379,9 @@ func RunMarkerTests(t *testing.T, dir string) { testenv.NeedsTool(t, "cgo") } config := fake.EditorConfig{ - Settings: test.settings, - Env: test.env, + Settings: test.settings, + CapabilitiesJSON: test.capabilities, + Env: test.env, } if _, ok := config.Settings["diagnosticsDelay"]; !ok { if config.Settings == nil { @@ -380,12 +389,15 @@ func RunMarkerTests(t *testing.T, dir string) { } config.Settings["diagnosticsDelay"] = "10ms" } + // inv: config.Settings != nil run := &markerTestRun{ - test: test, - env: newEnv(t, cache, test.files, test.proxyFiles, test.writeGoSum, config), - locations: make(map[expect.Identifier]protocol.Location), - diags: make(map[protocol.Location][]protocol.Diagnostic), + test: test, + env: newEnv(t, cache, test.files, test.proxyFiles, test.writeGoSum, config), + settings: config.Settings, + locations: make(map[expect.Identifier]protocol.Location), + diags: make(map[protocol.Location][]protocol.Diagnostic), + extraNotes: make(map[protocol.DocumentURI]map[string][]*expect.Note), } // TODO(rfindley): make it easier to clean up the regtest environment. defer run.env.Editor.Shutdown(context.Background()) // ignore error @@ -403,9 +415,19 @@ func RunMarkerTests(t *testing.T, dir string) { // Pre-process locations. var markers []marker for _, note := range test.notes { - mark := marker{run: run, note: note} + fn, ok := markerFuncs[note.Name] + if !ok { + // TODO(rfindley): simplify these deeply nested APIs. + uri := run.env.Sandbox.Workdir.URI(run.test.fset.File(note.Pos).Name()) + if run.extraNotes[uri] == nil { + run.extraNotes[uri] = make(map[string][]*expect.Note) + } + run.extraNotes[uri][note.Name] = append(run.extraNotes[uri][note.Name], note) + continue + } + mark := marker{run: run, note: note, fn: fn} switch note.Name { - case "loc": + case "loc": // as a special case, locations are collected before other markers mark.execute() default: markers = append(markers, mark) @@ -442,6 +464,15 @@ func RunMarkerTests(t *testing.T, dir string) { } } + // TODO(rfindley): use these for whole-file marker tests. + for uri, extras := range run.extraNotes { + for name, extra := range extras { + if len(extra) > 0 { + t.Errorf("%s: %d unused %q markers", run.env.Sandbox.Workdir.URIToPath(uri), len(extra), name) + } + } + } + formatted, err := formatTest(test) if err != nil { t.Errorf("formatTest: %v", err) @@ -478,6 +509,7 @@ func RunMarkerTests(t *testing.T, dir string) { type marker struct { run *markerTestRun note *expect.Note + fn markerFunc } // server returns the LSP server for the marker test run. @@ -499,26 +531,20 @@ func (mark marker) errorf(format string, args ...interface{}) { // execute invokes the marker's function with the arguments from note. func (mark marker) execute() { - fn, ok := markerFuncs[mark.note.Name] - if !ok { - mark.errorf("no marker function named %s", mark.note.Name) - return - } - // The first converter corresponds to the *Env argument. // All others must be converted from the marker syntax. args := []reflect.Value{reflect.ValueOf(mark)} var convert converter for i, in := range mark.note.Args { - if i < len(fn.converters) { - convert = fn.converters[i] - } else if !fn.variadic { + if i < len(mark.fn.converters) { + convert = mark.fn.converters[i] + } else if !mark.fn.variadic { goto arity // too many args } // Special handling for the blank identifier: treat it as the zero value. if ident, ok := in.(expect.Identifier); ok && ident == "_" { - zero := reflect.Zero(fn.paramTypes[i]) + zero := reflect.Zero(mark.fn.paramTypes[i]) args = append(args, zero) continue } @@ -530,16 +556,16 @@ func (mark marker) execute() { } args = append(args, reflect.ValueOf(out)) } - if len(args) < len(fn.converters) { + if len(args) < len(mark.fn.converters) { goto arity // too few args } - fn.fn.Call(args) + mark.fn.fn.Call(args) return arity: mark.errorf("got %d arguments to %s, want %d", - len(mark.note.Args), mark.note.Name, len(fn.converters)) + len(mark.note.Args), mark.note.Name, len(mark.fn.converters)) } // Supported marker functions. @@ -556,8 +582,9 @@ var markerFuncs = map[string]markerFunc{ "complete": makeMarkerFunc(completeMarker), "def": makeMarkerFunc(defMarker), "diag": makeMarkerFunc(diagMarker), - "hover": makeMarkerFunc(hoverMarker), + "foldingrange": makeMarkerFunc(foldingRangeMarker), "format": makeMarkerFunc(formatMarker), + "hover": makeMarkerFunc(hoverMarker), "implementation": makeMarkerFunc(implementationMarker), "loc": makeMarkerFunc(locMarker), "rename": makeMarkerFunc(renameMarker), @@ -573,16 +600,17 @@ var markerFuncs = map[string]markerFunc{ // See the documentation for RunMarkerTests for more information on the archive // format. type markerTest struct { - name string // relative path to the txtar file in the testdata dir - fset *token.FileSet // fileset used for parsing notes - content []byte // raw test content - archive *txtar.Archive // original test archive - settings map[string]interface{} // gopls settings - env map[string]string // editor environment - proxyFiles map[string][]byte // proxy content - files map[string][]byte // data files from the archive (excluding special files) - notes []*expect.Note // extracted notes from data files - golden map[string]*Golden // extracted golden content, by identifier name + name string // relative path to the txtar file in the testdata dir + fset *token.FileSet // fileset used for parsing notes + content []byte // raw test content + archive *txtar.Archive // original test archive + settings map[string]interface{} // gopls settings + capabilities []byte // content of capabilities.json file + env map[string]string // editor environment + proxyFiles map[string][]byte // proxy content + files map[string][]byte // data files from the archive (excluding special files) + notes []*expect.Note // extracted notes from data files + golden map[string]*Golden // extracted golden content, by identifier name skipReason string // the skip reason extracted from the "skip" archive file flags []string // flags extracted from the special "flags" archive file. @@ -742,6 +770,9 @@ func loadMarkerTest(name string, content []byte) (*markerTest, error) { return nil, err } + case file.Name == "capabilities.json": + test.capabilities = file.Data // lazily unmarshalled by the editor + case file.Name == "env": test.env = make(map[string]string) fields := strings.Fields(string(file.Data)) @@ -818,7 +849,7 @@ func formatTest(test *markerTest) ([]byte, error) { switch file.Name { // Preserve configuration files exactly as they were. They must have parsed // if we got this far. - case "skip", "flags", "settings.json", "env": + case "skip", "flags", "settings.json", "capabilities.json", "env": arch.Files = append(arch.Files, file) default: if _, ok := test.files[file.Name]; ok { // ordinary file @@ -903,13 +934,18 @@ type markerFunc struct { // A markerTestRun holds the state of one run of a marker test archive. type markerTestRun struct { - test *markerTest - env *Env + test *markerTest + env *Env + settings map[string]interface{} // Collected information. // Each @diag/@suggestedfix marker eliminates an entry from diags. locations map[expect.Identifier]protocol.Location diags map[protocol.Location][]protocol.Diagnostic // diagnostics by position; location end == start + + // Notes that weren't consumed by a marker. + // TODO(rfindley): use this for markers that must collect related notes, or delete it. + extraNotes map[protocol.DocumentURI]map[string][]*expect.Note } // sprintf returns a formatted string after applying pre-processing to @@ -1338,6 +1374,47 @@ func defMarker(mark marker, src, dst protocol.Location) { } } +func foldingRangeMarker(mark marker, g *Golden) { + env := mark.run.env + ranges, err := mark.server().FoldingRange(env.Ctx, &protocol.FoldingRangeParams{ + TextDocument: protocol.TextDocumentIdentifier{URI: mark.uri()}, + }) + if err != nil { + mark.errorf("foldingRange failed: %v", err) + return + } + var edits []protocol.TextEdit + insert := func(line, char uint32, text string) { + pos := protocol.Position{Line: line, Character: char} + edits = append(edits, protocol.TextEdit{ + Range: protocol.Range{ + Start: pos, + End: pos, + }, + NewText: text, + }) + } + for i, rng := range ranges { + insert(rng.StartLine, rng.StartCharacter, fmt.Sprintf("<%d kind=%q>", i, rng.Kind)) + insert(rng.EndLine, rng.EndCharacter, fmt.Sprintf("", i)) + } + filename := env.Sandbox.Workdir.URIToPath(mark.uri()) + mapper, err := env.Editor.Mapper(filename) + if err != nil { + mark.errorf("Editor.Mapper(%s) failed: %v", filename, err) + return + } + got, _, err := source.ApplyProtocolEdits(mapper, edits) + if err != nil { + mark.errorf("ApplyProtocolEdits failed: %v", err) + return + } + want, _ := g.Get(mark.run.env.T, "", got) + if diff := compare.Bytes(want, got); diff != "" { + mark.errorf("foldingRange mismatch:\n%s", diff) + } +} + // formatMarker implements the @format marker. func formatMarker(mark marker, golden *Golden) { edits, err := mark.server().Formatting(mark.run.env.Ctx, &protocol.DocumentFormattingParams{ diff --git a/gopls/internal/lsp/testdata/folding/a.go b/gopls/internal/lsp/testdata/folding/a.go deleted file mode 100644 index e07d7e0bf19..00000000000 --- a/gopls/internal/lsp/testdata/folding/a.go +++ /dev/null @@ -1,75 +0,0 @@ -package folding //@fold("package") - -import ( - "fmt" - _ "log" -) - -import _ "os" - -// bar is a function. -// With a multiline doc comment. -func bar() string { - /* This is a single line comment */ - switch { - case true: - if true { - fmt.Println("true") - } else { - fmt.Println("false") - } - case false: - fmt.Println("false") - default: - fmt.Println("default") - } - /* This is a multiline - block - comment */ - - /* This is a multiline - block - comment */ - // Followed by another comment. - _ = []int{ - 1, - 2, - 3, - } - _ = [2]string{"d", - "e", - } - _ = map[string]int{ - "a": 1, - "b": 2, - "c": 3, - } - type T struct { - f string - g int - h string - } - _ = T{ - f: "j", - g: 4, - h: "i", - } - x, y := make(chan bool), make(chan bool) - select { - case val := <-x: - if val { - fmt.Println("true from x") - } else { - fmt.Println("false from x") - } - case <-y: - fmt.Println("y") - default: - fmt.Println("default") - } - // This is a multiline comment - // that is not a doc comment. - return ` -this string -is not indented` -} diff --git a/gopls/internal/lsp/testdata/folding/a.go.golden b/gopls/internal/lsp/testdata/folding/a.go.golden deleted file mode 100644 index b04ca4dab3f..00000000000 --- a/gopls/internal/lsp/testdata/folding/a.go.golden +++ /dev/null @@ -1,722 +0,0 @@ --- foldingRange-0 -- -package folding //@fold("package") - -import (<>) - -import _ "os" - -// bar is a function.<> -func bar(<>) string {<>} - --- foldingRange-1 -- -package folding //@fold("package") - -import ( - "fmt" - _ "log" -) - -import _ "os" - -// bar is a function. -// With a multiline doc comment. -func bar() string { - /* This is a single line comment */ - switch {<>} - /* This is a multiline<> - - /* This is a multiline<> - _ = []int{<>} - _ = [2]string{<>} - _ = map[string]int{<>} - type T struct {<>} - _ = T{<>} - x, y := make(<>), make(<>) - select {<>} - // This is a multiline comment<> - return <> -} - --- foldingRange-2 -- -package folding //@fold("package") - -import ( - "fmt" - _ "log" -) - -import _ "os" - -// bar is a function. -// With a multiline doc comment. -func bar() string { - /* This is a single line comment */ - switch { - case true:<> - case false:<> - default:<> - } - /* This is a multiline - block - comment */ - - /* This is a multiline - block - comment */ - // Followed by another comment. - _ = []int{ - 1, - 2, - 3, - } - _ = [2]string{"d", - "e", - } - _ = map[string]int{ - "a": 1, - "b": 2, - "c": 3, - } - type T struct { - f string - g int - h string - } - _ = T{ - f: "j", - g: 4, - h: "i", - } - x, y := make(chan bool), make(chan bool) - select { - case val := <-x:<> - case <-y:<> - default:<> - } - // This is a multiline comment - // that is not a doc comment. - return ` -this string -is not indented` -} - --- foldingRange-3 -- -package folding //@fold("package") - -import ( - "fmt" - _ "log" -) - -import _ "os" - -// bar is a function. -// With a multiline doc comment. -func bar() string { - /* This is a single line comment */ - switch { - case true: - if true {<>} else {<>} - case false: - fmt.Println(<>) - default: - fmt.Println(<>) - } - /* This is a multiline - block - comment */ - - /* This is a multiline - block - comment */ - // Followed by another comment. - _ = []int{ - 1, - 2, - 3, - } - _ = [2]string{"d", - "e", - } - _ = map[string]int{ - "a": 1, - "b": 2, - "c": 3, - } - type T struct { - f string - g int - h string - } - _ = T{ - f: "j", - g: 4, - h: "i", - } - x, y := make(chan bool), make(chan bool) - select { - case val := <-x: - if val {<>} else {<>} - case <-y: - fmt.Println(<>) - default: - fmt.Println(<>) - } - // This is a multiline comment - // that is not a doc comment. - return ` -this string -is not indented` -} - --- foldingRange-4 -- -package folding //@fold("package") - -import ( - "fmt" - _ "log" -) - -import _ "os" - -// bar is a function. -// With a multiline doc comment. -func bar() string { - /* This is a single line comment */ - switch { - case true: - if true { - fmt.Println(<>) - } else { - fmt.Println(<>) - } - case false: - fmt.Println("false") - default: - fmt.Println("default") - } - /* This is a multiline - block - comment */ - - /* This is a multiline - block - comment */ - // Followed by another comment. - _ = []int{ - 1, - 2, - 3, - } - _ = [2]string{"d", - "e", - } - _ = map[string]int{ - "a": 1, - "b": 2, - "c": 3, - } - type T struct { - f string - g int - h string - } - _ = T{ - f: "j", - g: 4, - h: "i", - } - x, y := make(chan bool), make(chan bool) - select { - case val := <-x: - if val { - fmt.Println(<>) - } else { - fmt.Println(<>) - } - case <-y: - fmt.Println("y") - default: - fmt.Println("default") - } - // This is a multiline comment - // that is not a doc comment. - return ` -this string -is not indented` -} - --- foldingRange-comment-0 -- -package folding //@fold("package") - -import ( - "fmt" - _ "log" -) - -import _ "os" - -// bar is a function.<> -func bar() string { - /* This is a single line comment */ - switch { - case true: - if true { - fmt.Println("true") - } else { - fmt.Println("false") - } - case false: - fmt.Println("false") - default: - fmt.Println("default") - } - /* This is a multiline<> - - /* This is a multiline<> - _ = []int{ - 1, - 2, - 3, - } - _ = [2]string{"d", - "e", - } - _ = map[string]int{ - "a": 1, - "b": 2, - "c": 3, - } - type T struct { - f string - g int - h string - } - _ = T{ - f: "j", - g: 4, - h: "i", - } - x, y := make(chan bool), make(chan bool) - select { - case val := <-x: - if val { - fmt.Println("true from x") - } else { - fmt.Println("false from x") - } - case <-y: - fmt.Println("y") - default: - fmt.Println("default") - } - // This is a multiline comment<> - return ` -this string -is not indented` -} - --- foldingRange-imports-0 -- -package folding //@fold("package") - -import (<>) - -import _ "os" - -// bar is a function. -// With a multiline doc comment. -func bar() string { - /* This is a single line comment */ - switch { - case true: - if true { - fmt.Println("true") - } else { - fmt.Println("false") - } - case false: - fmt.Println("false") - default: - fmt.Println("default") - } - /* This is a multiline - block - comment */ - - /* This is a multiline - block - comment */ - // Followed by another comment. - _ = []int{ - 1, - 2, - 3, - } - _ = [2]string{"d", - "e", - } - _ = map[string]int{ - "a": 1, - "b": 2, - "c": 3, - } - type T struct { - f string - g int - h string - } - _ = T{ - f: "j", - g: 4, - h: "i", - } - x, y := make(chan bool), make(chan bool) - select { - case val := <-x: - if val { - fmt.Println("true from x") - } else { - fmt.Println("false from x") - } - case <-y: - fmt.Println("y") - default: - fmt.Println("default") - } - // This is a multiline comment - // that is not a doc comment. - return ` -this string -is not indented` -} - --- foldingRange-lineFolding-0 -- -package folding //@fold("package") - -import (<> -) - -import _ "os" - -// bar is a function.<> -func bar() string {<> -} - --- foldingRange-lineFolding-1 -- -package folding //@fold("package") - -import ( - "fmt" - _ "log" -) - -import _ "os" - -// bar is a function. -// With a multiline doc comment. -func bar() string { - /* This is a single line comment */ - switch {<> - } - /* This is a multiline<> - - /* This is a multiline<> - _ = []int{<>, - } - _ = [2]string{"d", - "e", - } - _ = map[string]int{<>, - } - type T struct {<> - } - _ = T{<>, - } - x, y := make(chan bool), make(chan bool) - select {<> - } - // This is a multiline comment<> - return <> -} - --- foldingRange-lineFolding-2 -- -package folding //@fold("package") - -import ( - "fmt" - _ "log" -) - -import _ "os" - -// bar is a function. -// With a multiline doc comment. -func bar() string { - /* This is a single line comment */ - switch { - case true:<> - case false:<> - default:<> - } - /* This is a multiline - block - comment */ - - /* This is a multiline - block - comment */ - // Followed by another comment. - _ = []int{ - 1, - 2, - 3, - } - _ = [2]string{"d", - "e", - } - _ = map[string]int{ - "a": 1, - "b": 2, - "c": 3, - } - type T struct { - f string - g int - h string - } - _ = T{ - f: "j", - g: 4, - h: "i", - } - x, y := make(chan bool), make(chan bool) - select { - case val := <-x:<> - case <-y:<> - default:<> - } - // This is a multiline comment - // that is not a doc comment. - return ` -this string -is not indented` -} - --- foldingRange-lineFolding-3 -- -package folding //@fold("package") - -import ( - "fmt" - _ "log" -) - -import _ "os" - -// bar is a function. -// With a multiline doc comment. -func bar() string { - /* This is a single line comment */ - switch { - case true: - if true {<> - } else {<> - } - case false: - fmt.Println("false") - default: - fmt.Println("default") - } - /* This is a multiline - block - comment */ - - /* This is a multiline - block - comment */ - // Followed by another comment. - _ = []int{ - 1, - 2, - 3, - } - _ = [2]string{"d", - "e", - } - _ = map[string]int{ - "a": 1, - "b": 2, - "c": 3, - } - type T struct { - f string - g int - h string - } - _ = T{ - f: "j", - g: 4, - h: "i", - } - x, y := make(chan bool), make(chan bool) - select { - case val := <-x: - if val {<> - } else {<> - } - case <-y: - fmt.Println("y") - default: - fmt.Println("default") - } - // This is a multiline comment - // that is not a doc comment. - return ` -this string -is not indented` -} - --- foldingRange-lineFolding-comment-0 -- -package folding //@fold("package") - -import ( - "fmt" - _ "log" -) - -import _ "os" - -// bar is a function.<> -func bar() string { - /* This is a single line comment */ - switch { - case true: - if true { - fmt.Println("true") - } else { - fmt.Println("false") - } - case false: - fmt.Println("false") - default: - fmt.Println("default") - } - /* This is a multiline<> - - /* This is a multiline<> - _ = []int{ - 1, - 2, - 3, - } - _ = [2]string{"d", - "e", - } - _ = map[string]int{ - "a": 1, - "b": 2, - "c": 3, - } - type T struct { - f string - g int - h string - } - _ = T{ - f: "j", - g: 4, - h: "i", - } - x, y := make(chan bool), make(chan bool) - select { - case val := <-x: - if val { - fmt.Println("true from x") - } else { - fmt.Println("false from x") - } - case <-y: - fmt.Println("y") - default: - fmt.Println("default") - } - // This is a multiline comment<> - return ` -this string -is not indented` -} - --- foldingRange-lineFolding-imports-0 -- -package folding //@fold("package") - -import (<> -) - -import _ "os" - -// bar is a function. -// With a multiline doc comment. -func bar() string { - /* This is a single line comment */ - switch { - case true: - if true { - fmt.Println("true") - } else { - fmt.Println("false") - } - case false: - fmt.Println("false") - default: - fmt.Println("default") - } - /* This is a multiline - block - comment */ - - /* This is a multiline - block - comment */ - // Followed by another comment. - _ = []int{ - 1, - 2, - 3, - } - _ = [2]string{"d", - "e", - } - _ = map[string]int{ - "a": 1, - "b": 2, - "c": 3, - } - type T struct { - f string - g int - h string - } - _ = T{ - f: "j", - g: 4, - h: "i", - } - x, y := make(chan bool), make(chan bool) - select { - case val := <-x: - if val { - fmt.Println("true from x") - } else { - fmt.Println("false from x") - } - case <-y: - fmt.Println("y") - default: - fmt.Println("default") - } - // This is a multiline comment - // that is not a doc comment. - return ` -this string -is not indented` -} - diff --git a/gopls/internal/lsp/testdata/folding/bad.go.golden b/gopls/internal/lsp/testdata/folding/bad.go.golden deleted file mode 100644 index ab274f75ac6..00000000000 --- a/gopls/internal/lsp/testdata/folding/bad.go.golden +++ /dev/null @@ -1,81 +0,0 @@ --- foldingRange-0 -- -package folding //@fold("package") - -import (<>) - -import (<>) - -// badBar is a function. -func badBar(<>) string {<>} - --- foldingRange-1 -- -package folding //@fold("package") - -import ( "fmt" - _ "log" -) - -import ( - _ "os" ) - -// badBar is a function. -func badBar() string { x := true - if x {<>} else {<>} - return -} - --- foldingRange-2 -- -package folding //@fold("package") - -import ( "fmt" - _ "log" -) - -import ( - _ "os" ) - -// badBar is a function. -func badBar() string { x := true - if x { - // This is the only foldable thing in this file when lineFoldingOnly - fmt.Println(<>) - } else { - fmt.Println(<>) } - return -} - --- foldingRange-imports-0 -- -package folding //@fold("package") - -import (<>) - -import (<>) - -// badBar is a function. -func badBar() string { x := true - if x { - // This is the only foldable thing in this file when lineFoldingOnly - fmt.Println("true") - } else { - fmt.Println("false") } - return -} - --- foldingRange-lineFolding-0 -- -package folding //@fold("package") - -import ( "fmt" - _ "log" -) - -import ( - _ "os" ) - -// badBar is a function. -func badBar() string { x := true - if x {<> - } else { - fmt.Println("false") } - return -} - diff --git a/gopls/internal/lsp/testdata/folding/bad.go.in b/gopls/internal/lsp/testdata/folding/bad.go.in deleted file mode 100644 index 84fcb740f40..00000000000 --- a/gopls/internal/lsp/testdata/folding/bad.go.in +++ /dev/null @@ -1,18 +0,0 @@ -package folding //@fold("package") - -import ( "fmt" - _ "log" -) - -import ( - _ "os" ) - -// badBar is a function. -func badBar() string { x := true - if x { - // This is the only foldable thing in this file when lineFoldingOnly - fmt.Println("true") - } else { - fmt.Println("false") } - return -} diff --git a/gopls/internal/lsp/testdata/summary.txt.golden b/gopls/internal/lsp/testdata/summary.txt.golden index 7375b821e69..30e5112dae9 100644 --- a/gopls/internal/lsp/testdata/summary.txt.golden +++ b/gopls/internal/lsp/testdata/summary.txt.golden @@ -9,7 +9,6 @@ FuzzyCompletionsCount = 8 RankedCompletionsCount = 174 CaseSensitiveCompletionsCount = 4 DiagnosticsCount = 23 -FoldingRangesCount = 2 SemanticTokenCount = 3 SuggestedFixCount = 80 MethodExtractionCount = 8 diff --git a/gopls/internal/lsp/testdata/summary_go1.21.txt.golden b/gopls/internal/lsp/testdata/summary_go1.21.txt.golden index 619c25ba757..7796082ba96 100644 --- a/gopls/internal/lsp/testdata/summary_go1.21.txt.golden +++ b/gopls/internal/lsp/testdata/summary_go1.21.txt.golden @@ -9,7 +9,6 @@ FuzzyCompletionsCount = 8 RankedCompletionsCount = 174 CaseSensitiveCompletionsCount = 4 DiagnosticsCount = 24 -FoldingRangesCount = 2 SemanticTokenCount = 3 SuggestedFixCount = 80 MethodExtractionCount = 8 diff --git a/gopls/internal/lsp/tests/tests.go b/gopls/internal/lsp/tests/tests.go index 41c9bad4aaf..deabe90bff5 100644 --- a/gopls/internal/lsp/tests/tests.go +++ b/gopls/internal/lsp/tests/tests.go @@ -74,7 +74,6 @@ type DeepCompletions = map[span.Span][]Completion type FuzzyCompletions = map[span.Span][]Completion type CaseSensitiveCompletions = map[span.Span][]Completion type RankCompletions = map[span.Span][]Completion -type FoldingRanges = []span.Span type SemanticTokens = []span.Span type SuggestedFixes = map[span.Span][]SuggestedFix type MethodExtractions = map[span.Span]span.Span @@ -102,7 +101,6 @@ type Data struct { FuzzyCompletions FuzzyCompletions CaseSensitiveCompletions CaseSensitiveCompletions RankCompletions RankCompletions - FoldingRanges FoldingRanges SemanticTokens SemanticTokens SuggestedFixes SuggestedFixes MethodExtractions MethodExtractions @@ -144,7 +142,6 @@ type Tests interface { FuzzyCompletion(*testing.T, span.Span, Completion, CompletionItems) CaseSensitiveCompletion(*testing.T, span.Span, Completion, CompletionItems) RankCompletion(*testing.T, span.Span, Completion, CompletionItems) - FoldingRanges(*testing.T, span.Span) SemanticTokens(*testing.T, span.Span) SuggestedFix(*testing.T, span.Span, []SuggestedFix, int) MethodExtraction(*testing.T, span.Span, span.Span) @@ -435,7 +432,6 @@ func load(t testing.TB, mode string, dir string) *Data { "casesensitive": datum.collectCompletions(CompletionCaseSensitive), "rank": datum.collectCompletions(CompletionRank), "snippet": datum.collectCompletionSnippets, - "fold": datum.collectFoldingRanges, "semantic": datum.collectSemanticTokens, "godef": datum.collectDefinitions, "typdef": datum.collectTypeDefinitions, @@ -617,16 +613,6 @@ func Run(t *testing.T, tests Tests, data *Data) { } }) - t.Run("FoldingRange", func(t *testing.T) { - t.Helper() - for _, spn := range data.FoldingRanges { - t.Run(uriName(spn.URI()), func(t *testing.T) { - t.Helper() - tests.FoldingRanges(t, spn) - }) - } - }) - t.Run("SemanticTokens", func(t *testing.T) { t.Helper() for _, spn := range data.SemanticTokens { @@ -832,7 +818,6 @@ func checkData(t *testing.T, data *Data) { fmt.Fprintf(buf, "RankedCompletionsCount = %v\n", countCompletions(data.RankCompletions)) fmt.Fprintf(buf, "CaseSensitiveCompletionsCount = %v\n", countCompletions(data.CaseSensitiveCompletions)) fmt.Fprintf(buf, "DiagnosticsCount = %v\n", diagnosticsCount) - fmt.Fprintf(buf, "FoldingRangesCount = %v\n", len(data.FoldingRanges)) fmt.Fprintf(buf, "SemanticTokenCount = %v\n", len(data.SemanticTokens)) fmt.Fprintf(buf, "SuggestedFixCount = %v\n", len(data.SuggestedFixes)) fmt.Fprintf(buf, "MethodExtractionCount = %v\n", len(data.MethodExtractions)) @@ -1003,10 +988,6 @@ func (data *Data) collectCompletionItems(pos token.Pos, label, detail, kind stri } } -func (data *Data) collectFoldingRanges(spn span.Span) { - data.FoldingRanges = append(data.FoldingRanges, spn) -} - func (data *Data) collectAddImports(spn span.Span, imp string) { data.AddImport[spn.URI()] = imp } diff --git a/gopls/internal/regtest/marker/testdata/foldingrange/a.txt b/gopls/internal/regtest/marker/testdata/foldingrange/a.txt new file mode 100644 index 00000000000..6210fc25251 --- /dev/null +++ b/gopls/internal/regtest/marker/testdata/foldingrange/a.txt @@ -0,0 +1,154 @@ +This test checks basic behavior of textDocument/foldingRange. + +-- a.go -- +package folding //@foldingrange(raw) + +import ( + "fmt" + _ "log" +) + +import _ "os" + +// bar is a function. +// With a multiline doc comment. +func bar() string { + /* This is a single line comment */ + switch { + case true: + if true { + fmt.Println("true") + } else { + fmt.Println("false") + } + case false: + fmt.Println("false") + default: + fmt.Println("default") + } + /* This is a multiline + block + comment */ + + /* This is a multiline + block + comment */ + // Followed by another comment. + _ = []int{ + 1, + 2, + 3, + } + _ = [2]string{"d", + "e", + } + _ = map[string]int{ + "a": 1, + "b": 2, + "c": 3, + } + type T struct { + f string + g int + h string + } + _ = T{ + f: "j", + g: 4, + h: "i", + } + x, y := make(chan bool), make(chan bool) + select { + case val := <-x: + if val { + fmt.Println("true from x") + } else { + fmt.Println("false from x") + } + case <-y: + fmt.Println("y") + default: + fmt.Println("default") + } + // This is a multiline comment + // that is not a doc comment. + return ` +this string +is not indented` +} +-- @raw -- +package folding //@foldingrange(raw) + +import (<0 kind="imports"> + "fmt" + _ "log" +) + +import _ "os" + +// bar is a function.<1 kind="comment"> +// With a multiline doc comment. +func bar(<2 kind="">) string {<3 kind=""> + /* This is a single line comment */ + switch {<4 kind=""> + case true:<5 kind=""> + if true {<6 kind=""> + fmt.Println(<7 kind="">"true") + } else {<8 kind=""> + fmt.Println(<9 kind="">"false") + } + case false:<10 kind=""> + fmt.Println(<11 kind="">"false") + default:<12 kind=""> + fmt.Println(<13 kind="">"default") + } + /* This is a multiline<14 kind="comment"> + block + comment */ + + /* This is a multiline<15 kind="comment"> + block + comment */ + // Followed by another comment. + _ = []int{<16 kind=""> + 1, + 2, + 3, + } + _ = [2]string{<17 kind="">"d", + "e", + } + _ = map[string]int{<18 kind=""> + "a": 1, + "b": 2, + "c": 3, + } + type T struct {<19 kind=""> + f string + g int + h string + } + _ = T{<20 kind=""> + f: "j", + g: 4, + h: "i", + } + x, y := make(<21 kind="">chan bool), make(<22 kind="">chan bool) + select {<23 kind=""> + case val := <-x:<24 kind=""> + if val {<25 kind=""> + fmt.Println(<26 kind="">"true from x") + } else {<27 kind=""> + fmt.Println(<28 kind="">"false from x") + } + case <-y:<29 kind=""> + fmt.Println(<30 kind="">"y") + default:<31 kind=""> + fmt.Println(<32 kind="">"default") + } + // This is a multiline comment<33 kind="comment"> + // that is not a doc comment. + return <34 kind="">` +this string +is not indented` +} diff --git a/gopls/internal/regtest/marker/testdata/foldingrange/a_lineonly.txt b/gopls/internal/regtest/marker/testdata/foldingrange/a_lineonly.txt new file mode 100644 index 00000000000..0c532e760f1 --- /dev/null +++ b/gopls/internal/regtest/marker/testdata/foldingrange/a_lineonly.txt @@ -0,0 +1,163 @@ +This test checks basic behavior of the textDocument/foldingRange, when the +editor only supports line folding. + +-- capabilities.json -- +{ + "textDocument": { + "foldingRange": { + "lineFoldingOnly": true + } + } +} +-- a.go -- +package folding //@foldingrange(raw) + +import ( + "fmt" + _ "log" +) + +import _ "os" + +// bar is a function. +// With a multiline doc comment. +func bar() string { + /* This is a single line comment */ + switch { + case true: + if true { + fmt.Println("true") + } else { + fmt.Println("false") + } + case false: + fmt.Println("false") + default: + fmt.Println("default") + } + /* This is a multiline + block + comment */ + + /* This is a multiline + block + comment */ + // Followed by another comment. + _ = []int{ + 1, + 2, + 3, + } + _ = [2]string{"d", + "e", + } + _ = map[string]int{ + "a": 1, + "b": 2, + "c": 3, + } + type T struct { + f string + g int + h string + } + _ = T{ + f: "j", + g: 4, + h: "i", + } + x, y := make(chan bool), make(chan bool) + select { + case val := <-x: + if val { + fmt.Println("true from x") + } else { + fmt.Println("false from x") + } + case <-y: + fmt.Println("y") + default: + fmt.Println("default") + } + // This is a multiline comment + // that is not a doc comment. + return ` +this string +is not indented` +} +-- @raw -- +package folding //@foldingrange(raw) + +import (<0 kind="imports"> + "fmt" + _ "log" +) + +import _ "os" + +// bar is a function.<1 kind="comment"> +// With a multiline doc comment. +func bar() string {<2 kind=""> + /* This is a single line comment */ + switch {<3 kind=""> + case true:<4 kind=""> + if true {<5 kind=""> + fmt.Println("true") + } else {<6 kind=""> + fmt.Println("false") + } + case false:<7 kind=""> + fmt.Println("false") + default:<8 kind=""> + fmt.Println("default") + } + /* This is a multiline<9 kind="comment"> + block + comment */ + + /* This is a multiline<10 kind="comment"> + block + comment */ + // Followed by another comment. + _ = []int{<11 kind=""> + 1, + 2, + 3, + } + _ = [2]string{"d", + "e", + } + _ = map[string]int{<12 kind=""> + "a": 1, + "b": 2, + "c": 3, + } + type T struct {<13 kind=""> + f string + g int + h string + } + _ = T{<14 kind=""> + f: "j", + g: 4, + h: "i", + } + x, y := make(chan bool), make(chan bool) + select {<15 kind=""> + case val := <-x:<16 kind=""> + if val {<17 kind=""> + fmt.Println("true from x") + } else {<18 kind=""> + fmt.Println("false from x") + } + case <-y:<19 kind=""> + fmt.Println("y") + default:<20 kind=""> + fmt.Println("default") + } + // This is a multiline comment<21 kind="comment"> + // that is not a doc comment. + return <22 kind="">` +this string +is not indented` +} diff --git a/gopls/internal/regtest/marker/testdata/foldingrange/bad.txt b/gopls/internal/regtest/marker/testdata/foldingrange/bad.txt new file mode 100644 index 00000000000..f9f14a4fa7d --- /dev/null +++ b/gopls/internal/regtest/marker/testdata/foldingrange/bad.txt @@ -0,0 +1,41 @@ +This test verifies behavior of textDocument/foldingRange in the presence of +unformatted syntax. + +-- a.go -- +package folding //@foldingrange(raw) + +import ( "fmt" + _ "log" +) + +import ( + _ "os" ) + +// badBar is a function. +func badBar() string { x := true + if x { + // This is the only foldable thing in this file when lineFoldingOnly + fmt.Println("true") + } else { + fmt.Println("false") } + return "" +} +-- @raw -- +package folding //@foldingrange(raw) + +import (<0 kind="imports"> "fmt" + _ "log" +) + +import (<1 kind="imports"> + _ "os" ) + +// badBar is a function. +func badBar(<2 kind="">) string {<3 kind=""> x := true + if x {<4 kind=""> + // This is the only foldable thing in this file when lineFoldingOnly + fmt.Println(<5 kind="">"true") + } else {<6 kind=""> + fmt.Println(<7 kind="">"false") } + return "" +} From fbb89100a637ecc20d37c0c79e484137b89364ad Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Fri, 8 Sep 2023 11:27:54 -0400 Subject: [PATCH 071/178] go/types/internal/play: show TypeAndValue.mode Change-Id: Id3efd7f56a965bb89d837b0de2d262257c9017dd Reviewed-on: https://go-review.googlesource.com/c/tools/+/526975 LUCI-TryBot-Result: Go LUCI Run-TryBot: Alan Donovan Reviewed-by: Robert Findley gopls-CI: kokoro TryBot-Result: Gopher Robot --- go/types/internal/play/play.go | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/go/types/internal/play/play.go b/go/types/internal/play/play.go index d5a164e5eba..ace7fddc7ab 100644 --- a/go/types/internal/play/play.go +++ b/go/types/internal/play/play.go @@ -137,9 +137,26 @@ func handleSelectJSON(w http.ResponseWriter, req *http.Request) { // Expression type information if innermostExpr != nil { if tv, ok := pkg.TypesInfo.Types[innermostExpr]; ok { - // TODO(adonovan): show tv.mode. - // e.g. IsVoid IsType IsBuiltin IsValue IsNil Addressable Assignable HasOk - fmt.Fprintf(out, "%T has type %v", innermostExpr, tv.Type) + var modes []string + for _, mode := range []struct { + name string + condition func(types.TypeAndValue) bool + }{ + {"IsVoid", types.TypeAndValue.IsVoid}, + {"IsType", types.TypeAndValue.IsType}, + {"IsBuiltin", types.TypeAndValue.IsBuiltin}, + {"IsValue", types.TypeAndValue.IsValue}, + {"IsNil", types.TypeAndValue.IsNil}, + {"Addressable", types.TypeAndValue.Addressable}, + {"Assignable", types.TypeAndValue.Assignable}, + {"HasOk", types.TypeAndValue.HasOk}, + } { + if mode.condition(tv) { + modes = append(modes, mode.name) + } + } + fmt.Fprintf(out, "%T has type %v and mode %s", + innermostExpr, tv.Type, modes) if tv.Value != nil { fmt.Fprintf(out, " and constant value %v", tv.Value) } From 882bb14c1266124aa0fbd61993e67de1672c132f Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Fri, 8 Sep 2023 14:03:05 -0400 Subject: [PATCH 072/178] go/analysis/unitchecker: revert subtle change to vendoring CL 510215 introduced a bug by "composing" two anonymous functions, though in reality the inocuous simplification overlooked that there are actually three steps: 1. vendoring resolution 2. importer.ForCompiler internals 3. PackageFile lookup and in doing so reordered them (1 3 2). Fixes golang/go#62519 Change-Id: I99914d895a8617165c0a4457653f28fdc667756a Reviewed-on: https://go-review.googlesource.com/c/tools/+/526419 Reviewed-by: Robert Findley Run-TryBot: Alan Donovan TryBot-Result: Gopher Robot --- go/analysis/unitchecker/unitchecker.go | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/go/analysis/unitchecker/unitchecker.go b/go/analysis/unitchecker/unitchecker.go index 4ff45feb4ce..53c3f4a806c 100644 --- a/go/analysis/unitchecker/unitchecker.go +++ b/go/analysis/unitchecker/unitchecker.go @@ -193,13 +193,7 @@ type factImporter = func(pkgPath string) ([]byte, error) // The defaults honor a Config in a manner compatible with 'go vet'. var ( makeTypesImporter = func(cfg *Config, fset *token.FileSet) types.Importer { - return importer.ForCompiler(fset, cfg.Compiler, func(importPath string) (io.ReadCloser, error) { - // Resolve import path to package path (vendoring, etc) - path, ok := cfg.ImportMap[importPath] - if !ok { - return nil, fmt.Errorf("can't resolve import %q", path) - } - + compilerImporter := importer.ForCompiler(fset, cfg.Compiler, func(path string) (io.ReadCloser, error) { // path is a resolved package path, not an import path. file, ok := cfg.PackageFile[path] if !ok { @@ -210,6 +204,13 @@ var ( } return os.Open(file) }) + return importerFunc(func(importPath string) (*types.Package, error) { + path, ok := cfg.ImportMap[importPath] // resolve vendoring, etc + if !ok { + return nil, fmt.Errorf("can't resolve import %q", path) + } + return compilerImporter.Import(path) + }) } exportTypes = func(*Config, *token.FileSet, *types.Package) error { @@ -433,3 +434,7 @@ type result struct { diagnostics []analysis.Diagnostic err error } + +type importerFunc func(path string) (*types.Package, error) + +func (f importerFunc) Import(path string) (*types.Package, error) { return f(path) } From fb4bd11ef6feea8f26ff371d1fcca356511e2e26 Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Wed, 6 Sep 2023 15:44:41 -0400 Subject: [PATCH 073/178] gopls/internal/lsp/cache: move Option management to the Server In order to move toward tracking options by Folder, not view, move them into the Server. This will also help us fix bugs related to configuration lifecycle events. For golang/go#57979 Updates golang/go#42814 Change-Id: Id281cad20697756138a7bdc67f718a7468a04d4a Reviewed-on: https://go-review.googlesource.com/c/tools/+/526417 LUCI-TryBot-Result: Go LUCI gopls-CI: kokoro Reviewed-by: Alan Donovan --- gopls/internal/lsp/cache/cache.go | 8 +--- gopls/internal/lsp/cache/session.go | 40 ++++------------ gopls/internal/lsp/cache/view.go | 30 ++++++++---- gopls/internal/lsp/cmd/capabilities_test.go | 2 +- gopls/internal/lsp/cmd/cmd.go | 2 +- gopls/internal/lsp/code_action.go | 2 +- gopls/internal/lsp/command.go | 2 +- gopls/internal/lsp/completion_test.go | 34 ++++++++------ gopls/internal/lsp/debug/serve.go | 13 ++---- gopls/internal/lsp/general.go | 52 ++++++++++++++------- gopls/internal/lsp/lsp_test.go | 5 +- gopls/internal/lsp/lsprpc/lsprpc.go | 8 ++-- gopls/internal/lsp/semantic.go | 8 ++-- gopls/internal/lsp/server.go | 11 ++++- gopls/internal/lsp/text_synchronization.go | 4 +- gopls/internal/lsp/workspace.go | 22 +++++---- gopls/internal/lsp/workspace_symbol.go | 4 +- 17 files changed, 132 insertions(+), 115 deletions(-) diff --git a/gopls/internal/lsp/cache/cache.go b/gopls/internal/lsp/cache/cache.go index 473d2513b51..b1cdfcef16b 100644 --- a/gopls/internal/lsp/cache/cache.go +++ b/gopls/internal/lsp/cache/cache.go @@ -11,7 +11,6 @@ import ( "sync/atomic" "time" - "golang.org/x/tools/gopls/internal/lsp/source" "golang.org/x/tools/internal/event" "golang.org/x/tools/internal/gocommand" "golang.org/x/tools/internal/memoize" @@ -56,17 +55,12 @@ type Cache struct { // The provided optionsOverrides may be nil. // // TODO(rfindley): move this to session.go. -func NewSession(ctx context.Context, c *Cache, optionsOverrides func(*source.Options)) *Session { +func NewSession(ctx context.Context, c *Cache) *Session { index := atomic.AddInt64(&sessionIndex, 1) - options := source.DefaultOptions().Clone() - if optionsOverrides != nil { - optionsOverrides(options) - } s := &Session{ id: strconv.FormatInt(index, 10), cache: c, gocmdRunner: &gocommand.Runner{}, - options: options, overlayFS: newOverlayFS(c), parseCache: newParseCache(1 * time.Minute), // keep recently parsed files for a minute, to optimize typing CPU } diff --git a/gopls/internal/lsp/cache/session.go b/gopls/internal/lsp/cache/session.go index af5190ba1f4..edfb78ec979 100644 --- a/gopls/internal/lsp/cache/session.go +++ b/gopls/internal/lsp/cache/session.go @@ -33,12 +33,9 @@ type Session struct { cache *Cache // shared cache gocmdRunner *gocommand.Runner // limits go command concurrency - optionsMu sync.Mutex - options *source.Options - viewMu sync.Mutex views []*View - viewMap map[span.URI]*View // map of URI->best view + viewMap map[span.URI]*View // file->best view parseCache *parseCache @@ -54,20 +51,6 @@ func (s *Session) GoCommandRunner() *gocommand.Runner { return s.gocmdRunner } -// Options returns a copy of the SessionOptions for this session. -func (s *Session) Options() *source.Options { - s.optionsMu.Lock() - defer s.optionsMu.Unlock() - return s.options -} - -// SetOptions sets the options of this session to new values. -func (s *Session) SetOptions(options *source.Options) { - s.optionsMu.Lock() - defer s.optionsMu.Unlock() - s.options = options -} - // Shutdown the session and all views it has created. func (s *Session) Shutdown(ctx context.Context) { var views []*View @@ -293,6 +276,7 @@ func bestViewForURI(uri span.URI, views []*View) *View { func (s *Session) RemoveView(view *View) { s.viewMu.Lock() defer s.viewMu.Unlock() + i := s.dropView(view) if i == -1 { // error reported elsewhere return @@ -302,18 +286,11 @@ func (s *Session) RemoveView(view *View) { s.views = removeElement(s.views, i) } -// updateView recreates the view with the given options. +// updateViewLocked recreates the view with the given options. // // If the resulting error is non-nil, the view may or may not have already been // dropped from the session. -func (s *Session) updateView(ctx context.Context, view *View, options *source.Options) (*View, error) { - s.viewMu.Lock() - defer s.viewMu.Unlock() - - return s.updateViewLocked(ctx, view, options) -} - -func (s *Session) updateViewLocked(ctx context.Context, view *View, options *source.Options) (*View, error) { +func (s *Session) updateViewLocked(ctx context.Context, view *View, options *source.Options) error { // Preserve the snapshot ID if we are recreating the view. view.snapshotMu.Lock() if view.snapshot == nil { @@ -325,7 +302,7 @@ func (s *Session) updateViewLocked(ctx context.Context, view *View, options *sou i := s.dropView(view) if i == -1 { - return nil, fmt.Errorf("view %q not found", view.id) + return fmt.Errorf("view %q not found", view.id) } v, snapshot, release, err := s.createView(ctx, view.name, view.folder, options, seqID) @@ -334,7 +311,7 @@ func (s *Session) updateViewLocked(ctx context.Context, view *View, options *sou // this should not happen and is very bad, but we still need to clean // up the view array if it happens s.views = removeElement(s.views, i) - return nil, err + return err } defer release() @@ -350,7 +327,7 @@ func (s *Session) updateViewLocked(ctx context.Context, view *View, options *sou // substitute the new view into the array where the old view was s.views[i] = v - return v, nil + return nil } // removeElement removes the ith element from the slice replacing it with the last element. @@ -457,8 +434,7 @@ func (s *Session) DidModifyFiles(ctx context.Context, changes []source.FileModif } if info != view.workspaceInformation { - _, err := s.updateViewLocked(ctx, view, view.Options()) - if err != nil { + if err := s.updateViewLocked(ctx, view, view.Options()); err != nil { // More catastrophic failure. The view may or may not still exist. // The best we can do is log and move on. event.Error(ctx, "recreating view", err) diff --git a/gopls/internal/lsp/cache/view.go b/gopls/internal/lsp/cache/view.go index a1c7e6bab6a..a71fc1d6acd 100644 --- a/gopls/internal/lsp/cache/view.go +++ b/gopls/internal/lsp/cache/view.go @@ -482,21 +482,35 @@ func minorOptionsChange(a, b *source.Options) bool { return reflect.DeepEqual(aBuildFlags, bBuildFlags) } -// SetViewOptions sets the options of the given view to new values. Calling -// this may cause the view to be invalidated and a replacement view added to -// the session. If so the new view will be returned, otherwise the original one -// will be returned. -func (s *Session) SetViewOptions(ctx context.Context, v *View, options *source.Options) (*View, error) { +// SetFolderOptions updates the options of each View associated with the folder +// of the given URI. +// +// Calling this may cause each related view to be invalidated and a replacement +// view added to the session. +func (s *Session) SetFolderOptions(ctx context.Context, uri span.URI, options *source.Options) error { + s.viewMu.Lock() + defer s.viewMu.Unlock() + + for _, v := range s.views { + if v.folder == uri { + if err := s.setViewOptions(ctx, v, options); err != nil { + return err + } + } + } + return nil +} + +func (s *Session) setViewOptions(ctx context.Context, v *View, options *source.Options) error { // no need to rebuild the view if the options were not materially changed v.optionsMu.Lock() if minorOptionsChange(v.options, options) { v.options = options v.optionsMu.Unlock() - return v, nil + return nil } v.optionsMu.Unlock() - newView, err := s.updateView(ctx, v, options) - return newView, err + return s.updateViewLocked(ctx, v, options) } // viewEnv returns a string describing the environment of a newly created view. diff --git a/gopls/internal/lsp/cmd/capabilities_test.go b/gopls/internal/lsp/cmd/capabilities_test.go index 6d4e32f0fe2..ddf56851937 100644 --- a/gopls/internal/lsp/cmd/capabilities_test.go +++ b/gopls/internal/lsp/cmd/capabilities_test.go @@ -49,7 +49,7 @@ func TestCapabilities(t *testing.T) { // Send an initialize request to the server. ctx := context.Background() client := newClient(app, nil) - server := lsp.NewServer(cache.NewSession(ctx, cache.New(nil), app.options), client) + server := lsp.NewServer(cache.NewSession(ctx, cache.New(nil)), client, app.options) result, err := server.Initialize(ctx, params) if err != nil { t.Fatal(err) diff --git a/gopls/internal/lsp/cmd/cmd.go b/gopls/internal/lsp/cmd/cmd.go index 340073d8a5a..2e1e6229edd 100644 --- a/gopls/internal/lsp/cmd/cmd.go +++ b/gopls/internal/lsp/cmd/cmd.go @@ -316,7 +316,7 @@ func (app *Application) connect(ctx context.Context, onProgress func(*protocol.P switch { case app.Remote == "": client := newClient(app, onProgress) - server := lsp.NewServer(cache.NewSession(ctx, cache.New(nil), app.options), client) + server := lsp.NewServer(cache.NewSession(ctx, cache.New(nil)), client, app.options) conn := newConnection(server, client) if err := conn.initialize(protocol.WithClient(ctx, client), app.options); err != nil { return nil, err diff --git a/gopls/internal/lsp/code_action.go b/gopls/internal/lsp/code_action.go index bef4e34d68f..555131ea796 100644 --- a/gopls/internal/lsp/code_action.go +++ b/gopls/internal/lsp/code_action.go @@ -299,7 +299,7 @@ func (s *Server) findMatchingDiagnostics(uri span.URI, pd protocol.Diagnostic) [ func (s *Server) getSupportedCodeActions() []protocol.CodeActionKind { allCodeActionKinds := make(map[protocol.CodeActionKind]struct{}) - for _, kinds := range s.session.Options().SupportedCodeActions { + for _, kinds := range s.Options().SupportedCodeActions { for kind := range kinds { allCodeActionKinds[kind] = struct{}{} } diff --git a/gopls/internal/lsp/command.go b/gopls/internal/lsp/command.go index 79d3a5c35d6..84a7f49b945 100644 --- a/gopls/internal/lsp/command.go +++ b/gopls/internal/lsp/command.go @@ -43,7 +43,7 @@ func (s *Server) executeCommand(ctx context.Context, params *protocol.ExecuteCom defer done() var found bool - for _, name := range s.session.Options().SupportedCommands { + for _, name := range s.Options().SupportedCommands { if name == params.Command { found = true break diff --git a/gopls/internal/lsp/completion_test.go b/gopls/internal/lsp/completion_test.go index 1fc7304fc43..1da7ade563d 100644 --- a/gopls/internal/lsp/completion_test.go +++ b/gopls/internal/lsp/completion_test.go @@ -144,20 +144,8 @@ func expected(t *testing.T, test tests.Completion, items tests.CompletionItems) func (r *runner) callCompletion(t *testing.T, src span.Span, options func(*source.Options)) []protocol.CompletionItem { t.Helper() - - view, err := r.server.session.ViewOf(src.URI()) - if err != nil { - t.Fatal(err) - } - original := view.Options() - modified := view.Options().Clone() - options(modified) - view, err = r.server.session.SetViewOptions(r.ctx, view, modified) - if err != nil { - t.Error(err) - return nil - } - defer r.server.session.SetViewOptions(r.ctx, view, original) + cleanup := r.toggleOptions(t, src.URI(), options) + defer cleanup() list, err := r.server.Completion(r.ctx, &protocol.CompletionParams{ TextDocumentPositionParams: protocol.TextDocumentPositionParams{ @@ -175,3 +163,21 @@ func (r *runner) callCompletion(t *testing.T, src span.Span, options func(*sourc } return list.Items } + +func (r *runner) toggleOptions(t *testing.T, uri span.URI, options func(*source.Options)) (reset func()) { + view, err := r.server.session.ViewOf(uri) + if err != nil { + t.Fatal(err) + } + folder := view.Folder() + + original := view.Options() + modified := view.Options().Clone() + options(modified) + if err = r.server.session.SetFolderOptions(r.ctx, folder, modified); err != nil { + t.Fatal(err) + } + return func() { + r.server.session.SetFolderOptions(r.ctx, folder, original) + } +} diff --git a/gopls/internal/lsp/debug/serve.go b/gopls/internal/lsp/debug/serve.go index 246a943e458..c0bfd9e7b50 100644 --- a/gopls/internal/lsp/debug/serve.go +++ b/gopls/internal/lsp/debug/serve.go @@ -701,9 +701,10 @@ Unknown page } return s }, - "options": func(s *cache.Session) []sessionOption { - return showOptions(s.Options()) - }, + // TODO(rfindley): re-enable option inspection. + // "options": func(s *cache.Session) []sessionOption { + // return showOptions(s.Options()) + // }, }) var MainTmpl = template.Must(template.Must(BaseTemplate.Clone()).Parse(` @@ -837,12 +838,6 @@ From: {{template "cachelink" .Cache.ID}}
  • {{.FileIdentity.URI}}
    • {{end}} -

    Options

    -{{range options .}} -

    {{.Name}} {{.Type}}

    -

    default: {{.Default}}

    -{{if ne .Default .Current}}

    current: {{.Current}}

    {{end}} -{{end}} {{end}} `)) diff --git a/gopls/internal/lsp/general.go b/gopls/internal/lsp/general.go index 9c10d9377a1..e2f8ddbbf67 100644 --- a/gopls/internal/lsp/general.go +++ b/gopls/internal/lsp/general.go @@ -57,8 +57,10 @@ func (s *Server) initialize(ctx context.Context, params *protocol.ParamInitializ } s.progress.SetSupportsWorkDoneProgress(params.Capabilities.Window.WorkDoneProgress) - options := s.session.Options() - defer func() { s.session.SetOptions(options) }() + options := s.Options().Clone() + // TODO(rfindley): remove the error return from handleOptionResults, and + // eliminate this defer. + defer func() { s.SetOptions(options) }() if err := s.handleOptionResults(ctx, source.SetOptions(options, params.InitializationOptions)); err != nil { return nil, err @@ -170,8 +172,8 @@ See https://github.com/golang/go/issues/45732 for more information.`, Range: &protocol.Or_SemanticTokensOptions_range{Value: true}, Full: &protocol.Or_SemanticTokensOptions_full{Value: true}, Legend: protocol.SemanticTokensLegend{ - TokenTypes: nonNilSliceString(s.session.Options().SemanticTypes), - TokenModifiers: nonNilSliceString(s.session.Options().SemanticMods), + TokenTypes: nonNilSliceString(s.Options().SemanticTypes), + TokenModifiers: nonNilSliceString(s.Options().SemanticMods), }, }, SignatureHelpProvider: &protocol.SignatureHelpOptions{ @@ -215,9 +217,7 @@ func (s *Server) initialized(ctx context.Context, params *protocol.InitializedPa } s.notifications = nil - options := s.session.Options() - defer func() { s.session.SetOptions(options) }() - + options := s.Options() if err := s.addFolders(ctx, s.pendingFolders); err != nil { return err } @@ -348,7 +348,7 @@ func (s *Server) addFolders(ctx context.Context, folders []protocol.WorkspaceFol viewErrors := make(map[span.URI]error) var ndiagnose sync.WaitGroup // number of unfinished diagnose calls - if s.session.Options().VerboseWorkDoneProgress { + if s.Options().VerboseWorkDoneProgress { work := s.progress.Start(ctx, DiagnosticWorkTitle(FromInitialWorkspaceLoad), "Calculating diagnostics for initial workspace load...", nil, nil) defer func() { go func() { @@ -475,7 +475,7 @@ func equalURISet(m1, m2 map[string]struct{}) bool { // registrations to the client and updates s.watchedDirectories. // The caller must not subsequently mutate patterns. func (s *Server) registerWatchedDirectoriesLocked(ctx context.Context, patterns map[string]struct{}) error { - if !s.session.Options().DynamicWatchedFilesSupported { + if !s.Options().DynamicWatchedFilesSupported { return nil } s.watchedGlobPatterns = patterns @@ -503,9 +503,27 @@ func (s *Server) registerWatchedDirectoriesLocked(ctx context.Context, patterns return nil } -func (s *Server) fetchConfig(ctx context.Context, name string, folder span.URI, o *source.Options) error { - if !s.session.Options().ConfigurationSupported { - return nil +// Options returns the current server options. +// +// The caller must not modify the result. +func (s *Server) Options() *source.Options { + s.optionsMu.Lock() + defer s.optionsMu.Unlock() + return s.options +} + +// SetOptions sets the current server options. +// +// The caller must not subsequently modify the options. +func (s *Server) SetOptions(opts *source.Options) { + s.optionsMu.Lock() + defer s.optionsMu.Unlock() + s.options = opts +} + +func (s *Server) fetchFolderOptions(ctx context.Context, folder span.URI) (*source.Options, error) { + if opts := s.Options(); !opts.ConfigurationSupported { + return opts, nil } configs, err := s.client.Configuration(ctx, &protocol.ParamConfiguration{ Items: []protocol.ConfigurationItem{{ @@ -515,14 +533,16 @@ func (s *Server) fetchConfig(ctx context.Context, name string, folder span.URI, }, ) if err != nil { - return fmt.Errorf("failed to get workspace configuration from client (%s): %v", folder, err) + return nil, fmt.Errorf("failed to get workspace configuration from client (%s): %v", folder, err) } + + folderOpts := s.Options().Clone() for _, config := range configs { - if err := s.handleOptionResults(ctx, source.SetOptions(o, config)); err != nil { - return err + if err := s.handleOptionResults(ctx, source.SetOptions(folderOpts, config)); err != nil { + return nil, err } } - return nil + return folderOpts, nil } func (s *Server) eventuallyShowMessage(ctx context.Context, msg *protocol.ShowMessageParams) error { diff --git a/gopls/internal/lsp/lsp_test.go b/gopls/internal/lsp/lsp_test.go index 40ebb9d636b..285f78a7f40 100644 --- a/gopls/internal/lsp/lsp_test.go +++ b/gopls/internal/lsp/lsp_test.go @@ -51,10 +51,9 @@ func testLSP(t *testing.T, datum *tests.Data) { // instrumentation. ctx = debug.WithInstance(ctx, "", "off") - session := cache.NewSession(ctx, cache.New(nil), nil) + session := cache.NewSession(ctx, cache.New(nil)) options := source.DefaultOptions().Clone() tests.DefaultOptions(options) - session.SetOptions(options) options.SetEnvSlice(datum.Config.Env) view, snapshot, release, err := session.NewView(ctx, datum.Config.Dir, span.URIFromPath(datum.Config.Dir), options) if err != nil { @@ -113,7 +112,7 @@ func testLSP(t *testing.T, datum *tests.Data) { editRecv: make(chan map[span.URI][]byte, 1), } - r.server = NewServer(session, testClient{runner: r}) + r.server = NewServer(session, testClient{runner: r}, nil) tests.Run(t, r, datum) } diff --git a/gopls/internal/lsp/lsprpc/lsprpc.go b/gopls/internal/lsp/lsprpc/lsprpc.go index 6b02cf5aa65..ef2c3ed0c54 100644 --- a/gopls/internal/lsp/lsprpc/lsprpc.go +++ b/gopls/internal/lsp/lsprpc/lsprpc.go @@ -56,10 +56,10 @@ func NewStreamServer(cache *cache.Cache, daemon bool, optionsFunc func(*source.O func (s *StreamServer) Binder() *ServerBinder { newServer := func(ctx context.Context, client protocol.ClientCloser) protocol.Server { - session := cache.NewSession(ctx, s.cache, s.optionsOverrides) + session := cache.NewSession(ctx, s.cache) server := s.serverForTest if server == nil { - server = lsp.NewServer(session, client) + server = lsp.NewServer(session, client, s.optionsOverrides) if instance := debug.GetInstance(ctx); instance != nil { instance.AddService(server, session) } @@ -73,10 +73,10 @@ func (s *StreamServer) Binder() *ServerBinder { // incoming streams using a new lsp server. func (s *StreamServer) ServeStream(ctx context.Context, conn jsonrpc2.Conn) error { client := protocol.ClientDispatcher(conn) - session := cache.NewSession(ctx, s.cache, s.optionsOverrides) + session := cache.NewSession(ctx, s.cache) server := s.serverForTest if server == nil { - server = lsp.NewServer(session, client) + server = lsp.NewServer(session, client, s.optionsOverrides) if instance := debug.GetInstance(ctx); instance != nil { instance.AddService(server, session) } diff --git a/gopls/internal/lsp/semantic.go b/gopls/internal/lsp/semantic.go index 12ee8dae903..825e654c2cc 100644 --- a/gopls/internal/lsp/semantic.go +++ b/gopls/internal/lsp/semantic.go @@ -76,8 +76,8 @@ func (s *Server) computeSemanticTokens(ctx context.Context, td protocol.TextDocu ctx: ctx, metadataSource: snapshot, rng: rng, - tokTypes: s.session.Options().SemanticTypes, - tokMods: s.session.Options().SemanticMods, + tokTypes: snapshot.Options().SemanticTypes, + tokMods: snapshot.Options().SemanticMods, } add := func(line, start uint32, len uint32) { e.add(line, start, len, tokMacro, nil) @@ -108,8 +108,8 @@ func (s *Server) computeSemanticTokens(ctx context.Context, td protocol.TextDocu ti: pkg.GetTypesInfo(), pkg: pkg, fset: pkg.FileSet(), - tokTypes: s.session.Options().SemanticTypes, - tokMods: s.session.Options().SemanticMods, + tokTypes: snapshot.Options().SemanticTypes, + tokMods: snapshot.Options().SemanticMods, noStrings: snapshot.Options().NoSemanticString, noNumbers: snapshot.Options().NoSemanticNumber, } diff --git a/gopls/internal/lsp/server.go b/gopls/internal/lsp/server.go index 94275b96343..bddf86458ea 100644 --- a/gopls/internal/lsp/server.go +++ b/gopls/internal/lsp/server.go @@ -26,7 +26,11 @@ const concurrentAnalyses = 1 // NewServer creates an LSP server and binds it to handle incoming client // messages on the supplied stream. -func NewServer(session *cache.Session, client protocol.ClientCloser) *Server { +func NewServer(session *cache.Session, client protocol.ClientCloser, optionsOverrides func(*source.Options)) *Server { + options := source.DefaultOptions().Clone() + if optionsOverrides != nil { + optionsOverrides(options) + } return &Server{ diagnostics: map[span.URI]*fileReports{}, gcOptimizationDetails: make(map[source.PackageID]struct{}), @@ -36,6 +40,7 @@ func NewServer(session *cache.Session, client protocol.ClientCloser) *Server { client: client, diagnosticsSema: make(chan struct{}, concurrentAnalyses), progress: progress.NewTracker(client), + options: options, } } @@ -115,6 +120,10 @@ type Server struct { // terminated with the StopProfile command. ongoingProfileMu sync.Mutex ongoingProfile *os.File // if non-nil, an ongoing profile is writing to this file + + // Track most recently requested options. + optionsMu sync.Mutex + options *source.Options } func (s *Server) workDoneProgressCancel(ctx context.Context, params *protocol.WorkDoneProgressCancelParams) error { diff --git a/gopls/internal/lsp/text_synchronization.go b/gopls/internal/lsp/text_synchronization.go index 5584287a78c..087f36f3b9c 100644 --- a/gopls/internal/lsp/text_synchronization.go +++ b/gopls/internal/lsp/text_synchronization.go @@ -88,6 +88,8 @@ func (s *Server) didOpen(ctx context.Context, params *protocol.DidOpenTextDocume // views, but it won't because ViewOf only returns an error when there // are no views in the session. I don't know if that logic should go // here, or if we can continue to rely on that implementation detail. + // + // TODO(golang/go#57979): this will be generalized to a different view calculation. if _, err := s.session.ViewOf(uri); err != nil { dir := filepath.Dir(uri.Filename()) if err := s.addFolders(ctx, []protocol.WorkspaceFolder{{ @@ -239,7 +241,7 @@ func (s *Server) didModifyFiles(ctx context.Context, modifications []source.File wg.Add(1) defer wg.Done() - if s.session.Options().VerboseWorkDoneProgress { + if s.Options().VerboseWorkDoneProgress { work := s.progress.Start(ctx, DiagnosticWorkTitle(cause), "Calculating file diagnostics...", nil, nil) go func() { wg.Wait() diff --git a/gopls/internal/lsp/workspace.go b/gopls/internal/lsp/workspace.go index e5f813e730c..7260a69ce93 100644 --- a/gopls/internal/lsp/workspace.go +++ b/gopls/internal/lsp/workspace.go @@ -36,8 +36,8 @@ func (s *Server) addView(ctx context.Context, name string, uri span.URI) (source if state < serverInitialized { return nil, nil, fmt.Errorf("addView called before server initialized") } - options := s.session.Options().Clone() - if err := s.fetchConfig(ctx, name, uri, options); err != nil { + options, err := s.fetchFolderOptions(ctx, uri) + if err != nil { return nil, nil, err } _, snapshot, release, err := s.session.NewView(ctx, name, uri, options) @@ -49,22 +49,24 @@ func (s *Server) didChangeConfiguration(ctx context.Context, _ *protocol.DidChan defer done() // Apply any changes to the session-level settings. - options := s.session.Options().Clone() - if err := s.fetchConfig(ctx, "", "", options); err != nil { + options, err := s.fetchFolderOptions(ctx, "") + if err != nil { return err } - s.session.SetOptions(options) + s.SetOptions(options) - // Go through each view, getting and updating its configuration. + // Collect options for all workspace folders. + seen := make(map[span.URI]bool) for _, view := range s.session.Views() { - options := s.session.Options().Clone() - if err := s.fetchConfig(ctx, view.Name(), view.Folder(), options); err != nil { - return err + if seen[view.Folder()] { + continue } - _, err := s.session.SetViewOptions(ctx, view, options) + seen[view.Folder()] = true + options, err := s.fetchFolderOptions(ctx, view.Folder()) if err != nil { return err } + s.session.SetFolderOptions(ctx, view.Folder(), options) } // Now that all views have been updated: reset vulncheck diagnostics, rerun diff --git a/gopls/internal/lsp/workspace_symbol.go b/gopls/internal/lsp/workspace_symbol.go index 88b3e8865ae..451289f1cad 100644 --- a/gopls/internal/lsp/workspace_symbol.go +++ b/gopls/internal/lsp/workspace_symbol.go @@ -17,8 +17,8 @@ func (s *Server) symbol(ctx context.Context, params *protocol.WorkspaceSymbolPar defer done() views := s.session.Views() - matcher := s.session.Options().SymbolMatcher - style := s.session.Options().SymbolStyle + matcher := s.Options().SymbolMatcher + style := s.Options().SymbolStyle // TODO(rfindley): it looks wrong that we need to pass views here. // // Evidence: From 0a9721c3dd4248e8ead3c3e0912c175480975dac Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Wed, 6 Sep 2023 13:49:15 -0400 Subject: [PATCH 074/178] gopls/internal/lsp: move options into the snapshot Snapshots should be idempotent, and the fact that configuration changes do not cause snapshots to increment has been a long-standing source of bugs. After a fair bit of setup, this CL finally moves options onto the snapshot. The only remaining use of the options stored on the View is for "minorOptionsChange" detection. This is of questionable value, but I opted not to delete it in this CL (Chesterton's fence). There is still more cleanup to do (and tests to update), but that is deferred to later CLs. Fixes golang/go#61325 Fixes golang/go#42814 Change-Id: If82dc199af13ddaa3464b2a67a8bab2013161f26 Reviewed-on: https://go-review.googlesource.com/c/tools/+/526159 LUCI-TryBot-Result: Go LUCI gopls-CI: kokoro Reviewed-by: Alan Donovan --- gopls/internal/lsp/cache/analysis.go | 4 +- gopls/internal/lsp/cache/check.go | 4 +- gopls/internal/lsp/cache/imports.go | 14 ++- gopls/internal/lsp/cache/load.go | 12 +-- gopls/internal/lsp/cache/session.go | 17 +++- gopls/internal/lsp/cache/snapshot.go | 73 +++++++++----- gopls/internal/lsp/cache/view.go | 101 ++++++++------------ gopls/internal/lsp/cmd/capabilities_test.go | 4 +- gopls/internal/lsp/cmd/cmd.go | 13 +-- gopls/internal/lsp/cmd/info.go | 3 +- gopls/internal/lsp/completion_test.go | 5 +- gopls/internal/lsp/debug/serve.go | 2 - gopls/internal/lsp/general.go | 2 +- gopls/internal/lsp/lsp_test.go | 5 +- gopls/internal/lsp/lsprpc/lsprpc.go | 6 +- gopls/internal/lsp/server.go | 6 +- gopls/internal/lsp/source/options.go | 10 +- 17 files changed, 145 insertions(+), 136 deletions(-) diff --git a/gopls/internal/lsp/cache/analysis.go b/gopls/internal/lsp/cache/analysis.go index 5676a7814a2..dfb09951494 100644 --- a/gopls/internal/lsp/cache/analysis.go +++ b/gopls/internal/lsp/cache/analysis.go @@ -192,7 +192,7 @@ func (snapshot *snapshot) Analyze(ctx context.Context, pkgs map[PackageID]unit, toSrc := make(map[*analysis.Analyzer]*source.Analyzer) var enabled []*analysis.Analyzer // enabled subset + transitive requirements for _, a := range analyzers { - if a.IsEnabled(snapshot.view.Options()) { + if a.IsEnabled(snapshot.options) { toSrc[a.Analyzer] = a enabled = append(enabled, a.Analyzer) } @@ -309,7 +309,7 @@ func (snapshot *snapshot) Analyze(ctx context.Context, pkgs map[PackageID]unit, // Now that we have read all files, // we no longer need the snapshot. // (but options are needed for progress reporting) - options := snapshot.view.Options() + options := snapshot.options snapshot = nil // Progress reporting. If supported, gopls reports progress on analysis diff --git a/gopls/internal/lsp/cache/check.go b/gopls/internal/lsp/cache/check.go index b7267e983ec..438da1604e6 100644 --- a/gopls/internal/lsp/cache/check.go +++ b/gopls/internal/lsp/cache/check.go @@ -1345,8 +1345,8 @@ func (s *snapshot) typeCheckInputs(ctx context.Context, m *source.Metadata) (typ depsByImpPath: m.DepsByImpPath, goVersion: goVersion, - relatedInformation: s.view.Options().RelatedInformationSupported, - linkTarget: s.view.Options().LinkTarget, + relatedInformation: s.options.RelatedInformationSupported, + linkTarget: s.options.LinkTarget, moduleMode: s.view.moduleMode(), }, nil } diff --git a/gopls/internal/lsp/cache/imports.go b/gopls/internal/lsp/cache/imports.go index f22d42c21a8..028607608cc 100644 --- a/gopls/internal/lsp/cache/imports.go +++ b/gopls/internal/lsp/cache/imports.go @@ -54,15 +54,13 @@ func (s *importsState) runProcessEnvFunc(ctx context.Context, snapshot *snapshot // view.goEnv is immutable -- changes make a new view. Options can change. // We can't compare build flags directly because we may add -modfile. - snapshot.view.optionsMu.Lock() - localPrefix := snapshot.view.options.Local - currentBuildFlags := snapshot.view.options.BuildFlags - currentDirectoryFilters := snapshot.view.options.DirectoryFilters + localPrefix := snapshot.options.Local + currentBuildFlags := snapshot.options.BuildFlags + currentDirectoryFilters := snapshot.options.DirectoryFilters changed := !reflect.DeepEqual(currentBuildFlags, s.cachedBuildFlags) || - snapshot.view.options.VerboseOutput != (s.processEnv.Logf != nil) || + snapshot.options.VerboseOutput != (s.processEnv.Logf != nil) || modFileHash != s.cachedModFileHash || - !reflect.DeepEqual(snapshot.view.options.DirectoryFilters, s.cachedDirectoryFilters) - snapshot.view.optionsMu.Unlock() + !reflect.DeepEqual(snapshot.options.DirectoryFilters, s.cachedDirectoryFilters) // If anything relevant to imports has changed, clear caches and // update the processEnv. Clearing caches blocks on any background @@ -120,7 +118,7 @@ func populateProcessEnvFromSnapshot(ctx context.Context, pe *imports.ProcessEnv, ctx, done := event.Start(ctx, "cache.populateProcessEnvFromSnapshot") defer done() - if snapshot.view.Options().VerboseOutput { + if snapshot.options.VerboseOutput { pe.Logf = func(format string, args ...interface{}) { event.Log(ctx, fmt.Sprintf(format, args...)) } diff --git a/gopls/internal/lsp/cache/load.go b/gopls/internal/lsp/cache/load.go index df68ba7429d..331e0e7ebc7 100644 --- a/gopls/internal/lsp/cache/load.go +++ b/gopls/internal/lsp/cache/load.go @@ -75,7 +75,7 @@ func (s *snapshot) load(ctx context.Context, allowNetwork bool, scopes ...loadSc if err != nil { continue } - if isStandaloneFile(contents, s.view.Options().StandaloneTags) { + if isStandaloneFile(contents, s.options.StandaloneTags) { standalone = true query = append(query, uri.Filename()) } else { @@ -160,7 +160,7 @@ func (s *snapshot) load(ctx context.Context, allowNetwork bool, scopes ...loadSc } moduleErrs := make(map[string][]packages.Error) // module path -> errors - filterFunc := s.view.filterFunc() + filterFunc := s.filterFunc() newMetadata := make(map[PackageID]*source.Metadata) for _, pkg := range pkgs { // The Go command returns synthetic list results for module queries that @@ -178,7 +178,7 @@ func (s *snapshot) load(ctx context.Context, allowNetwork bool, scopes ...loadSc continue } - if !containsDir || s.view.Options().VerboseOutput { + if !containsDir || s.options.VerboseOutput { event.Log(ctx, eventName, append( source.SnapshotLabels(s), tag.Package.Of(pkg.ID), @@ -359,7 +359,7 @@ func (s *snapshot) applyCriticalErrorToFiles(ctx context.Context, msg string, fi for _, fh := range files { // Place the diagnostics on the package or module declarations. var rng protocol.Range - switch s.view.FileKind(fh) { + switch s.FileKind(fh) { case source.Go: if pgf, err := s.ParseGo(ctx, fh, source.ParseHeader); err == nil { // Check that we have a valid `package foo` range to use for positioning the error. @@ -616,7 +616,7 @@ func containsPackageLocked(s *snapshot, m *source.Metadata) bool { uris[uri] = struct{}{} } - filterFunc := s.view.filterFunc() + filterFunc := s.filterFunc() for uri := range uris { // Don't use view.contains here. go.work files may include modules // outside of the workspace folder. @@ -671,7 +671,7 @@ func containsFileInWorkspaceLocked(s *snapshot, m *source.Metadata) bool { // The package's files are in this view. It may be a workspace package. // Vendored packages are not likely to be interesting to the user. - if !strings.Contains(string(uri), "/vendor/") && s.view.contains(uri) { + if !strings.Contains(string(uri), "/vendor/") && s.contains(uri) { return true } } diff --git a/gopls/internal/lsp/cache/session.go b/gopls/internal/lsp/cache/session.go index edfb78ec979..e2869dd6e29 100644 --- a/gopls/internal/lsp/cache/session.go +++ b/gopls/internal/lsp/cache/session.go @@ -117,9 +117,9 @@ func (s *Session) createView(ctx context.Context, name string, folder span.URI, v := &View{ id: strconv.FormatInt(index, 10), gocmdRunner: s.gocmdRunner, + lastOptions: options, initialWorkspaceLoad: make(chan struct{}), initializationSema: make(chan struct{}, 1), - options: options, baseCtx: baseCtx, name: name, moduleUpgrades: map[span.URI]map[string]string{}, @@ -167,6 +167,7 @@ func (s *Session) createView(ctx context.Context, name string, folder span.URI, workspaceModFiles: wsModFiles, workspaceModFilesErr: wsModFilesErr, pkgIndex: typerefs.NewPackageIndex(), + options: options, } // Save one reference in the view. v.releaseSnapshot = v.snapshot.Acquire() @@ -255,9 +256,15 @@ func bestViewForURI(uri span.URI, views []*View) *View { } // TODO(rfindley): this should consider the workspace layout (i.e. // go.work). - if view.contains(uri) { + snapshot, release, err := view.getSnapshot() + if err != nil { + // view is shutdown + continue + } + if snapshot.contains(uri) { longest = view } + release() } if longest != nil { return longest @@ -420,7 +427,7 @@ func (s *Session) DidModifyFiles(ctx context.Context, changes []source.FileModif // synchronously to change processing? Can we assume that the env did not // change, and derive go.work using a combination of the configured // GOWORK value and filesystem? - info, err := s.getWorkspaceInformation(ctx, view.folder, view.Options()) + info, err := s.getWorkspaceInformation(ctx, view.folder, view.lastOptions) if err != nil { // Catastrophic failure, equivalent to a failure of session // initialization and therefore should almost never happen. One @@ -434,7 +441,7 @@ func (s *Session) DidModifyFiles(ctx context.Context, changes []source.FileModif } if info != view.workspaceInformation { - if err := s.updateViewLocked(ctx, view, view.Options()); err != nil { + if err := s.updateViewLocked(ctx, view, view.lastOptions); err != nil { // More catastrophic failure. The view may or may not still exist. // The best we can do is log and move on. event.Error(ctx, "recreating view", err) @@ -492,7 +499,7 @@ func (s *Session) DidModifyFiles(ctx context.Context, changes []source.FileModif var releases []func() viewToSnapshot := map[*View]*snapshot{} for view, changed := range views { - snapshot, release := view.invalidateContent(ctx, changed, forceReloadMetadata) + snapshot, release := view.invalidateContent(ctx, changed, nil, forceReloadMetadata) releases = append(releases, release) viewToSnapshot[view] = snapshot } diff --git a/gopls/internal/lsp/cache/snapshot.go b/gopls/internal/lsp/cache/snapshot.go index 4010c985747..9d659f046dc 100644 --- a/gopls/internal/lsp/cache/snapshot.go +++ b/gopls/internal/lsp/cache/snapshot.go @@ -187,6 +187,10 @@ type snapshot struct { // active packages. Running type checking batches in parallel after an // invalidation can cause redundant calculation of this shared state. typeCheckMu sync.Mutex + + // options holds the user configuration at the time this snapshot was + // created. + options *source.Options } var globalSnapshotID uint64 @@ -275,12 +279,39 @@ func (s *snapshot) View() source.View { return s.view } -func (s *snapshot) FileKind(h source.FileHandle) source.FileKind { - return s.view.FileKind(h) +func (s *snapshot) FileKind(fh source.FileHandle) source.FileKind { + // The kind of an unsaved buffer comes from the + // TextDocumentItem.LanguageID field in the didChange event, + // not from the file name. They may differ. + if o, ok := fh.(*Overlay); ok { + if o.kind != source.UnknownKind { + return o.kind + } + } + + fext := filepath.Ext(fh.URI().Filename()) + switch fext { + case ".go": + return source.Go + case ".mod": + return source.Mod + case ".sum": + return source.Sum + case ".work": + return source.Work + } + exts := s.options.TemplateExtensions + for _, ext := range exts { + if fext == ext || fext == "."+ext { + return source.Tmpl + } + } + // and now what? This should never happen, but it does for cgo before go1.15 + return source.Go } func (s *snapshot) Options() *source.Options { - return s.view.Options() // temporarily return view options. + return s.options // temporarily return view options. } func (s *snapshot) BackgroundContext() context.Context { @@ -306,7 +337,7 @@ func (s *snapshot) Templates() map[span.URI]source.FileHandle { tmpls := map[span.URI]source.FileHandle{} s.files.Range(func(k span.URI, fh source.FileHandle) { - if s.view.FileKind(fh) == source.Tmpl { + if s.FileKind(fh) == source.Tmpl { tmpls[k] = fh } }) @@ -354,8 +385,7 @@ func (s *snapshot) workspaceMode() workspaceMode { return mode } mode |= moduleMode - options := s.view.Options() - if options.TempModfile { + if s.options.TempModfile { mode |= tempModfile } return mode @@ -368,9 +398,6 @@ func (s *snapshot) workspaceMode() workspaceMode { // multiple modules in on config, so buildOverlay needs to filter overlays by // module. func (s *snapshot) config(ctx context.Context, inv *gocommand.Invocation) *packages.Config { - s.view.optionsMu.Lock() - verboseOutput := s.view.options.VerboseOutput - s.view.optionsMu.Unlock() cfg := &packages.Config{ Context: ctx, @@ -393,7 +420,7 @@ func (s *snapshot) config(ctx context.Context, inv *gocommand.Invocation) *packa panic("go/packages must not be used to parse files") }, Logf: func(format string, args ...interface{}) { - if verboseOutput { + if s.options.VerboseOutput { event.Log(ctx, fmt.Sprintf(format, args...)) } }, @@ -475,18 +502,16 @@ func (s *snapshot) RunGoCommands(ctx context.Context, allowNetwork bool, wd stri // it used only after call to tempModFile. Clarify that it is only // non-nil on success. func (s *snapshot) goCommandInvocation(ctx context.Context, flags source.InvocationFlags, inv *gocommand.Invocation) (tmpURI span.URI, updatedInv *gocommand.Invocation, cleanup func(), err error) { - s.view.optionsMu.Lock() - allowModfileModificationOption := s.view.options.AllowModfileModifications - allowNetworkOption := s.view.options.AllowImplicitNetworkAccess + allowModfileModificationOption := s.options.AllowModfileModifications + allowNetworkOption := s.options.AllowImplicitNetworkAccess // TODO(rfindley): this is very hard to follow, and may not even be doing the // right thing: should inv.Env really trample view.options? Do we ever invoke // this with a non-empty inv.Env? // // We should refactor to make it clearer that the correct env is being used. - inv.Env = append(append(append(os.Environ(), s.view.options.EnvSlice()...), inv.Env...), "GO111MODULE="+s.view.GO111MODULE()) - inv.BuildFlags = append([]string{}, s.view.options.BuildFlags...) - s.view.optionsMu.Unlock() + inv.Env = append(append(append(os.Environ(), s.options.EnvSlice()...), inv.Env...), "GO111MODULE="+s.view.GO111MODULE()) + inv.BuildFlags = append([]string{}, s.options.BuildFlags...) cleanup = func() {} // fallback // All logic below is for module mode. @@ -993,8 +1018,7 @@ func (s *snapshot) workspaceDirs(ctx context.Context) []string { // Code) that do not send notifications for individual files in a directory // when the entire directory is deleted. func (s *snapshot) watchSubdirs() bool { - opts := s.view.Options() - switch p := opts.SubdirWatchPatterns; p { + switch p := s.options.SubdirWatchPatterns; p { case source.SubdirWatchPatternsOn: return true case source.SubdirWatchPatternsOff: @@ -1007,7 +1031,7 @@ func (s *snapshot) watchSubdirs() bool { // requirements that client names do not change. We should update the VS // Code extension to set a default value of "subdirWatchPatterns" to "on", // so that this workaround is only temporary. - if opts.ClientInfo != nil && opts.ClientInfo.Name == "Visual Studio Code" { + if s.options.ClientInfo != nil && s.options.ClientInfo.Name == "Visual Studio Code" { return true } return false @@ -1505,7 +1529,7 @@ func (s *snapshot) reloadOrphanedOpenFiles(ctx context.Context) error { var files []*Overlay for _, o := range open { uri := o.URI() - if s.IsBuiltin(uri) || s.view.FileKind(o) != source.Go { + if s.IsBuiltin(uri) || s.FileKind(o) != source.Go { continue } if len(meta.ids[uri]) == 0 { @@ -1606,7 +1630,7 @@ func (s *snapshot) OrphanedFileDiagnostics(ctx context.Context) (map[span.URI]*s searchOverlays: for _, o := range s.overlays() { uri := o.URI() - if s.IsBuiltin(uri) || s.view.FileKind(o) != source.Go { + if s.IsBuiltin(uri) || s.FileKind(o) != source.Go { continue } md, err := s.MetadataForFile(ctx, uri) @@ -1805,7 +1829,7 @@ func inVendor(uri span.URI) bool { return found && strings.Contains(after, "/") } -func (s *snapshot) clone(ctx, bgCtx context.Context, changes map[span.URI]source.FileHandle, forceReloadMetadata bool) (*snapshot, func()) { +func (s *snapshot) clone(ctx, bgCtx context.Context, changes map[span.URI]source.FileHandle, newOptions *source.Options, forceReloadMetadata bool) (*snapshot, func()) { ctx, done := event.Start(ctx, "cache.snapshot.clone") defer done() @@ -1838,6 +1862,11 @@ func (s *snapshot) clone(ctx, bgCtx context.Context, changes map[span.URI]source workspaceModFilesErr: s.workspaceModFilesErr, importGraph: s.importGraph, pkgIndex: s.pkgIndex, + options: s.options, + } + + if newOptions != nil { + result.options = newOptions } // Create a lease on the new snapshot. diff --git a/gopls/internal/lsp/cache/view.go b/gopls/internal/lsp/cache/view.go index a71fc1d6acd..7c2eda1743b 100644 --- a/gopls/internal/lsp/cache/view.go +++ b/gopls/internal/lsp/cache/view.go @@ -46,8 +46,11 @@ type View struct { // name is the user-specified name of this view. name string - optionsMu sync.Mutex - options *source.Options + // lastOptions holds the most recent options on this view, used for detecting + // major changes. + // + // Guarded by Session.viewMu. + lastOptions *source.Options // Workspace information. The fields below are immutable, and together with // options define the build list. Any change to these fields results in a new @@ -412,48 +415,18 @@ func (v *View) Folder() span.URI { return v.folder } -func (v *View) Options() *source.Options { - v.optionsMu.Lock() - defer v.optionsMu.Unlock() - return v.options -} - -func (v *View) FileKind(fh source.FileHandle) source.FileKind { - // The kind of an unsaved buffer comes from the - // TextDocumentItem.LanguageID field in the didChange event, - // not from the file name. They may differ. - if o, ok := fh.(*Overlay); ok { - if o.kind != source.UnknownKind { - return o.kind - } - } - - fext := filepath.Ext(fh.URI().Filename()) - switch fext { - case ".go": - return source.Go - case ".mod": - return source.Mod - case ".sum": - return source.Sum - case ".work": - return source.Work - } - exts := v.Options().TemplateExtensions - for _, ext := range exts { - if fext == ext || fext == "."+ext { - return source.Tmpl - } - } - // and now what? This should never happen, but it does for cgo before go1.15 - return source.Go -} - func minorOptionsChange(a, b *source.Options) bool { // TODO(rfindley): this function detects whether a view should be recreated, // but this is also checked by the getWorkspaceInformation logic. // // We should eliminate this redundancy. + // + // Additionally, this function has existed for a long time, but git history + // suggests that it was added arbitrarily, not due to an actual performance + // problem. + // + // Especially now that we have optimized reinitialization of the session, we + // should consider just always creating a new view on any options change. // Check if any of the settings that modify our understanding of files have // been changed. @@ -503,13 +476,12 @@ func (s *Session) SetFolderOptions(ctx context.Context, uri span.URI, options *s func (s *Session) setViewOptions(ctx context.Context, v *View, options *source.Options) error { // no need to rebuild the view if the options were not materially changed - v.optionsMu.Lock() - if minorOptionsChange(v.options, options) { - v.options = options - v.optionsMu.Unlock() + if minorOptionsChange(v.lastOptions, options) { + _, release := v.invalidateContent(ctx, nil, options, false) + release() + v.lastOptions = options return nil } - v.optionsMu.Unlock() return s.updateViewLocked(ctx, v, options) } @@ -517,8 +489,8 @@ func (s *Session) setViewOptions(ctx context.Context, v *View, options *source.O // // It must not be called concurrently with any other view methods. func viewEnv(v *View) string { - env := v.options.EnvSlice() - buildFlags := append([]string{}, v.options.BuildFlags...) + env := v.snapshot.options.EnvSlice() + buildFlags := append([]string{}, v.snapshot.options.BuildFlags...) var buf bytes.Buffer fmt.Fprintf(&buf, `go info for %v @@ -567,13 +539,13 @@ func fileHasExtension(path string, suffixes []string) bool { // locateTemplateFiles ensures that the snapshot has mapped template files // within the workspace folder. func (s *snapshot) locateTemplateFiles(ctx context.Context) { - if len(s.view.Options().TemplateExtensions) == 0 { + if len(s.options.TemplateExtensions) == 0 { return } - suffixes := s.view.Options().TemplateExtensions + suffixes := s.options.TemplateExtensions searched := 0 - filterFunc := s.view.filterFunc() + filterFunc := s.filterFunc() err := filepath.WalkDir(s.view.folder.Filename(), func(path string, entry os.DirEntry, err error) error { if err != nil { return err @@ -607,33 +579,33 @@ func (s *snapshot) locateTemplateFiles(ctx context.Context) { } } -func (v *View) contains(uri span.URI) bool { +func (s *snapshot) contains(uri span.URI) bool { // If we've expanded the go dir to a parent directory, consider if the // expanded dir contains the uri. // TODO(rfindley): should we ignore the root here? It is not provided by the // user. It would be better to explicitly consider the set of active modules // wherever relevant. inGoDir := false - if source.InDir(v.goCommandDir.Filename(), v.folder.Filename()) { - inGoDir = source.InDir(v.goCommandDir.Filename(), uri.Filename()) + if source.InDir(s.view.goCommandDir.Filename(), s.view.folder.Filename()) { + inGoDir = source.InDir(s.view.goCommandDir.Filename(), uri.Filename()) } - inFolder := source.InDir(v.folder.Filename(), uri.Filename()) + inFolder := source.InDir(s.view.folder.Filename(), uri.Filename()) if !inGoDir && !inFolder { return false } - return !v.filterFunc()(uri) + return !s.filterFunc()(uri) } // filterFunc returns a func that reports whether uri is filtered by the currently configured // directoryFilters. -func (v *View) filterFunc() func(span.URI) bool { - filterer := buildFilterer(v.folder.Filename(), v.gomodcache, v.Options()) +func (s *snapshot) filterFunc() func(span.URI) bool { + filterer := buildFilterer(s.view.folder.Filename(), s.view.gomodcache, s.options) return func(uri span.URI) bool { // Only filter relative to the configured root directory. - if source.InDir(v.folder.Filename(), uri.Filename()) { - return pathExcludedByFilter(strings.TrimPrefix(uri.Filename(), v.folder.Filename()), filterer) + if source.InDir(s.view.folder.Filename(), uri.Filename()) { + return pathExcludedByFilter(strings.TrimPrefix(uri.Filename(), s.view.folder.Filename()), filterer) } return false } @@ -660,7 +632,12 @@ func (v *View) relevantChange(c source.FileModification) bool { // had neither test nor associated issue, and cited only emacs behavior, this // logic was deleted. - return v.contains(c.URI) + snapshot, release, err := v.getSnapshot() + if err != nil { + return false // view was shut down + } + defer release() + return snapshot.contains(c.URI) } func (v *View) markKnown(uri span.URI) { @@ -938,7 +915,9 @@ func (s *snapshot) loadWorkspace(ctx context.Context, firstAttempt bool) (loadEr // // invalidateContent returns a non-nil snapshot for the new content, along with // a callback which the caller must invoke to release that snapshot. -func (v *View) invalidateContent(ctx context.Context, changes map[span.URI]source.FileHandle, forceReloadMetadata bool) (*snapshot, func()) { +// +// newOptions may be nil, in which case options remain unchanged. +func (v *View) invalidateContent(ctx context.Context, changes map[span.URI]source.FileHandle, newOptions *source.Options, forceReloadMetadata bool) (*snapshot, func()) { // Detach the context so that content invalidation cannot be canceled. ctx = xcontext.Detach(ctx) @@ -960,7 +939,7 @@ func (v *View) invalidateContent(ctx context.Context, changes map[span.URI]sourc prevSnapshot.AwaitInitialized(ctx) // Save one lease of the cloned snapshot in the view. - v.snapshot, v.releaseSnapshot = prevSnapshot.clone(ctx, v.baseCtx, changes, forceReloadMetadata) + v.snapshot, v.releaseSnapshot = prevSnapshot.clone(ctx, v.baseCtx, changes, newOptions, forceReloadMetadata) prevReleaseSnapshot() v.destroy(prevSnapshot, "View.invalidateContent") diff --git a/gopls/internal/lsp/cmd/capabilities_test.go b/gopls/internal/lsp/cmd/capabilities_test.go index ddf56851937..02ff0d950f6 100644 --- a/gopls/internal/lsp/cmd/capabilities_test.go +++ b/gopls/internal/lsp/cmd/capabilities_test.go @@ -15,6 +15,7 @@ import ( "golang.org/x/tools/gopls/internal/lsp" "golang.org/x/tools/gopls/internal/lsp/cache" "golang.org/x/tools/gopls/internal/lsp/protocol" + "golang.org/x/tools/gopls/internal/lsp/source" "golang.org/x/tools/internal/testenv" ) @@ -49,7 +50,8 @@ func TestCapabilities(t *testing.T) { // Send an initialize request to the server. ctx := context.Background() client := newClient(app, nil) - server := lsp.NewServer(cache.NewSession(ctx, cache.New(nil)), client, app.options) + options := source.DefaultOptions(app.options) + server := lsp.NewServer(cache.NewSession(ctx, cache.New(nil)), client, options) result, err := server.Initialize(ctx, params) if err != nil { t.Fatal(err) diff --git a/gopls/internal/lsp/cmd/cmd.go b/gopls/internal/lsp/cmd/cmd.go index 2e1e6229edd..9a9d627880a 100644 --- a/gopls/internal/lsp/cmd/cmd.go +++ b/gopls/internal/lsp/cmd/cmd.go @@ -316,7 +316,8 @@ func (app *Application) connect(ctx context.Context, onProgress func(*protocol.P switch { case app.Remote == "": client := newClient(app, onProgress) - server := lsp.NewServer(cache.NewSession(ctx, cache.New(nil)), client, app.options) + options := source.DefaultOptions(app.options) + server := lsp.NewServer(cache.NewSession(ctx, cache.New(nil)), client, options) conn := newConnection(server, client) if err := conn.initialize(protocol.WithClient(ctx, client), app.options); err != nil { return nil, err @@ -326,10 +327,7 @@ func (app *Application) connect(ctx context.Context, onProgress func(*protocol.P case strings.HasPrefix(app.Remote, "internal@"): internalMu.Lock() defer internalMu.Unlock() - opts := source.DefaultOptions().Clone() - if app.options != nil { - app.options(opts) - } + opts := source.DefaultOptions(app.options) key := fmt.Sprintf("%s %v %v %v", app.wd, opts.PreferredContentFormat, opts.HierarchicalDocumentSymbolSupport, opts.SymbolMatcher) if c := internalConnections[key]; c != nil { return c, nil @@ -376,10 +374,7 @@ func (c *connection) initialize(ctx context.Context, options func(*source.Option params.Capabilities.Workspace.Configuration = true // Make sure to respect configured options when sending initialize request. - opts := source.DefaultOptions().Clone() - if options != nil { - options(opts) - } + opts := source.DefaultOptions(options) // If you add an additional option here, you must update the map key in connect. params.Capabilities.TextDocument.Hover = &protocol.HoverClientCapabilities{ ContentFormat: []protocol.MarkupKind{opts.PreferredContentFormat}, diff --git a/gopls/internal/lsp/cmd/info.go b/gopls/internal/lsp/cmd/info.go index d63e24b6bfc..b0f08bbef67 100644 --- a/gopls/internal/lsp/cmd/info.go +++ b/gopls/internal/lsp/cmd/info.go @@ -299,8 +299,7 @@ gopls also includes software made available under these licenses: ` func (l *licenses) Run(ctx context.Context, args ...string) error { - opts := source.DefaultOptions() - l.app.options(opts) + opts := source.DefaultOptions(l.app.options) txt := licensePreamble if opts.LicensesText == "" { txt += "(development gopls, license information not available)" diff --git a/gopls/internal/lsp/completion_test.go b/gopls/internal/lsp/completion_test.go index 1da7ade563d..48ec9ea094e 100644 --- a/gopls/internal/lsp/completion_test.go +++ b/gopls/internal/lsp/completion_test.go @@ -171,13 +171,12 @@ func (r *runner) toggleOptions(t *testing.T, uri span.URI, options func(*source. } folder := view.Folder() - original := view.Options() - modified := view.Options().Clone() + modified := r.server.Options().Clone() options(modified) if err = r.server.session.SetFolderOptions(r.ctx, folder, modified); err != nil { t.Fatal(err) } return func() { - r.server.session.SetFolderOptions(r.ctx, folder, original) + r.server.session.SetFolderOptions(r.ctx, folder, r.server.Options()) } } diff --git a/gopls/internal/lsp/debug/serve.go b/gopls/internal/lsp/debug/serve.go index c0bfd9e7b50..8aa2938c228 100644 --- a/gopls/internal/lsp/debug/serve.go +++ b/gopls/internal/lsp/debug/serve.go @@ -846,8 +846,6 @@ var ViewTmpl = template.Must(template.Must(BaseTemplate.Clone()).Parse(` {{define "body"}} Name: {{.Name}}
    Folder: {{.Folder}}
    -

    Environment

    -
      {{range .Options.Env}}
    • {{.}}
      • {{end}}
    {{end}} `)) diff --git a/gopls/internal/lsp/general.go b/gopls/internal/lsp/general.go index e2f8ddbbf67..61c16c0531a 100644 --- a/gopls/internal/lsp/general.go +++ b/gopls/internal/lsp/general.go @@ -623,7 +623,7 @@ func (s *Server) beginFileRequest(ctx context.Context, pURI protocol.DocumentURI release() return nil, nil, false, func() {}, err } - if expectKind != source.UnknownKind && view.FileKind(fh) != expectKind { + if expectKind != source.UnknownKind && snapshot.FileKind(fh) != expectKind { // Wrong kind of file. Nothing to do. release() return nil, nil, false, func() {}, nil diff --git a/gopls/internal/lsp/lsp_test.go b/gopls/internal/lsp/lsp_test.go index 285f78a7f40..88f6a3940f1 100644 --- a/gopls/internal/lsp/lsp_test.go +++ b/gopls/internal/lsp/lsp_test.go @@ -52,8 +52,7 @@ func testLSP(t *testing.T, datum *tests.Data) { ctx = debug.WithInstance(ctx, "", "off") session := cache.NewSession(ctx, cache.New(nil)) - options := source.DefaultOptions().Clone() - tests.DefaultOptions(options) + options := source.DefaultOptions(tests.DefaultOptions) options.SetEnvSlice(datum.Config.Env) view, snapshot, release, err := session.NewView(ctx, datum.Config.Dir, span.URIFromPath(datum.Config.Dir), options) if err != nil { @@ -112,7 +111,7 @@ func testLSP(t *testing.T, datum *tests.Data) { editRecv: make(chan map[span.URI][]byte, 1), } - r.server = NewServer(session, testClient{runner: r}, nil) + r.server = NewServer(session, testClient{runner: r}, options) tests.Run(t, r, datum) } diff --git a/gopls/internal/lsp/lsprpc/lsprpc.go b/gopls/internal/lsp/lsprpc/lsprpc.go index ef2c3ed0c54..7dc709291fb 100644 --- a/gopls/internal/lsp/lsprpc/lsprpc.go +++ b/gopls/internal/lsp/lsprpc/lsprpc.go @@ -59,7 +59,8 @@ func (s *StreamServer) Binder() *ServerBinder { session := cache.NewSession(ctx, s.cache) server := s.serverForTest if server == nil { - server = lsp.NewServer(session, client, s.optionsOverrides) + options := source.DefaultOptions(s.optionsOverrides) + server = lsp.NewServer(session, client, options) if instance := debug.GetInstance(ctx); instance != nil { instance.AddService(server, session) } @@ -76,7 +77,8 @@ func (s *StreamServer) ServeStream(ctx context.Context, conn jsonrpc2.Conn) erro session := cache.NewSession(ctx, s.cache) server := s.serverForTest if server == nil { - server = lsp.NewServer(session, client, s.optionsOverrides) + options := source.DefaultOptions(s.optionsOverrides) + server = lsp.NewServer(session, client, options) if instance := debug.GetInstance(ctx); instance != nil { instance.AddService(server, session) } diff --git a/gopls/internal/lsp/server.go b/gopls/internal/lsp/server.go index bddf86458ea..a236779962f 100644 --- a/gopls/internal/lsp/server.go +++ b/gopls/internal/lsp/server.go @@ -26,11 +26,7 @@ const concurrentAnalyses = 1 // NewServer creates an LSP server and binds it to handle incoming client // messages on the supplied stream. -func NewServer(session *cache.Session, client protocol.ClientCloser, optionsOverrides func(*source.Options)) *Server { - options := source.DefaultOptions().Clone() - if optionsOverrides != nil { - optionsOverrides(options) - } +func NewServer(session *cache.Session, client protocol.ClientCloser, options *source.Options) *Server { return &Server{ diagnostics: map[span.URI]*fileReports{}, gcOptimizationDetails: make(map[source.PackageID]struct{}), diff --git a/gopls/internal/lsp/source/options.go b/gopls/internal/lsp/source/options.go index 2b91f834d6a..a0802361bf2 100644 --- a/gopls/internal/lsp/source/options.go +++ b/gopls/internal/lsp/source/options.go @@ -81,7 +81,7 @@ var ( // DefaultOptions is the options that are used for Gopls execution independent // of any externally provided configuration (LSP initialization, command // invocation, etc.). -func DefaultOptions() *Options { +func DefaultOptions(overrides ...func(*Options)) *Options { optionsOnce.Do(func() { var commands []string for _, c := range command.Commands { @@ -189,7 +189,13 @@ func DefaultOptions() *Options { }, } }) - return defaultOptions + options := defaultOptions.Clone() + for _, override := range overrides { + if override != nil { + override(options) + } + } + return options } // Options holds various configuration that affects Gopls execution, organized From cd231d88751c4ebd186bd396e4b40eacbd02a54b Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Wed, 6 Sep 2023 22:30:49 -0400 Subject: [PATCH 075/178] gopls/internal/lsp: track didChangeConfiguration diagnostics for tests By tracking didChangeConfiguration diagnostics, we can fail tests early when diagnostics are expected following a configuration change. Change-Id: I662740333312b12ed589b1f8ce8b31c9937f8a6f Reviewed-on: https://go-review.googlesource.com/c/tools/+/526170 LUCI-TryBot-Result: Go LUCI Reviewed-by: Alan Donovan gopls-CI: kokoro --- gopls/internal/lsp/fake/editor.go | 5 +++- gopls/internal/lsp/regtest/expectation.go | 11 ++++---- gopls/internal/lsp/text_synchronization.go | 17 ++++++----- gopls/internal/lsp/workspace.go | 28 ++++++++----------- .../regtest/misc/configuration_test.go | 8 ++---- .../regtest/workspace/standalone_test.go | 5 ---- 6 files changed, 32 insertions(+), 42 deletions(-) diff --git a/gopls/internal/lsp/fake/editor.go b/gopls/internal/lsp/fake/editor.go index 6fd46c3f133..78db2cca68b 100644 --- a/gopls/internal/lsp/fake/editor.go +++ b/gopls/internal/lsp/fake/editor.go @@ -56,7 +56,7 @@ type Editor struct { // CallCounts tracks the number of protocol notifications of different types. type CallCounts struct { - DidOpen, DidChange, DidSave, DidChangeWatchedFiles, DidClose uint64 + DidOpen, DidChange, DidSave, DidChangeWatchedFiles, DidClose, DidChangeConfiguration uint64 } // buffer holds information about an open buffer in the editor. @@ -1367,6 +1367,9 @@ func (e *Editor) ChangeConfiguration(ctx context.Context, newConfig EditorConfig if err := e.Server.DidChangeConfiguration(ctx, ¶ms); err != nil { return err } + e.callsMu.Lock() + e.calls.DidChangeConfiguration++ + e.callsMu.Unlock() } return nil } diff --git a/gopls/internal/lsp/regtest/expectation.go b/gopls/internal/lsp/regtest/expectation.go index 0136870bc3a..922f9a0b8a1 100644 --- a/gopls/internal/lsp/regtest/expectation.go +++ b/gopls/internal/lsp/regtest/expectation.go @@ -312,11 +312,12 @@ func ShowMessageRequest(title string) Expectation { func (e *Env) DoneDiagnosingChanges() Expectation { stats := e.Editor.Stats() statsBySource := map[lsp.ModificationSource]uint64{ - lsp.FromDidOpen: stats.DidOpen, - lsp.FromDidChange: stats.DidChange, - lsp.FromDidSave: stats.DidSave, - lsp.FromDidChangeWatchedFiles: stats.DidChangeWatchedFiles, - lsp.FromDidClose: stats.DidClose, + lsp.FromDidOpen: stats.DidOpen, + lsp.FromDidChange: stats.DidChange, + lsp.FromDidSave: stats.DidSave, + lsp.FromDidChangeWatchedFiles: stats.DidChangeWatchedFiles, + lsp.FromDidClose: stats.DidClose, + lsp.FromDidChangeConfiguration: stats.DidChangeConfiguration, } var expected []lsp.ModificationSource diff --git a/gopls/internal/lsp/text_synchronization.go b/gopls/internal/lsp/text_synchronization.go index 087f36f3b9c..0dddab2b14c 100644 --- a/gopls/internal/lsp/text_synchronization.go +++ b/gopls/internal/lsp/text_synchronization.go @@ -20,28 +20,27 @@ import ( "golang.org/x/tools/internal/jsonrpc2" ) -// ModificationSource identifies the originating cause of a file modification. +// ModificationSource identifies the origin of a change. type ModificationSource int const ( - // FromDidOpen is a file modification caused by opening a file. + // FromDidOpen is from a didOpen notification. FromDidOpen = ModificationSource(iota) - // FromDidChange is a file modification caused by changing a file. + // FromDidChange is from a didChange notification. FromDidChange - // FromDidChangeWatchedFiles is a file modification caused by a change to a - // watched file. + // FromDidChangeWatchedFiles is from didChangeWatchedFiles notification. FromDidChangeWatchedFiles - // FromDidSave is a file modification caused by a file save. + // FromDidSave is from a didSave notification. FromDidSave - // FromDidClose is a file modification caused by closing a file. + // FromDidClose is from a didClose notification. FromDidClose - // TODO: add FromDidChangeConfiguration, once configuration changes cause a - // new snapshot to be created. + // FromDidChangeConfiguration is from a didChangeConfiguration notification. + FromDidChangeConfiguration // FromRegenerateCgo refers to file modifications caused by regenerating // the cgo sources for the workspace. diff --git a/gopls/internal/lsp/workspace.go b/gopls/internal/lsp/workspace.go index 7260a69ce93..d48e4f473cf 100644 --- a/gopls/internal/lsp/workspace.go +++ b/gopls/internal/lsp/workspace.go @@ -7,6 +7,7 @@ package lsp import ( "context" "fmt" + "sync" "golang.org/x/tools/gopls/internal/lsp/protocol" "golang.org/x/tools/gopls/internal/lsp/source" @@ -69,25 +70,12 @@ func (s *Server) didChangeConfiguration(ctx context.Context, _ *protocol.DidChan s.session.SetFolderOptions(ctx, view.Folder(), options) } - // Now that all views have been updated: reset vulncheck diagnostics, rerun - // diagnostics, and hope for the best... - // - // TODO(golang/go#60465): this not a reliable way to ensure the correctness - // of the resulting diagnostics below. A snapshot could still be in the - // process of diagnosing the workspace, and not observe the configuration - // changes above. - // - // The real fix is golang/go#42814: we should create a new snapshot on any - // change that could affect the derived results in that snapshot. However, we - // are currently (2023-05-26) on the verge of a release, and the proper fix - // is too risky a change. Since in the common case a configuration change is - // only likely to occur during a period of quiescence on the server, it is - // likely that the clearing below will have the desired effect. - s.clearDiagnosticSource(modVulncheckSource) - + var wg sync.WaitGroup for _, view := range s.session.Views() { view := view + wg.Add(1) go func() { + defer wg.Done() snapshot, release, err := view.Snapshot() if err != nil { return // view is shut down; no need to diagnose @@ -97,6 +85,14 @@ func (s *Server) didChangeConfiguration(ctx context.Context, _ *protocol.DidChan }() } + if s.Options().VerboseWorkDoneProgress { + work := s.progress.Start(ctx, DiagnosticWorkTitle(FromDidChangeConfiguration), "Calculating diagnostics...", nil, nil) + go func() { + wg.Wait() + work.End(ctx, "Done.") + }() + } + // An options change may have affected the detected Go version. s.checkViewGoVersions() diff --git a/gopls/internal/regtest/misc/configuration_test.go b/gopls/internal/regtest/misc/configuration_test.go index 853abcd7dff..3f9c5b941c3 100644 --- a/gopls/internal/regtest/misc/configuration_test.go +++ b/gopls/internal/regtest/misc/configuration_test.go @@ -42,10 +42,8 @@ var FooErr = errors.New("foo") cfg.Settings = map[string]interface{}{ "staticcheck": true, } - // TODO(rfindley): support waiting on diagnostics following a configuration - // change. env.ChangeConfiguration(cfg) - env.Await( + env.AfterChange( Diagnostics(env.AtRegexp("a/a.go", "var (FooErr)")), ) }) @@ -89,10 +87,8 @@ var ErrFoo = errors.New("foo") cfg.Settings = map[string]interface{}{ "staticcheck": true, } - // TODO(rfindley): support waiting on diagnostics following a configuration - // change. env.ChangeConfiguration(cfg) - env.Await( + env.AfterChange( Diagnostics(env.AtRegexp("a/a.go", "var (FooErr)")), ) }) diff --git a/gopls/internal/regtest/workspace/standalone_test.go b/gopls/internal/regtest/workspace/standalone_test.go index c9ce2f02924..3e0ea40345d 100644 --- a/gopls/internal/regtest/workspace/standalone_test.go +++ b/gopls/internal/regtest/workspace/standalone_test.go @@ -198,11 +198,6 @@ func main() {} "standaloneTags": []string{"ignore"}, } env.ChangeConfiguration(cfg) - - // TODO(golang/go#56158): gopls does not purge previously published - // diagnostice when configuration changes. - env.RegexpReplace("ignore.go", "arbitrary", "meaningless") - env.AfterChange( NoDiagnostics(ForFile("ignore.go")), Diagnostics(env.AtRegexp("standalone.go", "package (main)")), From 627959a8e32af98dce9ff3e65ef6f491d3dcb9f6 Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Fri, 8 Sep 2023 15:12:38 -0400 Subject: [PATCH 076/178] cmd/stringer: log more information in tests Also associate the stderr and stdout output of subprocesses more clearly with the specific test. For golang/go#62534. Change-Id: I6768f2d8d60e21d4d6465208c17b542691a3f803 Reviewed-on: https://go-review.googlesource.com/c/tools/+/526172 Commit-Queue: Bryan Mills TryBot-Result: Gopher Robot Run-TryBot: Bryan Mills Reviewed-by: Ian Lance Taylor Auto-Submit: Bryan Mills --- cmd/stringer/endtoend_test.go | 56 +++++++++++++++++++++-------------- cmd/stringer/golden_test.go | 50 +++++++++++++++++-------------- cmd/stringer/stringer.go | 5 +++- 3 files changed, 65 insertions(+), 46 deletions(-) diff --git a/cmd/stringer/endtoend_test.go b/cmd/stringer/endtoend_test.go index e7faa4810ab..d513c1b52ba 100644 --- a/cmd/stringer/endtoend_test.go +++ b/cmd/stringer/endtoend_test.go @@ -11,11 +11,11 @@ package main import ( "bytes" + "flag" "fmt" "go/build" "io" "os" - "os/exec" "path" "path/filepath" "strings" @@ -42,6 +42,11 @@ func TestMain(m *testing.M) { // command, and much less complicated and expensive to build and clean up. os.Setenv("STRINGER_TEST_IS_STRINGER", "1") + flag.Parse() + if testing.Verbose() { + os.Setenv("GOPACKAGESDEBUG", "true") + } + os.Exit(m.Run()) } @@ -74,11 +79,12 @@ func TestEndToEnd(t *testing.T) { // This file is used for tag processing in TestTags or TestConstValueChange, below. continue } - if name == "cgo.go" && !build.Default.CgoEnabled { - t.Logf("cgo is not enabled for %s", name) - continue - } - stringerCompileAndRun(t, t.TempDir(), stringer, typeName(name), name) + t.Run(name, func(t *testing.T) { + if name == "cgo.go" && !build.Default.CgoEnabled { + t.Skipf("cgo is not enabled for %s", name) + } + stringerCompileAndRun(t, t.TempDir(), stringer, typeName(name), name) + }) } } @@ -122,7 +128,7 @@ func TestTags(t *testing.T) { // - Versions of Go earlier than Go 1.11, do not support absolute directories as a pattern. // - When the current directory is inside a go module, the path will not be considered // a valid path to a package. - err := runInDir(dir, stringer, "-type", "Const", ".") + err := runInDir(t, dir, stringer, "-type", "Const", ".") if err != nil { t.Fatal(err) } @@ -137,7 +143,7 @@ func TestTags(t *testing.T) { if err != nil { t.Fatal(err) } - err = runInDir(dir, stringer, "-type", "Const", "-tags", "tag", ".") + err = runInDir(t, dir, stringer, "-type", "Const", "-tags", "tag", ".") if err != nil { t.Fatal(err) } @@ -162,12 +168,12 @@ func TestConstValueChange(t *testing.T) { } stringSource := filepath.Join(dir, "day_string.go") // Run stringer in the directory that contains the package files. - err = runInDir(dir, stringer, "-type", "Day", "-output", stringSource) + err = runInDir(t, dir, stringer, "-type", "Day", "-output", stringSource) if err != nil { t.Fatal(err) } // Run the binary in the temporary directory as a sanity check. - err = run("go", "run", stringSource, source) + err = run(t, "go", "run", stringSource, source) if err != nil { t.Fatal(err) } @@ -185,8 +191,8 @@ func TestConstValueChange(t *testing.T) { // output. An alternative might be to check that the error output // matches a set of possible error strings emitted by known // Go compilers. - fmt.Fprintf(os.Stderr, "Note: the following messages should indicate an out-of-bounds compiler error\n") - err = run("go", "build", stringSource, source) + t.Logf("Note: the following messages should indicate an out-of-bounds compiler error\n") + err = run(t, "go", "build", stringSource, source) if err == nil { t.Fatal("unexpected compiler success") } @@ -213,7 +219,6 @@ func stringerPath(t *testing.T) string { // stringerCompileAndRun runs stringer for the named file and compiles and // runs the target binary in directory dir. That binary will panic if the String method is incorrect. func stringerCompileAndRun(t *testing.T, dir, stringer, typeName, fileName string) { - t.Helper() t.Logf("run: %s %s\n", fileName, typeName) source := filepath.Join(dir, path.Base(fileName)) err := copy(source, filepath.Join("testdata", fileName)) @@ -222,12 +227,12 @@ func stringerCompileAndRun(t *testing.T, dir, stringer, typeName, fileName strin } stringSource := filepath.Join(dir, typeName+"_string.go") // Run stringer in temporary directory. - err = run(stringer, "-type", typeName, "-output", stringSource, source) + err = run(t, stringer, "-type", typeName, "-output", stringSource, source) if err != nil { t.Fatal(err) } // Run the binary in the temporary directory. - err = run("go", "run", stringSource, source) + err = run(t, "go", "run", stringSource, source) if err != nil { t.Fatal(err) } @@ -251,17 +256,24 @@ func copy(to, from string) error { // run runs a single command and returns an error if it does not succeed. // os/exec should have this function, to be honest. -func run(name string, arg ...string) error { - return runInDir(".", name, arg...) +func run(t testing.TB, name string, arg ...string) error { + t.Helper() + return runInDir(t, ".", name, arg...) } // runInDir runs a single command in directory dir and returns an error if // it does not succeed. -func runInDir(dir, name string, arg ...string) error { - cmd := exec.Command(name, arg...) +func runInDir(t testing.TB, dir, name string, arg ...string) error { + t.Helper() + cmd := testenv.Command(t, name, arg...) cmd.Dir = dir - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr cmd.Env = append(os.Environ(), "GO111MODULE=auto") - return cmd.Run() + out, err := cmd.CombinedOutput() + if len(out) > 0 { + t.Logf("%s", out) + } + if err != nil { + return fmt.Errorf("%v: %v", cmd, err) + } + return nil } diff --git a/cmd/stringer/golden_test.go b/cmd/stringer/golden_test.go index 250af05f903..a26eef35e36 100644 --- a/cmd/stringer/golden_test.go +++ b/cmd/stringer/golden_test.go @@ -453,28 +453,32 @@ func TestGolden(t *testing.T) { dir := t.TempDir() for _, test := range golden { - g := Generator{ - trimPrefix: test.trimPrefix, - lineComment: test.lineComment, - } - input := "package test\n" + test.input - file := test.name + ".go" - absFile := filepath.Join(dir, file) - err := os.WriteFile(absFile, []byte(input), 0644) - if err != nil { - t.Error(err) - } - - g.parsePackage([]string{absFile}, nil) - // Extract the name and type of the constant from the first line. - tokens := strings.SplitN(test.input, " ", 3) - if len(tokens) != 3 { - t.Fatalf("%s: need type declaration on first line", test.name) - } - g.generate(tokens[1]) - got := string(g.format()) - if got != test.output { - t.Errorf("%s: got(%d)\n====\n%q====\nexpected(%d)\n====%q", test.name, len(got), got, len(test.output), test.output) - } + test := test + t.Run(test.name, func(t *testing.T) { + g := Generator{ + trimPrefix: test.trimPrefix, + lineComment: test.lineComment, + logf: t.Logf, + } + input := "package test\n" + test.input + file := test.name + ".go" + absFile := filepath.Join(dir, file) + err := os.WriteFile(absFile, []byte(input), 0644) + if err != nil { + t.Fatal(err) + } + + g.parsePackage([]string{absFile}, nil) + // Extract the name and type of the constant from the first line. + tokens := strings.SplitN(test.input, " ", 3) + if len(tokens) != 3 { + t.Fatalf("%s: need type declaration on first line", test.name) + } + g.generate(tokens[1]) + got := string(g.format()) + if got != test.output { + t.Errorf("%s: got(%d)\n====\n%q====\nexpected(%d)\n====%q", test.name, len(got), got, len(test.output), test.output) + } + }) } } diff --git a/cmd/stringer/stringer.go b/cmd/stringer/stringer.go index 998d1a51bfd..2b19c93e8ea 100644 --- a/cmd/stringer/stringer.go +++ b/cmd/stringer/stringer.go @@ -188,6 +188,8 @@ type Generator struct { trimPrefix string lineComment bool + + logf func(format string, args ...interface{}) // test logging hook; nil when not testing } func (g *Generator) Printf(format string, args ...interface{}) { @@ -221,13 +223,14 @@ func (g *Generator) parsePackage(patterns []string, tags []string) { // in a separate pass? For later. Tests: false, BuildFlags: []string{fmt.Sprintf("-tags=%s", strings.Join(tags, " "))}, + Logf: g.logf, } pkgs, err := packages.Load(cfg, patterns...) if err != nil { log.Fatal(err) } if len(pkgs) != 1 { - log.Fatalf("error: %d packages found", len(pkgs)) + log.Fatalf("error: %d packages matching %v", len(pkgs), strings.Join(patterns, " ")) } g.addPackage(pkgs[0]) } From 0049711f4906ce1cf43e2da29953045d1a9fa240 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Mon, 11 Sep 2023 13:54:19 -0400 Subject: [PATCH 077/178] go/types/internal/play: show underlying and core types Change-Id: I8cc27bfc08b9a81d3e3baad271b51fd95e0feae3 Reviewed-on: https://go-review.googlesource.com/c/tools/+/527355 gopls-CI: kokoro Reviewed-by: Robert Findley LUCI-TryBot-Result: Go LUCI Run-TryBot: Alan Donovan TryBot-Result: Gopher Robot --- go/types/internal/play/play.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/go/types/internal/play/play.go b/go/types/internal/play/play.go index ace7fddc7ab..9e8a2aade1f 100644 --- a/go/types/internal/play/play.go +++ b/go/types/internal/play/play.go @@ -30,6 +30,7 @@ import ( "golang.org/x/tools/go/ast/astutil" "golang.org/x/tools/go/packages" "golang.org/x/tools/go/types/typeutil" + "golang.org/x/tools/internal/typeparams" ) // TODO(adonovan): @@ -155,10 +156,16 @@ func handleSelectJSON(w http.ResponseWriter, req *http.Request) { modes = append(modes, mode.name) } } - fmt.Fprintf(out, "%T has type %v and mode %s", + fmt.Fprintf(out, "%T has type %v, mode %s", innermostExpr, tv.Type, modes) + if tu := tv.Type.Underlying(); tu != tv.Type { + fmt.Fprintf(out, ", underlying type %v", tu) + } + if tc := typeparams.CoreType(tv.Type); tc != tv.Type { + fmt.Fprintf(out, ", core type %v", tc) + } if tv.Value != nil { - fmt.Fprintf(out, " and constant value %v", tv.Value) + fmt.Fprintf(out, ", and constant value %v", tv.Value) } fmt.Fprintf(out, "\n\n") } From 9886d9888c815ee68795c08e4331fcd663fbc441 Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Fri, 8 Sep 2023 10:10:54 -0400 Subject: [PATCH 078/178] all: get tests passing with 1.18 and 1.19 In order to replace gopls legacy builds on Kokoro with LUCI, it is easiest to get x/tools tests passing at older Go versions. This CL gets tests passing at 1.18 and 1.19, using various techniques: - Where it was easy to carve out some leave node of the package graph, just use go:build directives. - Add an internal/compat package with compatibility shims (much like the typeparams package, which is coincidentally now obsolete). It turns out, the only required shim is Appendf. Notably, some go/ssa and go/ssa/interp tests recurse infinitely due to a go/types bug, which was not investigated. Change-Id: Ib2e6cc8a804ead394a324a4598728b200b24091c Reviewed-on: https://go-review.googlesource.com/c/tools/+/526264 Reviewed-by: Alan Donovan LUCI-TryBot-Result: Go LUCI gopls-CI: kokoro --- cmd/bisect/main_test.go | 3 ++- go/analysis/passes/printf/printf_test.go | 3 +++ go/analysis/unitchecker/separate_test.go | 2 ++ go/analysis/unitchecker/unitchecker_test.go | 2 ++ go/callgraph/rta/rta.go | 3 ++- go/ssa/builder_test.go | 4 +--- go/ssa/interp/interp_go117_test.go | 12 ------------ go/ssa/interp/interp_test.go | 5 +++++ go/types/internal/play/play.go | 2 ++ internal/cmd/deadcode/deadcode.go | 2 ++ internal/cmd/deadcode/deadcode_test.go | 2 ++ internal/compat/appendf.go | 13 +++++++++++++ internal/compat/appendf_118.go | 13 +++++++++++++ internal/compat/doc.go | 7 +++++++ internal/refactor/inline/analyzer/analyzer.go | 2 ++ internal/refactor/inline/analyzer/analyzer_test.go | 2 ++ 16 files changed, 60 insertions(+), 17 deletions(-) delete mode 100644 go/ssa/interp/interp_go117_test.go create mode 100644 internal/compat/appendf.go create mode 100644 internal/compat/appendf_118.go create mode 100644 internal/compat/doc.go diff --git a/cmd/bisect/main_test.go b/cmd/bisect/main_test.go index bff1bf23c0c..7c10ff0fb4b 100644 --- a/cmd/bisect/main_test.go +++ b/cmd/bisect/main_test.go @@ -17,6 +17,7 @@ import ( "testing" "golang.org/x/tools/internal/bisect" + "golang.org/x/tools/internal/compat" "golang.org/x/tools/internal/diffp" "golang.org/x/tools/txtar" ) @@ -81,7 +82,7 @@ func Test(t *testing.T) { have[color] = true } if m.ShouldReport(uint64(i)) { - out = fmt.Appendf(out, "%s %s\n", color, bisect.Marker(uint64(i))) + out = compat.Appendf(out, "%s %s\n", color, bisect.Marker(uint64(i))) } } err = nil diff --git a/go/analysis/passes/printf/printf_test.go b/go/analysis/passes/printf/printf_test.go index 142afa14e89..ed857fe801c 100644 --- a/go/analysis/passes/printf/printf_test.go +++ b/go/analysis/passes/printf/printf_test.go @@ -9,10 +9,13 @@ import ( "golang.org/x/tools/go/analysis/analysistest" "golang.org/x/tools/go/analysis/passes/printf" + "golang.org/x/tools/internal/testenv" "golang.org/x/tools/internal/typeparams" ) func Test(t *testing.T) { + testenv.NeedsGo1Point(t, 19) // tests use fmt.Appendf + testdata := analysistest.TestData() printf.Analyzer.Flags.Set("funcs", "Warn,Warnf") diff --git a/go/analysis/unitchecker/separate_test.go b/go/analysis/unitchecker/separate_test.go index 37e74e481ec..cf0143f8203 100644 --- a/go/analysis/unitchecker/separate_test.go +++ b/go/analysis/unitchecker/separate_test.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build go1.19 + package unitchecker_test // This file illustrates separate analysis with an example. diff --git a/go/analysis/unitchecker/unitchecker_test.go b/go/analysis/unitchecker/unitchecker_test.go index 270a3582ccf..9f41c71f9a3 100644 --- a/go/analysis/unitchecker/unitchecker_test.go +++ b/go/analysis/unitchecker/unitchecker_test.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build go1.19 + package unitchecker_test import ( diff --git a/go/callgraph/rta/rta.go b/go/callgraph/rta/rta.go index 001965b6531..36fe93f6056 100644 --- a/go/callgraph/rta/rta.go +++ b/go/callgraph/rta/rta.go @@ -45,6 +45,7 @@ import ( "golang.org/x/tools/go/callgraph" "golang.org/x/tools/go/ssa" "golang.org/x/tools/go/types/typeutil" + "golang.org/x/tools/internal/compat" ) // A Result holds the results of Rapid Type Analysis, which includes the @@ -538,7 +539,7 @@ func fingerprint(mset *types.MethodSet) uint64 { for i := 0; i < mset.Len(); i++ { method := mset.At(i).Obj() sig := method.Type().(*types.Signature) - sum := crc32.ChecksumIEEE(fmt.Appendf(space[:], "%s/%d/%d", + sum := crc32.ChecksumIEEE(compat.Appendf(space[:], "%s/%d/%d", method.Id(), sig.Params().Len(), sig.Results().Len())) diff --git a/go/ssa/builder_test.go b/go/ssa/builder_test.go index 06a8ee63878..25b72fc1ec6 100644 --- a/go/ssa/builder_test.go +++ b/go/ssa/builder_test.go @@ -713,9 +713,7 @@ var indirect = R[int].M // TestTypeparamTest builds SSA over compilable examples in $GOROOT/test/typeparam/*.go. func TestTypeparamTest(t *testing.T) { - if !typeparams.Enabled { - return - } + testenv.NeedsGo1Point(t, 19) // fails with infinite recursion at 1.18 -- not investigated // Tests use a fake goroot to stub out standard libraries with delcarations in // testdata/src. Decreases runtime from ~80s to ~1s. diff --git a/go/ssa/interp/interp_go117_test.go b/go/ssa/interp/interp_go117_test.go deleted file mode 100644 index 58bbaa39c91..00000000000 --- a/go/ssa/interp/interp_go117_test.go +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2021 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build go1.17 -// +build go1.17 - -package interp_test - -func init() { - testdataTests = append(testdataTests, "slice2arrayptr.go") -} diff --git a/go/ssa/interp/interp_test.go b/go/ssa/interp/interp_test.go index b3edc9916e2..9728d6ec523 100644 --- a/go/ssa/interp/interp_test.go +++ b/go/ssa/interp/interp_test.go @@ -2,6 +2,10 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// This test fails at Go 1.18 due to infinite recursion in go/types. + +//go:build go1.19 + package interp_test // This test runs the SSA interpreter over sample Go programs. @@ -125,6 +129,7 @@ var testdataTests = []string{ "range.go", "recover.go", "reflect.go", + "slice2arrayptr.go", "static.go", "width32.go", diff --git a/go/types/internal/play/play.go b/go/types/internal/play/play.go index 9e8a2aade1f..80deff5090a 100644 --- a/go/types/internal/play/play.go +++ b/go/types/internal/play/play.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build go1.19 + // The play program is a playground for go/types: a simple web-based // text editor into which the user can enter a Go program, select a // region, and see type information about it. diff --git a/internal/cmd/deadcode/deadcode.go b/internal/cmd/deadcode/deadcode.go index f3388aa6161..ecb9f9f12a8 100644 --- a/internal/cmd/deadcode/deadcode.go +++ b/internal/cmd/deadcode/deadcode.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build go1.20 + package main import ( diff --git a/internal/cmd/deadcode/deadcode_test.go b/internal/cmd/deadcode/deadcode_test.go index 417b81606d6..ab8c81c86f0 100644 --- a/internal/cmd/deadcode/deadcode_test.go +++ b/internal/cmd/deadcode/deadcode_test.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build go1.20 + package main_test import ( diff --git a/internal/compat/appendf.go b/internal/compat/appendf.go new file mode 100644 index 00000000000..069d5171704 --- /dev/null +++ b/internal/compat/appendf.go @@ -0,0 +1,13 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.19 + +package compat + +import "fmt" + +func Appendf(b []byte, format string, a ...interface{}) []byte { + return fmt.Appendf(b, format, a...) +} diff --git a/internal/compat/appendf_118.go b/internal/compat/appendf_118.go new file mode 100644 index 00000000000..29af353cdaf --- /dev/null +++ b/internal/compat/appendf_118.go @@ -0,0 +1,13 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !go1.19 + +package compat + +import "fmt" + +func Appendf(b []byte, format string, a ...interface{}) []byte { + return append(b, fmt.Sprintf(format, a...)...) +} diff --git a/internal/compat/doc.go b/internal/compat/doc.go new file mode 100644 index 00000000000..59c667a37a2 --- /dev/null +++ b/internal/compat/doc.go @@ -0,0 +1,7 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The compat package implements API shims for backward compatibility at older +// Go versions. +package compat diff --git a/internal/refactor/inline/analyzer/analyzer.go b/internal/refactor/inline/analyzer/analyzer.go index 2356fa484e7..943776e14b2 100644 --- a/internal/refactor/inline/analyzer/analyzer.go +++ b/internal/refactor/inline/analyzer/analyzer.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build go1.20 + package analyzer import ( diff --git a/internal/refactor/inline/analyzer/analyzer_test.go b/internal/refactor/inline/analyzer/analyzer_test.go index 5ad85cfb821..05daac901f7 100644 --- a/internal/refactor/inline/analyzer/analyzer_test.go +++ b/internal/refactor/inline/analyzer/analyzer_test.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build go1.20 + package analyzer_test import ( From 7e848b2467599efb25410f921928e7aba1cc2a30 Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Fri, 8 Sep 2023 09:54:11 -0400 Subject: [PATCH 079/178] gopls/internal/lsp/regtest: port the highlight marker This CL ports the highlight marker from the legacy marker tests. This one was very straightforward: the only changes are as follows: - Use the more useful protocol.Location marker parameter - s/mark/loc in tests - add @diag annotations as the tests had type errors For golang/go#54845 Change-Id: Id43ec07fea0caf0fced4622a23c323ef193aa52b Reviewed-on: https://go-review.googlesource.com/c/tools/+/527396 gopls-CI: kokoro Auto-Submit: Robert Findley LUCI-TryBot-Result: Go LUCI Reviewed-by: Alan Donovan --- gopls/internal/lsp/lsp_test.go | 44 ----- gopls/internal/lsp/regtest/marker.go | 32 +++- .../lsp/testdata/highlights/highlights.go | 151 ----------------- .../lsp/testdata/highlights/issue60435.go | 14 -- .../internal/lsp/testdata/summary.txt.golden | 1 - .../lsp/testdata/summary_go1.21.txt.golden | 1 - gopls/internal/lsp/tests/tests.go | 26 +-- .../marker/testdata/highlight/highlight.txt | 158 ++++++++++++++++++ .../marker/testdata/highlight/issue60435.txt | 15 ++ 9 files changed, 206 insertions(+), 236 deletions(-) delete mode 100644 gopls/internal/lsp/testdata/highlights/highlights.go delete mode 100644 gopls/internal/lsp/testdata/highlights/issue60435.go create mode 100644 gopls/internal/regtest/marker/testdata/highlight/highlight.txt create mode 100644 gopls/internal/regtest/marker/testdata/highlight/issue60435.txt diff --git a/gopls/internal/lsp/lsp_test.go b/gopls/internal/lsp/lsp_test.go index 88f6a3940f1..4fefbb57b0d 100644 --- a/gopls/internal/lsp/lsp_test.go +++ b/gopls/internal/lsp/lsp_test.go @@ -496,50 +496,6 @@ func (r *runner) Definition(t *testing.T, _ span.Span, d tests.Definition) { } } -func (r *runner) Highlight(t *testing.T, src span.Span, spans []span.Span) { - m, err := r.data.Mapper(src.URI()) - if err != nil { - t.Fatal(err) - } - loc, err := m.SpanLocation(src) - if err != nil { - t.Fatal(err) - } - params := &protocol.DocumentHighlightParams{ - TextDocumentPositionParams: protocol.LocationTextDocumentPositionParams(loc), - } - highlights, err := r.server.DocumentHighlight(r.ctx, params) - if err != nil { - t.Fatalf("DocumentHighlight(%v) failed: %v", params, err) - } - var got []protocol.Range - for _, h := range highlights { - got = append(got, h.Range) - } - - var want []protocol.Range - for _, s := range spans { - rng, err := m.SpanRange(s) - if err != nil { - t.Fatalf("Mapper.SpanRange(%v) failed: %v", s, err) - } - want = append(want, rng) - } - - sortRanges := func(s []protocol.Range) { - sort.Slice(s, func(i, j int) bool { - return protocol.CompareRange(s[i], s[j]) < 0 - }) - } - - sortRanges(got) - sortRanges(want) - - if diff := cmp.Diff(want, got); diff != "" { - t.Errorf("DocumentHighlight(%v) mismatch (-want +got):\n%s", src, diff) - } -} - func (r *runner) InlayHints(t *testing.T, spn span.Span) { uri := spn.URI() filename := uri.Filename() diff --git a/gopls/internal/lsp/regtest/marker.go b/gopls/internal/lsp/regtest/marker.go index 927eb072d5a..76c2e882de8 100644 --- a/gopls/internal/lsp/regtest/marker.go +++ b/gopls/internal/lsp/regtest/marker.go @@ -176,6 +176,10 @@ var update = flag.Bool("update", false, "if set, update test data during marker // source. If the formatting request fails, the golden file must contain // the error message. // +// - highlight(src location, dsts ...location): makes a +// textDocument/highlight request at the given src location, which should +// highlight the provided dst locations. +// // - hover(src, dst location, g Golden): perform a textDocument/hover at the // src location, and checks that the result is the dst location, with hover // content matching "hover.md" in the golden data g. @@ -333,7 +337,6 @@ var update = flag.Bool("update", false, "if set, update test data during marker // - SemanticTokens // - FunctionExtractions // - MethodExtractions -// - Highlights // - Renames // - PrepareRenames // - InlayHints @@ -584,6 +587,7 @@ var markerFuncs = map[string]markerFunc{ "diag": makeMarkerFunc(diagMarker), "foldingrange": makeMarkerFunc(foldingRangeMarker), "format": makeMarkerFunc(formatMarker), + "highlight": makeMarkerFunc(highlightMarker), "hover": makeMarkerFunc(hoverMarker), "implementation": makeMarkerFunc(implementationMarker), "loc": makeMarkerFunc(locMarker), @@ -1449,6 +1453,32 @@ func formatMarker(mark marker, golden *Golden) { } } +func highlightMarker(mark marker, src protocol.Location, dsts ...protocol.Location) { + highlights := mark.run.env.DocumentHighlight(src) + var got []protocol.Range + for _, h := range highlights { + got = append(got, h.Range) + } + + var want []protocol.Range + for _, d := range dsts { + want = append(want, d.Range) + } + + sortRanges := func(s []protocol.Range) { + sort.Slice(s, func(i, j int) bool { + return protocol.CompareRange(s[i], s[j]) < 0 + }) + } + + sortRanges(got) + sortRanges(want) + + if diff := cmp.Diff(want, got); diff != "" { + mark.errorf("DocumentHighlight(%v) mismatch (-want +got):\n%s", src, diff) + } +} + // hoverMarker implements the @hover marker, running textDocument/hover at the // given src location and asserting that the resulting hover is over the dst // location (typically a span surrounding src), and that the markdown content diff --git a/gopls/internal/lsp/testdata/highlights/highlights.go b/gopls/internal/lsp/testdata/highlights/highlights.go deleted file mode 100644 index 55ae68aa124..00000000000 --- a/gopls/internal/lsp/testdata/highlights/highlights.go +++ /dev/null @@ -1,151 +0,0 @@ -package highlights - -import ( - "fmt" //@mark(fmtImp, "\"fmt\""),highlight(fmtImp, fmtImp, fmt1, fmt2, fmt3, fmt4) - h2 "net/http" //@mark(hImp, "h2"),highlight(hImp, hImp, hUse) - "sort" -) - -type F struct{ bar int } //@mark(barDeclaration, "bar"),highlight(barDeclaration, barDeclaration, bar1, bar2, bar3) - -func _() F { - return F{ - bar: 123, //@mark(bar1, "bar"),highlight(bar1, barDeclaration, bar1, bar2, bar3) - } -} - -var foo = F{bar: 52} //@mark(fooDeclaration, "foo"),mark(bar2, "bar"),highlight(fooDeclaration, fooDeclaration, fooUse),highlight(bar2, barDeclaration, bar1, bar2, bar3) - -func Print() { //@mark(printFunc, "Print"),highlight(printFunc, printFunc, printTest) - _ = h2.Client{} //@mark(hUse, "h2"),highlight(hUse, hImp, hUse) - - fmt.Println(foo) //@mark(fooUse, "foo"),highlight(fooUse, fooDeclaration, fooUse),mark(fmt1, "fmt"),highlight(fmt1, fmtImp, fmt1, fmt2, fmt3, fmt4) - fmt.Print("yo") //@mark(printSep, "Print"),highlight(printSep, printSep, print1, print2),mark(fmt2, "fmt"),highlight(fmt2, fmtImp, fmt1, fmt2, fmt3, fmt4) -} - -func (x *F) Inc() { //@mark(xRightDecl, "x"),mark(xLeftDecl, " *"),highlight(xRightDecl, xRightDecl, xUse),highlight(xLeftDecl, xRightDecl, xUse) - x.bar++ //@mark(xUse, "x"),mark(bar3, "bar"),highlight(xUse, xRightDecl, xUse),highlight(bar3, barDeclaration, bar1, bar2, bar3) -} - -func testFunctions() { - fmt.Print("main start") //@mark(print1, "Print"),highlight(print1, printSep, print1, print2),mark(fmt3, "fmt"),highlight(fmt3, fmtImp, fmt1, fmt2, fmt3, fmt4) - fmt.Print("ok") //@mark(print2, "Print"),highlight(print2, printSep, print1, print2),mark(fmt4, "fmt"),highlight(fmt4, fmtImp, fmt1, fmt2, fmt3, fmt4) - Print() //@mark(printTest, "Print"),highlight(printTest, printFunc, printTest) -} - -func toProtocolHighlight(rngs []int) []DocumentHighlight { //@mark(doc1, "DocumentHighlight"),mark(docRet1, "[]DocumentHighlight"),highlight(doc1, docRet1, doc1, doc2, doc3, result) - result := make([]DocumentHighlight, 0, len(rngs)) //@mark(doc2, "DocumentHighlight"),highlight(doc2, doc1, doc2, doc3) - for _, rng := range rngs { - result = append(result, DocumentHighlight{ //@mark(doc3, "DocumentHighlight"),highlight(doc3, doc1, doc2, doc3) - Range: rng, - }) - } - return result //@mark(result, "result") -} - -func testForLoops() { - for i := 0; i < 10; i++ { //@mark(forDecl1, "for"),highlight(forDecl1, forDecl1, brk1, cont1) - if i > 8 { - break //@mark(brk1, "break"),highlight(brk1, forDecl1, brk1, cont1) - } - if i < 2 { - for j := 1; j < 10; j++ { //@mark(forDecl2, "for"),highlight(forDecl2, forDecl2, cont2) - if j < 3 { - for k := 1; k < 10; k++ { //@mark(forDecl3, "for"),highlight(forDecl3, forDecl3, cont3) - if k < 3 { - continue //@mark(cont3, "continue"),highlight(cont3, forDecl3, cont3) - } - } - continue //@mark(cont2, "continue"),highlight(cont2, forDecl2, cont2) - } - } - continue //@mark(cont1, "continue"),highlight(cont1, forDecl1, brk1, cont1) - } - } - - arr := []int{} - for i := range arr { //@mark(forDecl4, "for"),highlight(forDecl4, forDecl4, brk4, cont4) - if i > 8 { - break //@mark(brk4, "break"),highlight(brk4, forDecl4, brk4, cont4) - } - if i < 4 { - continue //@mark(cont4, "continue"),highlight(cont4, forDecl4, brk4, cont4) - } - } - -Outer: - for i := 0; i < 10; i++ { //@mark(forDecl5, "for"),highlight(forDecl5, forDecl5, brk5, brk6, brk8) - break //@mark(brk5, "break"),highlight(brk5, forDecl5, brk5, brk6, brk8) - for { //@mark(forDecl6, "for"),highlight(forDecl6, forDecl6, cont5) - if i == 1 { - break Outer //@mark(brk6, "break Outer"),highlight(brk6, forDecl5, brk5, brk6, brk8) - } - switch i { //@mark(switch1, "switch"),highlight(switch1, switch1, brk7) - case 5: - break //@mark(brk7, "break"),highlight(brk7, switch1, brk7) - case 6: - continue //@mark(cont5, "continue"),highlight(cont5, forDecl6, cont5) - case 7: - break Outer //@mark(brk8, "break Outer"),highlight(brk8, forDecl5, brk5, brk6, brk8) - } - } - } -} - -func testSwitch() { - var i, j int - -L1: - for { //@mark(forDecl7, "for"),highlight(forDecl7, forDecl7, brk10, cont6) - L2: - switch i { //@mark(switch2, "switch"),highlight(switch2, switch2, brk11, brk12, brk13) - case 1: - switch j { //@mark(switch3, "switch"),highlight(switch3, switch3, brk9) - case 1: - break //@mark(brk9, "break"),highlight(brk9, switch3, brk9) - case 2: - break L1 //@mark(brk10, "break L1"),highlight(brk10, forDecl7, brk10, cont6) - case 3: - break L2 //@mark(brk11, "break L2"),highlight(brk11, switch2, brk11, brk12, brk13) - default: - continue //@mark(cont6, "continue"),highlight(cont6, forDecl7, brk10, cont6) - } - case 2: - break //@mark(brk12, "break"),highlight(brk12, switch2, brk11, brk12, brk13) - default: - break L2 //@mark(brk13, "break L2"),highlight(brk13, switch2, brk11, brk12, brk13) - } - } -} - -func testReturn() bool { //@mark(func1, "func"),mark(bool1, "bool"),highlight(func1, func1, fullRet11, fullRet12),highlight(bool1, bool1, false1, bool2, true1) - if 1 < 2 { - return false //@mark(ret11, "return"),mark(fullRet11, "return false"),mark(false1, "false"),highlight(ret11, func1, fullRet11, fullRet12) - } - candidates := []int{} - sort.SliceStable(candidates, func(i, j int) bool { //@mark(func2, "func"),mark(bool2, "bool"),highlight(func2, func2, fullRet2) - return candidates[i] > candidates[j] //@mark(ret2, "return"),mark(fullRet2, "return candidates[i] > candidates[j]"),highlight(ret2, func2, fullRet2) - }) - return true //@mark(ret12, "return"),mark(fullRet12, "return true"),mark(true1, "true"),highlight(ret12, func1, fullRet11, fullRet12) -} - -func testReturnFields() float64 { //@mark(retVal1, "float64"),highlight(retVal1, retVal1, retVal11, retVal21) - if 1 < 2 { - return 20.1 //@mark(retVal11, "20.1"),highlight(retVal11, retVal1, retVal11, retVal21) - } - z := 4.3 //@mark(zDecl, "z") - return z //@mark(retVal21, "z"),highlight(retVal21, retVal1, retVal11, zDecl, retVal21) -} - -func testReturnMultipleFields() (float32, string) { //@mark(retVal31, "float32"),mark(retVal32, "string"),highlight(retVal31, retVal31, retVal41, retVal51),highlight(retVal32, retVal32, retVal42, retVal52) - y := "im a var" //@mark(yDecl, "y"), - if 1 < 2 { - return 20.1, y //@mark(retVal41, "20.1"),mark(retVal42, "y"),highlight(retVal41, retVal31, retVal41, retVal51),highlight(retVal42, retVal32, yDecl, retVal42, retVal52) - } - return 4.9, "test" //@mark(retVal51, "4.9"),mark(retVal52, "\"test\""),highlight(retVal51, retVal31, retVal41, retVal51),highlight(retVal52, retVal32, retVal42, retVal52) -} - -func testReturnFunc() int32 { //@mark(retCall, "int32") - mulch := 1 //@mark(mulchDec, "mulch"),highlight(mulchDec, mulchDec, mulchRet) - return int32(mulch) //@mark(mulchRet, "mulch"),mark(retFunc, "int32"),mark(retTotal, "int32(mulch)"),highlight(mulchRet, mulchDec, mulchRet),highlight(retFunc, retCall, retFunc, retTotal) -} diff --git a/gopls/internal/lsp/testdata/highlights/issue60435.go b/gopls/internal/lsp/testdata/highlights/issue60435.go deleted file mode 100644 index de0070e5832..00000000000 --- a/gopls/internal/lsp/testdata/highlights/issue60435.go +++ /dev/null @@ -1,14 +0,0 @@ -package highlights - -import ( - "net/http" //@mark(httpImp, `"net/http"`) - "net/http/httptest" //@mark(httptestImp, `"net/http/httptest"`) -) - -// This is a regression test for issue 60435: -// Highlighting "net/http" shouldn't have any effect -// on an import path that contains it as a substring, -// such as httptest. - -var _ = httptest.NewRequest -var _ = http.NewRequest //@mark(here, "http"), highlight(here, here, httpImp) diff --git a/gopls/internal/lsp/testdata/summary.txt.golden b/gopls/internal/lsp/testdata/summary.txt.golden index 30e5112dae9..e7b1d1ed638 100644 --- a/gopls/internal/lsp/testdata/summary.txt.golden +++ b/gopls/internal/lsp/testdata/summary.txt.golden @@ -14,7 +14,6 @@ SuggestedFixCount = 80 MethodExtractionCount = 8 DefinitionsCount = 46 TypeDefinitionsCount = 18 -HighlightsCount = 70 InlayHintsCount = 5 RenamesCount = 48 PrepareRenamesCount = 7 diff --git a/gopls/internal/lsp/testdata/summary_go1.21.txt.golden b/gopls/internal/lsp/testdata/summary_go1.21.txt.golden index 7796082ba96..0fce9b32b6d 100644 --- a/gopls/internal/lsp/testdata/summary_go1.21.txt.golden +++ b/gopls/internal/lsp/testdata/summary_go1.21.txt.golden @@ -14,7 +14,6 @@ SuggestedFixCount = 80 MethodExtractionCount = 8 DefinitionsCount = 46 TypeDefinitionsCount = 18 -HighlightsCount = 70 InlayHintsCount = 5 RenamesCount = 48 PrepareRenamesCount = 7 diff --git a/gopls/internal/lsp/tests/tests.go b/gopls/internal/lsp/tests/tests.go index deabe90bff5..c8a1df772fb 100644 --- a/gopls/internal/lsp/tests/tests.go +++ b/gopls/internal/lsp/tests/tests.go @@ -13,7 +13,6 @@ import ( "go/ast" "go/token" "io" - "io/ioutil" "os" "path/filepath" "regexp" @@ -78,7 +77,6 @@ type SemanticTokens = []span.Span type SuggestedFixes = map[span.Span][]SuggestedFix type MethodExtractions = map[span.Span]span.Span type Definitions = map[span.Span]Definition -type Highlights = map[span.Span][]span.Span type Renames = map[span.Span]string type PrepareRenames = map[span.Span]*source.PrepareItem type InlayHints = []span.Span @@ -105,7 +103,6 @@ type Data struct { SuggestedFixes SuggestedFixes MethodExtractions MethodExtractions Definitions Definitions - Highlights Highlights Renames Renames InlayHints InlayHints PrepareRenames PrepareRenames @@ -146,7 +143,6 @@ type Tests interface { SuggestedFix(*testing.T, span.Span, []SuggestedFix, int) MethodExtraction(*testing.T, span.Span, span.Span) Definition(*testing.T, span.Span, Definition) - Highlight(*testing.T, span.Span, []span.Span) InlayHints(*testing.T, span.Span) Rename(*testing.T, span.Span, string) PrepareRename(*testing.T, span.Span, *source.PrepareItem) @@ -282,7 +278,6 @@ func load(t testing.TB, mode string, dir string) *Data { RankCompletions: make(RankCompletions), CaseSensitiveCompletions: make(CaseSensitiveCompletions), Definitions: make(Definitions), - Highlights: make(Highlights), Renames: make(Renames), PrepareRenames: make(PrepareRenames), SuggestedFixes: make(SuggestedFixes), @@ -341,7 +336,7 @@ func load(t testing.TB, mode string, dir string) *Data { } else if index := strings.Index(fragment, overlayFileSuffix); index >= 0 { delete(files, fragment) partial := fragment[:index] + fragment[index+len(overlayFileSuffix):] - contents, err := ioutil.ReadFile(filepath.Join(dir, fragment)) + contents, err := os.ReadFile(filepath.Join(dir, fragment)) if err != nil { t.Fatal(err) } @@ -436,7 +431,6 @@ func load(t testing.TB, mode string, dir string) *Data { "godef": datum.collectDefinitions, "typdef": datum.collectTypeDefinitions, "hoverdef": datum.collectHoverDefinitions, - "highlight": datum.collectHighlights, "inlayHint": datum.collectInlayHints, "rename": datum.collectRenames, "prepare": datum.collectPrepareRenames, @@ -664,16 +658,6 @@ func Run(t *testing.T, tests Tests, data *Data) { } }) - t.Run("Highlight", func(t *testing.T) { - t.Helper() - for pos, locations := range data.Highlights { - t.Run(SpanName(pos), func(t *testing.T) { - t.Helper() - tests.Highlight(t, pos, locations) - }) - } - }) - t.Run("InlayHints", func(t *testing.T) { t.Helper() for _, src := range data.InlayHints { @@ -762,7 +746,7 @@ func Run(t *testing.T, tests Tests, data *Data) { sort.Slice(golden.Archive.Files, func(i, j int) bool { return golden.Archive.Files[i].Name < golden.Archive.Files[j].Name }) - if err := ioutil.WriteFile(golden.Filename, txtar.Format(golden.Archive), 0666); err != nil { + if err := os.WriteFile(golden.Filename, txtar.Format(golden.Archive), 0666); err != nil { t.Fatal(err) } } @@ -823,7 +807,6 @@ func checkData(t *testing.T, data *Data) { fmt.Fprintf(buf, "MethodExtractionCount = %v\n", len(data.MethodExtractions)) fmt.Fprintf(buf, "DefinitionsCount = %v\n", definitionCount) fmt.Fprintf(buf, "TypeDefinitionsCount = %v\n", typeDefinitionCount) - fmt.Fprintf(buf, "HighlightsCount = %v\n", len(data.Highlights)) fmt.Fprintf(buf, "InlayHintsCount = %v\n", len(data.InlayHints)) fmt.Fprintf(buf, "RenamesCount = %v\n", len(data.Renames)) fmt.Fprintf(buf, "PrepareRenamesCount = %v\n", len(data.PrepareRenames)) @@ -1073,11 +1056,6 @@ func (data *Data) collectDefinitionNames(src span.Span, name string) { data.Definitions[src] = d } -func (data *Data) collectHighlights(src span.Span, expected []span.Span) { - // Declaring a highlight in a test file: @highlight(src, expected1, expected2) - data.Highlights[src] = append(data.Highlights[src], expected...) -} - func (data *Data) collectInlayHints(src span.Span) { data.InlayHints = append(data.InlayHints, src) } diff --git a/gopls/internal/regtest/marker/testdata/highlight/highlight.txt b/gopls/internal/regtest/marker/testdata/highlight/highlight.txt new file mode 100644 index 00000000000..10b30259b10 --- /dev/null +++ b/gopls/internal/regtest/marker/testdata/highlight/highlight.txt @@ -0,0 +1,158 @@ +This test checks basic functionality of the textDocument/highlight request. + +-- highlights.go -- +package highlights + +import ( + "fmt" //@loc(fmtImp, "\"fmt\""),highlight(fmtImp, fmtImp, fmt1, fmt2, fmt3, fmt4) + h2 "net/http" //@loc(hImp, "h2"),highlight(hImp, hImp, hUse) + "sort" +) + +type F struct{ bar int } //@loc(barDeclaration, "bar"),highlight(barDeclaration, barDeclaration, bar1, bar2, bar3) + +func _() F { + return F{ + bar: 123, //@loc(bar1, "bar"),highlight(bar1, barDeclaration, bar1, bar2, bar3) + } +} + +var foo = F{bar: 52} //@loc(fooDeclaration, "foo"),loc(bar2, "bar"),highlight(fooDeclaration, fooDeclaration, fooUse),highlight(bar2, barDeclaration, bar1, bar2, bar3) + +func Print() { //@loc(printFunc, "Print"),highlight(printFunc, printFunc, printTest) + _ = h2.Client{} //@loc(hUse, "h2"),highlight(hUse, hImp, hUse) + + fmt.Println(foo) //@loc(fooUse, "foo"),highlight(fooUse, fooDeclaration, fooUse),loc(fmt1, "fmt"),highlight(fmt1, fmtImp, fmt1, fmt2, fmt3, fmt4) + fmt.Print("yo") //@loc(printSep, "Print"),highlight(printSep, printSep, print1, print2),loc(fmt2, "fmt"),highlight(fmt2, fmtImp, fmt1, fmt2, fmt3, fmt4) +} + +func (x *F) Inc() { //@loc(xRightDecl, "x"),loc(xLeftDecl, " *"),highlight(xRightDecl, xRightDecl, xUse),highlight(xLeftDecl, xRightDecl, xUse) + x.bar++ //@loc(xUse, "x"),loc(bar3, "bar"),highlight(xUse, xRightDecl, xUse),highlight(bar3, barDeclaration, bar1, bar2, bar3) +} + +func testFunctions() { + fmt.Print("main start") //@loc(print1, "Print"),highlight(print1, printSep, print1, print2),loc(fmt3, "fmt"),highlight(fmt3, fmtImp, fmt1, fmt2, fmt3, fmt4) + fmt.Print("ok") //@loc(print2, "Print"),highlight(print2, printSep, print1, print2),loc(fmt4, "fmt"),highlight(fmt4, fmtImp, fmt1, fmt2, fmt3, fmt4) + Print() //@loc(printTest, "Print"),highlight(printTest, printFunc, printTest) +} + +// DocumentHighlight is undefined, so its uses below are type errors. +// Nevertheless, document highlighting should still work. +//@diag(doc1, re"undefined|undeclared"), diag(doc2, re"undefined|undeclared"), diag(doc3, re"undefined|undeclared") + +func toProtocolHighlight(rngs []int) []DocumentHighlight { //@loc(doc1, "DocumentHighlight"),loc(docRet1, "[]DocumentHighlight"),highlight(doc1, docRet1, doc1, doc2, doc3, result) + result := make([]DocumentHighlight, 0, len(rngs)) //@loc(doc2, "DocumentHighlight"),highlight(doc2, doc1, doc2, doc3) + for _, rng := range rngs { + result = append(result, DocumentHighlight{ //@loc(doc3, "DocumentHighlight"),highlight(doc3, doc1, doc2, doc3) + Range: rng, + }) + } + return result //@loc(result, "result") +} + +func testForLoops() { + for i := 0; i < 10; i++ { //@loc(forDecl1, "for"),highlight(forDecl1, forDecl1, brk1, cont1) + if i > 8 { + break //@loc(brk1, "break"),highlight(brk1, forDecl1, brk1, cont1) + } + if i < 2 { + for j := 1; j < 10; j++ { //@loc(forDecl2, "for"),highlight(forDecl2, forDecl2, cont2) + if j < 3 { + for k := 1; k < 10; k++ { //@loc(forDecl3, "for"),highlight(forDecl3, forDecl3, cont3) + if k < 3 { + continue //@loc(cont3, "continue"),highlight(cont3, forDecl3, cont3) + } + } + continue //@loc(cont2, "continue"),highlight(cont2, forDecl2, cont2) + } + } + continue //@loc(cont1, "continue"),highlight(cont1, forDecl1, brk1, cont1) + } + } + + arr := []int{} + for i := range arr { //@loc(forDecl4, "for"),highlight(forDecl4, forDecl4, brk4, cont4) + if i > 8 { + break //@loc(brk4, "break"),highlight(brk4, forDecl4, brk4, cont4) + } + if i < 4 { + continue //@loc(cont4, "continue"),highlight(cont4, forDecl4, brk4, cont4) + } + } + +Outer: + for i := 0; i < 10; i++ { //@loc(forDecl5, "for"),highlight(forDecl5, forDecl5, brk5, brk6, brk8) + break //@loc(brk5, "break"),highlight(brk5, forDecl5, brk5, brk6, brk8) + for { //@loc(forDecl6, "for"),highlight(forDecl6, forDecl6, cont5), diag("for", re"unreachable") + if i == 1 { + break Outer //@loc(brk6, "break Outer"),highlight(brk6, forDecl5, brk5, brk6, brk8) + } + switch i { //@loc(switch1, "switch"),highlight(switch1, switch1, brk7) + case 5: + break //@loc(brk7, "break"),highlight(brk7, switch1, brk7) + case 6: + continue //@loc(cont5, "continue"),highlight(cont5, forDecl6, cont5) + case 7: + break Outer //@loc(brk8, "break Outer"),highlight(brk8, forDecl5, brk5, brk6, brk8) + } + } + } +} + +func testSwitch() { + var i, j int + +L1: + for { //@loc(forDecl7, "for"),highlight(forDecl7, forDecl7, brk10, cont6) + L2: + switch i { //@loc(switch2, "switch"),highlight(switch2, switch2, brk11, brk12, brk13) + case 1: + switch j { //@loc(switch3, "switch"),highlight(switch3, switch3, brk9) + case 1: + break //@loc(brk9, "break"),highlight(brk9, switch3, brk9) + case 2: + break L1 //@loc(brk10, "break L1"),highlight(brk10, forDecl7, brk10, cont6) + case 3: + break L2 //@loc(brk11, "break L2"),highlight(brk11, switch2, brk11, brk12, brk13) + default: + continue //@loc(cont6, "continue"),highlight(cont6, forDecl7, brk10, cont6) + } + case 2: + break //@loc(brk12, "break"),highlight(brk12, switch2, brk11, brk12, brk13) + default: + break L2 //@loc(brk13, "break L2"),highlight(brk13, switch2, brk11, brk12, brk13) + } + } +} + +func testReturn() bool { //@loc(func1, "func"),loc(bool1, "bool"),highlight(func1, func1, fullRet11, fullRet12),highlight(bool1, bool1, false1, bool2, true1) + if 1 < 2 { + return false //@loc(ret11, "return"),loc(fullRet11, "return false"),loc(false1, "false"),highlight(ret11, func1, fullRet11, fullRet12) + } + candidates := []int{} + sort.SliceStable(candidates, func(i, j int) bool { //@loc(func2, "func"),loc(bool2, "bool"),highlight(func2, func2, fullRet2) + return candidates[i] > candidates[j] //@loc(ret2, "return"),loc(fullRet2, "return candidates[i] > candidates[j]"),highlight(ret2, func2, fullRet2) + }) + return true //@loc(ret12, "return"),loc(fullRet12, "return true"),loc(true1, "true"),highlight(ret12, func1, fullRet11, fullRet12) +} + +func testReturnFields() float64 { //@loc(retVal1, "float64"),highlight(retVal1, retVal1, retVal11, retVal21) + if 1 < 2 { + return 20.1 //@loc(retVal11, "20.1"),highlight(retVal11, retVal1, retVal11, retVal21) + } + z := 4.3 //@loc(zDecl, "z") + return z //@loc(retVal21, "z"),highlight(retVal21, retVal1, retVal11, zDecl, retVal21) +} + +func testReturnMultipleFields() (float32, string) { //@loc(retVal31, "float32"),loc(retVal32, "string"),highlight(retVal31, retVal31, retVal41, retVal51),highlight(retVal32, retVal32, retVal42, retVal52) + y := "im a var" //@loc(yDecl, "y"), + if 1 < 2 { + return 20.1, y //@loc(retVal41, "20.1"),loc(retVal42, "y"),highlight(retVal41, retVal31, retVal41, retVal51),highlight(retVal42, retVal32, yDecl, retVal42, retVal52) + } + return 4.9, "test" //@loc(retVal51, "4.9"),loc(retVal52, "\"test\""),highlight(retVal51, retVal31, retVal41, retVal51),highlight(retVal52, retVal32, retVal42, retVal52) +} + +func testReturnFunc() int32 { //@loc(retCall, "int32") + mulch := 1 //@loc(mulchDec, "mulch"),highlight(mulchDec, mulchDec, mulchRet) + return int32(mulch) //@loc(mulchRet, "mulch"),loc(retFunc, "int32"),loc(retTotal, "int32(mulch)"),highlight(mulchRet, mulchDec, mulchRet),highlight(retFunc, retCall, retFunc, retTotal) +} diff --git a/gopls/internal/regtest/marker/testdata/highlight/issue60435.txt b/gopls/internal/regtest/marker/testdata/highlight/issue60435.txt new file mode 100644 index 00000000000..324e4b85e77 --- /dev/null +++ b/gopls/internal/regtest/marker/testdata/highlight/issue60435.txt @@ -0,0 +1,15 @@ +This is a regression test for issue 60435: +Highlighting "net/http" shouldn't have any effect +on an import path that contains it as a substring, +such as httptest. + +-- highlights.go -- +package highlights + +import ( + "net/http" //@loc(httpImp, `"net/http"`) + "net/http/httptest" //@loc(httptestImp, `"net/http/httptest"`) +) + +var _ = httptest.NewRequest +var _ = http.NewRequest //@loc(here, "http"), highlight(here, here, httpImp) From e6d89b486926d651f32f6648f06f563b546a4dda Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Mon, 11 Sep 2023 17:52:53 -0400 Subject: [PATCH 080/178] gopls/internal/lsp/regtest: parallelize marker tests The new marker tests were designed to be parallelizable, but I'd held off on doing so because I wanted to preserve an apples-to-apples comparison with the old tests, which could not be safely parallelized. However, the time has come where the new marker tests are getting too slow: they take 15-20s on my desktop. Making them parallel drops this to around 2.5s. For golang/go#54845 Change-Id: Ide58746d074040b07d639a9f78d43c58be413861 Reviewed-on: https://go-review.googlesource.com/c/tools/+/527397 Reviewed-by: Alan Donovan Auto-Submit: Robert Findley gopls-CI: kokoro LUCI-TryBot-Result: Go LUCI --- gopls/internal/lsp/regtest/marker.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gopls/internal/lsp/regtest/marker.go b/gopls/internal/lsp/regtest/marker.go index 76c2e882de8..993f9ed2c79 100644 --- a/gopls/internal/lsp/regtest/marker.go +++ b/gopls/internal/lsp/regtest/marker.go @@ -313,7 +313,7 @@ var update = flag.Bool("update", false, "if set, update test data during marker // internal/lsp/tests. // // Remaining TODO: -// - parallelize/optimize test execution +// - optimize test execution // - reorganize regtest packages (and rename to just 'test'?) // - Rename the files .txtar. // - Provide some means by which locations in the standard library @@ -359,7 +359,9 @@ func RunMarkerTests(t *testing.T, dir string) { cache := cache.New(nil) for _, test := range tests { + test := test t.Run(test.name, func(t *testing.T) { + t.Parallel() if test.skipReason != "" { t.Skip(test.skipReason) } From ebc1c299159f0dc2810a844095aa9339403f6a73 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Wed, 30 Aug 2023 16:36:35 -0400 Subject: [PATCH 081/178] internal/refactor/inline: parameter elimination This change is a redesign of the inliner to express the transformation in terms of Go syntax trees, not text splicing, enabling more complex analysis of cases at the cost of lossy comment handling. This change: 1. causes the inliner to eliminate each parameter that is safe to eliminate, either because it is unreferenced and its argument has no effects, or because it is safe to replace each occurrence of the parameter by the argument expression. The precise definition of "safe" is complex, and involves: - not changing the effects of the call, - not letting shadowing of identifiers within the callee changing the meaning or validity of the argument expression, - not removing the last reference to a local variable of the caller, - not removing significant implicit assignments during argument passing, and so on; see the notes in the code for the gory details. Parameter elimination requires that we analyze whether each parameter escapes or is assigned, as otherwise eliminating the parameter variable (or, more subtly, identifying it with a variable that belongs to the caller) may be incorrect. It also requires that we tabulate all references to each parameter so that we can edit them; and record the set of local names in scope at each such reference so we can detect when they might shadow a free variable of the substituted argument expression. 2. uses new strategies to reduce the call entirely, including: - when all parameters can be eliminated (as described above), and the body is a single { return expr }, it may be safe to replace the call by the substituted expr. (This subsumes the sole previous "no params no results" reduction strategy.) - when the callee function body is empty, the call may be completely eliminated. (Our friends in the Google C++ team tell us this was a common special case for automated inlining cleanups.) - when the callee does not contain defer, return, or labels, and is called from a statement, the call can be replaced by the callee function body, or the body statement itself if it is a singleton. The Inline function is split into 'inline' and 'imports' so that each strategy can return once finished, avoiding the awkward long forward jumps to the common tail. Also: - Log the decision making of each Inline operation so that clients can debug/understand why a given strategy was chosen. - Implement checks for implicit type conversions in argument and result passing. - Add helpers for computing purity, duplicability, and free variables of an argument expression, and a helper for computing the control complexity of the callee. - Better documentation. - No longer mutates borrowed syntax trees, plus test. Change-Id: I41640a41b83dd7a0d1f104f669dbf63911c6d175 Reviewed-on: https://go-review.googlesource.com/c/tools/+/524838 LUCI-TryBot-Result: Go LUCI Reviewed-by: Robert Findley gopls-CI: kokoro --- gopls/internal/lsp/source/inline.go | 4 +- .../marker/testdata/codeaction/inline.txt | 2 +- internal/refactor/inline/analyzer/analyzer.go | 2 +- .../inline/analyzer/testdata/src/a/a.go | 3 +- .../analyzer/testdata/src/a/a.go.golden | 5 +- .../analyzer/testdata/src/b/b.go.golden | 2 +- internal/refactor/inline/callee.go | 290 +++- internal/refactor/inline/inline.go | 1429 +++++++++++++---- internal/refactor/inline/inline_test.go | 100 +- .../refactor/inline/testdata/basic-err.txtar | 2 +- .../inline/testdata/basic-literal.txtar | 18 +- .../inline/testdata/basic-reduce.txtar | 33 +- .../refactor/inline/testdata/comments.txtar | 19 +- .../refactor/inline/testdata/crosspkg.txtar | 9 +- .../refactor/inline/testdata/dotimport.txtar | 6 +- .../refactor/inline/testdata/empty-body.txtar | 103 ++ .../inline/testdata/import-shadow.txtar | 6 +- .../refactor/inline/testdata/method.txtar | 51 +- .../inline/testdata/multistmt-body.txtar | 83 + .../inline/testdata/param-subst.txtar | 19 + .../inline/testdata/revdotimport.txtar | 3 +- internal/refactor/inline/util.go | 46 + 22 files changed, 1834 insertions(+), 401 deletions(-) create mode 100644 internal/refactor/inline/testdata/empty-body.txtar create mode 100644 internal/refactor/inline/testdata/multistmt-body.txtar create mode 100644 internal/refactor/inline/testdata/param-subst.txtar create mode 100644 internal/refactor/inline/util.go diff --git a/gopls/internal/lsp/source/inline.go b/gopls/internal/lsp/source/inline.go index 6a8d57d412a..74eeed5b117 100644 --- a/gopls/internal/lsp/source/inline.go +++ b/gopls/internal/lsp/source/inline.go @@ -100,7 +100,9 @@ func inlineCall(ctx context.Context, snapshot Snapshot, fh FileHandle, rng proto Call: call, Content: callerPGF.Src, } - got, err := inline.Inline(caller, callee) + + // Pass log.Printf here when debugging. + got, err := inline.Inline(nil, caller, callee) if err != nil { return nil, nil, err } diff --git a/gopls/internal/regtest/marker/testdata/codeaction/inline.txt b/gopls/internal/regtest/marker/testdata/codeaction/inline.txt index 8f1ea924864..134065f26b9 100644 --- a/gopls/internal/regtest/marker/testdata/codeaction/inline.txt +++ b/gopls/internal/regtest/marker/testdata/codeaction/inline.txt @@ -17,7 +17,7 @@ func add(x, y int) int { return x + y } package a func _() { - println(func(x, y int) int { return x + y }(1, 2)) //@codeaction("refactor.inline", "add", ")", inline) + println((1 + 2)) //@codeaction("refactor.inline", "add", ")", inline) } func add(x, y int) int { return x + y } diff --git a/internal/refactor/inline/analyzer/analyzer.go b/internal/refactor/inline/analyzer/analyzer.go index 943776e14b2..4758a4275fb 100644 --- a/internal/refactor/inline/analyzer/analyzer.go +++ b/internal/refactor/inline/analyzer/analyzer.go @@ -126,7 +126,7 @@ func run(pass *analysis.Pass) (interface{}, error) { Call: call, Content: content, } - got, err := inline.Inline(caller, callee) + got, err := inline.Inline(nil, caller, callee) if err != nil { pass.Reportf(call.Lparen, "%v", err) return diff --git a/internal/refactor/inline/analyzer/testdata/src/a/a.go b/internal/refactor/inline/analyzer/testdata/src/a/a.go index e661515b7c7..294278670f2 100644 --- a/internal/refactor/inline/analyzer/testdata/src/a/a.go +++ b/internal/refactor/inline/analyzer/testdata/src/a/a.go @@ -1,7 +1,8 @@ package a func f() { - One() // want `inline call of a.One` + One() // want `inline call of a.One` + new(T).Two() // want `inline call of \(a.T\).Two` } diff --git a/internal/refactor/inline/analyzer/testdata/src/a/a.go.golden b/internal/refactor/inline/analyzer/testdata/src/a/a.go.golden index fe9877b69c1..1a214fc9148 100644 --- a/internal/refactor/inline/analyzer/testdata/src/a/a.go.golden +++ b/internal/refactor/inline/analyzer/testdata/src/a/a.go.golden @@ -1,8 +1,9 @@ package a func f() { - _ = one // want `inline call of a.One` - func(_ T) int { return 2 }(*new(T)) // want `inline call of \(a.T\).Two` + _ = one // want `inline call of a.One` + + _ = 2 // want `inline call of \(a.T\).Two` } type T struct{} diff --git a/internal/refactor/inline/analyzer/testdata/src/b/b.go.golden b/internal/refactor/inline/analyzer/testdata/src/b/b.go.golden index 61b7bd9b349..b871b4b5100 100644 --- a/internal/refactor/inline/analyzer/testdata/src/b/b.go.golden +++ b/internal/refactor/inline/analyzer/testdata/src/b/b.go.golden @@ -5,5 +5,5 @@ import "a" func f() { a.One() // want `cannot inline call to a.One because body refers to non-exported one` - func(_ a.T) int { return 2 }(*new(a.T)) // want `inline call of \(a.T\).Two` + _ = 2 // want `inline call of \(a.T\).Two` } diff --git a/internal/refactor/inline/callee.go b/internal/refactor/inline/callee.go index 291971cf6d8..96ff827e034 100644 --- a/internal/refactor/inline/callee.go +++ b/internal/refactor/inline/callee.go @@ -30,25 +30,23 @@ func (callee *Callee) String() string { return callee.impl.Name } type gobCallee struct { Content []byte // file content, compacted to a single func decl - // syntax derived from compacted Content (not serialized) - fset *token.FileSet - decl *ast.FuncDecl - // results of type analysis (does not reach go/types data structures) - PkgPath string // package path of declaring package - Name string // user-friendly name for error messages - Unexported []string // names of free objects that are unexported - FreeRefs []freeRef // locations of references to free objects - FreeObjs []object // descriptions of free objects - BodyIsReturnExpr bool // function body is "return expr(s)" - ValidForCallStmt bool // => bodyIsReturnExpr and sole expr is f() or <-ch - NumResults int // number of results (according to type, not ast.FieldList) + PkgPath string // package path of declaring package + Name string // user-friendly name for error messages + Unexported []string // names of free objects that are unexported + FreeRefs []freeRef // locations of references to free objects + FreeObjs []object // descriptions of free objects + BodyIsReturnExpr bool // function body is "return expr(s)" with trivial conversion + ValidForCallStmt bool // => bodyIsReturnExpr and sole expr is f() or <-ch + NumResults int // number of results (according to type, not ast.FieldList) + Params []*paramInfo // information about receiver, params, and results } // A freeRef records a reference to a free object. Gob-serializable. +// (This means free relative to the FuncDecl as a whole, i.e. excluding parameters.) type freeRef struct { - Start, End int // Callee.content[start:end] is extent of the reference - Object int // index into Callee.freeObjs + Offset int // byte offset of the reference relative to the FuncDecl + Object int // index into Callee.freeObjs } // An object abstracts a free types.Object referenced by the callee. Gob-serializable. @@ -59,8 +57,6 @@ type object struct { ValidPos bool // Object.Pos().IsValid() } -func (callee *gobCallee) offset(pos token.Pos) int { return offsetOf(callee.fset, pos) } - // AnalyzeCallee analyzes a function that is a candidate for inlining // and returns a Callee that describes it. The Callee object, which is // serializable, can be passed to one or more subsequent calls to @@ -99,7 +95,8 @@ func AnalyzeCallee(fset *token.FileSet, pkg *types.Package, info *types.Info, de return nil, fmt.Errorf("cannot inline generic function %s: type parameters are not yet supported", name) } - // Record the location of all free references in the callee body. + // Record the location of all free references in the FuncDecl. + // (Parameters are not free by this definition.) var ( freeObjIndex = make(map[types.Object]int) freeObjs []object @@ -182,9 +179,9 @@ func AnalyzeCallee(fset *token.FileSet, pkg *types.Package, info *types.Info, de }) freeObjIndex[obj] = objidx } + freeRefs = append(freeRefs, freeRef{ - Start: offsetOf(fset, n.Pos()), - End: offsetOf(fset, n.End()), + Offset: int(n.Pos() - decl.Pos()), Object: objidx, }) } @@ -195,12 +192,37 @@ func AnalyzeCallee(fset *token.FileSet, pkg *types.Package, info *types.Info, de ast.Inspect(decl, visit) // Analyze callee body for "return results" form, where - // results is one or more expressions or an n-ary call. + // results is one or more expressions or an n-ary call, + // and the implied conversions are trivial. validForCallStmt := false - bodyIsReturnExpr := decl.Type.Results != nil && len(decl.Type.Results.List) > 0 && - len(decl.Body.List) == 1 && - is[*ast.ReturnStmt](decl.Body.List[0]) && - len(decl.Body.List[0].(*ast.ReturnStmt).Results) > 0 + bodyIsReturnExpr := func() bool { + if decl.Type.Results != nil && + len(decl.Type.Results.List) > 0 && + len(decl.Body.List) == 1 { + if ret, ok := decl.Body.List[0].(*ast.ReturnStmt); ok && len(ret.Results) > 0 { + // Don't reduce calls to functions whose + // return statement has non trivial conversions. + argType := func(i int) types.Type { + return info.TypeOf(ret.Results[i]) + } + if len(ret.Results) == 1 && sig.Results().Len() > 1 { + // Spread return: return f() where f.Results > 1. + tuple := info.TypeOf(ret.Results[0]).(*types.Tuple) + argType = func(i int) types.Type { + return tuple.At(i).Type() + } + } + for i := 0; i < sig.Results().Len(); i++ { + if !trivialConversion(argType(i), sig.Results().At(i)) { + return false + } + } + + return true + } + } + return false + }() if bodyIsReturnExpr { ret := decl.Body.List[0].(*ast.ReturnStmt) @@ -237,45 +259,26 @@ func AnalyzeCallee(fset *token.FileSet, pkg *types.Package, info *types.Info, de }() } + // Compact content to just the FuncDecl. + // // As a space optimization, we don't retain the complete // callee file content; all we need is "package _; func f() { ... }". // This reduces the size of analysis facts. // - // The FileSet file/line info is no longer meaningful - // and should not be used in error messages. - // But the FileSet offsets are valid w.r.t. the content. - // // (For ease of debugging we could insert a //line directive after // the package decl but it seems more trouble than it's worth.) - { - start, end := offsetOf(fset, decl.Pos()), offsetOf(fset, decl.End()) - - var compact bytes.Buffer - compact.WriteString("package _\n") - compact.Write(content[start:end]) - content = compact.Bytes() - - // Re-parse the compacted content. - var err error - decl, err = parseCompact(fset, content) - if err != nil { - return nil, err - } - - // (content, decl) are now updated. - - // Adjust the freeRefs offsets. - delta := int(offsetOf(fset, decl.Pos()) - start) - for i := range freeRefs { - freeRefs[i].Start += delta - freeRefs[i].End += delta - } + // + // Offsets in the callee information are "relocatable" + // since they are all relative to the FuncDecl. + content = append([]byte("package _\n"), + content[offsetOf(fset, decl.Pos()):offsetOf(fset, decl.End())]...) + // Sanity check: re-parse the compacted content. + if _, _, err := parseCompact(content); err != nil { + return nil, err } return &Callee{gobCallee{ Content: content, - fset: fset, - decl: decl, PkgPath: pkg.Path(), Name: name, Unexported: unexported, @@ -284,20 +287,183 @@ func AnalyzeCallee(fset *token.FileSet, pkg *types.Package, info *types.Info, de BodyIsReturnExpr: bodyIsReturnExpr, ValidForCallStmt: validForCallStmt, NumResults: sig.Results().Len(), + Params: analyzeParams(fset, info, decl), }}, nil } // parseCompact parses a Go source file of the form "package _\n func f() { ... }" // and returns the sole function declaration. -func parseCompact(fset *token.FileSet, content []byte) (*ast.FuncDecl, error) { +func parseCompact(content []byte) (*token.FileSet, *ast.FuncDecl, error) { + fset := token.NewFileSet() const mode = parser.ParseComments | parser.SkipObjectResolution | parser.AllErrors f, err := parser.ParseFile(fset, "callee.go", content, mode) if err != nil { - return nil, fmt.Errorf("internal error: cannot compact file: %v", err) + return nil, nil, fmt.Errorf("internal error: cannot compact file: %v", err) } - return f.Decls[0].(*ast.FuncDecl), nil + return fset, f.Decls[0].(*ast.FuncDecl), nil +} + +// A paramInfo records information about a callee receiver, parameter, or result variable. +type paramInfo struct { + Name string // parameter name (may be blank, or even "") + Kind string // one of {recv,param,result} + Assigned bool // parameter appears on left side of an assignment statement + Escapes bool // parameter has its address taken + Refs []int // FuncDecl-relative byte offset of parameter ref within body + Shadow map[string]bool // names shadowed at one of the above refs } +// analyzeParams computes information about parameters of function fn, +// including a simple "address taken" escape analysis. +// +// It returns a new array with an entry for each receiver, +// parameter, and result variable of function fn. +// +// The input must be well-typed. +func analyzeParams(fset *token.FileSet, info *types.Info, decl *ast.FuncDecl) (res []*paramInfo) { + fnobj, ok := info.Defs[decl.Name] + if !ok { + panic(fmt.Sprintf("%s: no func object for %q", + fset.Position(decl.Name.Pos()), decl.Name)) // ill-typed? + } + + paramInfos := make(map[*types.Var]*paramInfo) + { + sig := fnobj.Type().(*types.Signature) + newParamInfo := func(param *types.Var, kind string) *paramInfo { + info := ¶mInfo{Name: param.Name(), Kind: kind} + res = append(res, info) + paramInfos[param] = info + return info + } + if sig.Recv() != nil { + newParamInfo(sig.Recv(), "recv") + } + for i := 0; i < sig.Params().Len(); i++ { + newParamInfo(sig.Params().At(i), "param") + } + for i := 0; i < sig.Results().Len(); i++ { + newParamInfo(sig.Results().At(i), "result") + } + } + + // lvalue is called for each address-taken expression or LHS of assignment. + // Supported forms are: x, (x), x[i], x.f, *x, T{}. + var lvalue func(e ast.Expr, escapes bool) + lvalue = func(e ast.Expr, escapes bool) { + switch e := e.(type) { + case *ast.Ident: + if v, ok := info.Uses[e].(*types.Var); ok { + if info := paramInfos[v]; info != nil { + // e is a use of parameter v. + if escapes { + info.Escapes = true + } else { + info.Assigned = true + } + } + } + case *ast.ParenExpr: + lvalue(e.X, escapes) + case *ast.IndexExpr: + // TODO(adonovan): support generics without assuming e.X has a core type. + // Consider: + // + // func Index[T interface{ [3]int | []int }](t T, i int) *int { + // return &t[i] + // } + // + // We must traverse the normal terms and check + // whether any of them is an array. + if _, ok := info.TypeOf(e.X).Underlying().(*types.Array); ok { + lvalue(e.X, escapes) // &a[i] on array + } + case *ast.SelectorExpr: + if _, ok := info.TypeOf(e.X).Underlying().(*types.Struct); ok { + lvalue(e.X, escapes) // &s.f on struct + } + case *ast.StarExpr: + // *ptr indirects an existing pointer + case *ast.CompositeLit: + // &T{...} creates a new variable + default: + panic(fmt.Sprintf("&x on %T", e)) // unreachable in well-typed code + } + } + + // Search function body for operations &x, x.f(), and x = y + // where x is a parameter. Each of these treats x as an address. + // + // Also record locations of all references to parameters. + // And record the set of intervening definitions for each parameter. + if decl.Body != nil { + var stack []ast.Node + stack = append(stack, decl.Type) // for scope of function itself + ast.Inspect(decl.Body, func(n ast.Node) bool { + if n != nil { + stack = append(stack, n) // push + } else { + stack = stack[:len(stack)-1] // pop + } + + switch n := n.(type) { + case *ast.Ident: + if v, ok := info.Uses[n].(*types.Var); ok { + if pinfo, ok := paramInfos[v]; ok { + // Record location of ref to parameter. + offset := int(n.Pos() - decl.Pos()) + pinfo.Refs = append(pinfo.Refs, offset) + + // Find set of names shadowed within body + // (excluding the parameter itself). + // If these names are free in the arg expression, + // we can't substitute the parameter. + for _, n := range stack { + if scope, ok := info.Scopes[n]; ok { + for _, name := range scope.Names() { + if name != pinfo.Name { + if pinfo.Shadow == nil { + pinfo.Shadow = make(map[string]bool) + } + pinfo.Shadow[name] = true + } + } + } + } + } + } + + case *ast.UnaryExpr: + if n.Op == token.AND { + lvalue(n.X, true) // &x + } + + case *ast.CallExpr: + // implicit &x in method call x.f(), + // where x has type T and method is (*T).f + if sel, ok := n.Fun.(*ast.SelectorExpr); ok { + if seln, ok := info.Selections[sel]; ok && + seln.Kind() == types.MethodVal && + !seln.Indirect() && + is[*types.Pointer](seln.Obj().Type().(*types.Signature).Recv().Type()) { + lvalue(sel.X, true) // &x.f + } + } + + case *ast.AssignStmt: + for _, lhs := range n.Lhs { + lvalue(lhs, false) + } + } + return true + }) + } + + return res +} + +// -- callee helpers -- + // deref removes a pointer type constructor from the core type of t. func deref(t types.Type) types.Type { if ptr, ok := typeparams.CoreType(t).(*types.Pointer); ok { @@ -336,15 +502,5 @@ func (callee *Callee) GobEncode() ([]byte, error) { } func (callee *Callee) GobDecode(data []byte) error { - if err := gob.NewDecoder(bytes.NewReader(data)).Decode(&callee.impl); err != nil { - return err - } - fset := token.NewFileSet() - decl, err := parseCompact(fset, callee.impl.Content) - if err != nil { - return err - } - callee.impl.fset = fset - callee.impl.decl = decl - return nil + return gob.NewDecoder(bytes.NewReader(data)).Decode(&callee.impl) } diff --git a/internal/refactor/inline/inline.go b/internal/refactor/inline/inline.go index 9167498fc35..b48b43ec8cb 100644 --- a/internal/refactor/inline/inline.go +++ b/internal/refactor/inline/inline.go @@ -18,7 +18,19 @@ // caller and callee belong to the same token.FileSet or // types.Importer realms. // -// In general, inlining consists of modifying a function or method +// There are many aspects to a function call. It is the only construct +// that can simultaneously bind multiple variables of different +// explicit types, with implicit assignment conversions. (Neither var +// nor := declarations can do that.) It defines the scope of control +// labels, of return statements, and of defer statements. Arguments +// and results of function calls may be tuples even though tuples are +// not first-class values in Go, and a tuple-valued call expression +// may be "spread" across the argument list of a call or the operands +// of a return statement. All these unique features mean that in the +// general case, not everything that can be expressed by a function +// call can be expressed without one. +// +// So, in general, inlining consists of modifying a function or method // call expression f(a1, ..., an) so that the name of the function f // is replaced ("literalized") by a literal copy of the function // declaration, with free identifiers suitably modified to use the @@ -33,20 +45,36 @@ // reflection over the call stack, but this exception to the rule is // explicitly allowed.) // -// In some special cases it is possible to entirely replace ("reduce") -// the call by a copy of the function's body in which parameters have -// been replaced by arguments, but this is surprisingly tricky for a -// number of reasons, some of which are listed here for illustration: +// In a number of special cases it is possible to entirely replace +// ("reduce") the call by a copy of the function's body in which +// parameters have been replaced by arguments. The inliner supports a +// small number of reduction strategies, and we expect this set to +// grow. Nonetheless, sound reduction is surprisingly tricky. The +// following section lists some of the challenges. +// +// - All effects of the call argument expressions must be preserved, +// both in their number (they must not be eliminated or repeated), +// and in their order (both with respect to other arguments, and any +// effects in the callee function). +// +// This must be the case even if the corresponding parameters are +// never referenced, are referenced multiple times, referenced in +// a different order from the arguments, or referenced within a +// nested function that may be executed an arbitrary number of +// times. // -// - Any effects of the call argument expressions must be preserved, -// even if the corresponding parameters are never referenced, or are -// referenced multiple times, or are referenced in a different order -// from the arguments. +// Currently, parameter replacement is not applied to arguments +// with effects, but with further analysis of the sequence of +// strict effects within the callee we could relax this constraint. // // - Even an argument expression as simple as ptr.x may not be // referentially transparent, because another argument may have the // effect of changing the value of ptr. // +// This constraint could be relaxed by some kind of alias or +// escape analysis that proves that ptr cannot be mutated during +// the call. +// // - Although constants are referentially transparent, as a matter of // style we do not wish to duplicate literals that are referenced // multiple times in the body because this undoes proper factoring. @@ -54,18 +82,30 @@ // // - If the function body consists of statements other than just // "return expr", in some contexts it may be syntactically -// impossible to replace the call expression by the body statements. -// Consider "} else if x := f(); cond { ... }". -// (Go has no equivalent to Lisp's progn or Rust's blocks.) +// impossible to reduce the call. Consider: +// +// } else if x := f(); cond { ... } +// +// Go has no equivalent to Lisp's progn or Rust's blocks, +// nor ML's let expressions (let param = arg in body); +// its closest equivalent is func(param){body}(arg). +// Reduction strategies must therefore consider the syntactic +// context of the call. // // - Similarly, without the equivalent of Rust-style blocks and // first-class tuples, there is no general way to reduce a call // to a function such as -// > func(params)(args)(results) { stmts; return body } +// +// func(params)(args)(results) { stmts; return expr } +// // to an expression such as -// > { var params = args; stmts; body } +// +// { var params = args; stmts; expr } +// // or even a statement such as -// > results = { var params = args; stmts; body } +// +// results = { var params = args; stmts; expr } +// // Consequently the declaration and scope of the result variables, // and the assignment and control-flow implications of the return // statement, must be dealt with by cases. @@ -83,7 +123,9 @@ // - If a parameter or result variable is updated by an assignment // within the function body, it cannot always be safely replaced // by a variable in the caller. For example, given -// > func f(a int) int { a++; return a } +// +// func f(a int) int { a++; return a } +// // The call y = f(x) cannot be replaced by { x++; y = x } because // this would change the value of the caller's variable x. // Only if the caller is finished with x is this safe. @@ -99,13 +141,33 @@ // be α-renamed. // // - Given -// > func f() uint8 { return 0 } -// > var x any = f() +// +// func f() uint8 { return 0 } +// +// var x any = f() +// // reducing the call to var x any = 0 is unsound because it -// discards the implicit conversion. We may need to make each -// argument->parameter and return->result assignment conversion -// implicit if the types differ. Assignments to variadic -// parameters may need to explicitly construct a slice. +// discards the implicit conversion to uint8. We may need to make +// each argument-to-parameter conversion explicit if the types +// differ. Assignments to variadic parameters may need to +// explicitly construct a slice. +// +// An analogous problem applies to the implicit assignments in +// return statements: +// +// func g() any { return f() } +// +// Replacing the call f() with 0 would silently lose a +// conversion to uint8 and change the behavior of the program. +// +// - When inlining a call f(1, x, g()) where those parameters are +// unreferenced, we should be able to avoid evaluating 1 and x +// since they are pure and thus have no effect. But x may be the +// last reference to a local variable in the caller, so removing +// it would cause a compilation error. Argument elimination must +// avoid making the caller's local variables unreferenced (or must +// be prepared to eliminate the declaration too---this is where an +// iterative framework for simplification would really help). // // More complex callee functions are inlinable with more elaborate and // invasive changes to the statements surrounding the call expression. @@ -120,8 +182,10 @@ // random in the corpus, inlines them, and checks that the // result is either a sensible error or a valid transformation. // -// - Eliminate parameters that are unreferenced in the callee -// and whose argument expression is side-effect free. +// - Compute precisely (not conservatively) when parameter +// elimination would remove the last reference to a caller local +// variable, and blank out the local instead of retreating from +// the elimination. // // - Afford the client more control such as a limit on the total // increase in line count, or a refusal to inline using the @@ -141,7 +205,9 @@ // cannot declare all the parameters and initialize them to their // arguments in one go if they have varied types. Instead, // one must use multiple specs such as: -// > { var x int = 1; var y int32 = 2; body ...} +// +// { var x int = 1; var y int32 = 2; body ...} +// // but this means that the initializer expression for y is // within the scope of x, so it may require α-renaming. // @@ -159,12 +225,19 @@ // by their instantiations. // // - Support inlining of calls to function literals such as: -// > f := func(...) { ...} -// > f() +// +// f := func(...) { ... } +// +// f() +// // including recursive ones: -// > var f func(...) -// > f = func(...) { ...f...} -// > f() +// +// var f func(...) +// +// f = func(...) { ...f...} +// +// f() +// // But note that the existing algorithm makes widespread assumptions // that the callee is a package-level function or method. // @@ -173,7 +246,7 @@ // - Allow non-'go' build systems such as Bazel/Blaze a chance to // decide whether an import is accessible using logic other than // "/internal/" path segments. This could be achieved by returning -// the list of added import paths. +// the list of added import paths instead of a text diff. // // - Inlining a function from another module may change the // effective version of the Go language spec that governs it. We @@ -195,18 +268,18 @@ import ( "bytes" "fmt" "go/ast" + "go/format" + "go/parser" "go/token" "go/types" - "log" pathpkg "path" "reflect" - "sort" + "strconv" "strings" "golang.org/x/tools/go/ast/astutil" "golang.org/x/tools/go/types/typeutil" "golang.org/x/tools/imports" - "golang.org/x/tools/internal/typeparams" ) // A Caller describes the function call and its enclosing context. @@ -221,18 +294,178 @@ type Caller struct { Content []byte } -func (caller *Caller) offset(pos token.Pos) int { return offsetOf(caller.Fset, pos) } - // Inline inlines the called function (callee) into the function call (caller) // and returns the updated, formatted content of the caller source file. -func Inline(caller *Caller, callee_ *Callee) ([]byte, error) { - callee := &callee_.impl +// +// Inline does not mutate any part of Caller or Callee. +// +// The caller may supply a log function to observe the decision-making process. +// +// TODO(adonovan): provide an API for clients that want structured +// output: a list of import additions and deletions plus one or more +// localized diffs (or even AST transformations, though ownership and +// mutation are tricky) near the call site. +func Inline(logf func(string, ...any), caller *Caller, callee *Callee) ([]byte, error) { + if logf == nil { + logf = func(string, ...any) {} // discard + } + logf("inline %s @ %v", + formatNode(caller.Fset, caller.Call), + caller.Fset.Position(caller.Call.Lparen)) - // -- check caller -- + res, err := inline(logf, caller, &callee.impl) + if err != nil { + return nil, err + } + // Replace the call (or some node that encloses it) by new syntax. + assert(res.old != nil, "old is nil") + assert(res.new != nil, "new is nil") + + // Don't call replaceNode(caller.File, res.old, res.new) + // as it mutates the caller's syntax tree. + // Instead, splice the file, replacing the extent of the "old" + // node by a formatting of the "new" node, and re-parse. + // We'll fix up the imports on this new tree, and format again. + var f *ast.File + { + start := offsetOf(caller.Fset, res.old.Pos()) + end := offsetOf(caller.Fset, res.old.End()) + // TODO(adonovan): might it make more sense to use + // callee.Fset when formatting res.new?? + newFile := string(caller.Content[:start]) + + formatNode(caller.Fset, res.new) + + string(caller.Content[end:]) + const mode = parser.ParseComments | parser.SkipObjectResolution | parser.AllErrors + var err error + f, err = parser.ParseFile(caller.Fset, "callee.go", newFile, mode) + if err != nil { + // Something has gone very wrong. + logf("failed to parse <<%s>>", newFile) // debugging + return nil, err + } + } + + // Add new imports. + // + // Insert new imports after last existing import, + // to avoid migration of pre-import comments. + // The imports will be organized below. + if len(res.newImports) > 0 { + var importDecl *ast.GenDecl + if len(f.Imports) > 0 { + // Append specs to existing import decl + importDecl = f.Decls[0].(*ast.GenDecl) + } else { + // Insert new import decl. + importDecl = &ast.GenDecl{Tok: token.IMPORT} + f.Decls = prepend[ast.Decl](importDecl, f.Decls...) + } + for _, spec := range res.newImports { + // Check that all imports (in particular, the new ones) are accessible. + // TODO(adonovan): allow customization of the accessibility relation + // (e.g. for Bazel). + path, _ := strconv.Unquote(spec.Path.Value) + // TODO(adonovan): better segment hygiene. + if i := strings.Index(path, "/internal/"); i >= 0 { + if !strings.HasPrefix(caller.Types.Path(), path[:i]) { + return nil, fmt.Errorf("can't inline function %v as its body refers to inaccessible package %q", callee.impl.Name, path) + } + } + importDecl.Specs = append(importDecl.Specs, spec) + } + } + + // Remove imports that are no longer referenced. + // + // It ought to be possible to compute the set of PkgNames used + // by the "old" code, compute the free identifiers of the + // "new" code using a syntax-only (no go/types) algorithm, and + // see if the reduction in the number of uses of any PkgName + // equals the number of times it appears in caller.Info.Uses, + // indicating that it is no longer referenced by res.new. + // + // However, the notorious ambiguity of resolving T{F: 0} makes this + // unreliable: without types, we can't tell whether F refers to + // a field of struct T, or a package-level const/var of a + // dot-imported (!) package. + // + // So, for now, we run imports.Process, which is + // unsatisfactory as it has to run the go command, and it + // looks at the user's module cache state--unnecessarily, + // since this step cannot add new imports. + // + // TODO(adonovan): replace with a simpler implementation since + // all the necessary imports are present but merely untidy. + // That will be faster, and also less prone to nondeterminism + // if there are bugs in our logic for import maintenance. + // + // However, golang.org/x/tools/internal/imports.ApplyFixes is + // too simple as it requires the caller to have figured out + // all the logical edits. In our case, we know all the new + // imports that are needed (see newImports), each of which can + // be specified as: + // + // &imports.ImportFix{ + // StmtInfo: imports.ImportInfo{path, name, + // IdentName: name, + // FixType: imports.AddImport, + // } + // + // but we don't know which imports are made redundant by the + // inlining itself. For example, inlining a call to + // fmt.Println may make the "fmt" import redundant. + // + // Also, both imports.Process and internal/imports.ApplyFixes + // reformat the entire file, which is not ideal for clients + // such as gopls. (That said, the point of a canonical format + // is arguably that any tool can reformat as needed without + // this being inconvenient.) + // + // We could invoke imports.Process and parse its result, + // compare against the original AST, compute a list of import + // fixes, and return that too. + var out bytes.Buffer + if err := format.Node(&out, caller.Fset, f); err != nil { + return nil, err + } + formatted, err := imports.Process("output", out.Bytes(), nil) + if err != nil { + logf("cannot reformat: %v <<%s>>", err, &out) + return nil, err // cannot reformat (a bug?) + } + return formatted, nil +} + +type result struct { + newImports []*ast.ImportSpec + old, new ast.Node // e.g. replace call expr by callee function body expression +} + +// inline returns a pair of an old node (the call, or something +// enclosing it) and a new node (its replacement, which may be a +// combination of caller, callee, and new nodes), along with the set +// of new imports needed. +// +// TODO(adonovan): rethink the 'result' interface. The assumption of a +// one-to-one replacement seems fragile. One can easily imagine the +// transformation replacing the call and adding new variable +// declarations, for example, or replacing a call statement by zero or +// many statements.) +// +// TODO(adonovan): in earlier drafts, the transformation was expressed +// by splicing substrings of the two source files because syntax +// trees don't preserve comments faithfully (see #20744), but such +// transformations don't compose. The current implementation is +// tree-based but is very lossy wrt comments. It would make a good +// candidate for evaluating an alternative fully self-contained tree +// representation, such as any proposed solution to #20744, or even +// dst or some private fork of go/ast.) +func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*result, error) { // Inlining of dynamic calls is not currently supported, - // even for local closure calls. - if typeutil.StaticCallee(caller.Info, caller.Call) == nil { + // even for local closure calls. (This would be a lot of work.) + calleeSymbol := typeutil.StaticCallee(caller.Info, caller.Call) + if calleeSymbol == nil { // e.g. interface method return nil, fmt.Errorf("cannot inline: not a static function call") } @@ -249,7 +482,8 @@ func Inline(caller *Caller, callee_ *Callee) ([]byte, error) { // syntax path enclosing Call, innermost first (Path[0]=Call) callerPath, _ := astutil.PathEnclosingInterval(caller.File, caller.Call.Pos(), caller.Call.End()) - callerLookup := func(name string, pos token.Pos) types.Object { + callerLookup := func(name string) types.Object { + pos := caller.Call.Pos() for _, n := range callerPath { // The function body scope (containing not just params) // is associated with FuncDecl.Type, not FuncDecl.Body. @@ -279,7 +513,7 @@ func Inline(caller *Caller, callee_ *Callee) ([]byte, error) { } // localImportName returns the local name for a given imported package path. - var newImports []string + var newImports []*ast.ImportSpec localImportName := func(path string) string { name, ok := importMap[path] if !ok { @@ -294,7 +528,7 @@ func Inline(caller *Caller, callee_ *Callee) ([]byte, error) { // use the package's declared name. base := pathpkg.Base(path) name = base - for n := 0; callerLookup(name, caller.Call.Pos()) != nil; n++ { + for n := 0; callerLookup(name) != nil; n++ { name = fmt.Sprintf("%s%d", base, n) } @@ -303,19 +537,25 @@ func Inline(caller *Caller, callee_ *Callee) ([]byte, error) { // the package name or the last segment of path. // This requires that we tabulate (path, declared name, local name) // triples for each package referenced by the callee. - newImports = append(newImports, fmt.Sprintf("%s %q", name, path)) + newImports = append(newImports, &ast.ImportSpec{ + Name: makeIdent(name), + Path: &ast.BasicLit{ + Kind: token.STRING, + Value: strconv.Quote(path), + }, + }) importMap[path] = name } return name } // Compute the renaming of the callee's free identifiers. - objRenames := make([]string, len(callee.FreeObjs)) // "" => no rename + objRenames := make([]ast.Expr, len(callee.FreeObjs)) // nil => no change for i, obj := range callee.FreeObjs { // obj is a free object of the callee. // // Possible cases are: - // - nil or a builtin + // - builtin function, type, or value (e.g. nil, zero) // => check not shadowed in caller. // - package-level var/func/const/types // => same package: check not shadowed in caller. @@ -328,33 +568,31 @@ func Inline(caller *Caller, callee_ *Callee) ([]byte, error) { // // There can be no free references to labels, fields, or methods. - var newName string + var newName ast.Expr if obj.Kind == "pkgname" { // Use locally appropriate import, creating as needed. - newName = localImportName(obj.PkgPath) // imported package + newName = makeIdent(localImportName(obj.PkgPath)) // imported package } else if !obj.ValidPos { - // Built-in function, type, or nil: check not shadowed at caller. - found := callerLookup(obj.Name, caller.Call.Pos()) // can't fail + // Built-in function, type, or value (e.g. nil, zero): + // check not shadowed at caller. + found := callerLookup(obj.Name) // always finds something if found.Pos().IsValid() { return nil, fmt.Errorf("cannot inline because built-in %q is shadowed in caller by a %s (line %d)", obj.Name, objectKind(found), caller.Fset.Position(found.Pos()).Line) } - newName = obj.Name - } else { // Must be reference to package-level var/func/const/type, // since type parameters are not yet supported. - newName = obj.Name qualify := false if obj.PkgPath == callee.PkgPath { // reference within callee package if samePkg { // Caller and callee are in same package. // Check caller has not shadowed the decl. - found := callerLookup(obj.Name, caller.Call.Pos()) // can't fail + found := callerLookup(obj.Name) // can't fail if !isPkgLevel(found) { return nil, fmt.Errorf("cannot inline because %q is shadowed in caller by a %s (line %d)", obj.Name, objectKind(found), @@ -374,271 +612,717 @@ func Inline(caller *Caller, callee_ *Callee) ([]byte, error) { // Form a qualified identifier, pkg.Name. if qualify { pkgName := localImportName(obj.PkgPath) - newName = pkgName + "." + newName + newName = &ast.SelectorExpr{ + X: makeIdent(pkgName), + Sel: makeIdent(obj.Name), + } } } objRenames[i] = newName } - // Compute edits to inlined callee. - type edit struct { - start, end int // byte offsets wrt callee.content - new string - } - var edits []edit - - // Give explicit blank "_" names to all method parameters - // (including receiver) since we will make the receiver a regular - // parameter and one cannot mix named and unnamed parameters. - // e.g. func (T) f(int, string) -> (_ T, _ int, _ string) - if callee.decl.Recv != nil { - ensureNamed := func(params *ast.FieldList) { - for _, param := range params.List { - if param.Names == nil { - offset := callee.offset(param.Type.Pos()) - edits = append(edits, edit{ - start: offset, - end: offset, - new: "_ ", - }) - } - } - } - ensureNamed(callee.decl.Recv) - ensureNamed(callee.decl.Type.Params) + res := &result{ + newImports: newImports, + } + + // Parse callee function declaration. + calleeFset, calleeDecl, err := parseCompact(callee.Content) + if err != nil { + return nil, err // "can't happen" + } + + // replaceCalleeID replaces an identifier in the callee. + replaceCalleeID := func(offset int, repl ast.Expr) { + id := findIdent(calleeDecl, calleeDecl.Pos()+token.Pos(offset)) + logf("- replace id %q @ #%d to %q", id.Name, offset, formatNode(calleeFset, repl)) + replaceNode(calleeDecl, id, repl) } // Generate replacements for each free identifier. + // (The same tree may be spliced in multiple times, resulting in a DAG.) for _, ref := range callee.FreeRefs { - if repl := objRenames[ref.Object]; repl != "" { - edits = append(edits, edit{ - start: ref.Start, - end: ref.End, - new: repl, - }) + if repl := objRenames[ref.Object]; repl != nil { + replaceCalleeID(ref.Offset, repl) } } - // Edits are non-overlapping but insertions and edits may be coincident. - // Preserve original order. - sort.SliceStable(edits, func(i, j int) bool { - return edits[i].start < edits[j].start - }) + sig := calleeSymbol.Type().(*types.Signature) - // Check that all imports (in particular, the new ones) are accessible. - // TODO(adonovan): allow customization of the accessibility relation (e.g. for Bazel). - for path := range importMap { - // TODO(adonovan): better segment hygiene. - if i := strings.Index(path, "/internal/"); i >= 0 { - if !strings.HasPrefix(caller.Types.Path(), path[:i]) { - return nil, fmt.Errorf("can't inline function %v as its body refers to inaccessible package %q", callee.Name, path) - } - } + // Gather effective argument tuple, including receiver. + // + // If the receiver argument and parameter have + // different pointerness, make the "&" or "*" explicit. + // + // Beware that: + // + // - a method can only be called through a selection, but only + // the first of these two forms needs special treatment: + // + // expr.f(args) -> ([&*]expr, args) MethodVal + // T.f(recv, args) -> ( expr, args) MethodExpr + // + // - the presence of a value in receiver-position in the call + // is a property of the caller, not the callee. A method + // (calleeDecl.Recv != nil) may be called like an ordinary + // function. + // + // - the types.Signatures seen by the caller (from + // StaticCallee) and by the callee (from decl type) + // differ in this case. + // + // In a spread call f(g()), the sole ordinary argument g(), + // always last in args, has a tuple type. + // + // We compute type-based predicates like pure, duplicable, + // freevars, etc, now, before we start modifying things. + type argument struct { + expr ast.Expr + typ types.Type // may be tuple for sole non-receiver arg in spread call + pure bool // expr has no effects + duplicable bool // expr may be duplicated + freevars map[string]bool // free names of expr } + var args []*argument // effective arguments; nil => eliminated + if calleeDecl.Recv != nil { + sel := astutil.Unparen(caller.Call.Fun).(*ast.SelectorExpr) + if caller.Info.Selections[sel].Kind() == types.MethodVal { + // Move receiver argument recv.f(args) to argument list f(&recv, args). + arg := &argument{ + expr: sel.X, + typ: caller.Info.TypeOf(sel.X), + pure: pure(caller.Info, sel.X), + duplicable: duplicable(caller.Info, sel.X), + freevars: freevars(caller.Info, sel.X), + } + args = append(args, arg) - // The transformation is expressed by splicing substrings of - // the two source files, because syntax trees don't preserve - // comments faithfully (see #20744). - var out bytes.Buffer + // Make * or & explicit. + // + // We do this after we've computed the type-based + // predicates (pure et al) above, as they won't + // work on synthetic syntax. + argIsPtr := arg.typ != deref(arg.typ) + paramIsPtr := is[*types.Pointer](sig.Recv().Type()) + if !argIsPtr && paramIsPtr { + // &recv + arg.expr = &ast.UnaryExpr{Op: token.AND, X: arg.expr} + arg.typ = types.NewPointer(arg.typ) + } else if argIsPtr && !paramIsPtr { + // *recv + arg.expr = &ast.StarExpr{X: arg.expr} + arg.typ = deref(arg.typ) - // 'replace' emits to out the specified range of the callee, - // applying all edits that fall completely within it. - replace := func(start, end int) { - off := start - for _, edit := range edits { - if start <= edit.start && edit.end <= end { - out.Write(callee.Content[off:edit.start]) - out.WriteString(edit.new) - off = edit.end + // Technically *recv is non-pure and + // non-duplicable, as side effects + // could change the pointer between + // multiple reads. But unfortunately + // this really degrades many of our tests. + // + // TODO(adonovan): improve the precision + // purity and duplicability. + // For example, *new(T) is actually pure. + // And *ptr, where ptr doesn't escape and + // has no assignments other than its decl, + // is also pure; this is very common. + // + // arg.pure = false + // arg.duplicable = false } } - out.Write(callee.Content[off:end]) + } + for _, arg := range caller.Call.Args { + args = append(args, &argument{ + expr: arg, + typ: caller.Info.TypeOf(arg), + pure: pure(caller.Info, arg), + duplicable: duplicable(caller.Info, arg), + freevars: freevars(caller.Info, arg), + }) } - // Insert new imports after last existing import, - // to avoid migration of pre-import comments. - // The imports will be organized later. + // Parameter elimination + // + // Consider each parameter and its corresponding argument in turn + // and evaluate these conditions: + // + // - the parameter is neither address-taken nor assigned; + // - the argument is pure; + // - if the parameter refcount is zero, the argument must + // not contain the last use of a local var; + // - if the parameter refcount is > 1, the argument must be duplicable; + // - the argument (or types.Default(argument) if it's untyped) has + // the same type as the parameter. + // + // If all conditions are met then the parameter can be eliminated + // and each reference to it replaced by the argument. + var eliminatedParams []bool // (recv, params...) { - offset := caller.offset(caller.File.Name.End()) // after package decl - if len(caller.File.Imports) > 0 { - // It's tempting to insert the new import after the last ImportSpec, - // but that may not be at the end of the import decl. - // Consider: import ( "a"; "b" ‸ ) - for _, decl := range caller.File.Decls { - if decl, ok := decl.(*ast.GenDecl); ok && decl.Tok == token.IMPORT { - offset = caller.offset(decl.End()) // after import decl + // Gather effective parameter objects, + // including the receiver if any. + var paramObjs []*types.Var + if sig.Recv() != nil { + paramObjs = append(paramObjs, sig.Recv()) + } + for i := 0; i < sig.Params().Len(); i++ { + paramObjs = append(paramObjs, sig.Params().At(i)) + } + + // In most calls, args and paramObjs correspond. + // + // Edge case: in a variadic call, len(args) >= len(params)-1. + // + // Edge case: in a spread call f(g()) where g is n-ary (n > 1), + // len(args) = 1 and len(params) = n, + // unless (corner case!) f is variadic, + // in which case both are again 1. + // + // TODO(adonovan): support elimination of variadic parameters, + // and of spread arguments. + + eliminatedParams = make([]bool, len(paramObjs)) + nextParam: + for i, param := range callee.Params { + if param.Kind == "result" { + break // end of parameters + } + if sig.Variadic() && i == len(paramObjs)-1 { + // final ...T parameter + // TODO(adonovan): decouple the param and arg parts of this + // loop so that we can handle variadic cases. + logf("keeping param %q: variadic elimination not yet supported", + param.Name) + continue + } + if param.Escapes { + logf("keeping param %q: escapes from callee", param.Name) + continue + } + if param.Assigned { + logf("keeping param %q: assigned by callee", param.Name) + continue // callee needs the parameter variable + } + + // Check argument against parameter. + // + // Beware: don't use types.Info on arg since + // the syntax may be synthetic (not created by parser) + // and thus lacking positions and types; + // do it earlier (see pure/duplicable/freevars). + arg := args[i] + if is[*types.Tuple](arg.typ) { + // TODO(adonovan): handle elimination of spread arguments. + logf("keeping param %q: argument %s is spread", + param.Name, formatNode(caller.Fset, arg.expr)) + continue + } + if !arg.pure { + logf("keeping param %q: argument %s is impure", + param.Name, formatNode(caller.Fset, arg.expr)) + continue // unsafe to change order or cardinality of effects + } + if len(param.Refs) > 1 && !arg.duplicable { + logf("keeping param %q: argument is not duplicable", param.Name) + continue // incorrect or poor style to duplicate an expression + } + if len(param.Refs) == 0 { + // Eliminating an unreferenced parameter might + // remove the last reference to a caller local var. + for free := range arg.freevars { + if v, ok := callerLookup(free).(*types.Var); ok { + // TODO(adonovan): be more precise and check + // that v is defined within the body of the caller + // function (if any) and is indeed referenced + // only by the call. + logf("keeping param %q: arg contains perhaps the last reference to possible caller local %v @ %v", + param.Name, v, caller.Fset.Position(v.Pos())) + continue nextParam + } } } + + // Check that eliminating the parameter wouldn't materially + // change the type. + // + // (We don't simply wrap the argument in an explicit conversion + // to the parameter type because that could increase allocation + // in the number of (e.g.) string -> any conversions. + // Even when Uses = 1, the sole ref might be in a lambda that + // is multiply executed.) + if len(param.Refs) > 0 && !trivialConversion(args[i].typ, paramObjs[i]) { + logf("keeping param %q: argument passing converts %s to type %s", + param.Name, args[i].typ, paramObjs[i].Type()) + continue // implicit conversion is significant + } + + // Check for shadowing. + // + // Consider inlining a call f(z, 1) to + // func f(x, y int) int { z := y; return x + y + z }: + // we can't replace x in the body by z (or any + // expression that has z as a free identifier) + // because there's an intervening declaration of z + // that would shadow the caller's one. + for free := range arg.freevars { + if param.Shadow[free] { + logf("keeping param %q: cannot replace with argument as it has free ref to %s that is shadowed", param.Name, free) + continue nextParam // shadowing conflict + } + } + + // It is safe to eliminate param and replace it with arg. + // No additional parens are required around arg for + // the supported "pure" expressions. + logf("replacing parameter %q by argument %q", + param.Name, formatNode(caller.Fset, arg.expr)) + for _, ref := range param.Refs { + replaceCalleeID(ref, arg.expr) + } + eliminatedParams[i] = true + args[i] = nil } - out.Write(caller.Content[:offset]) - out.WriteString("\n") - for _, imp := range newImports { - fmt.Fprintf(&out, "import %s\n", imp) + } + + var remainingArgs []ast.Expr + for _, arg := range args { + if arg != nil { + remainingArgs = append(remainingArgs, arg.expr) } - out.Write(caller.Content[offset:caller.offset(caller.Call.Pos())]) } - // Special case: a call to a function whose body consists only - // of "return expr" may be replaced by the expression, so long as: + // -- let the inlining strategies begin -- + + // Special case: eliminate a call to a function whose body is empty. + // (=> callee has no results and caller is a statement.) // - // (a) There are no receiver or parameter argument expressions - // whose side effects must be considered. - // (b) There are no named parameter or named result variables - // that could potentially escape. + // func f(params) {} + // f(args) + // => _, _ = args // - // TODO(adonovan): expand this special case to cover more scenarios. - // Consider each parameter in turn. If: - // - the parameter does not escape and is never assigned; - // - its argument is pure (no effects or panics--basically just idents and literals) - // and referentially transparent (not new(T) or &T{...}) or referenced at most once; and - // - the argument and parameter have the same type - // then the parameter can be eliminated and each reference - // to it replaced by the argument. - // If: - // - all parameters can be so replaced; - // - and the body is just "return expr"; - // - and the result vars are unnamed or never referenced (and thus cannot escape); - // then the call expression can be replaced by its body expression. - if callee.BodyIsReturnExpr && - callee.decl.Recv == nil && // no receiver arg effects to consider - len(caller.Call.Args) == 0 && // no argument effects to consider - !hasNamedVars(callee.decl.Type.Params) && // no param vars escape - !hasNamedVars(callee.decl.Type.Results) { // no result vars escape - - // A single return operand inlined to an expression - // context may need parens. Otherwise: - // func two() int { return 1+1 } - // print(-two()) => print(-1+1) // oops! - parens := callee.NumResults == 1 - - // If the call is a standalone statement, but the - // callee body is not suitable as a standalone statement - // (f() or <-ch), explicitly discard the results: - // _, _ = expr - if isCallStmt(callerPath) { - parens = false - - if !callee.ValidForCallStmt { - for i := 0; i < callee.NumResults; i++ { - if i > 0 { - out.WriteString(", ") - } - out.WriteString("_") + if len(calleeDecl.Body.List) == 0 { + logf("strategy: reduce call to empty body") + + // Evaluate the arguments for effects and delete the call entirely. + stmt := callStmt(callerPath) // cannot fail + res.old = stmt + if nargs := len(remainingArgs); nargs > 0 { + // Emit "_, _ = args" to discard results. + // Make correction for spread calls + // f(g()) or x.f(g()) where g() is a tuple. + if last := args[len(args)-1]; last != nil { + if tuple, ok := last.typ.(*types.Tuple); ok { + nargs += tuple.Len() - 1 } - out.WriteString(" = ") } + res.new = &ast.AssignStmt{ + Lhs: blanks[ast.Expr](nargs), + Tok: token.ASSIGN, + Rhs: remainingArgs, + } + } else { + // No remaining arguments: delete call statement entirely + res.new = &ast.EmptyStmt{} } + return res, nil + } - // Emit the body expression(s). - for i, res := range callee.decl.Body.List[0].(*ast.ReturnStmt).Results { - if i > 0 { - out.WriteString(", ") + // Attempt to reduce parameterless calls + // whose result variables do not escape. + if func() bool { + for i, param := range callee.Params { + if param.Kind != "result" { // recv or param + if !eliminatedParams[i] { + logf("param %q not eliminated", param.Name) + return false + } + } else if param.Escapes { + logf("result variable %s escapes", param.Name) + return false } - if parens { - out.WriteString("(") + } + return true + }() { + logf("all params eliminated and no result vars escape") + + // Special case: parameterless call to { return expr(s) }. + // + // If: + // - the body is just "return expr" with trivial implicit conversions, + // - all parameters have been eliminated, and + // - no result var escapes, + // then the call expression can be replaced by the + // callee's body expression, suitably substituted. + if callee.BodyIsReturnExpr { + logf("strategy: reduce parameterless call to { return expr }") + + results := calleeDecl.Body.List[0].(*ast.ReturnStmt).Results + + clearPositions(calleeDecl.Body) + + context := callContext(callerPath) + if stmt, ok := context.(*ast.ExprStmt); ok { + logf("call in statement context") + + if callee.ValidForCallStmt { + logf("callee body is valid as statement") + // Replace the statement with the callee expr. + res.old = caller.Call + res.new = results[0] // Inv: len(results) == 1 + } else { + logf("callee body is not valid as statement") + // The call is a standalone statement, but the + // callee body is not suitable as a standalone statement + // (f() or <-ch), explicitly discard the results: + // _, _ = expr + res.old = stmt + res.new = &ast.AssignStmt{ + Lhs: blanks[ast.Expr](callee.NumResults), + Tok: token.ASSIGN, + Rhs: results, + } + } + + } else if callee.NumResults == 1 { + logf("call in expression context") + + // A single return operand inlined to a unary + // expression context may need parens. Otherwise: + // func two() int { return 1+1 } + // print(-two()) => print(-1+1) // oops! + // + // TODO(adonovan): do better by analyzing 'context' + // to see whether ambiguity is possible. + // For example, if the context is x[y:z], then + // the x subtree is subject to precedence ambiguity + // (replacing x by p+q would give p+q[y:z] which is wrong) + // but the y and z subtrees are safe. + res.old = caller.Call + res.new = &ast.ParenExpr{X: results[0]} + + } else { + logf("call in spread context") + + // The call returns multiple results but is + // not a standalone call statement. It must + // be the RHS of a spread assignment: + // var x, y = f() + // x, y := f() + // x, y = f() + // or the sole argument to a spread call: + // printf(f()) + res.old = context + switch context := context.(type) { + case *ast.AssignStmt: + // Inv: the call is in Rhs[0], not Lhs. + assign := shallowCopy(context) + assign.Rhs = results + res.new = assign + case *ast.ValueSpec: + // Inv: the call is in Values[0], not Names. + spec := shallowCopy(context) + spec.Values = results + res.new = spec + case *ast.CallExpr: + // Inv: the Call is Args[0], not Fun. + call := shallowCopy(context) + call.Args = results + res.new = call + default: + return nil, fmt.Errorf("internal error: unexpected context %T for spread call", context) + } } - replace(callee.offset(res.Pos()), callee.offset(res.End())) - if parens { - out.WriteString(")") + return res, nil + } + + // Special case: parameterless call to void function + // + // Inlining: + // f(args) + // where: + // func f(params) { stmts } + // reduces to: + // { stmts } + // so long as: + // - callee is a void function (no returns) + // - callee does not use defer + // - there is no label conflict between caller and callee + // - all parameters have been eliminated. + // + // If there is only a single statement, the braces are omitted. + body := calleeDecl.Body + if stmt := callStmt(callerPath); stmt != nil && !hasControl(body) { + logf("strategy: reduce parameterless call to { stmt } from a call stmt") + + var repl ast.Stmt = body + if len(body.List) == 1 { + repl = body.List[0] // omit braces around singleton statement } + clearPositions(repl) + res.old = stmt + res.new = repl + return res, nil } - goto rest + + // TODO(adonovan): parameterless call to { stmt; return expr } + // from one of these contexts: + // x, y = f() + // x, y := f() + // var x, y = f() + // => + // var (x T1, y T2); { stmts; x, y = expr } + // + // Because the params are no longer declared simultaneously + // we need to check that (for example) x ∉ freevars(T2), + // in addition to the usual checks for arg/result conversions, + // complex control, etc. + // Also test cases where expr is an n-ary call (spread returns). + + // TODO(adonovan): replace a call f(a1, a2) + // to func f(x T1, y T2) {body} by + // { var x T1 = a1 + // var y T2 = a2 + // body } + // if x ∉ freevars(a2) or freevars(T2), and so on, + // plus the usual checks for return conversions (if any), + // complex control, etc. + + // TODO(adonovan): a parameterless tail-call to a + // function with a single final return, no defer/label + // control, and no trivial return conversions, can be + // inlined to the body of the function (perhaps even + // omitting the braces.) } - // Emit a function literal in place of the callee name, - // with appropriate replacements. - out.WriteString("func (") - if recv := callee.decl.Recv; recv != nil { - // Move method receiver to head of ordinary parameters. - replace(callee.offset(recv.Opening+1), callee.offset(recv.Closing)) - if len(callee.decl.Type.Params.List) > 0 { - out.WriteString(", ") + // Infallible general case: literalization. + logf("strategy: literalization") + + // Modify callee's FuncDecl.Type.Params to remove eliminated + // parameters and move the receiver (if any) to the head of + // the ordinary parameters. + // + // The logic is fiddly because of the three forms of ast.Field: + // func(int), func(x int), func(x, y int) + // + // Also, ensure that all remaining parameters are named + // to avoid a mix of named/unnamed when joining (recv, params...). + // func (T) f(int, bool) -> (_ T, _ int, _ bool) + { + paramIdx := 0 // index in original parameter list (incl. receiver) + var newParams []*ast.Field + filterParams := func(field *ast.Field) { + var names []*ast.Ident + if field.Names == nil { + // Unnamed parameter field (e.g. func f(int) + if !eliminatedParams[paramIdx] { + // Give it an explicit name "_" since we will + // make the receiver (if any) a regular parameter + // and one cannot mix named and unnamed parameters. + names = blanks[*ast.Ident](1) + } + paramIdx++ + } else { + // Named parameter field e.g. func f(x, y int) + // Remove eliminated parameters in place. + // If all were eliminated, delete field. + for _, id := range field.Names { + if !eliminatedParams[paramIdx] { + names = append(names, id) + } + paramIdx++ + } + } + if names != nil { + newParams = append(newParams, &ast.Field{ + Names: names, + Type: field.Type, + }) + } + } + if calleeDecl.Recv != nil { + filterParams(calleeDecl.Recv.List[0]) + calleeDecl.Recv = nil + } + for _, field := range calleeDecl.Type.Params.List { + filterParams(field) } + calleeDecl.Type.Params.List = newParams } - replace(callee.offset(callee.decl.Type.Params.Opening+1), - callee.offset(callee.decl.End())) + // Emit a new call to a function literal in place of + // the callee name, with appropriate replacements. + newCall := &ast.CallExpr{ + Fun: &ast.FuncLit{ + Type: calleeDecl.Type, + Body: calleeDecl.Body, + }, + Args: remainingArgs, + } + clearPositions(newCall.Fun) + res.old = caller.Call + res.new = newCall + return res, nil +} - // Emit call arguments. - out.WriteString("(") - if callee.decl.Recv != nil { - // Move receiver argument x.f(...) to argument list f(x, ...). - recv := astutil.Unparen(caller.Call.Fun).(*ast.SelectorExpr).X +// -- predicates over expressions -- - // If the receiver argument and parameter have - // different pointerness, make the "&" or "*" explicit. - argPtr := is[*types.Pointer](typeparams.CoreType(caller.Info.TypeOf(recv))) - paramPtr := is[*ast.StarExpr](callee.decl.Recv.List[0].Type) - if !argPtr && paramPtr { - out.WriteString("&") - } else if argPtr && !paramPtr { - out.WriteString("*") +// freevars returns the names of all free identifiers of e: +// those lexically referenced by it but not defined within it. +// (Fields and methods are not included.) +func freevars(info *types.Info, e ast.Expr) map[string]bool { + free := make(map[string]bool) + ast.Inspect(e, func(n ast.Node) bool { + if id, ok := n.(*ast.Ident); ok { + // The isField check is so that we don't treat T{f: 0} as a ref to f. + if obj, ok := info.Uses[id]; ok && !within(obj.Pos(), e) && !isField(obj) { + free[obj.Name()] = true + } } + return true + }) + return free +} - out.Write(caller.Content[caller.offset(recv.Pos()):caller.offset(recv.End())]) +// pure reports whether the expression is pure, that is, +// has no side effects nor potential to panic. +// +// Beware that pure does not imply referentially transparent: for +// example, new(T) is a pure expression but it returns a different +// value each time it is evaluated. (One could say that is has effects +// on the memory allocator.) +// +// TODO(adonovan): +// - add a unit test of this function. +// - "potential to panic": I'm not sure this is an important +// criterion. We should be allowed to assume that good programs +// don't rely on runtime panics for correct behavior. +// - Should a binary + operator be considered pure? For strings, it +// allocates memory, but so does a composite literal and that's pure +// (but not duplicable). We need clearer definitions here. +func pure(info *types.Info, e ast.Expr) bool { + switch e := e.(type) { + case *ast.ParenExpr: + return pure(info, e.X) + case *ast.Ident: + return true + case *ast.FuncLit: + return true + case *ast.BasicLit: + return true + case *ast.UnaryExpr: // + - ! ^ & but not <- + return e.Op != token.ARROW && pure(info, e.X) + case *ast.CallExpr: + // A conversion is considered pure + if info.Types[e.Fun].IsType() { + // TODO(adonovan): fix: reject the newly allowed + // conversions between T[] and *[k]T, as they may panic. + return pure(info, e.Args[0]) + } - if len(caller.Call.Args) > 0 { - out.WriteString(", ") + // Call to these built-ins are pure if their arguments are pure. + if id, ok := astutil.Unparen(e.Fun).(*ast.Ident); ok { + if b, ok := info.ObjectOf(id).(*types.Builtin); ok { + switch b.Name() { + case "len", "cap", "complex", "imag", "real", "make", "new", "max", "min": + for _, arg := range e.Args { + if !pure(info, arg) { + return false + } + } + return true + } + } } - } - // Append ordinary args, sans initial "(". - out.Write(caller.Content[caller.offset(caller.Call.Lparen+1):caller.offset(caller.Call.End())]) - // Append rest of caller file. -rest: - out.Write(caller.Content[caller.offset(caller.Call.End()):]) + return false + case *ast.KeyValueExpr: + // map {key: value} or struct {field: value} + return pure(info, e.Key) && pure(info, e.Value) + case *ast.CompositeLit: + // T{x: 0} is pure (though it may imply + // an allocation, so it is not duplicable). + for _, elt := range e.Elts { + if !pure(info, elt) { + return false + } + } + return true + case *ast.SelectorExpr: + if sel, ok := info.Selections[e]; ok { + // A field or method selection x.f is pure + // if it does not indirect a pointer. + return !sel.Indirect() + } + // A qualified identifier pkg.Name is pure. + return true + case *ast.StarExpr: + return false // *ptr may panic + default: + return false + } +} - // Reformat, and organize imports. - // - // TODO(adonovan): this looks at the user's cache state. - // Replace with a simpler implementation since - // all the necessary imports are present but merely untidy. - // That will be faster, and also less prone to nondeterminism - // if there are bugs in our logic for import maintenance. - // - // However, golang.org/x/tools/internal/imports.ApplyFixes is - // too simple as it requires the caller to have figured out - // all the logical edits. In our case, we know all the new - // imports that are needed (see newImports), each of which can - // be specified as: - // - // &imports.ImportFix{ - // StmtInfo: imports.ImportInfo{path, name, - // IdentName: name, - // FixType: imports.AddImport, - // } - // - // but we don't know which imports are made redundant by the - // inlining itself. For example, inlining a call to - // fmt.Println may make the "fmt" import redundant. - // - // Also, both imports.Process and internal/imports.ApplyFixes - // reformat the entire file, which is not ideal for clients - // such as gopls. (That said, the point of a canonical format - // is arguably that any tool can reformat as needed without - // this being inconvenient.) - res, err := imports.Process("output", out.Bytes(), nil) - if err != nil { - if false { // debugging - log.Printf("cannot reformat: %v <<%s>>", err, &out) +// duplicable reports whether it is appropriate for the expression to +// be freely duplicated. +// +// Given the declaration +// +// func f(x T) T { return x + g() + x } +// +// an argument y is considered duplicable if we would wish to see a +// call f(y) simplified to y+g()+y. This is true for identifiers, +// integer literals, unary negation, and selectors x.f where x is not +// a pointer. But we would not wish to duplicate expressions that: +// - have side effects (e.g. nearly all calls), +// - are not referentially transparent (e.g. &T{}, ptr.field), or +// - are long (e.g. "huge string literal"). +func duplicable(info *types.Info, e ast.Expr) bool { + switch e := e.(type) { + case *ast.ParenExpr: + return duplicable(info, e.X) + case *ast.Ident: + return true + case *ast.BasicLit: + return e.Kind == token.INT + case *ast.UnaryExpr: // e.g. +1, -1 + return (e.Op == token.ADD || e.Op == token.SUB) && duplicable(info, e.X) + case *ast.CallExpr: + // Don't treat a conversion T(x) as duplicable even + // if x is duplicable because it could duplicate + // allocations. There may be cases to tease apart here. + return false + case *ast.SelectorExpr: + if sel, ok := info.Selections[e]; ok { + // A field or method selection x.f is referentially + // transparent if it does not indirect a pointer. + return !sel.Indirect() } - return nil, err // cannot reformat (a bug?) + // A qualified identifier pkg.Name is referentially transparent. + return true + default: + return false } - return res, nil } -// -- helpers -- +// -- inline helpers -- -func is[T any](x any) bool { - _, ok := x.(T) - return ok +func assert(cond bool, msg string) { + if !cond { + panic(msg) + } } -func within(pos token.Pos, n ast.Node) bool { - return n.Pos() <= pos && pos <= n.End() +// blanks returns a slice of n > 0 blank identifiers. +func blanks[E ast.Expr](n int) []E { + if n == 0 { + panic("blanks(0)") + } + res := make([]E, n) + for i := range res { + res[i] = any(makeIdent("_")).(E) // ugh + } + return res } -func offsetOf(fset *token.FileSet, pos token.Pos) int { - return fset.PositionFor(pos, false).Offset +func makeIdent(name string) *ast.Ident { + return &ast.Ident{Name: name} } // importedPkgName returns the PkgName object declared by an ImportSpec. @@ -658,31 +1342,214 @@ func isPkgLevel(obj types.Object) bool { return obj.Pkg().Scope().Lookup(obj.Name()) == obj } -// objectKind returns an object's kind (e.g. var, func, const, typename). -func objectKind(obj types.Object) string { - return strings.TrimPrefix(strings.ToLower(reflect.TypeOf(obj).String()), "*types.") +// callContext returns the node immediately enclosing the call +// (specified as a PathEnclosingInterval), ignoring parens. +func callContext(callPath []ast.Node) ast.Node { + _ = callPath[0].(*ast.CallExpr) // sanity check + for _, n := range callPath[1:] { + if !is[*ast.ParenExpr](n) { + return n + } + } + return nil } -// isCallStmt reports whether the function call (specified -// as a PathEnclosingInterval) appears within an ExprStmt. -func isCallStmt(callPath []ast.Node) bool { - _ = callPath[0].(*ast.CallExpr) - for _, n := range callPath[1:] { +// callStmt reports whether the function call (specified +// as a PathEnclosingInterval) appears within an ExprStmt, +// and returns it if so. +func callStmt(callPath []ast.Node) *ast.ExprStmt { + stmt, _ := callContext(callPath).(*ast.ExprStmt) + return stmt +} + +// hasControl reports whether the body of the function uses control +// constructs such as defer, return, or labels. +// +// TODO(adonovan): refine disposition w.r.t. return: +// a single, unnested final return may be ok for some callers. +func hasControl(body *ast.BlockStmt) (res bool) { + ast.Inspect(body, func(n ast.Node) bool { switch n.(type) { - case *ast.ParenExpr: - continue - case *ast.ExprStmt: - return true + case *ast.FuncLit: + return false // prune traversal + case *ast.DeferStmt, *ast.LabeledStmt, *ast.ReturnStmt: + res = true + } + return true + }) + return +} + +// replaceNode performs a destructive update of the tree rooted at +// root, replacing each occurrence of "from" with "to". If to is nil and +// the element is within a slice, the slice element is removed. +// +// The root itself cannot be replaced; an attempt will panic. +// +// This function must not be called on the caller's syntax tree. +// +// TODO(adonovan): polish this up and move it to astutil package. +// TODO(adonovan): needs a unit test. +func replaceNode(root ast.Node, from, to ast.Node) { + if from == nil { + panic("from == nil") + } + if reflect.ValueOf(from).IsNil() { + panic(fmt.Sprintf("from == (%T)(nil)", from)) + } + if from == root { + panic("from == root") + } + found := false + var parent reflect.Value // parent variable of interface type, containing a pointer + var visit func(reflect.Value) + visit = func(v reflect.Value) { + switch v.Kind() { + case reflect.Ptr: + if v.Interface() == from { + found = true + + // If v is a struct field or array element + // (e.g. Field.Comment or Field.Names[i]) + // then it is addressable (a pointer variable). + // + // But if it was the value an interface + // (e.g. *ast.Ident within ast.Node) + // then it is non-addressable, and we need + // to set the enclosing interface (parent). + if !v.CanAddr() { + v = parent + } + + // to=nil => use zero value + var toV reflect.Value + if to != nil { + toV = reflect.ValueOf(to) + } else { + toV = reflect.Zero(v.Type()) // e.g. ast.Expr(nil) + } + v.Set(toV) + + } else if !v.IsNil() { + switch v.Interface().(type) { + case *ast.Object, *ast.Scope: + // Skip fields of types potentially involved in cycles. + default: + visit(v.Elem()) + } + } + + case reflect.Struct: + for i := 0; i < v.Type().NumField(); i++ { + visit(v.Field(i)) + } + + case reflect.Slice: + compact := false + for i := 0; i < v.Len(); i++ { + visit(v.Index(i)) + if v.Index(i).IsNil() { + compact = true + } + } + if compact { + // Elements were deleted. Eliminate nils. + // (Do this is a second pass to avoid + // unnecessary writes in the common case.) + j := 0 + for i := 0; i < v.Len(); i++ { + if !v.Index(i).IsNil() { + v.Index(j).Set(v.Index(i)) + j++ + } + } + v.SetLen(j) + } + case reflect.Interface: + parent = v + visit(v.Elem()) + + case reflect.Array, reflect.Chan, reflect.Func, reflect.Map, reflect.UnsafePointer: + panic(v) // unreachable in AST + default: + // bool, string, number: nop } - break + parent = reflect.Value{} + } + visit(reflect.ValueOf(root)) + if !found { + panic(fmt.Sprintf("%T not found", from)) } - return false } -// hasNamedVars reports whether a function parameter tuple uses named variables. +// clearPositions destroys token.Pos information within the tree rooted at root, +// as positions in callee trees may cause caller comments to be emitted prematurely. +// +// In general it isn't safe to clear a valid Pos because some of them +// (e.g. CallExpr.Ellipsis, TypeSpec.Assign) are significant to +// go/printer, so this function sets each non-zero Pos to 1, which +// suffices to avoid advancing the printer's comment cursor. +// +// This function mutates its argument; do not invoke on caller syntax. // -// TODO(adonovan): this is a placeholder for a more complex analysis to detect -// whether inlining might cause named param/result variables to escape. -func hasNamedVars(tuple *ast.FieldList) bool { - return tuple != nil && len(tuple.List) > 0 && tuple.List[0].Names != nil +// TODO(adonovan): remove this horrendous workaround when #20744 is finally fixed. +func clearPositions(root ast.Node) { + posType := reflect.TypeOf(token.NoPos) + ast.Inspect(root, func(n ast.Node) bool { + if n != nil { + v := reflect.ValueOf(n).Elem() // deref the pointer to struct + fields := v.Type().NumField() + for i := 0; i < fields; i++ { + f := v.Field(i) + if f.Type() == posType { + // Clearing Pos arbitrarily is destructive, + // as its presence may be semantically significant + // (e.g. CallExpr.Ellipsis, TypeSpec.Assign) + // or affect formatting preferences (e.g. GenDecl.Lparen). + if f.Interface() != token.NoPos { + f.Set(reflect.ValueOf(token.Pos(1))) + } + } + } + } + return true + }) +} + +// findIdent returns the Ident beneath root that has the given pos. +func findIdent(root ast.Node, pos token.Pos) *ast.Ident { + // TODO(adonovan): opt: skip subtrees that don't contain pos. + var found *ast.Ident + ast.Inspect(root, func(n ast.Node) bool { + if found != nil { + return false + } + if id, ok := n.(*ast.Ident); ok { + if id.Pos() == pos { + found = id + } + } + return true + }) + if found == nil { + panic(fmt.Sprintf("findIdent %d not found", pos)) + } + return found +} + +func prepend[T any](elem T, slice ...T) []T { + return append([]T{elem}, slice...) +} + +func formatNode(fset *token.FileSet, n ast.Node) string { + var out strings.Builder + if err := format.Node(&out, fset, n); err != nil { + out.WriteString(err.Error()) + } + return out.String() +} + +func shallowCopy[T any](ptr *T) *T { + copy := *ptr + return © } diff --git a/internal/refactor/inline/inline_test.go b/internal/refactor/inline/inline_test.go index f77d2851f17..2fcd7b582bc 100644 --- a/internal/refactor/inline/inline_test.go +++ b/internal/refactor/inline/inline_test.go @@ -6,14 +6,18 @@ package inline_test import ( "bytes" + "crypto/sha256" + "encoding/binary" "encoding/gob" "fmt" "go/ast" "go/token" "os" "path/filepath" + "reflect" "regexp" "testing" + "unsafe" "golang.org/x/tools/go/ast/astutil" "golang.org/x/tools/go/expect" @@ -121,8 +125,7 @@ func Test(t *testing.T) { t.Errorf("%s: @inline(rx, want): want file name (to assert success) or error message regexp (to assert failure)", posn) continue } - t.Log("doInlineNote", posn) - if err := doInlineNote(pkg, file, content, pattern, posn, want); err != nil { + if err := doInlineNote(t.Logf, pkg, file, content, pattern, posn, want); err != nil { t.Errorf("%s: @inline(%v, %v): %v", posn, note.Args[0], note.Args[1], err) continue } @@ -141,7 +144,7 @@ func Test(t *testing.T) { // Finally it checks that, on success, the transformed file is equal // to want (a []byte), or on failure that the error message matches // want (a *Regexp). -func doInlineNote(pkg *packages.Package, file *ast.File, content []byte, pattern *regexp.Regexp, posn token.Position, want any) error { +func doInlineNote(logf func(string, ...any), pkg *packages.Package, file *ast.File, content []byte, pattern *regexp.Regexp, posn token.Position, want any) error { // Find extent of pattern match within commented line. var startPos, endPos token.Pos { @@ -284,7 +287,16 @@ func doInlineNote(pkg *packages.Package, file *ast.File, content []byte, pattern return nil, fmt.Errorf("internal error: gob decoding failed: %v", err) } - return inline.Inline(caller, callee) + // Check that the operation didn't mutate the tree. + pre := deepHash(caller.File) + defer func() { + post := deepHash(caller.File) + if pre != post { + panic("Inline mutated caller.File") + } + }() + + return inline.Inline(logf, caller, callee) }() if err != nil { if wantRE, ok := want.(*regexp.Regexp); ok { @@ -323,3 +335,83 @@ func extractTxtar(ar *txtar.Archive, dir string) error { } return nil } + +// deepHash computes a cryptographic hash of an ast.Node so that +// if the data structure is mutated, the hash changes. +// It assumes Go variables do not change address. +// +// TODO(adonovan): consider publishing this in the astutil package. +// +// TODO(adonovan): consider a variant that reports where in the tree +// the mutation occurred (obviously at a cost in space). +func deepHash(n ast.Node) [sha256.Size]byte { + seen := make(map[unsafe.Pointer]bool) // to break cycles + + hasher := sha256.New() + le := binary.LittleEndian + writeUint64 := func(v uint64) { + var bs [8]byte + le.PutUint64(bs[:], v) + hasher.Write(bs[:]) + } + + var visit func(reflect.Value) + visit = func(v reflect.Value) { + switch v.Kind() { + case reflect.Ptr: + ptr := v.UnsafePointer() + writeUint64(uint64(uintptr(ptr))) + if !v.IsNil() { + if !seen[ptr] { + seen[ptr] = true + // Skip types we don't handle yet, but don't care about. + switch v.Interface().(type) { + case *ast.Scope: + return // involves a map + } + + visit(v.Elem()) + } + } + + case reflect.Struct: + for i := 0; i < v.Type().NumField(); i++ { + visit(v.Field(i)) + } + + case reflect.Slice: + ptr := v.UnsafePointer() + // We may encounter different slices at the same address, + // so don't mark ptr as "seen". + writeUint64(uint64(uintptr(ptr))) + writeUint64(uint64(v.Len())) + writeUint64(uint64(v.Cap())) + for i := 0; i < v.Len(); i++ { + visit(v.Index(i)) + } + + case reflect.Interface: + if v.IsNil() { + writeUint64(0) + } else { + rtype := reflect.ValueOf(v.Type()).UnsafePointer() + writeUint64(uint64(uintptr(rtype))) + visit(v.Elem()) + } + + case reflect.Array, reflect.Chan, reflect.Func, reflect.Map, reflect.UnsafePointer: + panic(v) // unreachable in AST + + default: // bool, string, number + if v.Kind() == reflect.String { // proper framing + writeUint64(uint64(v.Len())) + } + binary.Write(hasher, le, v.Interface()) + } + } + visit(reflect.ValueOf(n)) + + var hash [sha256.Size]byte + hasher.Sum(hash[:0]) + return hash +} diff --git a/internal/refactor/inline/testdata/basic-err.txtar b/internal/refactor/inline/testdata/basic-err.txtar index 18e0eb7adb3..c289e9bb544 100644 --- a/internal/refactor/inline/testdata/basic-err.txtar +++ b/internal/refactor/inline/testdata/basic-err.txtar @@ -19,6 +19,6 @@ package a import "io" -var _ = func(err error) string { return err.Error() }(io.EOF) //@ inline(re"getError", getError) +var _ = (io.EOF.Error()) //@ inline(re"getError", getError) func getError(err error) string { return err.Error() } diff --git a/internal/refactor/inline/testdata/basic-literal.txtar b/internal/refactor/inline/testdata/basic-literal.txtar index 50bac33456a..c5e95960e20 100644 --- a/internal/refactor/inline/testdata/basic-literal.txtar +++ b/internal/refactor/inline/testdata/basic-literal.txtar @@ -1,19 +1,27 @@ -Most basic test of inlining by literalization. +Basic tests of inlining by literalization. + +recover() is an example of a function with effects, +so it (currently) defeats reduction. But note that the +other parameter of 'add' is eliminated. -- go.mod -- module testdata go 1.12 --- a/a.go -- +-- a/a1.go -- package a -var _ = add(1, 2) //@ inline(re"add", add) +func _() { + add(recover().(int), 2) //@ inline(re"add", add1) +} func add(x, y int) int { return x + y } --- add -- +-- add1 -- package a -var _ = func(x, y int) int { return x + y }(1, 2) //@ inline(re"add", add) +func _() { + func(x int) int { return x + 2 }(recover().(int)) //@ inline(re"add", add1) +} func add(x, y int) int { return x + y } diff --git a/internal/refactor/inline/testdata/basic-reduce.txtar b/internal/refactor/inline/testdata/basic-reduce.txtar index 9eedbc05f1e..46e44b77625 100644 --- a/internal/refactor/inline/testdata/basic-reduce.txtar +++ b/internal/refactor/inline/testdata/basic-reduce.txtar @@ -4,7 +4,7 @@ Most basic test of inlining by reduction. module testdata go 1.12 --- a/a.go -- +-- a/a0.go -- package a var _ = zero() //@ inline(re"zero", zero) @@ -17,3 +17,34 @@ package a var _ = (0) //@ inline(re"zero", zero) func zero() int { return 0 } + +-- a/a1.go -- +package a + +func _() { + one := 1 + add(one, 2) //@ inline(re"add", add1) +} + +func add(x, y int) int { return x + y } + +-- add1 -- +package a + +func _() { + one := 1 + _ = one + 2 //@ inline(re"add", add1) +} + +func add(x, y int) int { return x + y } + +-- a/a2.go -- +package a + +var _ = add(len(""), 2) //@ inline(re"add", add2) + +-- add2 -- +package a + +var _ = (len("") + 2) //@ inline(re"add", add2) + diff --git a/internal/refactor/inline/testdata/comments.txtar b/internal/refactor/inline/testdata/comments.txtar index 0482e919a48..189e9a20e8d 100644 --- a/internal/refactor/inline/testdata/comments.txtar +++ b/internal/refactor/inline/testdata/comments.txtar @@ -1,5 +1,11 @@ -Inlining, whether by literalization or reduction, -preserves comments in the callee. +Test of (lack of) comment preservation by inlining, +whether by literalization or reduction. + +Comment handling was better in an earlier implementation +based on byte-oriented file surgery; switching to AST +manipulation (though better in all other respects) was +a regression. The underlying problem of AST comment fidelity +is Go issue #20744. -- go.mod -- module testdata @@ -22,12 +28,7 @@ func f() { package a func _() { - func() { - // a - /* b */ - g() /* c */ - // d - }() //@ inline(re"f", f) + g() //@ inline(re"f", f) } func f() { @@ -50,7 +51,7 @@ func g() int { return 1 /*hello*/ + /*there*/ 1 } package a func _() { - println((1 /*hello*/ + /*there*/ 1)) //@ inline(re"g", g) + println((1 + 1)) //@ inline(re"g", g) } func g() int { return 1 /*hello*/ + /*there*/ 1 } diff --git a/internal/refactor/inline/testdata/crosspkg.txtar b/internal/refactor/inline/testdata/crosspkg.txtar index 43dc63f32ea..7c0704be819 100644 --- a/internal/refactor/inline/testdata/crosspkg.txtar +++ b/internal/refactor/inline/testdata/crosspkg.txtar @@ -46,7 +46,6 @@ package a import ( "fmt" "testdata/b" - c "testdata/c" ) @@ -54,8 +53,8 @@ import ( func A() { fmt.Println() - func() { c.C() }() //@ inline(re"B1", b1result) - b.B2() //@ inline(re"B2", b2result) + c.C() //@ inline(re"B1", b1result) + b.B2() //@ inline(re"B2", b2result) } -- b2result -- @@ -72,6 +71,6 @@ import ( func A() { fmt.Println() - b.B1() //@ inline(re"B1", b1result) - func() { fmt.Println() }() //@ inline(re"B2", b2result) + b.B1() //@ inline(re"B1", b1result) + fmt.Println() //@ inline(re"B2", b2result) } diff --git a/internal/refactor/inline/testdata/dotimport.txtar b/internal/refactor/inline/testdata/dotimport.txtar index 7e886afdb94..8ca5f05cda7 100644 --- a/internal/refactor/inline/testdata/dotimport.txtar +++ b/internal/refactor/inline/testdata/dotimport.txtar @@ -28,8 +28,10 @@ func _() { -- result -- package c -import a "testdata/a" +import ( + a "testdata/a" +) func _() { - func() { a.A() }() //@ inline(re"B", result) + a.A() //@ inline(re"B", result) } diff --git a/internal/refactor/inline/testdata/empty-body.txtar b/internal/refactor/inline/testdata/empty-body.txtar new file mode 100644 index 00000000000..77311c33702 --- /dev/null +++ b/internal/refactor/inline/testdata/empty-body.txtar @@ -0,0 +1,103 @@ +Test of elimination of calls to functions with completely empty bodies. +The arguments must still be evaluated and their results discarded. +The number of discard blanks must match the type, not the syntax (see 2-ary f). +If there are no arguments, the entire call is eliminated. + +We cannot eliminate some pure argument expressions because they +may contain the last reference to a local variable. + +-- go.mod -- +module testdata +go 1.12 + +-- a/a0.go -- +package a + +func _() { + empty() //@ inline(re"empty", empty0) +} + +func empty(...any) {} + +-- empty0 -- +package a + +func _() { + //@ inline(re"empty", empty0) +} + +func empty(...any) {} + +-- a/a1.go -- +package a + +func _(ch chan int) { + empty(f()) //@ inline(re"empty", empty1) +} + +func f() (int, int) + +-- empty1 -- +package a + +func _(ch chan int) { + _, _ = f() //@ inline(re"empty", empty1) +} + +func f() (int, int) + +-- a/a2.go -- +package a + +func _(ch chan int) { + empty(-1, ch, len(""), g(), <-ch) //@ inline(re"empty", empty2) +} + +func g() int + +-- empty2 -- +package a + +func _(ch chan int) { + _, _, _, _, _ = -1, ch, len(""), g(), <-ch //@ inline(re"empty", empty2) +} + +func g() int + +-- a/a3.go -- +package a + +func _() { + new(T).empty() //@ inline(re"empty", empty3) +} + +type T int + +func (T) empty() int {} + +-- empty3 -- +package a + +func _() { + //@ inline(re"empty", empty3) +} + +type T int + +func (T) empty() int {} + +-- a/a4.go -- +package a + +func _() { + var x T + x.empty() //@ inline(re"empty", empty4) +} + +-- empty4 -- +package a + +func _() { + var x T + _ = x //@ inline(re"empty", empty4) +} diff --git a/internal/refactor/inline/testdata/import-shadow.txtar b/internal/refactor/inline/testdata/import-shadow.txtar index 913c9cbe01a..e7ce5a26e07 100644 --- a/internal/refactor/inline/testdata/import-shadow.txtar +++ b/internal/refactor/inline/testdata/import-shadow.txtar @@ -31,11 +31,13 @@ func C() {} -- result -- package a -import c1 "testdata/c" +import ( + c1 "testdata/c" +) func A() { const c = 1 type c0 int - func() { c1.C() }() //@ inline(re"B", result) + c1.C() //@ inline(re"B", result) } diff --git a/internal/refactor/inline/testdata/method.txtar b/internal/refactor/inline/testdata/method.txtar index a4e02d575ca..511a90e4703 100644 --- a/internal/refactor/inline/testdata/method.txtar +++ b/internal/refactor/inline/testdata/method.txtar @@ -1,6 +1,7 @@ Test of inlining a method call. -The call to (*T).g0 implicitly takes the address &x. +The call to (*T).g0 implicitly takes the address &x, and +the call to T.h implictly dereferences the argument *ptr. The f1/g1 methods have parameters, exercising the splicing of the receiver into the parameter list. @@ -14,7 +15,7 @@ go 1.12 package a type T int -func (T) f0() {} +func (T) f0() { println() } func _(x T) { x.f0() //@ inline(re"f0", f0) @@ -25,16 +26,16 @@ package a type T int -func (T) f0() {} +func (T) f0() { println() } func _(x T) { - func(_ T) {}(x) //@ inline(re"f0", f0) + func(_ T) { println() }(x) //@ inline(re"f0", f0) } -- a/g0.go -- package a -func (recv *T) g0() {} +func (recv *T) g0() { println() } func _(x T) { x.g0() //@ inline(re"g0", g0) @@ -43,16 +44,16 @@ func _(x T) { -- g0 -- package a -func (recv *T) g0() {} +func (recv *T) g0() { println() } func _(x T) { - func(recv *T) {}(&x) //@ inline(re"g0", g0) + func(recv *T) { println() }(&x) //@ inline(re"g0", g0) } -- a/f1.go -- package a -func (T) f1(int, int) {} +func (T) f1(int, int) { println() } func _(x T) { x.f1(1, 2) //@ inline(re"f1", f1) @@ -61,16 +62,16 @@ func _(x T) { -- f1 -- package a -func (T) f1(int, int) {} +func (T) f1(int, int) { println() } func _(x T) { - func(_ T, _ int, _ int) {}(x, 1, 2) //@ inline(re"f1", f1) + func(_ T) { println() }(x) //@ inline(re"f1", f1) } -- a/g1.go -- package a -func (recv *T) g1(int, int) {} +func (recv *T) g1(int, int) { println() } func _(x T) { x.g1(1, 2) //@ inline(re"g1", g1) @@ -79,10 +80,10 @@ func _(x T) { -- g1 -- package a -func (recv *T) g1(int, int) {} +func (recv *T) g1(int, int) { println() } func _(x T) { - func(recv *T, _ int, _ int) {}(&x, 1, 2) //@ inline(re"g1", g1) + func(recv *T) { println() }(&x) //@ inline(re"g1", g1) } -- a/h.go -- @@ -91,7 +92,8 @@ package a func (T) h() int { return 1 } func _() { - new(T).h() //@ inline(re"h", h) + var ptr *T + ptr.h() //@ inline(re"h", h) } -- h -- @@ -100,5 +102,24 @@ package a func (T) h() int { return 1 } func _() { - func(_ T) int { return 1 }(*new(T)) //@ inline(re"h", h) + var ptr *T + func(_ T) int { return 1 }(*ptr) //@ inline(re"h", h) +} + +-- a/i.go -- +package a + +func (T) i() int { return 1 } + +func _() { + (*T).i(nil) //@ inline(re"i", i) +} + +-- i -- +package a + +func (T) i() int { return 1 } + +func _() { + _ = 1 //@ inline(re"i", i) } diff --git a/internal/refactor/inline/testdata/multistmt-body.txtar b/internal/refactor/inline/testdata/multistmt-body.txtar new file mode 100644 index 00000000000..b0e82f21715 --- /dev/null +++ b/internal/refactor/inline/testdata/multistmt-body.txtar @@ -0,0 +1,83 @@ +Tests of reduction of calls to multi-statement bodies. + +a1: literalized, because replacing parameter x with +argument z would cause a shadowing conflict. + +a2: reduced (no shadowing). + +a3: literalized, because of the return statement. + +-- go.mod -- +module testdata +go 1.12 + +-- a/a1.go -- +package a + +func _() { + z := 1 + f(z, 2) //@ inline(re"f", out1) +} + +func f(x, y int) { + z := 1 + print(x + y + z) +} + +-- out1 -- +package a + +func _() { + z := 1 + func(x int) { z := 1; print(x + 2 + z) }(z) //@ inline(re"f", out1) +} + +func f(x, y int) { + z := 1 + print(x + y + z) +} + +-- a/a2.go -- +package a + +func _() { + a := 1 + f(a, 2) //@ inline(re"f", out2) +} + +-- out2 -- +package a + +func _() { + a := 1 + { + z := 1 + print(a + 2 + z) + } //@ inline(re"f", out2) +} + +-- a/a3.go -- +package a + +func _() { + a := 1 + g(a, 2) //@ inline(re"g", out3) +} + +func g(x, y int) int { + z := 1 + return x + y + z +} + +-- out3 -- +package a + +func _() { + a := 1 + func() int { z := 1; return a + 2 + z }() //@ inline(re"g", out3) +} + +func g(x, y int) int { + z := 1 + return x + y + z +} diff --git a/internal/refactor/inline/testdata/param-subst.txtar b/internal/refactor/inline/testdata/param-subst.txtar new file mode 100644 index 00000000000..28e5effe712 --- /dev/null +++ b/internal/refactor/inline/testdata/param-subst.txtar @@ -0,0 +1,19 @@ +Test of parameter substitution. + +-- go.mod -- +module testdata +go 1.12 + +-- a/a0.go -- +package a + +var _ = add(2, 1+1) //@ inline(re"add", add) + +func add(x, y int) int { return x + 2*y } + +-- add -- +package a + +var _ = func(y int) int { return 2 + 2*y }(1 + 1) //@ inline(re"add", add) + +func add(x, y int) int { return x + 2*y } \ No newline at end of file diff --git a/internal/refactor/inline/testdata/revdotimport.txtar b/internal/refactor/inline/testdata/revdotimport.txtar index f8b895e9218..3838793754d 100644 --- a/internal/refactor/inline/testdata/revdotimport.txtar +++ b/internal/refactor/inline/testdata/revdotimport.txtar @@ -33,11 +33,10 @@ package c import ( . "testdata/a" - a "testdata/a" ) func _() { A() - func() { a.A() }() //@ inline(re"B", result) + a.A() //@ inline(re"B", result) } diff --git a/internal/refactor/inline/util.go b/internal/refactor/inline/util.go new file mode 100644 index 00000000000..4b6d45efa10 --- /dev/null +++ b/internal/refactor/inline/util.go @@ -0,0 +1,46 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package inline + +// This file defines various common helpers. + +import ( + "go/ast" + "go/token" + "go/types" + "reflect" + "strings" +) + +func is[T any](x any) bool { + _, ok := x.(T) + return ok +} + +func offsetOf(fset *token.FileSet, pos token.Pos) int { + return fset.PositionFor(pos, false).Offset +} + +// objectKind returns an object's kind (e.g. var, func, const, typename). +func objectKind(obj types.Object) string { + return strings.TrimPrefix(strings.ToLower(reflect.TypeOf(obj).String()), "*types.") +} + +// within reports whether pos is within the half-open interval [n.Pos, n.End). +func within(pos token.Pos, n ast.Node) bool { + return n.Pos() <= pos && pos < n.End() +} + +// trivialConversion reports whether it is safe to omit the implicit +// value-to-variable conversion that occurs in argument passing or +// result return. The only case currently allowed is converting from +// untyped constant to its default type (e.g. 0 to int). +// +// The reason for this check is that converting from A to B to C may +// yield a different result than converting A directly to C: consider +// 0 to int32 to any. +func trivialConversion(val types.Type, obj *types.Var) bool { + return types.Identical(types.Default(val), obj.Type()) +} From 33355ea86cd1c998790126844ca29a3e45e8dd93 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Fri, 8 Sep 2023 16:35:01 -0400 Subject: [PATCH 082/178] internal/refactor/inline: add parameterless tailcall strategy This change adds a new strategy: Inlining: return f(args) where: func f(params) (results) { ...body... } reduces to: ...body... so long as all parameters have been eliminated, all returns in the body have trivial assignment conversions, and there are no conflicts between caller and callee's control labels, and all result variables are unreferenced by the body. Plus a test. Change-Id: Ic92e8305c97b9e92ad4084a0a39abe218c1e7190 Reviewed-on: https://go-review.googlesource.com/c/tools/+/526102 Reviewed-by: Robert Findley LUCI-TryBot-Result: Go LUCI gopls-CI: kokoro --- internal/refactor/inline/callee.go | 55 ++++++ internal/refactor/inline/inline.go | 163 +++++++++++++----- .../refactor/inline/testdata/tailcall.txtar | 122 +++++++++++++ 3 files changed, 301 insertions(+), 39 deletions(-) create mode 100644 internal/refactor/inline/testdata/tailcall.txtar diff --git a/internal/refactor/inline/callee.go b/internal/refactor/inline/callee.go index 96ff827e034..e8f452d337a 100644 --- a/internal/refactor/inline/callee.go +++ b/internal/refactor/inline/callee.go @@ -40,6 +40,10 @@ type gobCallee struct { ValidForCallStmt bool // => bodyIsReturnExpr and sole expr is f() or <-ch NumResults int // number of results (according to type, not ast.FieldList) Params []*paramInfo // information about receiver, params, and results + HasDefer bool // uses defer + TotalReturns int // number of return statements + TrivialReturns int // number of return statements with trivial result conversions + Labels []string // names of all control labels } // A freeRef records a reference to a free object. Gob-serializable. @@ -259,6 +263,53 @@ func AnalyzeCallee(fset *token.FileSet, pkg *types.Package, info *types.Info, de }() } + // Record information about control flow in the callee + // (but not any nested functions). + var ( + hasDefer = false + totalReturns = 0 + trivialReturns = 0 + labels []string + ) + ast.Inspect(decl.Body, func(n ast.Node) bool { + switch n := n.(type) { + case *ast.FuncLit: + return false // prune traversal + case *ast.DeferStmt: + hasDefer = true + case *ast.LabeledStmt: + labels = append(labels, n.Label.Name) + case *ast.ReturnStmt: + totalReturns++ + + // Are implicit assignment conversions + // to result variables all trivial? + trivial := true + if len(n.Results) > 0 { + argType := func(i int) types.Type { + return info.TypeOf(n.Results[i]) + } + if len(n.Results) == 1 && sig.Results().Len() > 1 { + // Spread return: return f() where f.Results > 1. + tuple := info.TypeOf(n.Results[0]).(*types.Tuple) + argType = func(i int) types.Type { + return tuple.At(i).Type() + } + } + for i := 0; i < sig.Results().Len(); i++ { + if !trivialConversion(argType(i), sig.Results().At(i)) { + trivial = false + break + } + } + } + if trivial { + trivialReturns++ + } + } + return true + }) + // Compact content to just the FuncDecl. // // As a space optimization, we don't retain the complete @@ -288,6 +339,10 @@ func AnalyzeCallee(fset *token.FileSet, pkg *types.Package, info *types.Info, de ValidForCallStmt: validForCallStmt, NumResults: sig.Results().Len(), Params: analyzeParams(fset, info, decl), + HasDefer: hasDefer, + TotalReturns: totalReturns, + TrivialReturns: trivialReturns, + Labels: labels, }}, nil } diff --git a/internal/refactor/inline/inline.go b/internal/refactor/inline/inline.go index b48b43ec8cb..1778d5ea518 100644 --- a/internal/refactor/inline/inline.go +++ b/internal/refactor/inline/inline.go @@ -369,7 +369,7 @@ func Inline(logf func(string, ...any), caller *Caller, callee *Callee) ([]byte, // TODO(adonovan): better segment hygiene. if i := strings.Index(path, "/internal/"); i >= 0 { if !strings.HasPrefix(caller.Types.Path(), path[:i]) { - return nil, fmt.Errorf("can't inline function %v as its body refers to inaccessible package %q", callee.impl.Name, path) + return nil, fmt.Errorf("can't inline function %v as its body refers to inaccessible package %q", callee, path) } } importDecl.Specs = append(importDecl.Specs, spec) @@ -891,6 +891,11 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // -- let the inlining strategies begin -- + // TODO(adonovan): split this huge function into a sequence of + // function calls with an error sentinel that means "try the + // next strategy", and make sure each strategy writes to the + // log the reason it didn't match. + // Special case: eliminate a call to a function whose body is empty. // (=> callee has no results and caller is a statement.) // @@ -927,23 +932,24 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // Attempt to reduce parameterless calls // whose result variables do not escape. - if func() bool { - for i, param := range callee.Params { - if param.Kind != "result" { // recv or param - if !eliminatedParams[i] { - logf("param %q not eliminated", param.Name) - return false - } - } else if param.Escapes { - logf("result variable %s escapes", param.Name) + if forall(callee.Params, func(i int, param *paramInfo) bool { + if param.Kind != "result" { // recv or param + if !eliminatedParams[i] { + logf("param %q not eliminated", param.Name) return false } + } else if param.Escapes { + logf("result variable %s escapes", param.Name) + return false } return true - }() { + }) { logf("all params eliminated and no result vars escape") - // Special case: parameterless call to { return expr(s) }. + // Special case: parameterless call to { return exprs }. + // + // => reduce to: exprs (if legal) + // or: _, _ = expr (otherwise) // // If: // - the body is just "return expr" with trivial implicit conversions, @@ -1033,6 +1039,44 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu return res, nil } + // Special case: parameterless tail-call. + // + // Inlining: + // return f(args) + // where: + // func f(params) (results) { ...body... } + // reduces to: + // ...body... + // so long as: + // - all parameters are eliminated; + // - call is a tail-call; + // - all returns in body have trivial result conversions; + // - there is no label conflict; + // - no result variable is referenced by name. + // + // The body may use defer, arbitrary control flow, and + // multiple returns. + // + // TODO(adonovan): omit the braces if the sets of + // names in the two blocks are disjoint. + if ret, ok := callContext(callerPath).(*ast.ReturnStmt); ok && + len(ret.Results) == 1 && + callee.TrivialReturns == callee.TotalReturns && + !hasLabelConflict(callerPath, callee.Labels) && + forall(callee.Params, func(i int, p *paramInfo) bool { + // all result vars are unreferenced + return p.Kind != "result" || len(p.Refs) == 0 + }) { + logf("strategy: reduce parameterless tail-call") + res.old = ret + res.new = calleeDecl.Body + clearPositions(calleeDecl.Body) + return res, nil + } + + // Special case: parameterless call to void function + // + // Inlining: // Special case: parameterless call to void function // // Inlining: @@ -1048,13 +1092,16 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // - all parameters have been eliminated. // // If there is only a single statement, the braces are omitted. - body := calleeDecl.Body - if stmt := callStmt(callerPath); stmt != nil && !hasControl(body) { + if stmt := callStmt(callerPath); stmt != nil && + !callee.HasDefer && + !hasLabelConflict(callerPath, callee.Labels) && + callee.TotalReturns == 0 { logf("strategy: reduce parameterless call to { stmt } from a call stmt") + body := calleeDecl.Body var repl ast.Stmt = body if len(body.List) == 1 { - repl = body.List[0] // omit braces around singleton statement + repl = body.List[0] // singleton: omit braces } clearPositions(repl) res.old = stmt @@ -1084,12 +1131,6 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // if x ∉ freevars(a2) or freevars(T2), and so on, // plus the usual checks for return conversions (if any), // complex control, etc. - - // TODO(adonovan): a parameterless tail-call to a - // function with a single final return, no defer/label - // control, and no trivial return conversions, can be - // inlined to the body of the function (perhaps even - // omitting the braces.) } // Infallible general case: literalization. @@ -1354,6 +1395,48 @@ func callContext(callPath []ast.Node) ast.Node { return nil } +// hasLabelConflict reports whether the set of labels of the function +// enclosing the call (specified as a PathEnclosingInterval) +// intersects with the set of callee labels. +func hasLabelConflict(callPath []ast.Node, calleeLabels []string) bool { + var callerBody *ast.BlockStmt + switch f := callerFunc(callPath).(type) { + case *ast.FuncDecl: + callerBody = f.Body + case *ast.FuncLit: + callerBody = f.Body + } + conflict := false + if callerBody != nil { + ast.Inspect(callerBody, func(n ast.Node) bool { + switch n := n.(type) { + case *ast.FuncLit: + return false // prune traversal + case *ast.LabeledStmt: + for _, label := range calleeLabels { + if label == n.Label.Name { + conflict = true + } + } + } + return true + }) + } + return conflict +} + +// callerFunc returns the innermost Func{Decl,Lit} node enclosing the +// call (specified as a PathEnclosingInterval). +func callerFunc(callPath []ast.Node) ast.Node { + _ = callPath[0].(*ast.CallExpr) // sanity check + for _, n := range callPath[1:] { + if is[*ast.FuncDecl](n) || is[*ast.FuncLit](n) { + return n + } + } + return nil +} + // callStmt reports whether the function call (specified // as a PathEnclosingInterval) appears within an ExprStmt, // and returns it if so. @@ -1362,24 +1445,6 @@ func callStmt(callPath []ast.Node) *ast.ExprStmt { return stmt } -// hasControl reports whether the body of the function uses control -// constructs such as defer, return, or labels. -// -// TODO(adonovan): refine disposition w.r.t. return: -// a single, unnested final return may be ok for some callers. -func hasControl(body *ast.BlockStmt) (res bool) { - ast.Inspect(body, func(n ast.Node) bool { - switch n.(type) { - case *ast.FuncLit: - return false // prune traversal - case *ast.DeferStmt, *ast.LabeledStmt, *ast.ReturnStmt: - res = true - } - return true - }) - return -} - // replaceNode performs a destructive update of the tree rooted at // root, replacing each occurrence of "from" with "to". If to is nil and // the element is within a slice, the slice element is removed. @@ -1553,3 +1618,23 @@ func shallowCopy[T any](ptr *T) *T { copy := *ptr return © } + +// ∀ +func forall[T any](list []T, f func(i int, x T) bool) bool { + for i, x := range list { + if !f(i, x) { + return false + } + } + return true +} + +// ∃ +func exists[T any](list []T, f func(i int, x T) bool) bool { + for i, x := range list { + if f(i, x) { + return true + } + } + return false +} diff --git a/internal/refactor/inline/testdata/tailcall.txtar b/internal/refactor/inline/testdata/tailcall.txtar new file mode 100644 index 00000000000..64f5f9735a0 --- /dev/null +++ b/internal/refactor/inline/testdata/tailcall.txtar @@ -0,0 +1,122 @@ +Reduction of parameterless tail-call to functions. + +1. a0 (sum) is reduced, despite the complexity of the callee. + +2. a1 (conflict) is not reduced, because the caller and callee have + intersecting sets of labels. + +3. a2 (usesResult) is not reduced, because it refers to a result variable. + +-- go.mod -- +module testdata +go 1.12 + +-- a/a0.go -- +package a + +func _() int { + return sum(1, 2) //@ inline(re"sum", sum) +} + +func sum(lo, hi int) int { + total := 0 +start: + for i := lo; i <= hi; i++ { + total += i + if i == 6 { + goto start + } else if i == 7 { + return -1 + } + } + return total +} + +-- sum -- +package a + +func _() int { + { + total := 0 + start: + for i := 1; i <= 2; i++ { + total += i + if i == 6 { + goto start + } else if i == 7 { + return -1 + } + } + return total + } //@ inline(re"sum", sum) +} + +func sum(lo, hi int) int { + total := 0 +start: + for i := lo; i <= hi; i++ { + total += i + if i == 6 { + goto start + } else if i == 7 { + return -1 + } + } + return total +} + +-- a/a1.go -- +package a + +func _() int { + hello: + return conflict(1, 2) //@ inline(re"conflict", conflict) + goto hello +} + +func conflict(lo, hi int) int { +hello: + return lo + hi +} + +-- conflict -- +package a + +func _() int { +hello: + return func() int { + hello: + return 1 + 2 + }() //@ inline(re"conflict", conflict) + goto hello +} + +func conflict(lo, hi int) int { +hello: + return lo + hi +} + +-- a/a2.go -- +package a + +func _() int { + return usesResult(1, 2) //@ inline(re"usesResult", usesResult) +} + +func usesResult(lo, hi int) (z int) { + z = y + x + return +} + +-- usesResult -- +package a + +func _() int { + return func() (z int) { z = y + x; return }() //@ inline(re"usesResult", usesResult) +} + +func usesResult(lo, hi int) (z int) { + z = y + x + return +} + From cf5aad9d898a74d0f6cd041af86762f1122b926a Mon Sep 17 00:00:00 2001 From: "Hana (Hyang-Ah) Kim" Date: Tue, 5 Sep 2023 10:38:23 -0400 Subject: [PATCH 083/178] gopls: update to use the new golang.org/x/vuln API The new x/vuln API (golang.org/x/vuln/scan) takes args like the `govulncheck` command line tool, and in json mode, it outputs results in the new-line delimited JSON stream format. The stream is meant to be processed with the handlers like golang.org/x/vuln/internal/govulncheck. Unfortunately, none of the involved types and functions are exported, so we copy necessary ones into gopls/internal/vulncheck/osv and gopls/internal/vulncheck/govulncheck. The copier script (gopls/internal/vulncheck/copier.go) does the copying. 'gopls vulncheck' is like 'govulncheck -mode=source -scan=symbol'. This is implemented in gopls/internal/vulncheck/scan.Main. The 'Run govulncheck' codelens invokes the internal gopls command, which runs 'gopls vulncheck' in a separate process, processes the JSON output stream from stdout, and converts it into the gopls internal data structure gopls/internal/vulncheck.Result. This vulncheck.Result is currently stored in the view and used to produce the Hover and Diagnostics for contents in go.mod file. The old golang.org/x/vuln/client API is gone. To query vuln.go.dev, we need to use `govulncheck -mode=query @`. This outputs OSV entries that apply to the given module version. This usage is not publicly documented; see https://go.googlesource.com/vuln/+/v1.0.1/internal/scan/query.go The query command allows us multiple module@versions in batch. However, we keep using multiple queries (one per each module version), running in parallel. I want to revisit this after we figure out whether we need in-memory caching of the queried OSV entry results. gopls/internal/vulncheck/vulntest is a test utility that parses yaml files in txtar, and coverts to on-disk local vuln DB format. The vuln DB format and the osv package also went through major changes. See https://go.dev/security/vuln/database and gopls/internal/vulncheck/osv for the latest schema and types. File-based vuln db can implement the full schema or can be a directory containing a collection of JSON files. See the loading logic in https://go.googlesource.com/vulndb/+/7fdbb06ec0fe/internal/database/load.go#55 for reference. This CL also rearranges the packages and types. The Result type and analysis modes types were moved from gopls/internal/govulncheck to gopls/internal/vulncheck. Main and VulnerablePackages were moved from gopls/internal/vulncheck to gopls/internal/vulncheck/scan. I originally wanted to consolidate gopls/internal/vulncheck and gopls/internal/vulncheck/scan. But the dependency on gopls/internal/lsp/source from VulnerablePackages and RunGovulncheck causes circular dependency, so they can't be merged. While doing the conversion, we remove - the in-memory OSV entry cache: the previous implementation implemented the cache interface which is now gone from the API. Let's see if we still need some kind of in-memory caching, or Gopls's internal debouncing logic around diagnostics and govulncheck API's own caching (if any) is sufficient. - RelatedInfo from govulncheck diagnostics. Previously we converted the entry points to RelatedInfo of diagnostics. The new API can output many more entry points for each vulnerability. That makes hover/diagnostics UI flooded with RelatedInfo and prevents users from finding important information. Let's avoid using related info. Instead, users can still find the entry points from the text output result or from the future UI enhancement in vscode. There is "Summary" in the new OSV entry API. We now avoid the long description of each OSV entry in favor of the Summary field, when populating the hover message. Change-Id: I2f9610a90272f8806064e655c05b74aa54e29265 Reviewed-on: https://go-review.googlesource.com/c/tools/+/523507 Commit-Queue: Hyang-Ah Hana Kim Reviewed-by: Suzy Mueller Reviewed-by: Alan Donovan TryBot-Result: Gopher Robot Run-TryBot: Hyang-Ah Hana Kim Auto-Submit: Hyang-Ah Hana Kim --- gopls/doc/commands.md | 4 +- gopls/doc/settings.md | 2 +- gopls/go.mod | 6 +- gopls/go.sum | 8 +- gopls/internal/govulncheck/types_118.go | 43 -- gopls/internal/govulncheck/types_not118.go | 126 ----- gopls/internal/govulncheck/vulncache.go | 105 ---- gopls/internal/lsp/cache/mod_vuln.go | 35 +- gopls/internal/lsp/cache/session.go | 4 +- gopls/internal/lsp/cache/view.go | 10 +- gopls/internal/lsp/cache/view_test.go | 14 +- gopls/internal/lsp/cmd/usage/vulncheck.hlp | 4 - gopls/internal/lsp/cmd/vulncheck.go | 45 +- gopls/internal/lsp/command.go | 75 +-- gopls/internal/lsp/command/interface.go | 6 +- gopls/internal/lsp/mod/diagnostics.go | 206 ++++---- gopls/internal/lsp/mod/hover.go | 188 +++---- gopls/internal/lsp/source/api_json.go | 6 +- gopls/internal/lsp/source/view.go | 8 +- gopls/internal/regtest/misc/vuln_test.go | 88 ++-- gopls/internal/vulncheck/command.go | 337 ------------- gopls/internal/vulncheck/copier.go | 142 ++++++ .../vulncheck/govulncheck/govulncheck.go | 160 ++++++ .../internal/vulncheck/govulncheck/handler.go | 61 +++ .../vulncheck/govulncheck/jsonhandler.go | 46 ++ gopls/internal/vulncheck/osv/osv.go | 240 +++++++++ gopls/internal/vulncheck/scan/command.go | 471 ++++++++++++++++++ .../{govulncheck => vulncheck/scan}/util.go | 10 +- .../semver/semver.go | 8 + .../semver/semver_test.go | 0 .../{govulncheck => vulncheck}/types.go | 20 +- gopls/internal/vulncheck/vulncheck.go | 25 - gopls/internal/vulncheck/vulntest/db.go | 129 ++--- gopls/internal/vulncheck/vulntest/db_test.go | 72 ++- gopls/internal/vulncheck/vulntest/report.go | 15 +- .../vulntest/testdata/GO-2020-0001.json | 50 ++ 36 files changed, 1581 insertions(+), 1188 deletions(-) delete mode 100644 gopls/internal/govulncheck/types_118.go delete mode 100644 gopls/internal/govulncheck/types_not118.go delete mode 100644 gopls/internal/govulncheck/vulncache.go delete mode 100644 gopls/internal/vulncheck/command.go create mode 100644 gopls/internal/vulncheck/copier.go create mode 100644 gopls/internal/vulncheck/govulncheck/govulncheck.go create mode 100644 gopls/internal/vulncheck/govulncheck/handler.go create mode 100644 gopls/internal/vulncheck/govulncheck/jsonhandler.go create mode 100644 gopls/internal/vulncheck/osv/osv.go create mode 100644 gopls/internal/vulncheck/scan/command.go rename gopls/internal/{govulncheck => vulncheck/scan}/util.go (79%) rename gopls/internal/{govulncheck => vulncheck}/semver/semver.go (88%) rename gopls/internal/{govulncheck => vulncheck}/semver/semver_test.go (100%) rename gopls/internal/{govulncheck => vulncheck}/types.go (70%) delete mode 100644 gopls/internal/vulncheck/vulncheck.go create mode 100644 gopls/internal/vulncheck/vulntest/testdata/GO-2020-0001.json diff --git a/gopls/doc/commands.md b/gopls/doc/commands.md index eff548a442c..1a8c4dc99f8 100644 --- a/gopls/doc/commands.md +++ b/gopls/doc/commands.md @@ -117,7 +117,7 @@ Args: Result: ``` -map[golang.org/x/tools/gopls/internal/lsp/protocol.DocumentURI]*golang.org/x/tools/gopls/internal/govulncheck.Result +map[golang.org/x/tools/gopls/internal/lsp/protocol.DocumentURI]*golang.org/x/tools/gopls/internal/vulncheck.Result ``` ### **Toggle gc_details** @@ -307,7 +307,7 @@ Args: } ``` -### **Run govulncheck.** +### **Run vulncheck.** Identifier: `gopls.run_govulncheck` Run vulnerability check (`govulncheck`). diff --git a/gopls/doc/settings.md b/gopls/doc/settings.md index 781b7124dbe..5bc692ee05b 100644 --- a/gopls/doc/settings.md +++ b/gopls/doc/settings.md @@ -529,7 +529,7 @@ Runs `go generate` for a given directory. Identifier: `regenerate_cgo` Regenerates cgo definitions. -### **Run govulncheck.** +### **Run vulncheck.** Identifier: `run_govulncheck` diff --git a/gopls/go.mod b/gopls/go.mod index d04addfe084..f17636979f7 100644 --- a/gopls/go.mod +++ b/gopls/go.mod @@ -12,8 +12,8 @@ require ( golang.org/x/sys v0.12.0 golang.org/x/telemetry v0.0.0-20230822160736-17171dbf1d76 golang.org/x/text v0.13.0 - golang.org/x/tools v0.9.4-0.20230601214343-86c93e8732cc - golang.org/x/vuln v0.0.0-20230110180137-6ad3e3d07815 + golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 + golang.org/x/vuln v1.0.1 gopkg.in/yaml.v3 v3.0.1 honnef.co/go/tools v0.4.5 mvdan.cc/gofumpt v0.4.0 @@ -23,8 +23,8 @@ require ( require ( github.com/BurntSushi/toml v1.2.1 // indirect github.com/google/safehtml v0.1.0 // indirect - golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect golang.org/x/exp/typeparams v0.0.0-20221212164502-fae10dda9338 // indirect + ) replace golang.org/x/tools => ../ diff --git a/gopls/go.sum b/gopls/go.sum index 2782d7303f0..0b3aa549a7e 100644 --- a/gopls/go.sum +++ b/gopls/go.sum @@ -1,6 +1,5 @@ github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -31,8 +30,6 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= -golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= -golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= golang.org/x/exp/typeparams v0.0.0-20221212164502-fae10dda9338 h1:2O2DON6y3XMJiQRAS1UWU+54aec2uopH3x7MAiqGW6Y= golang.org/x/exp/typeparams v0.0.0-20221212164502-fae10dda9338/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= @@ -55,8 +52,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/vuln v0.0.0-20230110180137-6ad3e3d07815 h1:A9kONVi4+AnuOr1dopsibH6hLi1Huy54cbeJxnq4vmU= -golang.org/x/vuln v0.0.0-20230110180137-6ad3e3d07815/go.mod h1:XJiVExZgoZfrrxoTeVsFYrSSk1snhfpOEC95JL+A4T0= +golang.org/x/vuln v1.0.1 h1:KUas02EjQK5LTuIx1OylBQdKKZ9jeugs+HiqO5HormU= +golang.org/x/vuln v1.0.1/go.mod h1:bb2hMwln/tqxg32BNY4CcxHWtHXuYa3SbIBmtsyjxtM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= @@ -71,6 +68,5 @@ honnef.co/go/tools v0.4.5 h1:YGD4H+SuIOOqsyoLOpZDWcieM28W47/zRO7f+9V3nvo= honnef.co/go/tools v0.4.5/go.mod h1:GUV+uIBCLpdf0/v6UhHHG/yzI/z6qPskBeQCjcNB96k= mvdan.cc/gofumpt v0.4.0 h1:JVf4NN1mIpHogBj7ABpgOyZc65/UUOkKQFkoURsz4MM= mvdan.cc/gofumpt v0.4.0/go.mod h1:PljLOHDeZqgS8opHRKLzp2It2VBuSdteAgqUfzMTxlQ= -mvdan.cc/unparam v0.0.0-20211214103731-d0ef000c54e5 h1:Jh3LAeMt1eGpxomyu3jVkmVZWW2MxZ1qIIV2TZ/nRio= mvdan.cc/xurls/v2 v2.4.0 h1:tzxjVAj+wSBmDcF6zBB7/myTy3gX9xvi8Tyr28AuQgc= mvdan.cc/xurls/v2 v2.4.0/go.mod h1:+GEjq9uNjqs8LQfM9nVnM8rff0OQ5Iash5rzX+N1CSg= diff --git a/gopls/internal/govulncheck/types_118.go b/gopls/internal/govulncheck/types_118.go deleted file mode 100644 index 7b354d622a8..00000000000 --- a/gopls/internal/govulncheck/types_118.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build go1.18 -// +build go1.18 - -// Package govulncheck provides an experimental govulncheck API. -package govulncheck - -import ( - "golang.org/x/vuln/exp/govulncheck" -) - -var ( - // Source reports vulnerabilities that affect the analyzed packages. - Source = govulncheck.Source - - // DefaultCache constructs cache for a vulnerability database client. - DefaultCache = govulncheck.DefaultCache -) - -type ( - // Config is the configuration for Main. - Config = govulncheck.Config - - // Vuln represents a single OSV entry. - Vuln = govulncheck.Vuln - - // Module represents a specific vulnerability relevant to a - // single module or package. - Module = govulncheck.Module - - // Package is a Go package with known vulnerable symbols. - Package = govulncheck.Package - - // CallStacks contains a representative call stack for each - // vulnerable symbol that is called. - CallStack = govulncheck.CallStack - - // StackFrame represents a call stack entry. - StackFrame = govulncheck.StackFrame -) diff --git a/gopls/internal/govulncheck/types_not118.go b/gopls/internal/govulncheck/types_not118.go deleted file mode 100644 index faf5a7055b5..00000000000 --- a/gopls/internal/govulncheck/types_not118.go +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build !go1.18 -// +build !go1.18 - -package govulncheck - -import ( - "go/token" - - "golang.org/x/vuln/osv" -) - -// Vuln represents a single OSV entry. -type Vuln struct { - // OSV contains all data from the OSV entry for this vulnerability. - OSV *osv.Entry - - // Modules contains all of the modules in the OSV entry where a - // vulnerable package is imported by the target source code or binary. - // - // For example, a module M with two packages M/p1 and M/p2, where only p1 - // is vulnerable, will appear in this list if and only if p1 is imported by - // the target source code or binary. - Modules []*Module -} - -func (v *Vuln) IsCalled() bool { - return false -} - -// Module represents a specific vulnerability relevant to a single module. -type Module struct { - // Path is the module path of the module containing the vulnerability. - // - // Importable packages in the standard library will have the path "stdlib". - Path string - - // FoundVersion is the module version where the vulnerability was found. - FoundVersion string - - // FixedVersion is the module version where the vulnerability was - // fixed. If there are multiple fixed versions in the OSV report, this will - // be the latest fixed version. - // - // This is empty if a fix is not available. - FixedVersion string - - // Packages contains all the vulnerable packages in OSV entry that are - // imported by the target source code or binary. - // - // For example, given a module M with two packages M/p1 and M/p2, where - // both p1 and p2 are vulnerable, p1 and p2 will each only appear in this - // list they are individually imported by the target source code or binary. - Packages []*Package -} - -// Package is a Go package with known vulnerable symbols. -type Package struct { - // Path is the import path of the package containing the vulnerability. - Path string - - // CallStacks contains a representative call stack for each - // vulnerable symbol that is called. - // - // For vulnerabilities found from binary analysis, only CallStack.Symbol - // will be provided. - // - // For non-affecting vulnerabilities reported from the source mode - // analysis, this will be empty. - CallStacks []CallStack -} - -// CallStacks contains a representative call stack for a vulnerable -// symbol. -type CallStack struct { - // Symbol is the name of the detected vulnerable function - // or method. - // - // This follows the naming convention in the OSV report. - Symbol string - - // Summary is a one-line description of the callstack, used by the - // default govulncheck mode. - // - // Example: module3.main calls github.com/shiyanhui/dht.DHT.Run - Summary string - - // Frames contains an entry for each stack in the call stack. - // - // Frames are sorted starting from the entry point to the - // imported vulnerable symbol. The last frame in Frames should match - // Symbol. - Frames []*StackFrame -} - -// StackFrame represents a call stack entry. -type StackFrame struct { - // PackagePath is the import path. - PkgPath string - - // FuncName is the function name. - FuncName string - - // RecvType is the fully qualified receiver type, - // if the called symbol is a method. - // - // The client can create the final symbol name by - // prepending RecvType to FuncName. - RecvType string - - // Position describes an arbitrary source position - // including the file, line, and column location. - // A Position is valid if the line number is > 0. - Position token.Position -} - -func (sf *StackFrame) Name() string { - return "" -} - -func (sf *StackFrame) Pos() string { - return "" -} diff --git a/gopls/internal/govulncheck/vulncache.go b/gopls/internal/govulncheck/vulncache.go deleted file mode 100644 index a259f027336..00000000000 --- a/gopls/internal/govulncheck/vulncache.go +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build go1.18 -// +build go1.18 - -package govulncheck - -import ( - "sync" - "time" - - vulnc "golang.org/x/vuln/client" - "golang.org/x/vuln/osv" -) - -// inMemoryCache is an implementation of the [client.Cache] interface -// that "decorates" another instance of that interface to provide -// an additional layer of (memory-based) caching. -type inMemoryCache struct { - mu sync.Mutex - underlying vulnc.Cache - db map[string]*db -} - -var _ vulnc.Cache = &inMemoryCache{} - -type db struct { - retrieved time.Time - index vulnc.DBIndex - entry map[string][]*osv.Entry -} - -// NewInMemoryCache returns a new memory-based cache that decorates -// the provided cache (file-based, perhaps). -func NewInMemoryCache(underlying vulnc.Cache) *inMemoryCache { - return &inMemoryCache{ - underlying: underlying, - db: make(map[string]*db), - } -} - -func (c *inMemoryCache) lookupDBLocked(dbName string) *db { - cached := c.db[dbName] - if cached == nil { - cached = &db{entry: make(map[string][]*osv.Entry)} - c.db[dbName] = cached - } - return cached -} - -// ReadIndex returns the index for dbName from the cache, or returns zero values -// if it is not present. -func (c *inMemoryCache) ReadIndex(dbName string) (vulnc.DBIndex, time.Time, error) { - c.mu.Lock() - defer c.mu.Unlock() - cached := c.lookupDBLocked(dbName) - - if cached.retrieved.IsZero() { - // First time ReadIndex is called. - index, retrieved, err := c.underlying.ReadIndex(dbName) - if err != nil { - return index, retrieved, err - } - cached.index, cached.retrieved = index, retrieved - } - return cached.index, cached.retrieved, nil -} - -// WriteIndex puts the index and retrieved time into the cache. -func (c *inMemoryCache) WriteIndex(dbName string, index vulnc.DBIndex, retrieved time.Time) error { - c.mu.Lock() - defer c.mu.Unlock() - cached := c.lookupDBLocked(dbName) - cached.index, cached.retrieved = index, retrieved - // TODO(hyangah): shouldn't we invalidate all cached entries? - return c.underlying.WriteIndex(dbName, index, retrieved) -} - -// ReadEntries returns the vulndb entries for path from the cache. -func (c *inMemoryCache) ReadEntries(dbName, path string) ([]*osv.Entry, error) { - c.mu.Lock() - defer c.mu.Unlock() - cached := c.lookupDBLocked(dbName) - entries, ok := cached.entry[path] - if !ok { - // cache miss - entries, err := c.underlying.ReadEntries(dbName, path) - if err != nil { - return entries, err - } - cached.entry[path] = entries - } - return entries, nil -} - -// WriteEntries puts the entries for path into the cache. -func (c *inMemoryCache) WriteEntries(dbName, path string, entries []*osv.Entry) error { - c.mu.Lock() - defer c.mu.Unlock() - cached := c.lookupDBLocked(dbName) - cached.entry[path] = entries - return c.underlying.WriteEntries(dbName, path, entries) -} diff --git a/gopls/internal/lsp/cache/mod_vuln.go b/gopls/internal/lsp/cache/mod_vuln.go index dcd58bfa94a..8c635c181bf 100644 --- a/gopls/internal/lsp/cache/mod_vuln.go +++ b/gopls/internal/lsp/cache/mod_vuln.go @@ -6,45 +6,29 @@ package cache import ( "context" - "os" - "golang.org/x/tools/gopls/internal/govulncheck" - "golang.org/x/tools/gopls/internal/lsp/source" "golang.org/x/tools/gopls/internal/span" "golang.org/x/tools/gopls/internal/vulncheck" + "golang.org/x/tools/gopls/internal/vulncheck/scan" "golang.org/x/tools/internal/memoize" ) // ModVuln returns import vulnerability analysis for the given go.mod URI. // Concurrent requests are combined into a single command. -func (s *snapshot) ModVuln(ctx context.Context, modURI span.URI) (*govulncheck.Result, error) { +func (s *snapshot) ModVuln(ctx context.Context, modURI span.URI) (*vulncheck.Result, error) { s.mu.Lock() entry, hit := s.modVulnHandles.Get(modURI) s.mu.Unlock() type modVuln struct { - result *govulncheck.Result + result *vulncheck.Result err error } // Cache miss? if !hit { - // If the file handle is an overlay, it may not be written to disk. - // The go.mod file has to be on disk for vulncheck to work. - // - // TODO(hyangah): use overlays for vulncheck. - fh, err := s.ReadFile(ctx, modURI) - if err != nil { - return nil, err - } - if _, ok := fh.(*Overlay); ok { - if info, _ := os.Stat(modURI.Filename()); info == nil { - return nil, source.ErrNoModOnDisk - } - } - handle := memoize.NewPromise("modVuln", func(ctx context.Context, arg interface{}) interface{} { - result, err := modVulnImpl(ctx, arg.(*snapshot), modURI) + result, err := scan.VulnerablePackages(ctx, arg.(*snapshot)) return modVuln{result, err} }) @@ -62,14 +46,3 @@ func (s *snapshot) ModVuln(ctx context.Context, modURI span.URI) (*govulncheck.R res := v.(modVuln) return res.result, res.err } - -func modVulnImpl(ctx context.Context, s *snapshot, uri span.URI) (*govulncheck.Result, error) { - if vulncheck.VulnerablePackages == nil { - return &govulncheck.Result{}, nil - } - fh, err := s.ReadFile(ctx, uri) - if err != nil { - return nil, err - } - return vulncheck.VulnerablePackages(ctx, s, fh) -} diff --git a/gopls/internal/lsp/cache/session.go b/gopls/internal/lsp/cache/session.go index e2869dd6e29..7346e24f82a 100644 --- a/gopls/internal/lsp/cache/session.go +++ b/gopls/internal/lsp/cache/session.go @@ -13,10 +13,10 @@ import ( "sync/atomic" "golang.org/x/tools/gopls/internal/bug" - "golang.org/x/tools/gopls/internal/govulncheck" "golang.org/x/tools/gopls/internal/lsp/source" "golang.org/x/tools/gopls/internal/lsp/source/typerefs" "golang.org/x/tools/gopls/internal/span" + "golang.org/x/tools/gopls/internal/vulncheck" "golang.org/x/tools/internal/event" "golang.org/x/tools/internal/gocommand" "golang.org/x/tools/internal/imports" @@ -123,7 +123,7 @@ func (s *Session) createView(ctx context.Context, name string, folder span.URI, baseCtx: baseCtx, name: name, moduleUpgrades: map[span.URI]map[string]string{}, - vulns: map[span.URI]*govulncheck.Result{}, + vulns: map[span.URI]*vulncheck.Result{}, parseCache: s.parseCache, fs: s.overlayFS, workspaceInformation: info, diff --git a/gopls/internal/lsp/cache/view.go b/gopls/internal/lsp/cache/view.go index 7c2eda1743b..ed1e2fe56ef 100644 --- a/gopls/internal/lsp/cache/view.go +++ b/gopls/internal/lsp/cache/view.go @@ -24,10 +24,10 @@ import ( "golang.org/x/mod/modfile" "golang.org/x/mod/semver" exec "golang.org/x/sys/execabs" - "golang.org/x/tools/gopls/internal/govulncheck" "golang.org/x/tools/gopls/internal/lsp/protocol" "golang.org/x/tools/gopls/internal/lsp/source" "golang.org/x/tools/gopls/internal/span" + "golang.org/x/tools/gopls/internal/vulncheck" "golang.org/x/tools/internal/event" "golang.org/x/tools/internal/gocommand" "golang.org/x/tools/internal/imports" @@ -66,7 +66,7 @@ type View struct { // vulns maps each go.mod file's URI to its known vulnerabilities. vulnsMu sync.Mutex - vulns map[span.URI]*govulncheck.Result + vulns map[span.URI]*vulncheck.Result // parseCache holds an LRU cache of recently parsed files. parseCache *parseCache @@ -1125,8 +1125,8 @@ func (v *View) ClearModuleUpgrades(modfile span.URI) { const maxGovulncheckResultAge = 1 * time.Hour // Invalidate results older than this limit. var timeNow = time.Now // for testing -func (v *View) Vulnerabilities(modfiles ...span.URI) map[span.URI]*govulncheck.Result { - m := make(map[span.URI]*govulncheck.Result) +func (v *View) Vulnerabilities(modfiles ...span.URI) map[span.URI]*vulncheck.Result { + m := make(map[span.URI]*vulncheck.Result) now := timeNow() v.vulnsMu.Lock() defer v.vulnsMu.Unlock() @@ -1147,7 +1147,7 @@ func (v *View) Vulnerabilities(modfiles ...span.URI) map[span.URI]*govulncheck.R return m } -func (v *View) SetVulnerabilities(modfile span.URI, vulns *govulncheck.Result) { +func (v *View) SetVulnerabilities(modfile span.URI, vulns *vulncheck.Result) { v.vulnsMu.Lock() defer v.vulnsMu.Unlock() diff --git a/gopls/internal/lsp/cache/view_test.go b/gopls/internal/lsp/cache/view_test.go index 21b10b6a982..2b7249b69ab 100644 --- a/gopls/internal/lsp/cache/view_test.go +++ b/gopls/internal/lsp/cache/view_test.go @@ -12,10 +12,10 @@ import ( "time" "github.com/google/go-cmp/cmp" - "golang.org/x/tools/gopls/internal/govulncheck" "golang.org/x/tools/gopls/internal/lsp/fake" "golang.org/x/tools/gopls/internal/lsp/source" "golang.org/x/tools/gopls/internal/span" + "golang.org/x/tools/gopls/internal/vulncheck" ) func TestCaseInsensitiveFilesystem(t *testing.T) { @@ -216,19 +216,19 @@ func TestView_Vulnerabilities(t *testing.T) { now := time.Now() view := &View{ - vulns: make(map[span.URI]*govulncheck.Result), + vulns: make(map[span.URI]*vulncheck.Result), } file1, file2 := span.URIFromPath("f1/go.mod"), span.URIFromPath("f2/go.mod") - vuln1 := &govulncheck.Result{AsOf: now.Add(-(maxGovulncheckResultAge * 3) / 4)} // already ~3/4*maxGovulncheckResultAge old + vuln1 := &vulncheck.Result{AsOf: now.Add(-(maxGovulncheckResultAge * 3) / 4)} // already ~3/4*maxGovulncheckResultAge old view.SetVulnerabilities(file1, vuln1) - vuln2 := &govulncheck.Result{AsOf: now} // fresh. + vuln2 := &vulncheck.Result{AsOf: now} // fresh. view.SetVulnerabilities(file2, vuln2) t.Run("fresh", func(t *testing.T) { got := view.Vulnerabilities() - want := map[span.URI]*govulncheck.Result{ + want := map[span.URI]*vulncheck.Result{ file1: vuln1, file2: vuln2, } @@ -242,7 +242,7 @@ func TestView_Vulnerabilities(t *testing.T) { timeNow = func() time.Time { return now.Add(maxGovulncheckResultAge / 2) } t.Run("after30min", func(t *testing.T) { got := view.Vulnerabilities() - want := map[span.URI]*govulncheck.Result{ + want := map[span.URI]*vulncheck.Result{ file1: nil, // expired. file2: vuln2, } @@ -257,7 +257,7 @@ func TestView_Vulnerabilities(t *testing.T) { t.Run("after1hr", func(t *testing.T) { got := view.Vulnerabilities() - want := map[span.URI]*govulncheck.Result{ + want := map[span.URI]*vulncheck.Result{ file1: nil, file2: nil, } diff --git a/gopls/internal/lsp/cmd/usage/vulncheck.hlp b/gopls/internal/lsp/cmd/usage/vulncheck.hlp index 4fbe573e22a..1f5800f0ae1 100644 --- a/gopls/internal/lsp/cmd/usage/vulncheck.hlp +++ b/gopls/internal/lsp/cmd/usage/vulncheck.hlp @@ -11,7 +11,3 @@ Usage: Example: $ gopls vulncheck - -config - If true, the command reads a JSON-encoded package load configuration from stdin - -summary - If true, outputs a JSON-encoded govulnchecklib.Summary JSON diff --git a/gopls/internal/lsp/cmd/vulncheck.go b/gopls/internal/lsp/cmd/vulncheck.go index 5c851b66e78..bb53e0c82b3 100644 --- a/gopls/internal/lsp/cmd/vulncheck.go +++ b/gopls/internal/lsp/cmd/vulncheck.go @@ -6,34 +6,19 @@ package cmd import ( "context" - "encoding/json" "flag" "fmt" "os" - "golang.org/x/tools/go/packages" - vulnchecklib "golang.org/x/tools/gopls/internal/vulncheck" - "golang.org/x/tools/internal/tool" + "golang.org/x/tools/gopls/internal/vulncheck/scan" ) // vulncheck implements the vulncheck command. +// TODO(hakim): hide from the public. type vulncheck struct { - Config bool `flag:"config" help:"If true, the command reads a JSON-encoded package load configuration from stdin"` - AsSummary bool `flag:"summary" help:"If true, outputs a JSON-encoded govulnchecklib.Summary JSON"` - app *Application + app *Application } -type pkgLoadConfig struct { - // BuildFlags is a list of command-line flags to be passed through to - // the build system's query tool. - BuildFlags []string - - // If Tests is set, the loader includes related test packages. - Tests bool -} - -// TODO(hyangah): document pkgLoadConfig - func (v *vulncheck) Name() string { return "vulncheck" } func (v *vulncheck) Parent() string { return v.app.Name() } func (v *vulncheck) Usage() string { return "" } @@ -51,32 +36,10 @@ func (v *vulncheck) DetailedHelp(f *flag.FlagSet) { $ gopls vulncheck `) - printFlagDefaults(f) } func (v *vulncheck) Run(ctx context.Context, args ...string) error { - if vulnchecklib.Main == nil { - return fmt.Errorf("vulncheck command is available only in gopls compiled with go1.18 or newer") - } - - // TODO(hyangah): what's wrong with allowing multiple targets? - if len(args) > 1 { - return tool.CommandLineErrorf("vulncheck accepts at most one package pattern") - } - var cfg pkgLoadConfig - if v.Config { - if err := json.NewDecoder(os.Stdin).Decode(&cfg); err != nil { - return tool.CommandLineErrorf("failed to parse cfg: %v", err) - } - } - loadCfg := packages.Config{ - Context: ctx, - Tests: cfg.Tests, - BuildFlags: cfg.BuildFlags, - // inherit the current process's cwd and env. - } - - if err := vulnchecklib.Main(loadCfg, args...); err != nil { + if err := scan.Main(ctx, args...); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } diff --git a/gopls/internal/lsp/command.go b/gopls/internal/lsp/command.go index 84a7f49b945..99b0b384ffe 100644 --- a/gopls/internal/lsp/command.go +++ b/gopls/internal/lsp/command.go @@ -12,18 +12,15 @@ import ( "fmt" "io" "os" - "os/exec" "path/filepath" "runtime" "runtime/pprof" "sort" "strings" - "time" "golang.org/x/mod/modfile" "golang.org/x/tools/go/ast/astutil" "golang.org/x/tools/gopls/internal/bug" - "golang.org/x/tools/gopls/internal/govulncheck" "golang.org/x/tools/gopls/internal/lsp/cache" "golang.org/x/tools/gopls/internal/lsp/command" "golang.org/x/tools/gopls/internal/lsp/debug" @@ -32,6 +29,7 @@ import ( "golang.org/x/tools/gopls/internal/lsp/source" "golang.org/x/tools/gopls/internal/span" "golang.org/x/tools/gopls/internal/vulncheck" + "golang.org/x/tools/gopls/internal/vulncheck/scan" "golang.org/x/tools/internal/event" "golang.org/x/tools/internal/gocommand" "golang.org/x/tools/internal/tokeninternal" @@ -896,8 +894,8 @@ type pkgLoadConfig struct { Tests bool } -func (c *commandHandler) FetchVulncheckResult(ctx context.Context, arg command.URIArg) (map[protocol.DocumentURI]*govulncheck.Result, error) { - ret := map[protocol.DocumentURI]*govulncheck.Result{} +func (c *commandHandler) FetchVulncheckResult(ctx context.Context, arg command.URIArg) (map[protocol.DocumentURI]*vulncheck.Result, error) { + ret := map[protocol.DocumentURI]*vulncheck.Result{} err := c.run(ctx, commandConfig{forURI: arg.URI}, func(ctx context.Context, deps commandDeps) error { if deps.snapshot.Options().Vulncheck == source.ModeVulncheckImports { for _, modfile := range deps.snapshot.ModFiles() { @@ -936,59 +934,22 @@ func (c *commandHandler) RunGovulncheck(ctx context.Context, args command.Vulnch }, func(ctx context.Context, deps commandDeps) error { tokenChan <- deps.work.Token() - opts := deps.snapshot.Options() - // quickly test if gopls is compiled to support govulncheck - // by checking vulncheck.Main. Alternatively, we can continue and - // let the `gopls vulncheck` command fail. This is lighter-weight. - if vulncheck.Main == nil { - return errors.New("vulncheck feature is not available") - } - - cmd := exec.CommandContext(ctx, os.Args[0], "vulncheck", "-config", args.Pattern) - cmd.Dir = filepath.Dir(args.URI.SpanURI().Filename()) - - var viewEnv []string - if e := opts.EnvSlice(); e != nil { - viewEnv = append(os.Environ(), e...) - } - cmd.Env = viewEnv - - // stdin: gopls vulncheck expects JSON-encoded configuration from STDIN when -config flag is set. - var stdin bytes.Buffer - cmd.Stdin = &stdin - - if err := json.NewEncoder(&stdin).Encode(pkgLoadConfig{ - BuildFlags: opts.BuildFlags, - // TODO(hyangah): add `tests` flag in command.VulncheckArgs - }); err != nil { - return fmt.Errorf("failed to pass package load config: %v", err) - } + workDoneWriter := progress.NewWorkDoneWriter(ctx, deps.work) + dir := filepath.Dir(args.URI.SpanURI().Filename()) + pattern := args.Pattern - // stderr: stream gopls vulncheck's STDERR as progress reports - er := progress.NewEventWriter(ctx, "vulncheck") - stderr := io.MultiWriter(er, progress.NewWorkDoneWriter(ctx, deps.work)) - cmd.Stderr = stderr - // TODO: can we stream stdout? - stdout, err := cmd.Output() + result, err := scan.RunGovulncheck(ctx, pattern, deps.snapshot, dir, workDoneWriter) if err != nil { - return fmt.Errorf("failed to run govulncheck: %v", err) - } - - var result govulncheck.Result - if err := json.Unmarshal(stdout, &result); err != nil { - // TODO: for easy debugging, log the failed stdout somewhere? - return fmt.Errorf("failed to parse govulncheck output: %v", err) + return err } - result.Mode = govulncheck.ModeGovulncheck - result.AsOf = time.Now() - deps.snapshot.View().SetVulnerabilities(args.URI.SpanURI(), &result) + deps.snapshot.View().SetVulnerabilities(args.URI.SpanURI(), result) c.s.diagnoseSnapshot(deps.snapshot, nil, false, 0) - vulns := result.Vulns - affecting := make([]string, 0, len(vulns)) - for _, v := range vulns { - if v.IsCalled() { - affecting = append(affecting, v.OSV.ID) + + affecting := make(map[string]bool, len(result.Entries)) + for _, finding := range result.Findings { + if len(finding.Trace) > 1 { // at least 2 frames if callstack exists (vulnerability, entry) + affecting[finding.OSV] = true } } if len(affecting) == 0 { @@ -997,10 +958,14 @@ func (c *commandHandler) RunGovulncheck(ctx context.Context, args command.Vulnch Message: "No vulnerabilities found", }) } - sort.Strings(affecting) + affectingOSVs := make([]string, 0, len(affecting)) + for id := range affecting { + affectingOSVs = append(affectingOSVs, id) + } + sort.Strings(affectingOSVs) return c.s.client.ShowMessage(ctx, &protocol.ShowMessageParams{ Type: protocol.Warning, - Message: fmt.Sprintf("Found %v", strings.Join(affecting, ", ")), + Message: fmt.Sprintf("Found %v", strings.Join(affectingOSVs, ", ")), }) }) if err != nil { diff --git a/gopls/internal/lsp/command/interface.go b/gopls/internal/lsp/command/interface.go index ef9d1fb5a96..261776d5ad1 100644 --- a/gopls/internal/lsp/command/interface.go +++ b/gopls/internal/lsp/command/interface.go @@ -17,8 +17,8 @@ package command import ( "context" - "golang.org/x/tools/gopls/internal/govulncheck" "golang.org/x/tools/gopls/internal/lsp/protocol" + "golang.org/x/tools/gopls/internal/vulncheck" ) // Interface defines the interface gopls exposes for the @@ -160,7 +160,7 @@ type Interface interface { // runner. StopProfile(context.Context, StopProfileArgs) (StopProfileResult, error) - // RunGovulncheck: Run govulncheck. + // RunGovulncheck: Run vulncheck. // // Run vulnerability check (`govulncheck`). RunGovulncheck(context.Context, VulncheckArgs) (RunVulncheckResult, error) @@ -168,7 +168,7 @@ type Interface interface { // FetchVulncheckResult: Get known vulncheck result // // Fetch the result of latest vulnerability check (`govulncheck`). - FetchVulncheckResult(context.Context, URIArg) (map[protocol.DocumentURI]*govulncheck.Result, error) + FetchVulncheckResult(context.Context, URIArg) (map[protocol.DocumentURI]*vulncheck.Result, error) // MemStats: fetch memory statistics // diff --git a/gopls/internal/lsp/mod/diagnostics.go b/gopls/internal/lsp/mod/diagnostics.go index 43fc0a24481..9f901206988 100644 --- a/gopls/internal/lsp/mod/diagnostics.go +++ b/gopls/internal/lsp/mod/diagnostics.go @@ -17,13 +17,12 @@ import ( "golang.org/x/mod/modfile" "golang.org/x/mod/semver" "golang.org/x/sync/errgroup" - "golang.org/x/tools/gopls/internal/govulncheck" "golang.org/x/tools/gopls/internal/lsp/command" "golang.org/x/tools/gopls/internal/lsp/protocol" "golang.org/x/tools/gopls/internal/lsp/source" "golang.org/x/tools/gopls/internal/span" + "golang.org/x/tools/gopls/internal/vulncheck/govulncheck" "golang.org/x/tools/internal/event" - "golang.org/x/vuln/osv" ) // Diagnostics returns diagnostics from parsing the modules in the workspace. @@ -61,7 +60,6 @@ func VulnerabilityDiagnostics(ctx context.Context, snapshot source.Snapshot) (ma } func collectDiagnostics(ctx context.Context, snapshot source.Snapshot, diagFn func(context.Context, source.Snapshot, source.FileHandle) ([]*source.Diagnostic, error)) (map[span.URI][]*source.Diagnostic, error) { - g, ctx := errgroup.WithContext(ctx) cpulimit := runtime.GOMAXPROCS(0) g.SetLimit(cpulimit) @@ -199,7 +197,7 @@ func ModVulnerabilityDiagnostics(ctx context.Context, snapshot source.Snapshot, } diagSource = source.Vulncheck } - if vs == nil || len(vs.Vulns) == 0 { + if vs == nil || len(vs.Findings) == 0 { return nil, nil } @@ -208,20 +206,17 @@ func ModVulnerabilityDiagnostics(ctx context.Context, snapshot source.Snapshot, // must not happen return nil, err // TODO: bug report } - type modVuln struct { - mod *govulncheck.Module - vuln *govulncheck.Vuln - } - vulnsByModule := make(map[string][]modVuln) - for _, vuln := range vs.Vulns { - for _, mod := range vuln.Modules { - vulnsByModule[mod.Path] = append(vulnsByModule[mod.Path], modVuln{mod, vuln}) + vulnsByModule := make(map[string][]*govulncheck.Finding) + + for _, finding := range vs.Findings { + if vuln, typ := foundVuln(finding); typ == vulnCalled || typ == vulnImported { + vulnsByModule[vuln.Module] = append(vulnsByModule[vuln.Module], finding) } } - for _, req := range pm.File.Require { - vulns := vulnsByModule[req.Mod.Path] - if len(vulns) == 0 { + mod := req.Mod.Path + findings := vulnsByModule[mod] + if len(findings) == 0 { continue } // note: req.Syntax is the line corresponding to 'require', which means @@ -239,10 +234,8 @@ func ModVulnerabilityDiagnostics(ctx context.Context, snapshot source.Snapshot, // others to 'info' level diagnostics. // Fixes will include only the upgrades for warning level diagnostics. var warningFixes, infoFixes []source.SuggestedFix - var warning, info []string - var relatedInfo []protocol.DiagnosticRelatedInformation - for _, mv := range vulns { - mod, vuln := mv.mod, mv.vuln + var warningSet, infoSet = map[string]bool{}, map[string]bool{} + for _, finding := range findings { // It is possible that the source code was changed since the last // govulncheck run and information in the `vulns` info is stale. // For example, imagine that a user is in the middle of updating @@ -259,33 +252,42 @@ func ModVulnerabilityDiagnostics(ctx context.Context, snapshot source.Snapshot, // version in the require statement is equal to or higher than the // fixed version, skip generating a diagnostic about the vulnerability. // Eventually, the user has to rerun govulncheck. - if mod.FixedVersion != "" && semver.IsValid(req.Mod.Version) && semver.Compare(mod.FixedVersion, req.Mod.Version) <= 0 { + if finding.FixedVersion != "" && semver.IsValid(req.Mod.Version) && semver.Compare(finding.FixedVersion, req.Mod.Version) <= 0 { continue } - if !vuln.IsCalled() { - info = append(info, vuln.OSV.ID) - } else { - warning = append(warning, vuln.OSV.ID) - relatedInfo = append(relatedInfo, listRelatedInfo(ctx, snapshot, vuln)...) + switch _, typ := foundVuln(finding); typ { + case vulnImported: + infoSet[finding.OSV] = true + case vulnCalled: + warningSet[finding.OSV] = true } // Upgrade to the exact version we offer the user, not the most recent. - if fixedVersion := mod.FixedVersion; semver.IsValid(fixedVersion) && semver.Compare(req.Mod.Version, fixedVersion) < 0 { + if fixedVersion := finding.FixedVersion; semver.IsValid(fixedVersion) && semver.Compare(req.Mod.Version, fixedVersion) < 0 { cmd, err := getUpgradeCodeAction(fh, req, fixedVersion) if err != nil { return nil, err // TODO: bug report } sf := source.SuggestedFixFromCommand(cmd, protocol.QuickFix) - if !vuln.IsCalled() { + switch _, typ := foundVuln(finding); typ { + case vulnImported: infoFixes = append(infoFixes, sf) - } else { + case vulnCalled: warningFixes = append(warningFixes, sf) } } } - if len(warning) == 0 && len(info) == 0 { + if len(warningSet) == 0 && len(infoSet) == 0 { continue } + // Remove affecting osvs from the non-affecting osv list if any. + if len(warningSet) > 0 { + for k := range infoSet { + if warningSet[k] { + delete(infoSet, k) + } + } + } // Add an upgrade for module@latest. // TODO(suzmue): verify if latest is the same as fixedVersion. latest, err := getUpgradeCodeAction(fh, req, "latest") @@ -299,11 +301,8 @@ func ModVulnerabilityDiagnostics(ctx context.Context, snapshot source.Snapshot, if len(infoFixes) > 0 { infoFixes = append(infoFixes, sf) } - - sort.Strings(warning) - sort.Strings(info) - - if len(warning) > 0 { + if len(warningSet) > 0 { + warning := sortedKeys(warningSet) warningFixes = append(warningFixes, suggestRunOrResetGovulncheck) vulnDiagnostics = append(vulnDiagnostics, &source.Diagnostic{ URI: fh.URI(), @@ -312,10 +311,10 @@ func ModVulnerabilityDiagnostics(ctx context.Context, snapshot source.Snapshot, Source: diagSource, Message: getVulnMessage(req.Mod.Path, warning, true, diagSource == source.Govulncheck), SuggestedFixes: warningFixes, - Related: relatedInfo, }) } - if len(info) > 0 { + if len(infoSet) > 0 { + info := sortedKeys(infoSet) infoFixes = append(infoFixes, suggestRunOrResetGovulncheck) vulnDiagnostics = append(vulnDiagnostics, &source.Diagnostic{ URI: fh.URI(), @@ -350,41 +349,44 @@ func ModVulnerabilityDiagnostics(ctx context.Context, snapshot source.Snapshot, return vulnDiagnostics, nil // TODO: bug report } - stdlib := stdlibVulns[0].mod.FoundVersion - var warning, info []string - var relatedInfo []protocol.DiagnosticRelatedInformation - for _, mv := range stdlibVulns { - vuln := mv.vuln - stdlib = mv.mod.FoundVersion - if !vuln.IsCalled() { - info = append(info, vuln.OSV.ID) - } else { - warning = append(warning, vuln.OSV.ID) - relatedInfo = append(relatedInfo, listRelatedInfo(ctx, snapshot, vuln)...) + var warningSet, infoSet = map[string]bool{}, map[string]bool{} + for _, finding := range stdlibVulns { + switch _, typ := foundVuln(finding); typ { + case vulnImported: + infoSet[finding.OSV] = true + case vulnCalled: + warningSet[finding.OSV] = true } } - if len(warning) > 0 { + if len(warningSet) > 0 { + warning := sortedKeys(warningSet) fixes := []source.SuggestedFix{suggestRunOrResetGovulncheck} vulnDiagnostics = append(vulnDiagnostics, &source.Diagnostic{ URI: fh.URI(), Range: rng, Severity: protocol.SeverityWarning, Source: diagSource, - Message: getVulnMessage(stdlib, warning, true, diagSource == source.Govulncheck), + Message: getVulnMessage("go", warning, true, diagSource == source.Govulncheck), SuggestedFixes: fixes, - Related: relatedInfo, }) + + // remove affecting osvs from the non-affecting osv list if any. + for k := range infoSet { + if warningSet[k] { + delete(infoSet, k) + } + } } - if len(info) > 0 { + if len(infoSet) > 0 { + info := sortedKeys(infoSet) fixes := []source.SuggestedFix{suggestRunOrResetGovulncheck} vulnDiagnostics = append(vulnDiagnostics, &source.Diagnostic{ URI: fh.URI(), Range: rng, Severity: protocol.SeverityInformation, Source: diagSource, - Message: getVulnMessage(stdlib, info, false, diagSource == source.Govulncheck), + Message: getVulnMessage("go", info, false, diagSource == source.Govulncheck), SuggestedFixes: fixes, - Related: relatedInfo, }) } } @@ -392,6 +394,46 @@ func ModVulnerabilityDiagnostics(ctx context.Context, snapshot source.Snapshot, return vulnDiagnostics, nil } +type vulnFindingType int + +const ( + vulnUnknown vulnFindingType = iota + vulnCalled + vulnImported + vulnRequired +) + +// foundVuln returns the frame info describing discovered vulnerable symbol/package/module +// and how this vulnerability affects the analyzed package or module. +func foundVuln(finding *govulncheck.Finding) (*govulncheck.Frame, vulnFindingType) { + // finding.Trace is sorted from the imported vulnerable symbol to + // the entry point in the callstack. + // If Function is set, then Package must be set. Module will always be set. + // If Function is set it was found in the call graph, otherwise if Package is set + // it was found in the import graph, otherwise it was found in the require graph. + // See the documentation of govulncheck.Finding. + if len(finding.Trace) == 0 { // this shouldn't happen, but just in case... + return nil, vulnUnknown + } + vuln := finding.Trace[0] + if vuln.Package == "" { + return vuln, vulnRequired + } + if vuln.Function == "" { + return vuln, vulnImported + } + return vuln, vulnCalled +} + +func sortedKeys(m map[string]bool) []string { + ret := make([]string, 0, len(m)) + for k := range m { + ret = append(ret, k) + } + sort.Strings(ret) + return ret +} + // suggestGovulncheckAction returns a code action that suggests either run govulncheck // for more accurate investigation (if the present vulncheck diagnostics are based on // analysis less accurate than govulncheck) or reset the existing govulncheck result @@ -446,66 +488,12 @@ func getVulnMessage(mod string, vulns []string, used, fromGovulncheck bool) stri return b.String() } -func listRelatedInfo(ctx context.Context, snapshot source.Snapshot, vuln *govulncheck.Vuln) []protocol.DiagnosticRelatedInformation { - var ri []protocol.DiagnosticRelatedInformation - for _, m := range vuln.Modules { - for _, p := range m.Packages { - for _, c := range p.CallStacks { - if len(c.Frames) == 0 { - continue - } - entry := c.Frames[0] - pos := entry.Position - if pos.Filename == "" { - continue // token.Position Filename is an optional field. - } - uri := span.URIFromPath(pos.Filename) - startPos := protocol.Position{ - Line: uint32(pos.Line) - 1, - // We need to read the file contents to precisesly map - // token.Position (pos) to the UTF16-based column offset - // protocol.Position requires. That can be expensive. - // We need this related info to just help users to open - // the entry points of the callstack and once the file is - // open, we will compute the precise location based on the - // open file contents. So, use the beginning of the line - // as the position here instead of precise UTF16-based - // position computation. - Character: 0, - } - ri = append(ri, protocol.DiagnosticRelatedInformation{ - Location: protocol.Location{ - URI: protocol.URIFromSpanURI(uri), - Range: protocol.Range{ - Start: startPos, - End: startPos, - }, - }, - Message: fmt.Sprintf("[%v] %v -> %v.%v", vuln.OSV.ID, entry.Name(), p.Path, c.Symbol), - }) - } - } - } - return ri -} - -func formatMessage(v *govulncheck.Vuln) string { - details := []byte(v.OSV.Details) - // Remove any new lines that are not preceded or followed by a new line. - for i, r := range details { - if r == '\n' && i > 0 && details[i-1] != '\n' && i+1 < len(details) && details[i+1] != '\n' { - details[i] = ' ' - } - } - return strings.TrimSpace(strings.Replace(string(details), "\n\n", "\n\n ", -1)) -} - // href returns the url for the vulnerability information. // Eventually we should retrieve the url embedded in the osv.Entry. // While vuln.go.dev is under development, this always returns // the page in pkg.go.dev. -func href(vuln *osv.Entry) string { - return fmt.Sprintf("https://pkg.go.dev/vuln/%s", vuln.ID) +func href(vulnID string) string { + return fmt.Sprintf("https://pkg.go.dev/vuln/%s", vulnID) } func getUpgradeCodeAction(fh source.FileHandle, req *modfile.Require, version string) (protocol.Command, error) { diff --git a/gopls/internal/lsp/mod/hover.go b/gopls/internal/lsp/mod/hover.go index bc754dcb911..b39993b2924 100644 --- a/gopls/internal/lsp/mod/hover.go +++ b/gopls/internal/lsp/mod/hover.go @@ -13,9 +13,11 @@ import ( "golang.org/x/mod/modfile" "golang.org/x/mod/semver" - "golang.org/x/tools/gopls/internal/govulncheck" "golang.org/x/tools/gopls/internal/lsp/protocol" "golang.org/x/tools/gopls/internal/lsp/source" + "golang.org/x/tools/gopls/internal/vulncheck" + "golang.org/x/tools/gopls/internal/vulncheck/govulncheck" + "golang.org/x/tools/gopls/internal/vulncheck/osv" "golang.org/x/tools/internal/event" ) @@ -90,7 +92,7 @@ func hoverOnRequireStatement(ctx context.Context, pm *source.ParsedModule, offse } fromGovulncheck = false } - affecting, nonaffecting := lookupVulns(vs, req.Mod.Path, req.Mod.Version) + affecting, nonaffecting, osvs := lookupVulns(vs, req.Mod.Path, req.Mod.Version) // Get the `go mod why` results for the given file. why, err := snapshot.ModWhy(ctx, fh) @@ -113,7 +115,7 @@ func hoverOnRequireStatement(ctx context.Context, pm *source.ParsedModule, offse isPrivate := snapshot.View().IsGoPrivatePath(req.Mod.Path) header := formatHeader(req.Mod.Path, options) explanation = formatExplanation(explanation, req, options, isPrivate) - vulns := formatVulnerabilities(req.Mod.Path, affecting, nonaffecting, options, fromGovulncheck) + vulns := formatVulnerabilities(affecting, nonaffecting, osvs, options, fromGovulncheck) return &protocol.Hover{ Contents: protocol.MarkupContent{ @@ -149,9 +151,9 @@ func hoverOnModuleStatement(ctx context.Context, pm *source.ParsedModule, offset } modpath := "stdlib" goVersion := snapshot.View().GoVersionString() - affecting, nonaffecting := lookupVulns(vs, modpath, goVersion) + affecting, nonaffecting, osvs := lookupVulns(vs, modpath, goVersion) options := snapshot.Options() - vulns := formatVulnerabilities(modpath, affecting, nonaffecting, options, fromGovulncheck) + vulns := formatVulnerabilities(affecting, nonaffecting, osvs, options, fromGovulncheck) return &protocol.Hover{ Contents: protocol.MarkupContent{ @@ -174,50 +176,86 @@ func formatHeader(modpath string, options *source.Options) string { return b.String() } -func lookupVulns(vulns *govulncheck.Result, modpath, version string) (affecting, nonaffecting []*govulncheck.Vuln) { - if vulns == nil { - return nil, nil +func lookupVulns(vulns *vulncheck.Result, modpath, version string) (affecting, nonaffecting []*govulncheck.Finding, osvs map[string]*osv.Entry) { + if vulns == nil || len(vulns.Entries) == 0 { + return nil, nil, nil } - for _, vuln := range vulns.Vulns { - for _, mod := range vuln.Modules { - if mod.Path != modpath { - continue - } - // It is possible that the source code was changed since the last - // govulncheck run and information in the `vulns` info is stale. - // For example, imagine that a user is in the middle of updating - // problematic modules detected by the govulncheck run by applying - // quick fixes. Stale diagnostics can be confusing and prevent the - // user from quickly locating the next module to fix. - // Ideally we should rerun the analysis with the updated module - // dependencies or any other code changes, but we are not yet - // in the position of automatically triggering the analysis - // (govulncheck can take a while). We also don't know exactly what - // part of source code was changed since `vulns` was computed. - // As a heuristic, we assume that a user upgrades the affecting - // module to the version with the fix or the latest one, and if the - // version in the require statement is equal to or higher than the - // fixed version, skip the vulnerability information in the hover. - // Eventually, the user has to rerun govulncheck. - if mod.FixedVersion != "" && semver.IsValid(version) && semver.Compare(mod.FixedVersion, version) <= 0 { - continue - } - if vuln.IsCalled() { - affecting = append(affecting, vuln) - } else { - nonaffecting = append(nonaffecting, vuln) + for _, finding := range vulns.Findings { + vuln, typ := foundVuln(finding) + if vuln.Module != modpath { + continue + } + // It is possible that the source code was changed since the last + // govulncheck run and information in the `vulns` info is stale. + // For example, imagine that a user is in the middle of updating + // problematic modules detected by the govulncheck run by applying + // quick fixes. Stale diagnostics can be confusing and prevent the + // user from quickly locating the next module to fix. + // Ideally we should rerun the analysis with the updated module + // dependencies or any other code changes, but we are not yet + // in the position of automatically triggering the analysis + // (govulncheck can take a while). We also don't know exactly what + // part of source code was changed since `vulns` was computed. + // As a heuristic, we assume that a user upgrades the affecting + // module to the version with the fix or the latest one, and if the + // version in the require statement is equal to or higher than the + // fixed version, skip the vulnerability information in the hover. + // Eventually, the user has to rerun govulncheck. + if finding.FixedVersion != "" && semver.IsValid(version) && semver.Compare(finding.FixedVersion, version) <= 0 { + continue + } + switch typ { + case vulnCalled: + affecting = append(affecting, finding) + case vulnImported: + nonaffecting = append(nonaffecting, finding) + } + } + + // Remove affecting elements from nonaffecting. + // An OSV entry can appear in both lists if an OSV entry covers + // multiple packages imported but not all vulnerable symbols are used. + // The current wording of hover message doesn't clearly + // present this case well IMO, so let's skip reporting nonaffecting. + if len(affecting) > 0 && len(nonaffecting) > 0 { + affectingSet := map[string]bool{} + for _, f := range affecting { + affectingSet[f.OSV] = true + } + n := 0 + for _, v := range nonaffecting { + if !affectingSet[v.OSV] { + nonaffecting[n] = v + n++ } } + nonaffecting = nonaffecting[:n] + } + sort.Slice(nonaffecting, func(i, j int) bool { return nonaffecting[i].OSV < nonaffecting[j].OSV }) + sort.Slice(affecting, func(i, j int) bool { return affecting[i].OSV < affecting[j].OSV }) + return affecting, nonaffecting, vulns.Entries +} + +func fixedVersion(fixed string) string { + if fixed == "" { + return "No fix is available." } - sort.Slice(nonaffecting, func(i, j int) bool { return nonaffecting[i].OSV.ID < nonaffecting[j].OSV.ID }) - sort.Slice(affecting, func(i, j int) bool { return affecting[i].OSV.ID < affecting[j].OSV.ID }) - return affecting, nonaffecting + return "Fixed in " + fixed + "." } -func formatVulnerabilities(modPath string, affecting, nonaffecting []*govulncheck.Vuln, options *source.Options, fromGovulncheck bool) string { - if len(affecting) == 0 && len(nonaffecting) == 0 { +func formatVulnerabilities(affecting, nonaffecting []*govulncheck.Finding, osvs map[string]*osv.Entry, options *source.Options, fromGovulncheck bool) string { + if len(osvs) == 0 || (len(affecting) == 0 && len(nonaffecting) == 0) { return "" } + byOSV := func(findings []*govulncheck.Finding) map[string][]*govulncheck.Finding { + m := make(map[string][]*govulncheck.Finding) + for _, f := range findings { + m[f.OSV] = append(m[f.OSV], f) + } + return m + } + affectingByOSV := byOSV(affecting) + nonaffectingByOSV := byOSV(nonaffecting) // TODO(hyangah): can we use go templates to generate hover messages? // Then, we can use a different template for markdown case. @@ -225,22 +263,23 @@ func formatVulnerabilities(modPath string, affecting, nonaffecting []*govulnchec var b strings.Builder - if len(affecting) > 0 { + if len(affectingByOSV) > 0 { // TODO(hyangah): make the message more eyecatching (icon/codicon/color) - if len(affecting) == 1 { - b.WriteString(fmt.Sprintf("\n**WARNING:** Found %d reachable vulnerability.\n", len(affecting))) + if len(affectingByOSV) == 1 { + fmt.Fprintf(&b, "\n**WARNING:** Found %d reachable vulnerability.\n", len(affectingByOSV)) } else { - b.WriteString(fmt.Sprintf("\n**WARNING:** Found %d reachable vulnerabilities.\n", len(affecting))) + fmt.Fprintf(&b, "\n**WARNING:** Found %d reachable vulnerabilities.\n", len(affectingByOSV)) } } - for _, v := range affecting { - fix := fixedVersionInfo(v, modPath) - pkgs := vulnerablePkgsInfo(v, modPath, useMarkdown) + for id, findings := range affectingByOSV { + fix := fixedVersion(findings[0].FixedVersion) + pkgs := vulnerablePkgsInfo(findings, useMarkdown) + osvEntry := osvs[id] if useMarkdown { - fmt.Fprintf(&b, "- [**%v**](%v) %v%v%v\n", v.OSV.ID, href(v.OSV), formatMessage(v), pkgs, fix) + fmt.Fprintf(&b, "- [**%v**](%v) %v%v\n%v\n", id, href(id), osvEntry.Summary, pkgs, fix) } else { - fmt.Fprintf(&b, " - [%v] %v (%v) %v%v\n", v.OSV.ID, formatMessage(v), href(v.OSV), pkgs, fix) + fmt.Fprintf(&b, " - [%v] %v (%v) %v%v\n", id, osvEntry.Summary, href(id), pkgs, fix) } } if len(nonaffecting) > 0 { @@ -250,60 +289,41 @@ func formatVulnerabilities(modPath string, affecting, nonaffecting []*govulnchec fmt.Fprintf(&b, "\n**Note:** The project imports packages with known vulnerabilities. Use `govulncheck` to check if the project uses vulnerable symbols.\n") } } - for _, v := range nonaffecting { - fix := fixedVersionInfo(v, modPath) - pkgs := vulnerablePkgsInfo(v, modPath, useMarkdown) + for k, findings := range nonaffectingByOSV { + fix := fixedVersion(findings[0].FixedVersion) + pkgs := vulnerablePkgsInfo(findings, useMarkdown) + osvEntry := osvs[k] + if useMarkdown { - fmt.Fprintf(&b, "- [%v](%v) %v%v%v\n", v.OSV.ID, href(v.OSV), formatMessage(v), pkgs, fix) + fmt.Fprintf(&b, "- [%v](%v) %v%v\n%v\n", k, href(k), osvEntry.Summary, pkgs, fix) } else { - fmt.Fprintf(&b, " - [%v] %v (%v) %v%v\n", v.OSV.ID, formatMessage(v), href(v.OSV), pkgs, fix) + fmt.Fprintf(&b, " - [%v] %v (%v) %v\n%v\n", k, osvEntry.Summary, href(k), pkgs, fix) } } b.WriteString("\n") return b.String() } -func vulnerablePkgsInfo(v *govulncheck.Vuln, modPath string, useMarkdown bool) string { - var b bytes.Buffer - for _, m := range v.Modules { - if m.Path != modPath { - continue - } - if c := len(m.Packages); c == 1 { - b.WriteString("\n Vulnerable package is:") - } else if c > 1 { - b.WriteString("\n Vulnerable packages are:") - } - for _, pkg := range m.Packages { +func vulnerablePkgsInfo(findings []*govulncheck.Finding, useMarkdown bool) string { + var b strings.Builder + seen := map[string]bool{} + for _, f := range findings { + p := f.Trace[0].Package + if !seen[p] { + seen[p] = true if useMarkdown { b.WriteString("\n * `") } else { b.WriteString("\n ") } - b.WriteString(pkg.Path) + b.WriteString(p) if useMarkdown { b.WriteString("`") } } } - if b.Len() == 0 { - return "" - } return b.String() } -func fixedVersionInfo(v *govulncheck.Vuln, modPath string) string { - fix := "\n\n **No fix is available.**" - for _, m := range v.Modules { - if m.Path != modPath { - continue - } - if m.FixedVersion != "" { - fix = "\n\n Fixed in " + m.FixedVersion + "." - } - break - } - return fix -} func formatExplanation(text string, req *modfile.Require, options *source.Options, isPrivate bool) string { text = strings.TrimSuffix(text, "\n") diff --git a/gopls/internal/lsp/source/api_json.go b/gopls/internal/lsp/source/api_json.go index 60425db2c5c..a66ceeff78e 100644 --- a/gopls/internal/lsp/source/api_json.go +++ b/gopls/internal/lsp/source/api_json.go @@ -732,7 +732,7 @@ var GeneratedAPIJSON = &APIJSON{ Title: "Get known vulncheck result", Doc: "Fetch the result of latest vulnerability check (`govulncheck`).", ArgDoc: "{\n\t// The file URI.\n\t\"URI\": string,\n}", - ResultDoc: "map[golang.org/x/tools/gopls/internal/lsp/protocol.DocumentURI]*golang.org/x/tools/gopls/internal/govulncheck.Result", + ResultDoc: "map[golang.org/x/tools/gopls/internal/lsp/protocol.DocumentURI]*golang.org/x/tools/gopls/internal/vulncheck.Result", }, { Command: "gopls.gc_details", @@ -798,7 +798,7 @@ var GeneratedAPIJSON = &APIJSON{ }, { Command: "gopls.run_govulncheck", - Title: "Run govulncheck.", + Title: "Run vulncheck.", Doc: "Run vulnerability check (`govulncheck`).", ArgDoc: "{\n\t// Any document in the directory from which govulncheck will run.\n\t\"URI\": string,\n\t// Package pattern. E.g. \"\", \".\", \"./...\".\n\t\"Pattern\": string,\n}", ResultDoc: "{\n\t// Token holds the progress token for LSP workDone reporting of the vulncheck\n\t// invocation.\n\t\"Token\": interface{},\n}", @@ -891,7 +891,7 @@ var GeneratedAPIJSON = &APIJSON{ }, { Lens: "run_govulncheck", - Title: "Run govulncheck.", + Title: "Run vulncheck.", Doc: "Run vulnerability check (`govulncheck`).", }, { diff --git a/gopls/internal/lsp/source/view.go b/gopls/internal/lsp/source/view.go index 8aea01f113b..f4a81922b2b 100644 --- a/gopls/internal/lsp/source/view.go +++ b/gopls/internal/lsp/source/view.go @@ -22,12 +22,12 @@ import ( "golang.org/x/tools/go/analysis" "golang.org/x/tools/go/packages" "golang.org/x/tools/go/types/objectpath" - "golang.org/x/tools/gopls/internal/govulncheck" "golang.org/x/tools/gopls/internal/lsp/progress" "golang.org/x/tools/gopls/internal/lsp/protocol" "golang.org/x/tools/gopls/internal/lsp/safetoken" "golang.org/x/tools/gopls/internal/lsp/source/methodsets" "golang.org/x/tools/gopls/internal/span" + "golang.org/x/tools/gopls/internal/vulncheck" "golang.org/x/tools/internal/event/label" "golang.org/x/tools/internal/event/tag" "golang.org/x/tools/internal/gocommand" @@ -148,7 +148,7 @@ type Snapshot interface { // ModVuln returns import vulnerability analysis for the given go.mod URI. // Concurrent requests are combined into a single command. - ModVuln(ctx context.Context, modURI span.URI) (*govulncheck.Result, error) + ModVuln(ctx context.Context, modURI span.URI) (*vulncheck.Result, error) // GoModForFile returns the URI of the go.mod file for the given URI. GoModForFile(uri span.URI) span.URI @@ -391,11 +391,11 @@ type View interface { // Vulnerabilities returns known vulnerabilities for the given modfile. // TODO(suzmue): replace command.Vuln with a different type, maybe // https://pkg.go.dev/golang.org/x/vuln/cmd/govulncheck/govulnchecklib#Summary? - Vulnerabilities(modfile ...span.URI) map[span.URI]*govulncheck.Result + Vulnerabilities(modfile ...span.URI) map[span.URI]*vulncheck.Result // SetVulnerabilities resets the list of vulnerabilities that exists for the given modules // required by modfile. - SetVulnerabilities(modfile span.URI, vulncheckResult *govulncheck.Result) + SetVulnerabilities(modfile span.URI, vulncheckResult *vulncheck.Result) // GoVersion returns the configured Go version for this view. GoVersion() int diff --git a/gopls/internal/regtest/misc/vuln_test.go b/gopls/internal/regtest/misc/vuln_test.go index 41b2549bcb2..40baf8cb017 100644 --- a/gopls/internal/regtest/misc/vuln_test.go +++ b/gopls/internal/regtest/misc/vuln_test.go @@ -10,20 +10,19 @@ package misc import ( "context" "encoding/json" - "path/filepath" "sort" "strings" "testing" "github.com/google/go-cmp/cmp" - "golang.org/x/tools/gopls/internal/govulncheck" "golang.org/x/tools/gopls/internal/lsp/command" "golang.org/x/tools/gopls/internal/lsp/protocol" . "golang.org/x/tools/gopls/internal/lsp/regtest" "golang.org/x/tools/gopls/internal/lsp/source" "golang.org/x/tools/gopls/internal/lsp/tests/compare" "golang.org/x/tools/gopls/internal/vulncheck" + "golang.org/x/tools/gopls/internal/vulncheck/scan" "golang.org/x/tools/gopls/internal/vulncheck/vulntest" "golang.org/x/tools/internal/testenv" ) @@ -86,7 +85,7 @@ func F() { // build error incomplete env.Await( CompletedProgress(result.Token, &ws), ) - wantEndMsg, wantMsgPart := "failed", "failed to load packages due to errors" + wantEndMsg, wantMsgPart := "failed", "There are errors with the provided package patterns:" if ws.EndMsg != "failed" || !strings.Contains(ws.Msg, wantMsgPart) { t.Errorf("work status = %+v, want {EndMessage: %q, Message: %q}", ws, wantEndMsg, wantMsgPart) } @@ -100,14 +99,14 @@ modules: versions: - introduced: 1.0.0 - fixed: 1.0.4 - - introduced: 1.1.2 packages: - package: golang.org/amod/avuln symbols: - VulnData.Vuln1 - VulnData.Vuln2 description: > - vuln in amod + vuln in amod is found +summary: vuln in amod references: - href: pkg.go.dev/vuln/GO-2022-01 -- GO-2022-03.yaml -- @@ -121,7 +120,8 @@ modules: symbols: - nonExisting description: > - unaffecting vulnerability + unaffecting vulnerability is found +summary: unaffecting vulnerability -- GO-2022-02.yaml -- modules: - module: golang.org/bmod @@ -130,10 +130,11 @@ modules: symbols: - Vuln description: | - vuln in bmod + vuln in bmod is found. This is a long description of this vulnerability. +summary: vuln in bmod (no fix) references: - href: pkg.go.dev/vuln/GO-2022-03 -- GO-2022-04.yaml -- @@ -144,7 +145,8 @@ modules: symbols: - Vuln description: | - vuln in bmod/somtrhingelse + vuln in bmod/somethingelse is found +summary: vuln in bmod/somethingelse references: - href: pkg.go.dev/vuln/GO-2022-04 -- GOSTDLIB.yaml -- @@ -156,6 +158,7 @@ modules: - package: archive/zip symbols: - OpenReader +summary: vuln in GOSTDLIB references: - href: pkg.go.dev/vuln/GOSTDLIB ` @@ -193,7 +196,7 @@ func main() { // When fetchinging stdlib package vulnerability info, // behave as if our go version is go1.18 for this testing. // The default behavior is to run `go env GOVERSION` (which isn't mutable env var). - vulncheck.GoVersionForVulnTest: "go1.18", + scan.GoVersionForVulnTest: "go1.18", "_GOPLS_TEST_BINARY_RUN_AS_GOPLS": "true", // needed to run `gopls vulncheck`. }, Settings{ @@ -233,7 +236,7 @@ func main() { NoDiagnostics(ForFile("go.mod")), ) testFetchVulncheckResult(t, env, map[string]fetchVulncheckResult{ - "go.mod": {IDs: []string{"GOSTDLIB"}, Mode: govulncheck.ModeGovulncheck}}) + "go.mod": {IDs: []string{"GOSTDLIB"}, Mode: vulncheck.ModeGovulncheck}}) }) } @@ -269,7 +272,7 @@ func main() { "GOVULNDB": db.URI(), // When fetchinging stdlib package vulnerability info, // behave as if our go version is go1.18 for this testing. - vulncheck.GoVersionForVulnTest: "go1.18", + scan.GoVersionForVulnTest: "go1.18", "_GOPLS_TEST_BINARY_RUN_AS_GOPLS": "true", // needed to run `gopls vulncheck`. }, Settings{"ui.diagnostic.vulncheck": "Imports"}, @@ -282,7 +285,7 @@ func main() { testFetchVulncheckResult(t, env, map[string]fetchVulncheckResult{ "go.mod": { IDs: []string{"GOSTDLIB"}, - Mode: govulncheck.ModeImports, + Mode: vulncheck.ModeImports, }, }) }) @@ -290,13 +293,13 @@ func main() { type fetchVulncheckResult struct { IDs []string - Mode govulncheck.AnalysisMode + Mode vulncheck.AnalysisMode } func testFetchVulncheckResult(t *testing.T, env *Env, want map[string]fetchVulncheckResult) { t.Helper() - var result map[protocol.DocumentURI]*govulncheck.Result + var result map[protocol.DocumentURI]*vulncheck.Result fetchCmd, err := command.NewFetchVulncheckResultCommand("fetch", command.URIArg{ URI: env.Sandbox.Workdir.URI("go.mod"), }) @@ -313,14 +316,18 @@ func testFetchVulncheckResult(t *testing.T, env *Env, want map[string]fetchVulnc } got := map[string]fetchVulncheckResult{} for k, r := range result { - var osv []string - for _, v := range r.Vulns { - osv = append(osv, v.OSV.ID) + osv := map[string]bool{} + for _, v := range r.Findings { + osv[v.OSV] = true } - sort.Strings(osv) + ids := make([]string, 0, len(osv)) + for id := range osv { + ids = append(ids, id) + } + sort.Strings(ids) modfile := env.Sandbox.Workdir.RelPath(k.SpanURI().Filename()) got[modfile] = fetchVulncheckResult{ - IDs: osv, + IDs: ids, Mode: r.Mode, } } @@ -466,7 +473,7 @@ func vulnTestEnv(vulnsDB, proxyData string) (*vulntest.DB, []RunOption, error) { // When fetching stdlib package vulnerability info, // behave as if our go version is go1.18 for this testing. // The default behavior is to run `go env GOVERSION` (which isn't mutable env var). - vulncheck.GoVersionForVulnTest: "go1.18", + scan.GoVersionForVulnTest: "go1.18", "_GOPLS_TEST_BINARY_RUN_AS_GOPLS": "true", // needed to run `gopls vulncheck`. "GOSUMDB": "off", } @@ -494,7 +501,7 @@ func TestRunVulncheckPackageDiagnostics(t *testing.T) { testFetchVulncheckResult(t, env, map[string]fetchVulncheckResult{ "go.mod": { IDs: []string{"GO-2022-01", "GO-2022-02", "GO-2022-03"}, - Mode: govulncheck.ModeImports, + Mode: vulncheck.ModeImports, }, }) @@ -533,7 +540,7 @@ func TestRunVulncheckPackageDiagnostics(t *testing.T) { codeActions: []string{ "Run govulncheck to verify", }, - hover: []string{"GO-2022-02", "This is a long description of this vulnerability.", "No fix is available."}, + hover: []string{"GO-2022-02", "vuln in bmod (no fix)", "No fix is available."}, }, } @@ -645,12 +652,10 @@ func TestRunVulncheckWarning(t *testing.T) { ) testFetchVulncheckResult(t, env, map[string]fetchVulncheckResult{ - "go.mod": {IDs: []string{"GO-2022-01", "GO-2022-02", "GO-2022-03"}, Mode: govulncheck.ModeGovulncheck}, + "go.mod": {IDs: []string{"GO-2022-01", "GO-2022-02", "GO-2022-03"}, Mode: vulncheck.ModeGovulncheck}, }) env.OpenFile("x/x.go") - lineX := env.RegexpSearch("x/x.go", `c\.C1\(\)\.Vuln1\(\)`).Range.Start env.OpenFile("y/y.go") - lineY := env.RegexpSearch("y/y.go", `c\.C2\(\)\(\)`).Range.Start wantDiagnostics := map[string]vulnDiagExpectation{ "golang.org/amod": { applyAction: "Upgrade to v1.0.6", @@ -664,10 +669,6 @@ func TestRunVulncheckWarning(t *testing.T) { "Upgrade to latest", "Reset govulncheck result", }, - relatedInfo: []vulnRelatedInfo{ - {"x.go", uint32(lineX.Line), "[GO-2022-01]"}, // avuln.VulnData.Vuln1 - {"x.go", uint32(lineX.Line), "[GO-2022-01]"}, // avuln.VulnData.Vuln2 - }, }, { msg: "golang.org/amod has a vulnerability GO-2022-03 that is not used in the code.", @@ -696,15 +697,12 @@ func TestRunVulncheckWarning(t *testing.T) { codeActions: []string{ "Reset govulncheck result", // no fix, but we should give an option to reset. }, - relatedInfo: []vulnRelatedInfo{ - {"y.go", uint32(lineY.Line), "[GO-2022-02]"}, // bvuln.Vuln - }, }, }, codeActions: []string{ "Reset govulncheck result", // no fix, but we should give an option to reset. }, - hover: []string{"GO-2022-02", "This is a long description of this vulnerability.", "No fix is available."}, + hover: []string{"GO-2022-02", "vuln in bmod (no fix)", "No fix is available."}, }, } @@ -810,7 +808,7 @@ func TestGovulncheckInfo(t *testing.T) { ReadDiagnostics("go.mod", gotDiagnostics), ) - testFetchVulncheckResult(t, env, map[string]fetchVulncheckResult{"go.mod": {IDs: []string{"GO-2022-02"}, Mode: govulncheck.ModeGovulncheck}}) + testFetchVulncheckResult(t, env, map[string]fetchVulncheckResult{"go.mod": {IDs: []string{"GO-2022-02"}, Mode: vulncheck.ModeGovulncheck}}) // wantDiagnostics maps a module path in the require // section of a go.mod to diagnostics that will be returned // when running vulncheck. @@ -829,7 +827,7 @@ func TestGovulncheckInfo(t *testing.T) { codeActions: []string{ "Reset govulncheck result", }, - hover: []string{"GO-2022-02", "This is a long description of this vulnerability.", "No fix is available."}, + hover: []string{"GO-2022-02", "vuln in bmod (no fix)", "No fix is available."}, }, } @@ -886,10 +884,6 @@ func testVulnDiagnostics(t *testing.T, env *Env, pattern string, want vulnDiagEx if diag.Severity != w.severity || diag.Source != w.source { t.Errorf("incorrect (severity, source) for %q, want (%s, %s) got (%s, %s)\n", w.msg, w.severity, w.source, diag.Severity, diag.Source) } - sort.Slice(w.relatedInfo, func(i, j int) bool { return w.relatedInfo[i].less(w.relatedInfo[j]) }) - if got, want := summarizeRelatedInfo(diag.RelatedInformation), w.relatedInfo; !cmp.Equal(got, want) { - t.Errorf("related info for %q do not match, want %v, got %v\n", w.msg, want, got) - } // Check expected code actions appear. gotActions := env.CodeAction("go.mod", []protocol.Diagnostic{*diag}) if diff := diffCodeActions(gotActions, w.codeActions); diff != "" { @@ -910,22 +904,6 @@ func testVulnDiagnostics(t *testing.T, env *Env, pattern string, want vulnDiagEx return modPathDiagnostics } -// summarizeRelatedInfo converts protocol.DiagnosticRelatedInformation to vulnRelatedInfo -// that captures only the part that we want to test. -func summarizeRelatedInfo(rinfo []protocol.DiagnosticRelatedInformation) []vulnRelatedInfo { - var res []vulnRelatedInfo - for _, r := range rinfo { - filename := filepath.Base(r.Location.URI.SpanURI().Filename()) - message, _, _ := strings.Cut(r.Message, " ") - line := r.Location.Range.Start.Line - res = append(res, vulnRelatedInfo{filename, line, message}) - } - sort.Slice(res, func(i, j int) bool { - return res[i].less(res[j]) - }) - return res -} - type vulnRelatedInfo struct { Filename string Line uint32 diff --git a/gopls/internal/vulncheck/command.go b/gopls/internal/vulncheck/command.go deleted file mode 100644 index 4a3d3d2dcc0..00000000000 --- a/gopls/internal/vulncheck/command.go +++ /dev/null @@ -1,337 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build go1.18 -// +build go1.18 - -package vulncheck - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "log" - "os" - "sort" - "strings" - "sync" - - "golang.org/x/mod/semver" - "golang.org/x/sync/errgroup" - "golang.org/x/tools/go/packages" - "golang.org/x/tools/gopls/internal/govulncheck" - "golang.org/x/tools/gopls/internal/lsp/source" - "golang.org/x/vuln/client" - gvcapi "golang.org/x/vuln/exp/govulncheck" - "golang.org/x/vuln/osv" - "golang.org/x/vuln/vulncheck" -) - -func init() { - VulnerablePackages = vulnerablePackages -} - -func findGOVULNDB(env []string) []string { - for _, kv := range env { - if strings.HasPrefix(kv, "GOVULNDB=") { - return strings.Split(kv[len("GOVULNDB="):], ",") - } - } - if GOVULNDB := os.Getenv("GOVULNDB"); GOVULNDB != "" { - return strings.Split(GOVULNDB, ",") - } - return []string{"https://vuln.go.dev"} -} - -// GoVersionForVulnTest is an internal environment variable used in gopls -// testing to examine govulncheck behavior with a go version different -// than what `go version` returns in the system. -const GoVersionForVulnTest = "_GOPLS_TEST_VULNCHECK_GOVERSION" - -func init() { - Main = func(cfg packages.Config, patterns ...string) error { - // Set the mode that Source needs. - cfg.Mode = packages.NeedName | packages.NeedImports | packages.NeedTypes | - packages.NeedSyntax | packages.NeedTypesInfo | packages.NeedDeps | - packages.NeedModule - logf := log.New(os.Stderr, "", log.Ltime).Printf - logf("Loading packages...") - pkgs, err := packages.Load(&cfg, patterns...) - if err != nil { - logf("Failed to load packages: %v", err) - return err - } - if n := packages.PrintErrors(pkgs); n > 0 { - err := errors.New("failed to load packages due to errors") - logf("%v", err) - return err - } - logf("Loaded %d packages and their dependencies", len(pkgs)) - cache, err := govulncheck.DefaultCache() - if err != nil { - return err - } - cli, err := client.NewClient(findGOVULNDB(cfg.Env), client.Options{ - HTTPCache: cache, - }) - if err != nil { - return err - } - res, err := gvcapi.Source(context.Background(), &gvcapi.Config{ - Client: cli, - GoVersion: os.Getenv(GoVersionForVulnTest), - }, vulncheck.Convert(pkgs)) - if err != nil { - return err - } - affecting := 0 - for _, v := range res.Vulns { - if v.IsCalled() { - affecting++ - } - } - logf("Found %d affecting vulns and %d unaffecting vulns in imported packages", affecting, len(res.Vulns)-affecting) - if err := json.NewEncoder(os.Stdout).Encode(res); err != nil { - return err - } - return nil - } -} - -// semverToGoTag returns the Go standard library repository tag corresponding -// to semver, a version string without the initial "v". -// Go tags differ from standard semantic versions in a few ways, -// such as beginning with "go" instead of "v". -func semverToGoTag(v string) string { - if strings.HasPrefix(v, "v0.0.0") { - return "master" - } - // Special case: v1.0.0 => go1. - if v == "v1.0.0" { - return "go1" - } - if !semver.IsValid(v) { - return fmt.Sprintf("", v) - } - goVersion := semver.Canonical(v) - prerelease := semver.Prerelease(goVersion) - versionWithoutPrerelease := strings.TrimSuffix(goVersion, prerelease) - patch := strings.TrimPrefix(versionWithoutPrerelease, semver.MajorMinor(goVersion)+".") - if patch == "0" { - versionWithoutPrerelease = strings.TrimSuffix(versionWithoutPrerelease, ".0") - } - goVersion = fmt.Sprintf("go%s", strings.TrimPrefix(versionWithoutPrerelease, "v")) - if prerelease != "" { - // Go prereleases look like "beta1" instead of "beta.1". - // "beta1" is bad for sorting (since beta10 comes before beta9), so - // require the dot form. - i := finalDigitsIndex(prerelease) - if i >= 1 { - if prerelease[i-1] != '.' { - return fmt.Sprintf("", v) - } - // Remove the dot. - prerelease = prerelease[:i-1] + prerelease[i:] - } - goVersion += strings.TrimPrefix(prerelease, "-") - } - return goVersion -} - -// finalDigitsIndex returns the index of the first digit in the sequence of digits ending s. -// If s doesn't end in digits, it returns -1. -func finalDigitsIndex(s string) int { - // Assume ASCII (since the semver package does anyway). - var i int - for i = len(s) - 1; i >= 0; i-- { - if s[i] < '0' || s[i] > '9' { - break - } - } - if i == len(s)-1 { - return -1 - } - return i + 1 -} - -// vulnerablePackages queries the vulndb and reports which vulnerabilities -// apply to this snapshot. The result contains a set of packages, -// grouped by vuln ID and by module. -func vulnerablePackages(ctx context.Context, snapshot source.Snapshot, modfile source.FileHandle) (*govulncheck.Result, error) { - // We want to report the intersection of vulnerable packages in the vulndb - // and packages transitively imported by this module ('go list -deps all'). - // We use snapshot.AllMetadata to retrieve the list of packages - // as an approximation. - // - // TODO(hyangah): snapshot.AllMetadata is a superset of - // `go list all` - e.g. when the workspace has multiple main modules - // (multiple go.mod files), that can include packages that are not - // used by this module. Vulncheck behavior with go.work is not well - // defined. Figure out the meaning, and if we decide to present - // the result as if each module is analyzed independently, make - // gopls track a separate build list for each module and use that - // information instead of snapshot.AllMetadata. - metadata, err := snapshot.AllMetadata(ctx) - if err != nil { - return nil, err - } - - // TODO(hyangah): handle vulnerabilities in the standard library. - - // Group packages by modules since vuln db is keyed by module. - metadataByModule := map[source.PackagePath][]*source.Metadata{} - for _, md := range metadata { - mi := md.Module - modulePath := source.PackagePath("stdlib") - if mi != nil { - modulePath = source.PackagePath(mi.Path) - } - metadataByModule[modulePath] = append(metadataByModule[modulePath], md) - } - - // Request vuln entries from remote service. - fsCache, err := govulncheck.DefaultCache() - if err != nil { - return nil, err - } - cli, err := client.NewClient( - findGOVULNDB(snapshot.Options().EnvSlice()), - client.Options{HTTPCache: govulncheck.NewInMemoryCache(fsCache)}) - if err != nil { - return nil, err - } - // Keys are osv.Entry.IDs - vulnsResult := map[string]*govulncheck.Vuln{} - var ( - group errgroup.Group - mu sync.Mutex - ) - - goVersion := snapshot.Options().Env[GoVersionForVulnTest] - if goVersion == "" { - goVersion = snapshot.View().GoVersionString() - } - group.SetLimit(10) - stdlibModule := &packages.Module{ - Path: "stdlib", - Version: goVersion, - } - for path, mds := range metadataByModule { - path, mds := path, mds - group.Go(func() error { - effectiveModule := stdlibModule - if m := mds[0].Module; m != nil { - effectiveModule = m - } - for effectiveModule.Replace != nil { - effectiveModule = effectiveModule.Replace - } - ver := effectiveModule.Version - - // TODO(go.dev/issues/56312): batch these requests for efficiency. - vulns, err := cli.GetByModule(ctx, effectiveModule.Path) - if err != nil { - return err - } - if len(vulns) == 0 { // No known vulnerability. - return nil - } - - // set of packages in this module known to gopls. - // This will be lazily initialized when we need it. - var knownPkgs map[source.PackagePath]bool - - // Report vulnerabilities that affect packages of this module. - for _, entry := range vulns { - var vulnerablePkgs []*govulncheck.Package - - for _, a := range entry.Affected { - if a.Package.Ecosystem != osv.GoEcosystem || a.Package.Name != effectiveModule.Path { - continue - } - if !a.Ranges.AffectsSemver(ver) { - continue - } - for _, imp := range a.EcosystemSpecific.Imports { - if knownPkgs == nil { - knownPkgs = toPackagePathSet(mds) - } - if knownPkgs[source.PackagePath(imp.Path)] { - vulnerablePkgs = append(vulnerablePkgs, &govulncheck.Package{ - Path: imp.Path, - }) - } - } - } - if len(vulnerablePkgs) == 0 { - continue - } - mu.Lock() - vuln, ok := vulnsResult[entry.ID] - if !ok { - vuln = &govulncheck.Vuln{OSV: entry} - vulnsResult[entry.ID] = vuln - } - vuln.Modules = append(vuln.Modules, &govulncheck.Module{ - Path: string(path), - FoundVersion: ver, - FixedVersion: fixedVersion(effectiveModule.Path, entry.Affected), - Packages: vulnerablePkgs, - }) - mu.Unlock() - } - return nil - }) - } - if err := group.Wait(); err != nil { - return nil, err - } - - vulns := make([]*govulncheck.Vuln, 0, len(vulnsResult)) - for _, v := range vulnsResult { - vulns = append(vulns, v) - } - // Sort so the results are deterministic. - sort.Slice(vulns, func(i, j int) bool { - return vulns[i].OSV.ID < vulns[j].OSV.ID - }) - ret := &govulncheck.Result{ - Vulns: vulns, - Mode: govulncheck.ModeImports, - } - return ret, nil -} - -// toPackagePathSet transforms the metadata to a set of package paths. -func toPackagePathSet(mds []*source.Metadata) map[source.PackagePath]bool { - pkgPaths := make(map[source.PackagePath]bool, len(mds)) - for _, md := range mds { - pkgPaths[md.PkgPath] = true - } - return pkgPaths -} - -func fixedVersion(modulePath string, affected []osv.Affected) string { - fixed := govulncheck.LatestFixed(modulePath, affected) - if fixed != "" { - fixed = versionString(modulePath, fixed) - } - return fixed -} - -// versionString prepends a version string prefix (`v` or `go` -// depending on the modulePath) to the given semver-style version string. -func versionString(modulePath, version string) string { - if version == "" { - return "" - } - v := "v" + version - // These are internal Go module paths used by the vuln DB - // when listing vulns in standard library and the go command. - if modulePath == "stdlib" || modulePath == "toolchain" { - return semverToGoTag(v) - } - return v -} diff --git a/gopls/internal/vulncheck/copier.go b/gopls/internal/vulncheck/copier.go new file mode 100644 index 00000000000..ade5a5f6be2 --- /dev/null +++ b/gopls/internal/vulncheck/copier.go @@ -0,0 +1,142 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build ignore +// +build ignore + +//go:generate go run ./copier.go + +// Copier is a tool to automate copy of govulncheck's internal files. +// +// - copy golang.org/x/vuln/internal/osv/ to osv +// - copy golang.org/x/vuln/internal/govulncheck/ to govulncheck +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "go/parser" + "go/token" + "log" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + + "golang.org/x/tools/internal/edit" +) + +func main() { + log.SetPrefix("copier: ") + log.SetFlags(log.Lshortfile) + + srcMod := "golang.org/x/vuln" + srcModVers := "@latest" + srcDir, srcVer := downloadModule(srcMod + srcModVers) + + cfg := rewrite{ + banner: fmt.Sprintf("// Code generated by copying from %v@%v (go run copier.go); DO NOT EDIT.", srcMod, srcVer), + srcImportPath: "golang.org/x/vuln/internal", + dstImportPath: currentPackagePath(), + } + + copyFiles("osv", filepath.Join(srcDir, "internal", "osv"), cfg) + copyFiles("govulncheck", filepath.Join(srcDir, "internal", "govulncheck"), cfg) +} + +type rewrite struct { + // DO NOT EDIT marker to add at the beginning + banner string + // rewrite srcImportPath with dstImportPath + srcImportPath string + dstImportPath string +} + +func copyFiles(dst, src string, cfg rewrite) { + entries, err := os.ReadDir(src) + if err != nil { + log.Fatalf("failed to read dir: %v", err) + } + if err := os.MkdirAll(dst, 0777); err != nil { + log.Fatalf("failed to create dir: %v", err) + } + + for _, e := range entries { + fname := e.Name() + // we need only non-test go files. + if e.IsDir() || !strings.HasSuffix(fname, ".go") || strings.HasSuffix(fname, "_test.go") { + continue + } + data, err := os.ReadFile(filepath.Join(src, fname)) + if err != nil { + log.Fatal(err) + } + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, fname, data, parser.ParseComments|parser.ImportsOnly) + if err != nil { + log.Fatalf("parsing source module:\n%s", err) + } + + buf := edit.NewBuffer(data) + at := func(p token.Pos) int { + return fset.File(p).Offset(p) + } + + // Add banner right after the copyright statement (the first comment) + bannerInsert, banner := f.FileStart, cfg.banner + if len(f.Comments) > 0 && strings.HasPrefix(f.Comments[0].Text(), "Copyright ") { + bannerInsert = f.Comments[0].End() + banner = "\n\n" + banner + } + buf.Replace(at(bannerInsert), at(bannerInsert), banner) + + // Adjust imports + for _, spec := range f.Imports { + path, err := strconv.Unquote(spec.Path.Value) + if err != nil { + log.Fatal(err) + } + if strings.HasPrefix(path, cfg.srcImportPath) { + newPath := strings.Replace(path, cfg.srcImportPath, cfg.dstImportPath, 1) + buf.Replace(at(spec.Path.Pos()), at(spec.Path.End()), strconv.Quote(newPath)) + } + } + data = buf.Bytes() + + if err := os.WriteFile(filepath.Join(dst, fname), data, 0666); err != nil { + log.Fatal(err) + } + } +} + +func downloadModule(srcModVers string) (dir, ver string) { + var stdout, stderr bytes.Buffer + cmd := exec.Command("go", "mod", "download", "-json", srcModVers) + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + log.Fatalf("go mod download -json %s: %v\n%s%s", srcModVers, err, stderr.Bytes(), stdout.Bytes()) + } + var info struct { + Dir string + Version string + } + if err := json.Unmarshal(stdout.Bytes(), &info); err != nil { + log.Fatalf("go mod download -json %s: invalid JSON output: %v\n%s%s", srcModVers, err, stderr.Bytes(), stdout.Bytes()) + } + return info.Dir, info.Version +} + +func currentPackagePath() string { + var stdout, stderr bytes.Buffer + cmd := exec.Command("go", "list", ".") + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + log.Fatalf("go list: %v\n%s%s", err, stderr.Bytes(), stdout.Bytes()) + } + return strings.TrimSpace(stdout.String()) +} diff --git a/gopls/internal/vulncheck/govulncheck/govulncheck.go b/gopls/internal/vulncheck/govulncheck/govulncheck.go new file mode 100644 index 00000000000..fd0390703ae --- /dev/null +++ b/gopls/internal/vulncheck/govulncheck/govulncheck.go @@ -0,0 +1,160 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Code generated by copying from golang.org/x/vuln@v1.0.1 (go run copier.go); DO NOT EDIT. + +// Package govulncheck contains the JSON output structs for govulncheck. +package govulncheck + +import ( + "time" + + "golang.org/x/tools/gopls/internal/vulncheck/osv" +) + +const ( + // ProtocolVersion is the current protocol version this file implements + ProtocolVersion = "v1.0.0" +) + +// Message is an entry in the output stream. It will always have exactly one +// field filled in. +type Message struct { + Config *Config `json:"config,omitempty"` + Progress *Progress `json:"progress,omitempty"` + OSV *osv.Entry `json:"osv,omitempty"` + Finding *Finding `json:"finding,omitempty"` +} + +// Config must occur as the first message of a stream and informs the client +// about the information used to generate the findings. +// The only required field is the protocol version. +type Config struct { + // ProtocolVersion specifies the version of the JSON protocol. + ProtocolVersion string `json:"protocol_version"` + + // ScannerName is the name of the tool, for example, govulncheck. + // + // We expect this JSON format to be used by other tools that wrap + // govulncheck, which will have a different name. + ScannerName string `json:"scanner_name,omitempty"` + + // ScannerVersion is the version of the tool. + ScannerVersion string `json:"scanner_version,omitempty"` + + // DB is the database used by the tool, for example, + // vuln.go.dev. + DB string `json:"db,omitempty"` + + // LastModified is the last modified time of the data source. + DBLastModified *time.Time `json:"db_last_modified,omitempty"` + + // GoVersion is the version of Go used for analyzing standard library + // vulnerabilities. + GoVersion string `json:"go_version,omitempty"` + + // ScanLevel instructs govulncheck to analyze at a specific level of detail. + // Valid values include module, package and symbol. + ScanLevel ScanLevel `json:"scan_level,omitempty"` +} + +// Progress messages are informational only, intended to allow users to monitor +// the progress of a long running scan. +// A stream must remain fully valid and able to be interpreted with all progress +// messages removed. +type Progress struct { + // A time stamp for the message. + Timestamp *time.Time `json:"time,omitempty"` + + // Message is the progress message. + Message string `json:"message,omitempty"` +} + +// Vuln represents a single OSV entry. +type Finding struct { + // OSV is the id of the detected vulnerability. + OSV string `json:"osv,omitempty"` + + // FixedVersion is the module version where the vulnerability was + // fixed. This is empty if a fix is not available. + // + // If there are multiple fixed versions in the OSV report, this will + // be the fixed version in the latest range event for the OSV report. + // + // For example, if the range events are + // {introduced: 0, fixed: 1.0.0} and {introduced: 1.1.0}, the fixed version + // will be empty. + // + // For the stdlib, we will show the fixed version closest to the + // Go version that is used. For example, if a fix is available in 1.17.5 and + // 1.18.5, and the GOVERSION is 1.17.3, 1.17.5 will be returned as the + // fixed version. + FixedVersion string `json:"fixed_version,omitempty"` + + // Trace contains an entry for each frame in the trace. + // + // Frames are sorted starting from the imported vulnerable symbol + // until the entry point. The first frame in Frames should match + // Symbol. + // + // In binary mode, trace will contain a single-frame with no position + // information. + // + // When a package is imported but no vulnerable symbol is called, the trace + // will contain a single-frame with no symbol or position information. + Trace []*Frame `json:"trace,omitempty"` +} + +// Frame represents an entry in a finding trace. +type Frame struct { + // Module is the module path of the module containing this symbol. + // + // Importable packages in the standard library will have the path "stdlib". + Module string `json:"module"` + + // Version is the module version from the build graph. + Version string `json:"version,omitempty"` + + // Package is the import path. + Package string `json:"package,omitempty"` + + // Function is the function name. + Function string `json:"function,omitempty"` + + // Receiver is the receiver type if the called symbol is a method. + // + // The client can create the final symbol name by + // prepending Receiver to FuncName. + Receiver string `json:"receiver,omitempty"` + + // Position describes an arbitrary source position + // including the file, line, and column location. + // A Position is valid if the line number is > 0. + Position *Position `json:"position,omitempty"` +} + +// Position represents arbitrary source position. +type Position struct { + Filename string `json:"filename,omitempty"` // filename, if any + Offset int `json:"offset"` // byte offset, starting at 0 + Line int `json:"line"` // line number, starting at 1 + Column int `json:"column"` // column number, starting at 1 (byte count) +} + +// ScanLevel represents the detail level at which a scan occurred. +// This can be necessary to correctly interpret the findings, for instance if +// a scan is at symbol level and a finding does not have a symbol it means the +// vulnerability was imported but not called. If the scan however was at +// "package" level, that determination cannot be made. +type ScanLevel string + +const ( + scanLevelModule = "module" + scanLevelPackage = "package" + scanLevelSymbol = "symbol" +) + +// WantSymbols can be used to check whether the scan level is one that is able +// to generate symbols called findings. +func (l ScanLevel) WantSymbols() bool { return l == scanLevelSymbol } diff --git a/gopls/internal/vulncheck/govulncheck/handler.go b/gopls/internal/vulncheck/govulncheck/handler.go new file mode 100644 index 00000000000..4100910a3c3 --- /dev/null +++ b/gopls/internal/vulncheck/govulncheck/handler.go @@ -0,0 +1,61 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Code generated by copying from golang.org/x/vuln@v1.0.1 (go run copier.go); DO NOT EDIT. + +package govulncheck + +import ( + "encoding/json" + "io" + + "golang.org/x/tools/gopls/internal/vulncheck/osv" +) + +// Handler handles messages to be presented in a vulnerability scan output +// stream. +type Handler interface { + // Config communicates introductory message to the user. + Config(config *Config) error + + // Progress is called to display a progress message. + Progress(progress *Progress) error + + // OSV is invoked for each osv Entry in the stream. + OSV(entry *osv.Entry) error + + // Finding is called for each vulnerability finding in the stream. + Finding(finding *Finding) error +} + +// HandleJSON reads the json from the supplied stream and hands the decoded +// output to the handler. +func HandleJSON(from io.Reader, to Handler) error { + dec := json.NewDecoder(from) + for dec.More() { + msg := Message{} + // decode the next message in the stream + if err := dec.Decode(&msg); err != nil { + return err + } + // dispatch the message + var err error + if msg.Config != nil { + err = to.Config(msg.Config) + } + if msg.Progress != nil { + err = to.Progress(msg.Progress) + } + if msg.OSV != nil { + err = to.OSV(msg.OSV) + } + if msg.Finding != nil { + err = to.Finding(msg.Finding) + } + if err != nil { + return err + } + } + return nil +} diff --git a/gopls/internal/vulncheck/govulncheck/jsonhandler.go b/gopls/internal/vulncheck/govulncheck/jsonhandler.go new file mode 100644 index 00000000000..eb110a2aee9 --- /dev/null +++ b/gopls/internal/vulncheck/govulncheck/jsonhandler.go @@ -0,0 +1,46 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Code generated by copying from golang.org/x/vuln@v1.0.1 (go run copier.go); DO NOT EDIT. + +package govulncheck + +import ( + "encoding/json" + + "io" + + "golang.org/x/tools/gopls/internal/vulncheck/osv" +) + +type jsonHandler struct { + enc *json.Encoder +} + +// NewJSONHandler returns a handler that writes govulncheck output as json. +func NewJSONHandler(w io.Writer) Handler { + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + return &jsonHandler{enc: enc} +} + +// Config writes config block in JSON to the underlying writer. +func (h *jsonHandler) Config(config *Config) error { + return h.enc.Encode(Message{Config: config}) +} + +// Progress writes a progress message in JSON to the underlying writer. +func (h *jsonHandler) Progress(progress *Progress) error { + return h.enc.Encode(Message{Progress: progress}) +} + +// OSV writes an osv entry in JSON to the underlying writer. +func (h *jsonHandler) OSV(entry *osv.Entry) error { + return h.enc.Encode(Message{OSV: entry}) +} + +// Finding writes a finding in JSON to the underlying writer. +func (h *jsonHandler) Finding(finding *Finding) error { + return h.enc.Encode(Message{Finding: finding}) +} diff --git a/gopls/internal/vulncheck/osv/osv.go b/gopls/internal/vulncheck/osv/osv.go new file mode 100644 index 00000000000..08e18abf87d --- /dev/null +++ b/gopls/internal/vulncheck/osv/osv.go @@ -0,0 +1,240 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Code generated by copying from golang.org/x/vuln@v1.0.1 (go run copier.go); DO NOT EDIT. + +// Package osv implements the Go OSV vulnerability format +// (https://go.dev/security/vuln/database#schema), which is a subset of +// the OSV shared vulnerability format +// (https://ossf.github.io/osv-schema), with database and +// ecosystem-specific meanings and fields. +// +// As this package is intended for use with the Go vulnerability +// database, only the subset of features which are used by that +// database are implemented (for instance, only the SEMVER affected +// range type is implemented). +package osv + +import "time" + +// RangeType specifies the type of version range being recorded and +// defines the interpretation of the RangeEvent object's Introduced +// and Fixed fields. +// +// In this implementation, only the "SEMVER" type is supported. +// +// See https://ossf.github.io/osv-schema/#affectedrangestype-field. +type RangeType string + +// RangeTypeSemver indicates a semantic version as defined by +// SemVer 2.0.0, with no leading "v" prefix. +const RangeTypeSemver RangeType = "SEMVER" + +// Ecosystem identifies the overall library ecosystem. +// In this implementation, only the "Go" ecosystem is supported. +type Ecosystem string + +// GoEcosystem indicates the Go ecosystem. +const GoEcosystem Ecosystem = "Go" + +// Pseudo-module paths used to describe vulnerabilities +// in the Go standard library and toolchain. +const ( + // GoStdModulePath is the pseudo-module path string used + // to describe vulnerabilities in the Go standard library. + GoStdModulePath = "stdlib" + // GoCmdModulePath is the pseudo-module path string used + // to describe vulnerabilities in the go command. + GoCmdModulePath = "toolchain" +) + +// Module identifies the Go module containing the vulnerability. +// Note that this field is called "package" in the OSV specification. +// +// See https://ossf.github.io/osv-schema/#affectedpackage-field. +type Module struct { + // The Go module path. Required. + // For the Go standard library, this is "stdlib". + // For the Go toolchain, this is "toolchain." + Path string `json:"name"` + // The ecosystem containing the module. Required. + // This should always be "Go". + Ecosystem Ecosystem `json:"ecosystem"` +} + +// RangeEvent describes a single module version that either +// introduces or fixes a vulnerability. +// +// Exactly one of Introduced and Fixed must be present. Other range +// event types (e.g, "last_affected" and "limit") are not supported in +// this implementation. +// +// See https://ossf.github.io/osv-schema/#affectedrangesevents-fields. +type RangeEvent struct { + // Introduced is a version that introduces the vulnerability. + // A special value, "0", represents a version that sorts before + // any other version, and should be used to indicate that the + // vulnerability exists from the "beginning of time". + Introduced string `json:"introduced,omitempty"` + // Fixed is a version that fixes the vulnerability. + Fixed string `json:"fixed,omitempty"` +} + +// Range describes the affected versions of the vulnerable module. +// +// See https://ossf.github.io/osv-schema/#affectedranges-field. +type Range struct { + // Type is the version type that should be used to interpret the + // versions in Events. Required. + // In this implementation, only the "SEMVER" type is supported. + Type RangeType `json:"type"` + // Events is a list of versions representing the ranges in which + // the module is vulnerable. Required. + // The events should be sorted, and MUST represent non-overlapping + // ranges. + // There must be at least one RangeEvent containing a value for + // Introduced. + // See https://ossf.github.io/osv-schema/#examples for examples. + Events []RangeEvent `json:"events"` +} + +// Reference type is a reference (link) type. +type ReferenceType string + +const ( + // ReferenceTypeAdvisory is a published security advisory for + // the vulnerability. + ReferenceTypeAdvisory = ReferenceType("ADVISORY") + // ReferenceTypeArticle is an article or blog post describing the vulnerability. + ReferenceTypeArticle = ReferenceType("ARTICLE") + // ReferenceTypeReport is a report, typically on a bug or issue tracker, of + // the vulnerability. + ReferenceTypeReport = ReferenceType("REPORT") + // ReferenceTypeFix is a source code browser link to the fix (e.g., a GitHub commit). + ReferenceTypeFix = ReferenceType("FIX") + // ReferenceTypePackage is a home web page for the package. + ReferenceTypePackage = ReferenceType("PACKAGE") + // ReferenceTypeEvidence is a demonstration of the validity of a vulnerability claim. + ReferenceTypeEvidence = ReferenceType("EVIDENCE") + // ReferenceTypeWeb is a web page of some unspecified kind. + ReferenceTypeWeb = ReferenceType("WEB") +) + +// Reference is a reference URL containing additional information, +// advisories, issue tracker entries, etc., about the vulnerability. +// +// See https://ossf.github.io/osv-schema/#references-field. +type Reference struct { + // The type of reference. Required. + Type ReferenceType `json:"type"` + // The fully-qualified URL of the reference. Required. + URL string `json:"url"` +} + +// Affected gives details about a module affected by the vulnerability. +// +// See https://ossf.github.io/osv-schema/#affected-fields. +type Affected struct { + // The affected Go module. Required. + // Note that this field is called "package" in the OSV specification. + Module Module `json:"package"` + // The module version ranges affected by the vulnerability. + Ranges []Range `json:"ranges,omitempty"` + // Details on the affected packages and symbols within the module. + EcosystemSpecific EcosystemSpecific `json:"ecosystem_specific"` +} + +// Package contains additional information about an affected package. +// This is an ecosystem-specific field for the Go ecosystem. +type Package struct { + // Path is the package import path. Required. + Path string `json:"path,omitempty"` + // GOOS is the execution operating system where the symbols appear, if + // known. + GOOS []string `json:"goos,omitempty"` + // GOARCH specifies the execution architecture where the symbols appear, if + // known. + GOARCH []string `json:"goarch,omitempty"` + // Symbols is a list of function and method names affected by + // this vulnerability. Methods are listed as .. + // + // If included, only programs which use these symbols will be marked as + // vulnerable by `govulncheck`. If omitted, any program which imports this + // package will be marked vulnerable. + Symbols []string `json:"symbols,omitempty"` +} + +// EcosystemSpecific contains additional information about the vulnerable +// module for the Go ecosystem. +// +// See https://go.dev/security/vuln/database#schema. +type EcosystemSpecific struct { + // Packages is the list of affected packages within the module. + Packages []Package `json:"imports,omitempty"` +} + +// Entry represents a vulnerability in the Go OSV format, documented +// in https://go.dev/security/vuln/database#schema. +// It is a subset of the OSV schema (https://ossf.github.io/osv-schema). +// Only fields that are published in the Go Vulnerability Database +// are supported. +type Entry struct { + // SchemaVersion is the OSV schema version used to encode this + // vulnerability. + SchemaVersion string `json:"schema_version,omitempty"` + // ID is a unique identifier for the vulnerability. Required. + // The Go vulnerability database issues IDs of the form + // GO--. + ID string `json:"id"` + // Modified is the time the entry was last modified. Required. + Modified time.Time `json:"modified,omitempty"` + // Published is the time the entry should be considered to have + // been published. + Published time.Time `json:"published,omitempty"` + // Withdrawn is the time the entry should be considered to have + // been withdrawn. If the field is missing, then the entry has + // not been withdrawn. + Withdrawn *time.Time `json:"withdrawn,omitempty"` + // Aliases is a list of IDs for the same vulnerability in other + // databases. + Aliases []string `json:"aliases,omitempty"` + // Summary gives a one-line, English textual summary of the vulnerability. + // It is recommended that this field be kept short, on the order of no more + // than 120 characters. + Summary string `json:"summary,omitempty"` + // Details contains additional English textual details about the vulnerability. + Details string `json:"details"` + // Affected contains information on the modules and versions + // affected by the vulnerability. + Affected []Affected `json:"affected"` + // References contains links to more information about the + // vulnerability. + References []Reference `json:"references,omitempty"` + // Credits contains credits to entities that helped find or fix the + // vulnerability. + Credits []Credit `json:"credits,omitempty"` + // DatabaseSpecific contains additional information about the + // vulnerability, specific to the Go vulnerability database. + DatabaseSpecific *DatabaseSpecific `json:"database_specific,omitempty"` +} + +// Credit represents a credit for the discovery, confirmation, patch, or +// other event in the life cycle of a vulnerability. +// +// See https://ossf.github.io/osv-schema/#credits-fields. +type Credit struct { + // Name is the name, label, or other identifier of the individual or + // entity being credited. Required. + Name string `json:"name"` +} + +// DatabaseSpecific contains additional information about the +// vulnerability, specific to the Go vulnerability database. +// +// See https://go.dev/security/vuln/database#schema. +type DatabaseSpecific struct { + // The URL of the Go advisory for this vulnerability, of the form + // "https://pkg.go.dev/GO-YYYY-XXXX". + URL string `json:"url,omitempty"` +} diff --git a/gopls/internal/vulncheck/scan/command.go b/gopls/internal/vulncheck/scan/command.go new file mode 100644 index 00000000000..4a7a5262d8d --- /dev/null +++ b/gopls/internal/vulncheck/scan/command.go @@ -0,0 +1,471 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.18 +// +build go1.18 + +package scan + +import ( + "context" + "fmt" + "io" + "os" + "os/exec" + "sort" + "strings" + "sync" + "time" + + "golang.org/x/mod/semver" + "golang.org/x/sync/errgroup" + "golang.org/x/tools/go/packages" + "golang.org/x/tools/gopls/internal/lsp/source" + "golang.org/x/tools/gopls/internal/vulncheck" + "golang.org/x/tools/gopls/internal/vulncheck/govulncheck" + "golang.org/x/tools/gopls/internal/vulncheck/osv" + isem "golang.org/x/tools/gopls/internal/vulncheck/semver" + "golang.org/x/vuln/scan" +) + +// GoVersionForVulnTest is an internal environment variable used in gopls +// testing to examine govulncheck behavior with a go version different +// than what `go version` returns in the system. +const GoVersionForVulnTest = "_GOPLS_TEST_VULNCHECK_GOVERSION" + +// Main implements gopls vulncheck. +func Main(ctx context.Context, args ...string) error { + // wrapping govulncheck. + cmd := scan.Command(ctx, args...) + if err := cmd.Start(); err != nil { + return err + } + return cmd.Wait() +} + +// RunGovulncheck implements the codelens "Run Govulncheck" +// that runs 'gopls vulncheck' and converts the output to gopls's internal data +// used for diagnostics and hover message construction. +func RunGovulncheck(ctx context.Context, pattern string, snapshot source.Snapshot, dir string, log io.Writer) (*vulncheck.Result, error) { + vulncheckargs := []string{ + "vulncheck", "--", + "-json", + "-mode", "source", + "-scan", "symbol", + } + if dir != "" { + vulncheckargs = append(vulncheckargs, "-C", dir) + } + if db := getEnv(snapshot, "GOVULNDB"); db != "" { + vulncheckargs = append(vulncheckargs, "-db", db) + } + vulncheckargs = append(vulncheckargs, pattern) + // TODO: support -tags. need to compute tags args from opts.BuildFlags. + // TODO: support -test. + + ir, iw := io.Pipe() + handler := &govulncheckHandler{logger: log, osvs: map[string]*osv.Entry{}} + + var g errgroup.Group + // We run the govulncheck's analysis in a separate process as it can + // consume a lot of CPUs and memory, and terminates: a separate process + // is a perfect garbage collector and affords us ways to limit its resource usage. + g.Go(func() error { + defer iw.Close() + + cmd := exec.CommandContext(ctx, os.Args[0], vulncheckargs...) + cmd.Env = getEnvSlices(snapshot) + if goversion := getEnv(snapshot, GoVersionForVulnTest); goversion != "" { + // Let govulncheck API use a different Go version using the (undocumented) hook + // in https://go.googlesource.com/vuln/+/v1.0.1/internal/scan/run.go#76 + cmd.Env = append(cmd.Env, "GOVERSION="+goversion) + } + cmd.Stderr = log // stream vulncheck's STDERR as progress reports + cmd.Stdout = iw // let the other goroutine parses the result. + + if err := cmd.Start(); err != nil { + return fmt.Errorf("failed to start govulncheck: %v", err) + } + if err := cmd.Wait(); err != nil { + return fmt.Errorf("failed to run govulncheck: %v", err) + } + return nil + }) + g.Go(func() error { + return govulncheck.HandleJSON(ir, handler) + }) + if err := g.Wait(); err != nil { + return nil, fmt.Errorf("failed to read govulncheck output: %v", err) + } + + findings := handler.findings // sort so the findings in the result is deterministic. + sort.Slice(findings, func(i, j int) bool { + x, y := findings[i], findings[j] + if x.OSV != y.OSV { + return x.OSV < y.OSV + } + return x.Trace[0].Package < y.Trace[0].Package + }) + result := &vulncheck.Result{ + Mode: vulncheck.ModeGovulncheck, + AsOf: time.Now(), + Entries: handler.osvs, + Findings: findings, + } + return result, nil +} + +type govulncheckHandler struct { + logger io.Writer // forward progress reports to logger. + err error + + osvs map[string]*osv.Entry + findings []*govulncheck.Finding +} + +// Config implements vulncheck.Handler. +func (h *govulncheckHandler) Config(config *govulncheck.Config) error { + if config.GoVersion != "" { + fmt.Fprintf(h.logger, "Go: %v\n", config.GoVersion) + } + if config.ScannerName != "" { + scannerName := fmt.Sprintf("Scanner: %v", config.ScannerName) + if config.ScannerVersion != "" { + scannerName += "@" + config.ScannerVersion + } + fmt.Fprintln(h.logger, scannerName) + } + if config.DB != "" { + dbInfo := fmt.Sprintf("DB: %v", config.DB) + if config.DBLastModified != nil { + dbInfo += fmt.Sprintf(" (DB updated: %v)", config.DBLastModified.String()) + } + fmt.Fprintln(h.logger, dbInfo) + } + return nil +} + +// Finding implements vulncheck.Handler. +func (h *govulncheckHandler) Finding(finding *govulncheck.Finding) error { + h.findings = append(h.findings, finding) + return nil +} + +// OSV implements vulncheck.Handler. +func (h *govulncheckHandler) OSV(entry *osv.Entry) error { + h.osvs[entry.ID] = entry + return nil +} + +// Progress implements vulncheck.Handler. +func (h *govulncheckHandler) Progress(progress *govulncheck.Progress) error { + if progress.Message != "" { + fmt.Fprintf(h.logger, "%v\n", progress.Message) + } + return nil +} + +func getEnv(snapshot source.Snapshot, key string) string { + val, ok := snapshot.Options().Env[key] + if ok { + return val + } + return os.Getenv(key) +} + +func getEnvSlices(snapshot source.Snapshot) []string { + return append(os.Environ(), snapshot.Options().EnvSlice()...) +} + +// semverToGoTag returns the Go standard library repository tag corresponding +// to semver, a version string without the initial "v". +// Go tags differ from standard semantic versions in a few ways, +// such as beginning with "go" instead of "v". +func semverToGoTag(v string) string { + if strings.HasPrefix(v, "v0.0.0") { + return "master" + } + // Special case: v1.0.0 => go1. + if v == "v1.0.0" { + return "go1" + } + if !semver.IsValid(v) { + return fmt.Sprintf("", v) + } + goVersion := semver.Canonical(v) + prerelease := semver.Prerelease(goVersion) + versionWithoutPrerelease := strings.TrimSuffix(goVersion, prerelease) + patch := strings.TrimPrefix(versionWithoutPrerelease, semver.MajorMinor(goVersion)+".") + if patch == "0" { + versionWithoutPrerelease = strings.TrimSuffix(versionWithoutPrerelease, ".0") + } + goVersion = fmt.Sprintf("go%s", strings.TrimPrefix(versionWithoutPrerelease, "v")) + if prerelease != "" { + // Go prereleases look like "beta1" instead of "beta.1". + // "beta1" is bad for sorting (since beta10 comes before beta9), so + // require the dot form. + i := finalDigitsIndex(prerelease) + if i >= 1 { + if prerelease[i-1] != '.' { + return fmt.Sprintf("", v) + } + // Remove the dot. + prerelease = prerelease[:i-1] + prerelease[i:] + } + goVersion += strings.TrimPrefix(prerelease, "-") + } + return goVersion +} + +// finalDigitsIndex returns the index of the first digit in the sequence of digits ending s. +// If s doesn't end in digits, it returns -1. +func finalDigitsIndex(s string) int { + // Assume ASCII (since the semver package does anyway). + var i int + for i = len(s) - 1; i >= 0; i-- { + if s[i] < '0' || s[i] > '9' { + break + } + } + if i == len(s)-1 { + return -1 + } + return i + 1 +} + +// VulnerablePackages queries the vulndb and reports which vulnerabilities +// apply to this snapshot. The result contains a set of packages, +// grouped by vuln ID and by module. This implements the "import-based" +// vulnerability report on go.mod files. +func VulnerablePackages(ctx context.Context, snapshot source.Snapshot) (*vulncheck.Result, error) { + // TODO(hyangah): can we let 'govulncheck' take a package list + // used in the workspace and implement this function? + + // We want to report the intersection of vulnerable packages in the vulndb + // and packages transitively imported by this module ('go list -deps all'). + // We use snapshot.AllMetadata to retrieve the list of packages + // as an approximation. + // + // TODO(hyangah): snapshot.AllMetadata is a superset of + // `go list all` - e.g. when the workspace has multiple main modules + // (multiple go.mod files), that can include packages that are not + // used by this module. Vulncheck behavior with go.work is not well + // defined. Figure out the meaning, and if we decide to present + // the result as if each module is analyzed independently, make + // gopls track a separate build list for each module and use that + // information instead of snapshot.AllMetadata. + metadata, err := snapshot.AllMetadata(ctx) + if err != nil { + return nil, err + } + + // TODO(hyangah): handle vulnerabilities in the standard library. + + // Group packages by modules since vuln db is keyed by module. + metadataByModule := map[source.PackagePath][]*source.Metadata{} + for _, md := range metadata { + modulePath := source.PackagePath(osv.GoStdModulePath) + if mi := md.Module; mi != nil { + modulePath = source.PackagePath(mi.Path) + } + metadataByModule[modulePath] = append(metadataByModule[modulePath], md) + } + + var ( + mu sync.Mutex + // Keys are osv.Entry.ID + osvs = map[string]*osv.Entry{} + findings []*govulncheck.Finding + ) + + goVersion := snapshot.Options().Env[GoVersionForVulnTest] + if goVersion == "" { + goVersion = snapshot.View().GoVersionString() + } + + stdlibModule := &packages.Module{ + Path: osv.GoStdModulePath, + Version: goVersion, + } + + // GOVULNDB may point the test db URI. + db := getEnv(snapshot, "GOVULNDB") + + var group errgroup.Group + group.SetLimit(10) // limit govulncheck api runs + for _, mds := range metadataByModule { + mds := mds + group.Go(func() error { + effectiveModule := stdlibModule + if m := mds[0].Module; m != nil { + effectiveModule = m + } + for effectiveModule.Replace != nil { + effectiveModule = effectiveModule.Replace + } + ver := effectiveModule.Version + if ver == "" || !isem.Valid(ver) { + // skip invalid version strings. the underlying scan api is strict. + return nil + } + + // TODO(hyangah): batch these requests and add in-memory cache for efficiency. + vulns, err := osvsByModule(ctx, db, effectiveModule.Path+"@"+ver) + if err != nil { + return err + } + if len(vulns) == 0 { // No known vulnerability. + return nil + } + + // set of packages in this module known to gopls. + // This will be lazily initialized when we need it. + var knownPkgs map[source.PackagePath]bool + + // Report vulnerabilities that affect packages of this module. + for _, entry := range vulns { + var vulnerablePkgs []*govulncheck.Finding + fixed := fixedVersion(effectiveModule.Path, entry.Affected) + + for _, a := range entry.Affected { + if a.Module.Ecosystem != osv.GoEcosystem || a.Module.Path != effectiveModule.Path { + continue + } + for _, imp := range a.EcosystemSpecific.Packages { + if knownPkgs == nil { + knownPkgs = toPackagePathSet(mds) + } + if knownPkgs[source.PackagePath(imp.Path)] { + vulnerablePkgs = append(vulnerablePkgs, &govulncheck.Finding{ + OSV: entry.ID, + FixedVersion: fixed, + Trace: []*govulncheck.Frame{ + { + Module: effectiveModule.Path, + Version: effectiveModule.Version, + Package: imp.Path, + }, + }, + }) + } + } + } + if len(vulnerablePkgs) == 0 { + continue + } + mu.Lock() + osvs[entry.ID] = entry + findings = append(findings, vulnerablePkgs...) + mu.Unlock() + } + return nil + }) + } + if err := group.Wait(); err != nil { + return nil, err + } + + // Sort so the results are deterministic. + sort.Slice(findings, func(i, j int) bool { + x, y := findings[i], findings[j] + if x.OSV != y.OSV { + return x.OSV < y.OSV + } + return x.Trace[0].Package < y.Trace[0].Package + }) + ret := &vulncheck.Result{ + Entries: osvs, + Findings: findings, + Mode: vulncheck.ModeImports, + } + return ret, nil +} + +// toPackagePathSet transforms the metadata to a set of package paths. +func toPackagePathSet(mds []*source.Metadata) map[source.PackagePath]bool { + pkgPaths := make(map[source.PackagePath]bool, len(mds)) + for _, md := range mds { + pkgPaths[md.PkgPath] = true + } + return pkgPaths +} + +func fixedVersion(modulePath string, affected []osv.Affected) string { + fixed := LatestFixed(modulePath, affected) + if fixed != "" { + fixed = versionString(modulePath, fixed) + } + return fixed +} + +// versionString prepends a version string prefix (`v` or `go` +// depending on the modulePath) to the given semver-style version string. +func versionString(modulePath, version string) string { + if version == "" { + return "" + } + v := "v" + version + // These are internal Go module paths used by the vuln DB + // when listing vulns in standard library and the go command. + if modulePath == "stdlib" || modulePath == "toolchain" { + return semverToGoTag(v) + } + return v +} + +// osvsByModule runs a govulncheck database query. +func osvsByModule(ctx context.Context, db, moduleVersion string) ([]*osv.Entry, error) { + var args []string + args = append(args, "-mode=query", "-json") + if db != "" { + args = append(args, "-db="+db) + } + args = append(args, moduleVersion) + + ir, iw := io.Pipe() + handler := &osvReader{} + + var g errgroup.Group + g.Go(func() error { + defer iw.Close() // scan API doesn't close cmd.Stderr/cmd.Stdout. + cmd := scan.Command(ctx, args...) + cmd.Stdout = iw + // TODO(hakim): Do we need to set cmd.Env = getEnvSlices(), + // or is the process environment good enough? + if err := cmd.Start(); err != nil { + return err + } + return cmd.Wait() + }) + g.Go(func() error { + return govulncheck.HandleJSON(ir, handler) + }) + + if err := g.Wait(); err != nil { + return nil, err + } + return handler.entry, nil +} + +// osvReader implements govulncheck.Handler. +type osvReader struct { + entry []*osv.Entry +} + +func (h *osvReader) OSV(entry *osv.Entry) error { + h.entry = append(h.entry, entry) + return nil +} + +func (h *osvReader) Config(config *govulncheck.Config) error { + return nil +} + +func (h *osvReader) Finding(finding *govulncheck.Finding) error { + return nil +} + +func (h *osvReader) Progress(progress *govulncheck.Progress) error { + return nil +} diff --git a/gopls/internal/govulncheck/util.go b/gopls/internal/vulncheck/scan/util.go similarity index 79% rename from gopls/internal/govulncheck/util.go rename to gopls/internal/vulncheck/scan/util.go index 544fba2a593..2ea75a5183a 100644 --- a/gopls/internal/govulncheck/util.go +++ b/gopls/internal/vulncheck/scan/util.go @@ -5,12 +5,12 @@ //go:build go1.18 // +build go1.18 -package govulncheck +package scan import ( "golang.org/x/mod/semver" - isem "golang.org/x/tools/gopls/internal/govulncheck/semver" - "golang.org/x/vuln/osv" + "golang.org/x/tools/gopls/internal/vulncheck/osv" + isem "golang.org/x/tools/gopls/internal/vulncheck/semver" ) // LatestFixed returns the latest fixed version in the list of affected ranges, @@ -18,11 +18,11 @@ import ( func LatestFixed(modulePath string, as []osv.Affected) string { v := "" for _, a := range as { - if a.Package.Name != modulePath { + if a.Module.Path != modulePath { continue } for _, r := range a.Ranges { - if r.Type == osv.TypeSemver { + if r.Type == osv.RangeTypeSemver { for _, e := range r.Events { if e.Fixed != "" && (v == "" || semver.Compare(isem.CanonicalizeSemverPrefix(e.Fixed), isem.CanonicalizeSemverPrefix(v)) > 0) { diff --git a/gopls/internal/govulncheck/semver/semver.go b/gopls/internal/vulncheck/semver/semver.go similarity index 88% rename from gopls/internal/govulncheck/semver/semver.go rename to gopls/internal/vulncheck/semver/semver.go index 4ab298d137b..5cd1ee864d3 100644 --- a/gopls/internal/govulncheck/semver/semver.go +++ b/gopls/internal/vulncheck/semver/semver.go @@ -12,6 +12,8 @@ package semver import ( "regexp" "strings" + + "golang.org/x/mod/semver" ) // addSemverPrefix adds a 'v' prefix to s if it isn't already prefixed @@ -40,6 +42,12 @@ func CanonicalizeSemverPrefix(s string) string { return addSemverPrefix(removeSemverPrefix(s)) } +// Valid returns whether v is valid semver, allowing +// either a "v", "go" or no prefix. +func Valid(v string) bool { + return semver.IsValid(CanonicalizeSemverPrefix(v)) +} + var ( // Regexp for matching go tags. The groups are: // 1 the major.minor version diff --git a/gopls/internal/govulncheck/semver/semver_test.go b/gopls/internal/vulncheck/semver/semver_test.go similarity index 100% rename from gopls/internal/govulncheck/semver/semver_test.go rename to gopls/internal/vulncheck/semver/semver_test.go diff --git a/gopls/internal/govulncheck/types.go b/gopls/internal/vulncheck/types.go similarity index 70% rename from gopls/internal/govulncheck/types.go rename to gopls/internal/vulncheck/types.go index 2881cf4bc40..450cd961797 100644 --- a/gopls/internal/govulncheck/types.go +++ b/gopls/internal/vulncheck/types.go @@ -2,15 +2,25 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package govulncheck +// go:generate go run copier.go -import "time" +package vulncheck + +import ( + "time" + + gvc "golang.org/x/tools/gopls/internal/vulncheck/govulncheck" + "golang.org/x/tools/gopls/internal/vulncheck/osv" +) // Result is the result of vulnerability scanning. type Result struct { - // Vulns contains all vulnerabilities that are called or imported by - // the analyzed module. - Vulns []*Vuln `json:",omitempty"` + // Entries contains all vulnerabilities that are called or imported by + // the analyzed module. Keys are Entry.IDs. + Entries map[string]*osv.Entry + // Findings are vulnerabilities found by vulncheck or import-based analysis. + // Ordered by the OSV IDs and the package names. + Findings []*gvc.Finding // Mode contains the source of the vulnerability info. // Clients of the gopls.fetch_vulncheck_result command may need diff --git a/gopls/internal/vulncheck/vulncheck.go b/gopls/internal/vulncheck/vulncheck.go deleted file mode 100644 index 3c361bd01e4..00000000000 --- a/gopls/internal/vulncheck/vulncheck.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2022 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package vulncheck provides an analysis command -// that runs vulnerability analysis using data from -// golang.org/x/vuln/vulncheck. -// This package requires go1.18 or newer. -package vulncheck - -import ( - "context" - - "golang.org/x/tools/go/packages" - "golang.org/x/tools/gopls/internal/govulncheck" - "golang.org/x/tools/gopls/internal/lsp/source" -) - -// With go1.18+, this is swapped with the real implementation. -var Main func(cfg packages.Config, patterns ...string) error = nil - -// VulnerablePackages queries the vulndb and reports which vulnerabilities -// apply to this snapshot. The result contains a set of packages, -// grouped by vuln ID and by module. -var VulnerablePackages func(ctx context.Context, snapshot source.Snapshot, modfile source.FileHandle) (*govulncheck.Result, error) = nil diff --git a/gopls/internal/vulncheck/vulntest/db.go b/gopls/internal/vulncheck/vulntest/db.go index 511a47e1ba9..2e756e3ea33 100644 --- a/gopls/internal/vulncheck/vulntest/db.go +++ b/gopls/internal/vulncheck/vulntest/db.go @@ -21,9 +21,8 @@ import ( "time" "golang.org/x/tools/gopls/internal/span" + "golang.org/x/tools/gopls/internal/vulncheck/osv" "golang.org/x/tools/txtar" - "golang.org/x/vuln/client" - "golang.org/x/vuln/osv" ) // NewDatabase returns a read-only DB containing the provided @@ -64,7 +63,7 @@ type DB struct { // URI returns the file URI that can be used for VULNDB environment // variable. func (db *DB) URI() string { - u := span.URIFromPath(db.disk) + u := span.URIFromPath(filepath.Join(db.disk, "ID")) return string(u) } @@ -73,11 +72,6 @@ func (db *DB) Clean() error { return os.RemoveAll(db.disk) } -// NewClient returns a vuln DB client that works with the given DB. -func NewClient(db *DB) (client.Client, error) { - return client.NewClient([]string{db.URI()}, client.Options{}) -} - // // The following was selectively copied from golang.org/x/vulndb/internal/database // @@ -89,14 +83,6 @@ const ( // listed by their IDs. idDirectory = "ID" - // stdFileName is the name of the .json file in the vulndb repo - // that will contain info on standard library vulnerabilities. - stdFileName = "stdlib" - - // toolchainFileName is the name of the .json file in the vulndb repo - // that will contain info on toolchain (cmd/...) vulnerabilities. - toolchainFileName = "toolchain" - // cmdModule is the name of the module containing Go toolchain // binaries. cmdModule = "cmd" @@ -109,38 +95,15 @@ const ( func generateDB(ctx context.Context, txtarData []byte, jsonDir string, indent bool) error { archive := txtar.Parse(txtarData) - jsonVulns, entries, err := generateEntries(ctx, archive) + entries, err := generateEntries(ctx, archive) if err != nil { return err } - - index := make(client.DBIndex, len(jsonVulns)) - for modulePath, vulns := range jsonVulns { - epath, err := client.EscapeModulePath(modulePath) - if err != nil { - return err - } - if err := writeVulns(filepath.Join(jsonDir, epath), vulns, indent); err != nil { - return err - } - for _, v := range vulns { - if v.Modified.After(index[modulePath]) { - index[modulePath] = v.Modified - } - } - } - if err := writeJSON(filepath.Join(jsonDir, "index.json"), index, indent); err != nil { - return err - } - if err := writeAliasIndex(jsonDir, entries, indent); err != nil { - return err - } return writeEntriesByID(filepath.Join(jsonDir, idDirectory), entries, indent) } -func generateEntries(_ context.Context, archive *txtar.Archive) (map[string][]osv.Entry, []osv.Entry, error) { +func generateEntries(_ context.Context, archive *txtar.Archive) ([]osv.Entry, error) { now := time.Now() - jsonVulns := map[string][]osv.Entry{} var entries []osv.Entry for _, f := range archive.Files { if !strings.HasSuffix(f.Name, ".yaml") { @@ -148,17 +111,14 @@ func generateEntries(_ context.Context, archive *txtar.Archive) (map[string][]os } r, err := readReport(bytes.NewReader(f.Data)) if err != nil { - return nil, nil, err + return nil, err } name := strings.TrimSuffix(filepath.Base(f.Name), filepath.Ext(f.Name)) linkName := fmt.Sprintf("%s%s", dbURL, name) - entry, modulePaths := generateOSVEntry(name, linkName, now, *r) - for _, modulePath := range modulePaths { - jsonVulns[modulePath] = append(jsonVulns[modulePath], entry) - } + entry := generateOSVEntry(name, linkName, now, *r) entries = append(entries, entry) } - return jsonVulns, entries, nil + return entries, nil } func writeVulns(outPath string, vulns []osv.Entry, indent bool) error { @@ -173,27 +133,13 @@ func writeEntriesByID(idDir string, entries []osv.Entry, indent bool) error { if err := os.MkdirAll(idDir, 0755); err != nil { return fmt.Errorf("failed to create directory %q: %v", idDir, err) } - var idIndex []string for _, e := range entries { outPath := filepath.Join(idDir, e.ID+".json") if err := writeJSON(outPath, e, indent); err != nil { return err } - idIndex = append(idIndex, e.ID) } - // Write an index.json in the ID directory with a list of all the IDs. - return writeJSON(filepath.Join(idDir, "index.json"), idIndex, indent) -} - -// Write a JSON file containing a map from alias to GO IDs. -func writeAliasIndex(dir string, entries []osv.Entry, indent bool) error { - aliasToGoIDs := map[string][]string{} - for _, e := range entries { - for _, a := range e.Aliases { - aliasToGoIDs[a] = append(aliasToGoIDs[a], e.ID) - } - } - return writeJSON(filepath.Join(dir, "aliases.json"), aliasToGoIDs, indent) + return nil } func writeJSON(filename string, value any, indent bool) (err error) { @@ -214,45 +160,40 @@ func jsonMarshal(v any, indent bool) ([]byte, error) { // generateOSVEntry create an osv.Entry for a report. In addition to the report, it // takes the ID for the vuln and a URL that will point to the entry in the vuln DB. // It returns the osv.Entry and a list of module paths that the vuln affects. -func generateOSVEntry(id, url string, lastModified time.Time, r Report) (osv.Entry, []string) { +func generateOSVEntry(id, url string, lastModified time.Time, r Report) osv.Entry { entry := osv.Entry{ - ID: id, - Published: r.Published, - Modified: lastModified, - Withdrawn: r.Withdrawn, - Details: r.Description, + ID: id, + Published: r.Published, + Modified: lastModified, + Withdrawn: r.Withdrawn, + Summary: r.Summary, + Details: r.Description, + DatabaseSpecific: &osv.DatabaseSpecific{URL: url}, } moduleMap := make(map[string]bool) for _, m := range r.Modules { switch m.Module { case stdModule: - moduleMap[stdFileName] = true + moduleMap[osv.GoStdModulePath] = true case cmdModule: - moduleMap[toolchainFileName] = true + moduleMap[osv.GoCmdModulePath] = true default: moduleMap[m.Module] = true } - entry.Affected = append(entry.Affected, generateAffected(m, url)) + entry.Affected = append(entry.Affected, toAffected(m)) } for _, ref := range r.References { entry.References = append(entry.References, osv.Reference{ - Type: string(ref.Type), + Type: ref.Type, URL: ref.URL, }) } - - var modulePaths []string - for module := range moduleMap { - modulePaths = append(modulePaths, module) - } - // TODO: handle missing fields - Aliases - - return entry, modulePaths + return entry } -func generateAffectedRanges(versions []VersionRange) osv.Affects { - a := osv.AffectsRange{Type: osv.TypeSemver} +func AffectedRanges(versions []VersionRange) []osv.Range { + a := osv.Range{Type: osv.RangeTypeSemver} if len(versions) == 0 || versions[0].Introduced == "" { a.Events = append(a.Events, osv.RangeEvent{Introduced: "0"}) } @@ -264,15 +205,15 @@ func generateAffectedRanges(versions []VersionRange) osv.Affects { a.Events = append(a.Events, osv.RangeEvent{Fixed: v.Fixed.Canonical()}) } } - return osv.Affects{a} + return []osv.Range{a} } -func generateImports(m *Module) (imps []osv.EcosystemSpecificImport) { - for _, p := range m.Packages { +func toOSVPackages(pkgs []*Package) (imps []osv.Package) { + for _, p := range pkgs { syms := append([]string{}, p.Symbols...) syms = append(syms, p.DerivedSymbols...) sort.Strings(syms) - imps = append(imps, osv.EcosystemSpecificImport{ + imps = append(imps, osv.Package{ Path: p.Package, GOOS: p.GOOS, GOARCH: p.GOARCH, @@ -281,23 +222,23 @@ func generateImports(m *Module) (imps []osv.EcosystemSpecificImport) { } return imps } -func generateAffected(m *Module, url string) osv.Affected { + +func toAffected(m *Module) osv.Affected { name := m.Module switch name { case stdModule: - name = "stdlib" + name = osv.GoStdModulePath case cmdModule: - name = "toolchain" + name = osv.GoCmdModulePath } return osv.Affected{ - Package: osv.Package{ - Name: name, + Module: osv.Module{ + Path: name, Ecosystem: osv.GoEcosystem, }, - Ranges: generateAffectedRanges(m.Versions), - DatabaseSpecific: osv.DatabaseSpecific{URL: url}, + Ranges: AffectedRanges(m.Versions), EcosystemSpecific: osv.EcosystemSpecific{ - Imports: generateImports(m), + Packages: toOSVPackages(m.Packages), }, } } diff --git a/gopls/internal/vulncheck/vulntest/db_test.go b/gopls/internal/vulncheck/vulntest/db_test.go index 7d939421c94..d68ba08b1eb 100644 --- a/gopls/internal/vulncheck/vulntest/db_test.go +++ b/gopls/internal/vulncheck/vulntest/db_test.go @@ -10,52 +10,70 @@ package vulntest import ( "context" "encoding/json" + "flag" + "os" + "path/filepath" "testing" + "time" + + "github.com/google/go-cmp/cmp" + "golang.org/x/tools/gopls/internal/span" + "golang.org/x/tools/gopls/internal/vulncheck/osv" ) +var update = flag.Bool("update", false, "update golden files in testdata/") + func TestNewDatabase(t *testing.T) { ctx := context.Background() - in := []byte(` --- GO-2020-0001.yaml -- -modules: - - module: github.com/gin-gonic/gin - versions: - - fixed: 1.6.0 - packages: - - package: github.com/gin-gonic/gin - symbols: - - defaultLogFormatter -description: | - Something. -published: 2021-04-14T20:04:52Z -references: - - fix: https://github.com/gin-gonic/gin/pull/2237 -`) - db, err := NewDatabase(ctx, in) + in, err := os.ReadFile("testdata/report.yaml") if err != nil { t.Fatal(err) } - defer db.Clean() + in = append([]byte("-- GO-2020-0001.yaml --\n"), in...) - cli, err := NewClient(db) + db, err := NewDatabase(ctx, in) if err != nil { t.Fatal(err) } - got, err := cli.GetByID(ctx, "GO-2020-0001") + defer db.Clean() + dbpath := span.URIFromURI(db.URI()).Filename() + + // The generated JSON file will be in DB/GO-2022-0001.json. + got := readOSVEntry(t, filepath.Join(dbpath, "GO-2020-0001.json")) + got.Modified = time.Time{} + + if *update { + updateTestData(t, got, "testdata/GO-2020-0001.json") + } + + want := readOSVEntry(t, "testdata/GO-2020-0001.json") + want.Modified = time.Time{} + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } +} + +func updateTestData(t *testing.T, got *osv.Entry, fname string) { + content, err := json.MarshalIndent(got, "", "\t") if err != nil { t.Fatal(err) } - if got.ID != "GO-2020-0001" { - m, _ := json.Marshal(got) - t.Errorf("got %s\nwant GO-2020-0001 entry", m) + if err := os.WriteFile(fname, content, 0666); err != nil { + t.Fatal(err) } - gotAll, err := cli.GetByModule(ctx, "github.com/gin-gonic/gin") + t.Logf("updated %v", fname) +} + +func readOSVEntry(t *testing.T, filename string) *osv.Entry { + t.Helper() + content, err := os.ReadFile(filename) if err != nil { t.Fatal(err) } - if len(gotAll) != 1 || gotAll[0].ID != "GO-2020-0001" { - m, _ := json.Marshal(got) - t.Errorf("got %s\nwant GO-2020-0001 entry", m) + var entry osv.Entry + if err := json.Unmarshal(content, &entry); err != nil { + t.Fatal(err) } + return &entry } diff --git a/gopls/internal/vulncheck/vulntest/report.go b/gopls/internal/vulncheck/vulntest/report.go index e5595e8ba06..cbfd0aeb8ff 100644 --- a/gopls/internal/vulncheck/vulntest/report.go +++ b/gopls/internal/vulncheck/vulntest/report.go @@ -15,6 +15,7 @@ import ( "time" "golang.org/x/mod/semver" + "golang.org/x/tools/gopls/internal/vulncheck/osv" "gopkg.in/yaml.v3" ) @@ -36,10 +37,15 @@ func readReport(in io.Reader) (*Report, error) { } // Report represents a vulnerability report in the vulndb. -// Remember to update doc/format.md when this structure changes. +// See https://go.googlesource.com/vulndb/+/refs/heads/master/doc/format.md type Report struct { + ID string `yaml:",omitempty"` + Modules []*Module `yaml:",omitempty"` + // Summary is a short phrase describing the vulnerability. + Summary string `yaml:",omitempty"` + // Description is the CVE description from an existing CVE. If we are // assigning a CVE ID ourselves, use CVEMetadata.Description instead. Description string `yaml:",omitempty"` @@ -153,10 +159,7 @@ var ReferenceTypes = []ReferenceType{ // // For ease of typing, References are represented in the YAML as a // single-element mapping of type to URL. -type Reference struct { - Type ReferenceType `json:"type,omitempty"` - URL string `json:"url,omitempty"` -} +type Reference osv.Reference func (r *Reference) MarshalYAML() (interface{}, error) { return map[string]string{ @@ -170,7 +173,7 @@ func (r *Reference) UnmarshalYAML(n *yaml.Node) (err error) { fmt.Sprintf("line %d: report.Reference must contain a mapping with one value", n.Line), }} } - r.Type = ReferenceType(strings.ToUpper(n.Content[0].Value)) + r.Type = osv.ReferenceType(strings.ToUpper(n.Content[0].Value)) r.URL = n.Content[1].Value return nil } diff --git a/gopls/internal/vulncheck/vulntest/testdata/GO-2020-0001.json b/gopls/internal/vulncheck/vulntest/testdata/GO-2020-0001.json new file mode 100644 index 00000000000..db371bd6930 --- /dev/null +++ b/gopls/internal/vulncheck/vulntest/testdata/GO-2020-0001.json @@ -0,0 +1,50 @@ +{ + "id": "GO-2020-0001", + "modified": "0001-01-01T00:00:00Z", + "published": "0001-01-01T00:00:00Z", + "details": "The default Formatter for the Logger middleware (LoggerConfig.Formatter),\nwhich is included in the Default engine, allows attackers to inject arbitrary\nlog entries by manipulating the request path.\n", + "affected": [ + { + "package": { + "name": "github.com/gin-gonic/gin", + "ecosystem": "Go" + }, + "ranges": [ + { + "type": "SEMVER", + "events": [ + { + "introduced": "0" + }, + { + "fixed": "1.6.0" + } + ] + } + ], + "ecosystem_specific": { + "imports": [ + { + "path": "github.com/gin-gonic/gin", + "symbols": [ + "defaultLogFormatter" + ] + } + ] + } + } + ], + "references": [ + { + "type": "FIX", + "url": "https://github.com/gin-gonic/gin/pull/1234" + }, + { + "type": "FIX", + "url": "https://github.com/gin-gonic/gin/commit/abcdefg" + } + ], + "database_specific": { + "url": "https://pkg.go.dev/vuln/GO-2020-0001" + } +} \ No newline at end of file From 995ecf714dae527bf54d7052a3742e816f515a6a Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Mon, 11 Sep 2023 15:55:26 -0400 Subject: [PATCH 084/178] gopls/internal/lsp/source: recover from inliner panics The inliner assumes well-typed input, but gopls does not provide it. This change uses recover to turn panics into errors if the packages were ill-typed. Change-Id: I3efb653e21393c4ecca27fabd88ccf8751b223f0 Reviewed-on: https://go-review.googlesource.com/c/tools/+/527395 Reviewed-by: Robert Findley Auto-Submit: Alan Donovan gopls-CI: kokoro LUCI-TryBot-Result: Go LUCI --- gopls/internal/lsp/source/inline.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/gopls/internal/lsp/source/inline.go b/gopls/internal/lsp/source/inline.go index 74eeed5b117..a9a8e493259 100644 --- a/gopls/internal/lsp/source/inline.go +++ b/gopls/internal/lsp/source/inline.go @@ -12,10 +12,12 @@ import ( "go/ast" "go/token" "go/types" + "runtime/debug" "golang.org/x/tools/go/analysis" "golang.org/x/tools/go/ast/astutil" "golang.org/x/tools/go/types/typeutil" + "golang.org/x/tools/gopls/internal/bug" "golang.org/x/tools/gopls/internal/lsp/protocol" "golang.org/x/tools/gopls/internal/lsp/safetoken" "golang.org/x/tools/gopls/internal/span" @@ -56,7 +58,7 @@ loop: return call, fn, nil } -func inlineCall(ctx context.Context, snapshot Snapshot, fh FileHandle, rng protocol.Range) (*token.FileSet, *analysis.SuggestedFix, error) { +func inlineCall(ctx context.Context, snapshot Snapshot, fh FileHandle, rng protocol.Range) (_ *token.FileSet, _ *analysis.SuggestedFix, err error) { // Find enclosing static call. callerPkg, callerPGF, err := NarrowestPackageForFile(ctx, snapshot, fh.URI()) if err != nil { @@ -86,6 +88,21 @@ func inlineCall(ctx context.Context, snapshot Snapshot, fh FileHandle, rng proto if calleeDecl == nil { return nil, nil, fmt.Errorf("can't find callee") } + + // The inliner assumes that input is well-typed, + // but that is frequently not the case within gopls. + // Until we are able to harden the inliner, + // report panics as errors to avoid crashing the server. + bad := func(p Package) bool { return len(p.GetParseErrors())+len(p.GetTypeErrors()) > 0 } + if bad(calleePkg) || bad(callerPkg) { + defer func() { + if x := recover(); x != nil { + err = bug.Errorf("inlining failed unexpectedly: %v\nstack: %v", + x, debug.Stack()) + } + }() + } + callee, err := inline.AnalyzeCallee(calleePkg.FileSet(), calleePkg.GetTypes(), calleePkg.GetTypesInfo(), calleeDecl, calleePGF.Src) if err != nil { return nil, nil, err From 8d6ad4609255441a8f79ce73abc36141c24178b1 Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Mon, 11 Sep 2023 18:09:03 -0400 Subject: [PATCH 085/178] gopls/internal/regtest: port the codelens marker tests Add a @codelenses marker to the marker test framework, which compares codelens in the current document with those annotated by @codelens markers. This is the first time that a marker consumes other markers, and it seems to have worked out OK. Notably, the old codelens marker tests weren't even running (!) because the marker function exited early for all but go.mod files. For golang/go#54845 Change-Id: I01b6b3b6e5f3f98f4378ab1b745a1a926824966d Reviewed-on: https://go-review.googlesource.com/c/tools/+/527735 Reviewed-by: Alan Donovan LUCI-TryBot-Result: Go LUCI gopls-CI: kokoro Auto-Submit: Robert Findley --- gopls/internal/lsp/lsp_test.go | 21 +---- gopls/internal/lsp/regtest/marker.go | 78 +++++++++++++++++-- .../lsp/testdata/codelens/codelens_test.go | 16 ---- .../lsp/testdata/generate/generate.go | 4 - .../internal/lsp/testdata/summary.txt.golden | 1 - .../lsp/testdata/summary_go1.21.txt.golden | 1 - gopls/internal/lsp/tests/tests.go | 41 +--------- gopls/internal/lsp/tests/util.go | 61 --------------- .../marker/testdata/codelens/generate.txt | 9 +++ .../regtest/marker/testdata/codelens/test.txt | 31 ++++++++ 10 files changed, 115 insertions(+), 148 deletions(-) delete mode 100644 gopls/internal/lsp/testdata/codelens/codelens_test.go delete mode 100644 gopls/internal/lsp/testdata/generate/generate.go create mode 100644 gopls/internal/regtest/marker/testdata/codelens/generate.txt create mode 100644 gopls/internal/regtest/marker/testdata/codelens/test.txt diff --git a/gopls/internal/lsp/lsp_test.go b/gopls/internal/lsp/lsp_test.go index 4fefbb57b0d..9934fee2d37 100644 --- a/gopls/internal/lsp/lsp_test.go +++ b/gopls/internal/lsp/lsp_test.go @@ -35,8 +35,8 @@ func TestMain(m *testing.M) { } // TestLSP runs the marker tests in files beneath testdata/ using -// implementations of each of the marker operations (e.g. @codelens) that -// make LSP RPCs (e.g. textDocument/codeLens) to a gopls server. +// implementations of each of the marker operations that make LSP RPCs to a +// gopls server. func TestLSP(t *testing.T) { tests.RunTests(t, "testdata", true, testLSP) } @@ -210,23 +210,6 @@ func (r *runner) CallHierarchy(t *testing.T, spn span.Span, expectedCalls *tests } } -func (r *runner) CodeLens(t *testing.T, uri span.URI, want []protocol.CodeLens) { - if !strings.HasSuffix(uri.Filename(), "go.mod") { - return - } - got, err := r.server.codeLens(r.ctx, &protocol.CodeLensParams{ - TextDocument: protocol.TextDocumentIdentifier{ - URI: protocol.DocumentURI(uri), - }, - }) - if err != nil { - t.Fatal(err) - } - if diff := tests.DiffCodeLens(uri, want, got); diff != "" { - t.Errorf("%s: %s", uri, diff) - } -} - func (r *runner) Diagnostics(t *testing.T, uri span.URI, want []*source.Diagnostic) { // Get the diagnostics for this view if we have not done it before. v := r.server.session.ViewByName(r.data.Config.Dir) diff --git a/gopls/internal/lsp/regtest/marker.go b/gopls/internal/lsp/regtest/marker.go index 993f9ed2c79..422acc75c3c 100644 --- a/gopls/internal/lsp/regtest/marker.go +++ b/gopls/internal/lsp/regtest/marker.go @@ -148,6 +148,14 @@ var update = flag.Bool("update", false, "if set, update test data during marker // - codeactionerr(kind, start, end, wantError): specifies a codeaction that // fails with an error that matches the expectation. // +// - codelens(location, title): specifies that a codelens is expected at the +// given location, with given title. Must be used in conjunction with +// @codelenses. +// +// - codelenses(): specifies that textDocument/codeLens should be run for the +// current document, with results compared to the @codelens annotations in +// the current document. +// // - complete(location, ...labels): specifies expected completion results at // the given location. // @@ -322,7 +330,6 @@ var update = flag.Bool("update", false, "if set, update test data during marker // // Existing marker tests (in ../testdata) to port: // - CallHierarchy -// - CodeLens // - Diagnostics // - CompletionItems // - Completions @@ -584,6 +591,7 @@ var markerFuncs = map[string]markerFunc{ "acceptcompletion": makeMarkerFunc(acceptCompletionMarker), "codeaction": makeMarkerFunc(codeActionMarker), "codeactionerr": makeMarkerFunc(codeActionErrMarker), + "codelenses": makeMarkerFunc(codeLensesMarker), "complete": makeMarkerFunc(completeMarker), "def": makeMarkerFunc(defMarker), "diag": makeMarkerFunc(diagMarker), @@ -949,8 +957,9 @@ type markerTestRun struct { locations map[expect.Identifier]protocol.Location diags map[protocol.Location][]protocol.Diagnostic // diagnostics by position; location end == start - // Notes that weren't consumed by a marker. - // TODO(rfindley): use this for markers that must collect related notes, or delete it. + // Notes that weren't associated with a top-level marker func. They may be + // consumed by another marker (e.g. @codelenses collects @codelens markers). + // Any notes that aren't consumed are flagged as an error. extraNotes map[protocol.DocumentURI]map[string][]*expect.Note } @@ -981,6 +990,11 @@ func (mark marker) uri() protocol.DocumentURI { return mark.run.env.Sandbox.Workdir.URI(mark.run.test.fset.File(mark.note.Pos).Name()) } +// path returns the relative path to the file containing the marker. +func (mark marker) path() string { + return mark.run.env.Sandbox.Workdir.RelPath(mark.run.test.fset.File(mark.note.Pos).Name()) +} + // fmtLoc formats the given pos in the context of the test, using // archive-relative paths for files and including the line number in the full // archive file. @@ -1347,7 +1361,7 @@ func acceptCompletionMarker(mark marker, src protocol.Location, label string, go mark.errorf("Completion(...) did not return an item labeled %q", label) return } - filename := mark.run.env.Sandbox.Workdir.URIToPath(mark.uri()) + filename := mark.path() mapper, err := mark.run.env.Editor.Mapper(filename) if err != nil { mark.errorf("Editor.Mapper(%s) failed: %v", filename, err) @@ -1404,7 +1418,7 @@ func foldingRangeMarker(mark marker, g *Golden) { insert(rng.StartLine, rng.StartCharacter, fmt.Sprintf("<%d kind=%q>", i, rng.Kind)) insert(rng.EndLine, rng.EndCharacter, fmt.Sprintf("", i)) } - filename := env.Sandbox.Workdir.URIToPath(mark.uri()) + filename := mark.path() mapper, err := env.Editor.Mapper(filename) if err != nil { mark.errorf("Editor.Mapper(%s) failed: %v", filename, err) @@ -1431,7 +1445,7 @@ func formatMarker(mark marker, golden *Golden) { got = []byte(err.Error() + "\n") // all golden content is newline terminated } else { env := mark.run.env - filename := env.Sandbox.Workdir.URIToPath(mark.uri()) + filename := mark.path() mapper, err := env.Editor.Mapper(filename) if err != nil { mark.errorf("Editor.Mapper(%s) failed: %v", filename, err) @@ -1650,6 +1664,58 @@ func codeActionErrMarker(mark marker, actionKind string, start, end protocol.Loc wantErr.check(mark, err) } +// codeLensesMarker runs the @codelenses() marker, collecting @codelens marks +// in the current file and comparing with the result of the +// textDocument/codeLens RPC. +func codeLensesMarker(mark marker) { + type codeLens struct { + Range protocol.Range + Title string + } + + lenses := mark.run.env.CodeLens(mark.path()) + var got []codeLens + for _, lens := range lenses { + title := "" + if lens.Command != nil { + title = lens.Command.Title + } + got = append(got, codeLens{lens.Range, title}) + } + + var want []codeLens + mark.collectExtraNotes("codelens", makeMarkerFunc(func(mark marker, loc protocol.Location, title string) { + want = append(want, codeLens{loc.Range, title}) + })) + + for _, s := range [][]codeLens{got, want} { + sort.Slice(s, func(i, j int) bool { + li, lj := s[i], s[j] + if c := protocol.CompareRange(li.Range, lj.Range); c != 0 { + return c < 0 + } + return li.Title < lj.Title + }) + } + + if diff := cmp.Diff(want, got); diff != "" { + mark.errorf("codelenses: unexpected diff (-want +got):\n%s", diff) + } +} + +// collectExtraNotes runs the provided markerFunc for each extra note with the +// given name, and marks all matching notes as used. +func (mark marker) collectExtraNotes(name string, f markerFunc) { + uri := mark.uri() + notes := mark.run.extraNotes[uri][name] + delete(mark.run.extraNotes[uri], name) + + for _, note := range notes { + mark := marker{run: mark.run, note: note, fn: f} + mark.execute() + } +} + // suggestedfixMarker implements the @suggestedfix(location, regexp, // kind, golden) marker. It acts like @diag(location, regexp), to set // the expectation of a diagnostic, but then it applies the first code diff --git a/gopls/internal/lsp/testdata/codelens/codelens_test.go b/gopls/internal/lsp/testdata/codelens/codelens_test.go deleted file mode 100644 index f6c696416a8..00000000000 --- a/gopls/internal/lsp/testdata/codelens/codelens_test.go +++ /dev/null @@ -1,16 +0,0 @@ -package codelens //@codelens("package codelens", "run file benchmarks", "test") - -import "testing" - -func TestMain(m *testing.M) {} // no code lens for TestMain - -func TestFuncWithCodeLens(t *testing.T) { //@codelens("func", "run test", "test") -} - -func thisShouldNotHaveACodeLens(t *testing.T) { -} - -func BenchmarkFuncWithCodeLens(b *testing.B) { //@codelens("func", "run benchmark", "test") -} - -func helper() {} // expect no code lens diff --git a/gopls/internal/lsp/testdata/generate/generate.go b/gopls/internal/lsp/testdata/generate/generate.go deleted file mode 100644 index ae5e90d1a48..00000000000 --- a/gopls/internal/lsp/testdata/generate/generate.go +++ /dev/null @@ -1,4 +0,0 @@ -package generate - -//go:generate echo Hi //@ codelens("//go:generate", "run go generate", "generate"), codelens("//go:generate", "run go generate ./...", "generate") -//go:generate echo I shall have no CodeLens diff --git a/gopls/internal/lsp/testdata/summary.txt.golden b/gopls/internal/lsp/testdata/summary.txt.golden index e7b1d1ed638..c814ebb9322 100644 --- a/gopls/internal/lsp/testdata/summary.txt.golden +++ b/gopls/internal/lsp/testdata/summary.txt.golden @@ -1,6 +1,5 @@ -- summary -- CallHierarchyCount = 2 -CodeLensCount = 5 CompletionsCount = 264 CompletionSnippetCount = 115 UnimportedCompletionsCount = 5 diff --git a/gopls/internal/lsp/testdata/summary_go1.21.txt.golden b/gopls/internal/lsp/testdata/summary_go1.21.txt.golden index 0fce9b32b6d..2e78a46913a 100644 --- a/gopls/internal/lsp/testdata/summary_go1.21.txt.golden +++ b/gopls/internal/lsp/testdata/summary_go1.21.txt.golden @@ -1,6 +1,5 @@ -- summary -- CallHierarchyCount = 2 -CodeLensCount = 5 CompletionsCount = 263 CompletionSnippetCount = 115 UnimportedCompletionsCount = 5 diff --git a/gopls/internal/lsp/tests/tests.go b/gopls/internal/lsp/tests/tests.go index c8a1df772fb..87d2a2257bb 100644 --- a/gopls/internal/lsp/tests/tests.go +++ b/gopls/internal/lsp/tests/tests.go @@ -26,7 +26,6 @@ import ( "golang.org/x/tools/go/expect" "golang.org/x/tools/go/packages" "golang.org/x/tools/go/packages/packagestest" - "golang.org/x/tools/gopls/internal/lsp/command" "golang.org/x/tools/gopls/internal/lsp/protocol" "golang.org/x/tools/gopls/internal/lsp/safetoken" "golang.org/x/tools/gopls/internal/lsp/source" @@ -63,7 +62,6 @@ var UpdateGolden = flag.Bool("golden", false, "Update golden files") // These type names apparently avoid the need to repeat the // type in the field name and the make() expression. type CallHierarchy = map[span.Span]*CallHierarchyResult -type CodeLens = map[span.URI][]protocol.CodeLens type Diagnostics = map[span.URI][]*source.Diagnostic type CompletionItems = map[token.Pos]*completion.CompletionItem type Completions = map[span.Span][]Completion @@ -89,7 +87,6 @@ type Data struct { Config packages.Config Exported *packagestest.Exported CallHierarchy CallHierarchy - CodeLens CodeLens Diagnostics Diagnostics CompletionItems CompletionItems Completions Completions @@ -123,14 +120,13 @@ type Data struct { } // The Tests interface abstracts the LSP-based implementation of the marker -// test operators (such as @codelens) appearing in files beneath ../testdata/. +// test operators appearing in files beneath ../testdata/. // // TODO(adonovan): reduce duplication; see https://github.com/golang/go/issues/54845. // There is only one implementation (*runner in ../lsp_test.go), so // we can abolish the interface now. type Tests interface { CallHierarchy(*testing.T, span.Span, *CallHierarchyResult) - CodeLens(*testing.T, span.URI, []protocol.CodeLens) Diagnostics(*testing.T, span.URI, []*source.Diagnostic) Completion(*testing.T, span.Span, Completion, CompletionItems) CompletionSnippet(*testing.T, span.Span, CompletionSnippet, bool, CompletionItems) @@ -232,7 +228,6 @@ func DefaultOptions(o *source.Options) { source.Work: {}, source.Tmpl: {}, } - o.UserOptions.Codelenses[string(command.Test)] = true o.HoverKind = source.SynopsisDocumentation o.InsertTextFormat = protocol.SnippetTextFormat o.CompletionBudget = time.Minute @@ -267,7 +262,6 @@ func RunTests(t *testing.T, dataDir string, includeMultiModule bool, f func(*tes func load(t testing.TB, mode string, dir string) *Data { datum := &Data{ CallHierarchy: make(CallHierarchy), - CodeLens: make(CodeLens), Diagnostics: make(Diagnostics), CompletionItems: make(CompletionItems), Completions: make(Completions), @@ -417,7 +411,6 @@ func load(t testing.TB, mode string, dir string) *Data { // Collect any data that needs to be used by subsequent tests. if err := datum.Exported.Expect(map[string]interface{}{ - "codelens": datum.collectCodeLens, "diag": datum.collectDiagnostics, "item": datum.collectCompletionItems, "complete": datum.collectCompletions(CompletionDefault), @@ -579,20 +572,6 @@ func Run(t *testing.T, tests Tests, data *Data) { eachCompletion(t, data.RankCompletions, tests.RankCompletion) }) - t.Run("CodeLens", func(t *testing.T) { - t.Helper() - for uri, want := range data.CodeLens { - // Check if we should skip this URI if the -modfile flag is not available. - if shouldSkip(data, uri) { - continue - } - t.Run(uriName(uri), func(t *testing.T) { - t.Helper() - tests.CodeLens(t, uri, want) - }) - } - }) - t.Run("Diagnostics", func(t *testing.T) { t.Helper() for uri, want := range data.Diagnostics { @@ -785,15 +764,7 @@ func checkData(t *testing.T, data *Data) { return count } - countCodeLens := func(c map[span.URI][]protocol.CodeLens) (count int) { - for _, want := range c { - count += len(want) - } - return count - } - fmt.Fprintf(buf, "CallHierarchyCount = %v\n", len(data.CallHierarchy)) - fmt.Fprintf(buf, "CodeLensCount = %v\n", countCodeLens(data.CodeLens)) fmt.Fprintf(buf, "CompletionsCount = %v\n", countCompletions(data.Completions)) fmt.Fprintf(buf, "CompletionSnippetCount = %v\n", snippetCount) fmt.Fprintf(buf, "UnimportedCompletionsCount = %v\n", countCompletions(data.UnimportedCompletions)) @@ -893,16 +864,6 @@ func (data *Data) Golden(t *testing.T, tag, target string, update func() ([]byte return file.Data[:len(file.Data)-1] // drop the trailing \n } -func (data *Data) collectCodeLens(spn span.Span, title, cmd string) { - data.CodeLens[spn.URI()] = append(data.CodeLens[spn.URI()], protocol.CodeLens{ - Range: data.mustRange(spn), - Command: &protocol.Command{ - Title: title, - Command: cmd, - }, - }) -} - func (data *Data) collectDiagnostics(spn span.Span, msgSource, msgPattern, msgSeverity string) { severity := protocol.SeverityError switch msgSeverity { diff --git a/gopls/internal/lsp/tests/util.go b/gopls/internal/lsp/tests/util.go index 99a6393280e..67b939087c6 100644 --- a/gopls/internal/lsp/tests/util.go +++ b/gopls/internal/lsp/tests/util.go @@ -172,67 +172,6 @@ func inRange(p protocol.Position, r protocol.Range) bool { return false } -func DiffCodeLens(uri span.URI, want, got []protocol.CodeLens) string { - sortCodeLens(want) - sortCodeLens(got) - - if len(got) != len(want) { - return summarizeCodeLens(-1, uri, want, got, "different lengths got %v want %v", len(got), len(want)) - } - for i, w := range want { - g := got[i] - if w.Command.Command != g.Command.Command { - return summarizeCodeLens(i, uri, want, got, "incorrect Command Name got %v want %v", g.Command.Command, w.Command.Command) - } - if w.Command.Title != g.Command.Title { - return summarizeCodeLens(i, uri, want, got, "incorrect Command Title got %v want %v", g.Command.Title, w.Command.Title) - } - if protocol.ComparePosition(w.Range.Start, g.Range.Start) != 0 { - return summarizeCodeLens(i, uri, want, got, "incorrect Start got %v want %v", g.Range.Start, w.Range.Start) - } - if !protocol.IsPoint(g.Range) { // Accept any 'want' range if the codelens returns a zero-length range. - if protocol.ComparePosition(w.Range.End, g.Range.End) != 0 { - return summarizeCodeLens(i, uri, want, got, "incorrect End got %v want %v", g.Range.End, w.Range.End) - } - } - } - return "" -} - -func sortCodeLens(c []protocol.CodeLens) { - sort.Slice(c, func(i int, j int) bool { - if r := protocol.CompareRange(c[i].Range, c[j].Range); r != 0 { - return r < 0 - } - if c[i].Command.Command < c[j].Command.Command { - return true - } else if c[i].Command.Command == c[j].Command.Command { - return c[i].Command.Title < c[j].Command.Title - } else { - return false - } - }) -} - -func summarizeCodeLens(i int, uri span.URI, want, got []protocol.CodeLens, reason string, args ...interface{}) string { - msg := &bytes.Buffer{} - fmt.Fprint(msg, "codelens failed") - if i >= 0 { - fmt.Fprintf(msg, " at %d", i) - } - fmt.Fprint(msg, " because of ") - fmt.Fprintf(msg, reason, args...) - fmt.Fprint(msg, ":\nexpected:\n") - for _, d := range want { - fmt.Fprintf(msg, " %s:%v: %s | %s\n", uri, d.Range, d.Command.Command, d.Command.Title) - } - fmt.Fprintf(msg, "got:\n") - for _, d := range got { - fmt.Fprintf(msg, " %s:%v: %s | %s\n", uri, d.Range, d.Command.Command, d.Command.Title) - } - return msg.String() -} - func DiffSignatures(spn span.Span, want, got *protocol.SignatureHelp) string { decorate := func(f string, args ...interface{}) string { return fmt.Sprintf("invalid signature at %s: %s", spn, fmt.Sprintf(f, args...)) diff --git a/gopls/internal/regtest/marker/testdata/codelens/generate.txt b/gopls/internal/regtest/marker/testdata/codelens/generate.txt new file mode 100644 index 00000000000..086c961f07d --- /dev/null +++ b/gopls/internal/regtest/marker/testdata/codelens/generate.txt @@ -0,0 +1,9 @@ +This test exercises the "generate" codelens. + +-- generate.go -- +//@codelenses() + +package generate + +//go:generate echo Hi //@ codelens("//go:generate", "run go generate"), codelens("//go:generate", "run go generate ./...") +//go:generate echo I shall have no CodeLens diff --git a/gopls/internal/regtest/marker/testdata/codelens/test.txt b/gopls/internal/regtest/marker/testdata/codelens/test.txt new file mode 100644 index 00000000000..90782bddef9 --- /dev/null +++ b/gopls/internal/regtest/marker/testdata/codelens/test.txt @@ -0,0 +1,31 @@ +This file tests codelenses for test functions. + +TODO: for some reason these code lens have zero width. Does that affect their +utility/visibility in various LSP clients? + +-- settings.json -- +{ + "codelenses": { + "test": true + } +} + +-- p_test.go -- +//@codelenses() + +package codelens //@codelens(re"()package codelens", "run file benchmarks") + +import "testing" + +func TestMain(m *testing.M) {} // no code lens for TestMain + +func TestFuncWithCodeLens(t *testing.T) { //@codelens(re"()func", "run test") +} + +func thisShouldNotHaveACodeLens(t *testing.T) { +} + +func BenchmarkFuncWithCodeLens(b *testing.B) { //@codelens(re"()func", "run benchmark") +} + +func helper() {} // expect no code lens From ad827af96d4e28fafac7d4aaea42d16a0b836913 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Sun, 10 Sep 2023 00:33:44 -0400 Subject: [PATCH 086/178] internal/refactor/inline: add table-driven test This change adds a table-driven test of "one liner" callers, callees, and expected results/errors. The various strategies need much more extensive testing of minor variants than is practical with the .txtar approach. Also: - add assertions that types.Info is correctly populated; - opt: skip computation of unreferenced imports if there were none to begin with; - add fix and test for a bug in the "infallible" literalization strategy (spread calls to methods). - Clone syntax trees before splicing, otherwise we end up putting caller trees into the callee and then clearing their positions. - Move TODO comment re: var decl approach to parameter assignment. Change-Id: I44a9825ab87f12700423fb00aa7df97f81d12e81 Reviewed-on: https://go-review.googlesource.com/c/tools/+/527195 LUCI-TryBot-Result: Go LUCI Reviewed-by: Robert Findley gopls-CI: kokoro --- internal/refactor/inline/callee.go | 1 + internal/refactor/inline/inline.go | 154 +++++++++++--- internal/refactor/inline/inline_test.go | 261 ++++++++++++++++++++++-- internal/refactor/inline/util.go | 9 + 4 files changed, 372 insertions(+), 53 deletions(-) diff --git a/internal/refactor/inline/callee.go b/internal/refactor/inline/callee.go index e8f452d337a..8ccfeea5e3d 100644 --- a/internal/refactor/inline/callee.go +++ b/internal/refactor/inline/callee.go @@ -70,6 +70,7 @@ type object struct { // golang.org/x/tools/go/analysis framework: the inlining information // about a callee can be recorded as a "fact". func AnalyzeCallee(fset *token.FileSet, pkg *types.Package, info *types.Info, decl *ast.FuncDecl, content []byte) (*Callee, error) { + checkInfoFields(info) // The client is expected to have determined that the callee // is a function with a declaration (not a built-in or var). diff --git a/internal/refactor/inline/inline.go b/internal/refactor/inline/inline.go index 1778d5ea518..4b9e2fad8c6 100644 --- a/internal/refactor/inline/inline.go +++ b/internal/refactor/inline/inline.go @@ -310,7 +310,7 @@ func Inline(logf func(string, ...any), caller *Caller, callee *Callee) ([]byte, logf = func(string, ...any) {} // discard } logf("inline %s @ %v", - formatNode(caller.Fset, caller.Call), + debugFormatNode(caller.Fset, caller.Call), caller.Fset.Position(caller.Call.Lparen)) res, err := inline(logf, caller, &callee.impl) @@ -331,17 +331,19 @@ func Inline(logf func(string, ...any), caller *Caller, callee *Callee) ([]byte, { start := offsetOf(caller.Fset, res.old.Pos()) end := offsetOf(caller.Fset, res.old.End()) + var out bytes.Buffer + out.Write(caller.Content[:start]) // TODO(adonovan): might it make more sense to use // callee.Fset when formatting res.new?? - newFile := string(caller.Content[:start]) + - formatNode(caller.Fset, res.new) + - string(caller.Content[end:]) + if err := format.Node(&out, caller.Fset, res.new); err != nil { + return nil, err + } + out.Write(caller.Content[end:]) const mode = parser.ParseComments | parser.SkipObjectResolution | parser.AllErrors - var err error - f, err = parser.ParseFile(caller.Fset, "callee.go", newFile, mode) + f, err = parser.ParseFile(caller.Fset, "callee.go", &out, mode) if err != nil { // Something has gone very wrong. - logf("failed to parse <<%s>>", newFile) // debugging + logf("failed to parse <<%s>>", &out) // debugging return nil, err } } @@ -376,6 +378,12 @@ func Inline(logf func(string, ...any), caller *Caller, callee *Callee) ([]byte, } } + var out bytes.Buffer + if err := format.Node(&out, caller.Fset, f); err != nil { + return nil, err + } + newSrc := out.Bytes() + // Remove imports that are no longer referenced. // // It ought to be possible to compute the set of PkgNames used @@ -425,16 +433,17 @@ func Inline(logf func(string, ...any), caller *Caller, callee *Callee) ([]byte, // We could invoke imports.Process and parse its result, // compare against the original AST, compute a list of import // fixes, and return that too. - var out bytes.Buffer - if err := format.Node(&out, caller.Fset, f); err != nil { - return nil, err - } - formatted, err := imports.Process("output", out.Bytes(), nil) - if err != nil { - logf("cannot reformat: %v <<%s>>", err, &out) - return nil, err // cannot reformat (a bug?) + + // Recompute imports only if there were existing ones. + if len(f.Imports) > 0 { + formatted, err := imports.Process("output", newSrc, nil) + if err != nil { + logf("cannot reformat: %v <<%s>>", err, &out) + return nil, err // cannot reformat (a bug?) + } + newSrc = formatted } - return formatted, nil + return newSrc, nil } type result struct { @@ -462,6 +471,8 @@ type result struct { // representation, such as any proposed solution to #20744, or even // dst or some private fork of go/ast.) func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*result, error) { + checkInfoFields(caller.Info) + // Inlining of dynamic calls is not currently supported, // even for local closure calls. (This would be a lot of work.) calleeSymbol := typeutil.StaticCallee(caller.Info, caller.Call) @@ -632,9 +643,10 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu } // replaceCalleeID replaces an identifier in the callee. + // The replacement tree must not belong to the caller; use cloneNode as needed. replaceCalleeID := func(offset int, repl ast.Expr) { id := findIdent(calleeDecl, calleeDecl.Pos()+token.Pos(offset)) - logf("- replace id %q @ #%d to %q", id.Name, offset, formatNode(calleeFset, repl)) + logf("- replace id %q @ #%d to %q", id.Name, offset, debugFormatNode(calleeFset, repl)) replaceNode(calleeDecl, id, repl) } @@ -812,12 +824,12 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu if is[*types.Tuple](arg.typ) { // TODO(adonovan): handle elimination of spread arguments. logf("keeping param %q: argument %s is spread", - param.Name, formatNode(caller.Fset, arg.expr)) - continue + param.Name, debugFormatNode(caller.Fset, arg.expr)) + break // spread => last argument, but not last parameter } if !arg.pure { logf("keeping param %q: argument %s is impure", - param.Name, formatNode(caller.Fset, arg.expr)) + param.Name, debugFormatNode(caller.Fset, arg.expr)) continue // unsafe to change order or cardinality of effects } if len(param.Refs) > 1 && !arg.duplicable { @@ -872,10 +884,13 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // It is safe to eliminate param and replace it with arg. // No additional parens are required around arg for // the supported "pure" expressions. + // + // Because arg.expr belongs to the caller, + // we clone it before splicing it into the callee tree. logf("replacing parameter %q by argument %q", - param.Name, formatNode(caller.Fset, arg.expr)) + param.Name, debugFormatNode(caller.Fset, arg.expr)) for _, ref := range param.Refs { - replaceCalleeID(ref, arg.expr) + replaceCalleeID(ref, cloneNode(arg.expr).(ast.Expr)) } eliminatedParams[i] = true args[i] = nil @@ -889,6 +904,19 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu } } + // TODO(adonovan): eliminate all remaining parameters + // by replacing a call f(a1, a2) + // to func f(x T1, y T2) {body} by + // { var x T1 = a1 + // var y T2 = a2 + // body } + // if x ∉ freevars(a2) or freevars(T2), and so on, + // plus the usual checks for return conversions (if any), + // complex control, etc. + // + // If viable, use this with the reduction strategies below + // that produce a block (not a value). + // -- let the inlining strategies begin -- // TODO(adonovan): split this huge function into a sequence of @@ -1044,9 +1072,9 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // Inlining: // return f(args) // where: - // func f(params) (results) { ...body... } + // func f(params) (results) { body } // reduces to: - // ...body... + // { body } // so long as: // - all parameters are eliminated; // - call is a tail-call; @@ -1059,6 +1087,10 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // // TODO(adonovan): omit the braces if the sets of // names in the two blocks are disjoint. + // + // TODO(adonovan): add a strategy for a 'void tail + // call', i.e. a call statement prior to an (explicit + // or implicit) return. if ret, ok := callContext(callerPath).(*ast.ReturnStmt); ok && len(ret.Results) == 1 && callee.TrivialReturns == callee.TotalReturns && @@ -1122,15 +1154,21 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // in addition to the usual checks for arg/result conversions, // complex control, etc. // Also test cases where expr is an n-ary call (spread returns). + } - // TODO(adonovan): replace a call f(a1, a2) - // to func f(x T1, y T2) {body} by - // { var x T1 = a1 - // var y T2 = a2 - // body } - // if x ∉ freevars(a2) or freevars(T2), and so on, - // plus the usual checks for return conversions (if any), - // complex control, etc. + // Literalization isn't quite infallible. + // Consider a spread call to a method in which + // no parameters are eliminated, e.g. + // new(T).f(g()) + // where + // func (recv *T) f(x, y int) { body } + // func g() (int, int) + // This would be literalized to: + // func (recv *T, x, y int) { body }(new(T), g()), + // which is not a valid argument list because g() must appear alone. + // Reject this case for now. + if len(args) == 2 && args[0] != nil && args[1] != nil && is[*types.Tuple](args[1].typ) { + return nil, fmt.Errorf("can't yet inline spread call to method") } // Infallible general case: literalization. @@ -1547,6 +1585,54 @@ func replaceNode(root ast.Node, from, to ast.Node) { } } +// cloneNode returns a deep copy of a Node. +// It omits pointers to ast.{Scope,Object} variables. +func cloneNode(n ast.Node) ast.Node { + var clone func(x reflect.Value) reflect.Value + clone = func(x reflect.Value) reflect.Value { + switch x.Kind() { + case reflect.Ptr: + if x.IsNil() { + return x + } + // Skip fields of types potentially involved in cycles. + switch x.Interface().(type) { + case *ast.Object, *ast.Scope: + return reflect.Zero(x.Type()) + } + y := reflect.New(x.Type().Elem()) + y.Elem().Set(clone(x.Elem())) + return y + + case reflect.Struct: + y := reflect.New(x.Type()).Elem() + for i := 0; i < x.Type().NumField(); i++ { + y.Field(i).Set(clone(x.Field(i))) + } + return y + + case reflect.Slice: + y := reflect.MakeSlice(x.Type(), x.Len(), x.Cap()) + for i := 0; i < x.Len(); i++ { + y.Index(i).Set(clone(x.Index(i))) + } + return y + + case reflect.Interface: + y := reflect.New(x.Type()).Elem() + y.Set(clone(x.Elem())) + return y + + case reflect.Array, reflect.Chan, reflect.Func, reflect.Map, reflect.UnsafePointer: + panic(x) // unreachable in AST + + default: + return x // bool, string, number + } + } + return clone(reflect.ValueOf(n)).Interface().(ast.Node) +} + // clearPositions destroys token.Pos information within the tree rooted at root, // as positions in callee trees may cause caller comments to be emitted prematurely. // @@ -1606,7 +1692,9 @@ func prepend[T any](elem T, slice ...T) []T { return append([]T{elem}, slice...) } -func formatNode(fset *token.FileSet, n ast.Node) string { +// debugFormatNode formats a node or returns a formatting error. +// Its sloppy treatment of errors is appropriate only for logging. +func debugFormatNode(fset *token.FileSet, n ast.Node) string { var out strings.Builder if err := format.Node(&out, fset, n); err != nil { out.WriteString(err.Error()) diff --git a/internal/refactor/inline/inline_test.go b/internal/refactor/inline/inline_test.go index 2fcd7b582bc..11392e0b4ea 100644 --- a/internal/refactor/inline/inline_test.go +++ b/internal/refactor/inline/inline_test.go @@ -11,11 +11,14 @@ import ( "encoding/gob" "fmt" "go/ast" + "go/parser" "go/token" + "go/types" "os" "path/filepath" "reflect" "regexp" + "strings" "testing" "unsafe" @@ -29,8 +32,8 @@ import ( "golang.org/x/tools/txtar" ) -// Test executes test scenarios specified by files in testdata/*.txtar. -func Test(t *testing.T) { +// TestData executes test scenarios specified by files in testdata/*.txtar. +func TestData(t *testing.T) { testenv.NeedsGoPackages(t) files, err := filepath.Glob("testdata/*.txtar") @@ -277,25 +280,12 @@ func doInlineNote(logf func(string, ...any), pkg *packages.Package, file *ast.Fi return nil, err } - // Perform Gob transcoding so that it is exercised by the test. - var enc bytes.Buffer - if err := gob.NewEncoder(&enc).Encode(callee); err != nil { - return nil, fmt.Errorf("internal error: gob encoding failed: %v", err) - } - *callee = inline.Callee{} - if err := gob.NewDecoder(&enc).Decode(callee); err != nil { - return nil, fmt.Errorf("internal error: gob decoding failed: %v", err) + if err := checkTranscode(callee); err != nil { + return nil, err } - // Check that the operation didn't mutate the tree. - pre := deepHash(caller.File) - defer func() { - post := deepHash(caller.File) - if pre != post { - panic("Inline mutated caller.File") - } - }() - + check := checkNoMutation(caller.File) + defer check() return inline.Inline(logf, caller, callee) }() if err != nil { @@ -322,6 +312,237 @@ func doInlineNote(logf func(string, ...any), pkg *packages.Package, file *ast.Fi } +// TestTable is a table driven test, enabling more compact expression +// of single-package test cases than is possible with the txtar notation. +func TestTable(t *testing.T) { + // Each callee must declare a function or method named f, + // and each caller must call it. + const funcName = "f" + + var tests = []struct { + descr string + callee, caller string // Go source files (sans package decl) of caller, callee + want string // expected new portion of caller file, or "error: regexp" + }{ + { + "Basic", + `func f(x int) int { return x }`, + `var _ = f(0)`, + `var _ = (0)`, + }, + { + "Empty body, no arg effects.", + `func f(x, y int) {}`, + `func _() { f(1, 2) }`, + `func _() {}`, + }, + { + "Empty body, some arg effects.", + `func f(x, y, z int) {}`, + `func _() { f(1, recover().(int), 3) }`, + `func _() { _ = recover().(int) }`, + }, + { + "Tail call.", + `func f() int { return 1 }`, + `func _() int { return f() }`, + `func _() int { return (1) }`, + }, + { + "Void tail call.", + `func f() { println() }`, + `func _() { f() }`, + `func _() { println() }`, + }, + { + "Void tail call with defer.", // => literalized + `func f() { defer f(); println() }`, + `func _() { f() }`, + `func _() { func() { defer f(); println() }() }`, + }, + { + "Edge case: cannot literalize spread method call.", + `type I int + func g() (I, I) + func (r I) f(x, y I) I { + defer g() // force literalization + return x + y + r + }`, + `func _() I { return recover().(I).f(g()) }`, + `error: can't yet inline spread call to method`, + }, + // TODO(adonovan): improve coverage of the cross + // product of each strategy with the checklist of + // concerns enumerated in the package doc comment. + } + for _, test := range tests { + test := test + t.Run(test.descr, func(t *testing.T) { + fset := token.NewFileSet() + mustParse := func(filename string, content any) *ast.File { + f, err := parser.ParseFile(fset, filename, content, parser.ParseComments|parser.SkipObjectResolution) + if err != nil { + t.Fatalf("ParseFile: %v", err) + } + return f + } + + // Parse callee file and find first func decl named f. + calleeContent := "package p\n" + test.callee + calleeFile := mustParse("callee.go", calleeContent) + var decl *ast.FuncDecl + for _, d := range calleeFile.Decls { + if d, ok := d.(*ast.FuncDecl); ok && d.Name.Name == funcName { + decl = d + break + } + } + if decl == nil { + t.Fatalf("declaration of func %s not found: %s", funcName, test.callee) + } + + // Parse caller file and find first call to f(). + callerContent := "package p\n" + test.caller + callerFile := mustParse("caller.go", callerContent) + var call *ast.CallExpr + ast.Inspect(callerFile, func(n ast.Node) bool { + if n, ok := n.(*ast.CallExpr); ok { + switch fun := n.Fun.(type) { + case *ast.SelectorExpr: + if fun.Sel.Name == funcName { + call = n + } + case *ast.Ident: + if fun.Name == funcName { + call = n + } + } + } + return call == nil + }) + if call == nil { + t.Fatalf("call to %s not found: %s", funcName, test.caller) + } + + // Type check both files as one package. + info := &types.Info{ + Defs: make(map[*ast.Ident]types.Object), + Uses: make(map[*ast.Ident]types.Object), + Types: make(map[ast.Expr]types.TypeAndValue), + Implicits: make(map[ast.Node]types.Object), + Selections: make(map[*ast.SelectorExpr]*types.Selection), + Scopes: make(map[ast.Node]*types.Scope), + } + conf := &types.Config{Error: func(err error) { t.Error(err) }} + pkg, err := conf.Check("p", fset, []*ast.File{callerFile, calleeFile}, info) + if err != nil { + t.Fatal(err) + } + + // Analyze callee and inline call. + doIt := func() ([]byte, error) { + callee, err := inline.AnalyzeCallee(fset, pkg, info, decl, []byte(calleeContent)) + if err != nil { + return nil, err + } + if err := checkTranscode(callee); err != nil { + t.Fatal(err) + } + + caller := &inline.Caller{ + Fset: fset, + Types: pkg, + Info: info, + File: callerFile, + Call: call, + Content: []byte(callerContent), + } + check := checkNoMutation(caller.File) + defer check() + return inline.Inline(t.Logf, caller, callee) + } + gotContent, err := doIt() + + // Want error? + if rest := strings.TrimPrefix(test.want, "error: "); rest != test.want { + if err == nil { + t.Fatalf("unexpected sucess: want error matching %q", rest) + } + msg := err.Error() + if ok, err := regexp.MatchString(rest, msg); err != nil { + t.Fatalf("invalid regexp: %v", err) + } else if !ok { + t.Fatalf("wrong error: %s (want match for %q)", msg, rest) + } + return + } + + // Want success. + if err != nil { + t.Fatal(err) + } + + // Compute a single-hunk line-based diff. + srcLines := strings.Split(callerContent, "\n") + gotLines := strings.Split(string(gotContent), "\n") + for len(srcLines) > 0 && len(gotLines) > 0 && + srcLines[0] == gotLines[0] { + srcLines = srcLines[1:] + gotLines = gotLines[1:] + } + for len(srcLines) > 0 && len(gotLines) > 0 && + srcLines[len(srcLines)-1] == gotLines[len(gotLines)-1] { + srcLines = srcLines[:len(srcLines)-1] + gotLines = gotLines[:len(gotLines)-1] + } + got := strings.Join(gotLines, "\n") + + if strings.TrimSpace(got) != strings.TrimSpace(test.want) { + t.Errorf("\nInlining this call:\t%s\nof this callee: \t%s\nproduced:\n%s\nWant:\n\n%s", + test.caller, + test.callee, + got, + test.want) + } + + // Check that resulting code type-checks. + newCallerFile := mustParse("newcaller.go", gotContent) + if _, err := conf.Check("p", fset, []*ast.File{newCallerFile, calleeFile}, nil); err != nil { + t.Fatalf("modified source failed to typecheck: <<%s>>", gotContent) + } + }) + } +} + +// -- helpers -- + +// checkNoMutation returns a function that, when called, +// asserts that file was not modified since the checkNoMutation call. +func checkNoMutation(file *ast.File) func() { + pre := deepHash(file) + return func() { + post := deepHash(file) + if pre != post { + panic("Inline mutated caller.File") + } + } +} + +// checkTranscode replaces *callee by the results of gob-encoding and +// then decoding it, to test that these operations are lossless. +func checkTranscode(callee *inline.Callee) error { + // Perform Gob transcoding so that it is exercised by the test. + var enc bytes.Buffer + if err := gob.NewEncoder(&enc).Encode(callee); err != nil { + return fmt.Errorf("internal error: gob encoding failed: %v", err) + } + *callee = inline.Callee{} + if err := gob.NewDecoder(&enc).Decode(callee); err != nil { + return fmt.Errorf("internal error: gob decoding failed: %v", err) + } + return nil +} + // TODO(adonovan): publish this a helper (#61386). func extractTxtar(ar *txtar.Archive, dir string) error { for _, file := range ar.Files { @@ -344,7 +565,7 @@ func extractTxtar(ar *txtar.Archive, dir string) error { // // TODO(adonovan): consider a variant that reports where in the tree // the mutation occurred (obviously at a cost in space). -func deepHash(n ast.Node) [sha256.Size]byte { +func deepHash(n ast.Node) any { seen := make(map[unsafe.Pointer]bool) // to break cycles hasher := sha256.New() diff --git a/internal/refactor/inline/util.go b/internal/refactor/inline/util.go index 4b6d45efa10..87b0f7b0228 100644 --- a/internal/refactor/inline/util.go +++ b/internal/refactor/inline/util.go @@ -44,3 +44,12 @@ func within(pos token.Pos, n ast.Node) bool { func trivialConversion(val types.Type, obj *types.Var) bool { return types.Identical(types.Default(val), obj.Type()) } + +func checkInfoFields(info *types.Info) { + assert(info.Defs != nil, "types.Info.Defs is nil") + assert(info.Implicits != nil, "types.Info.Implicits is nil") + assert(info.Scopes != nil, "types.Info.Scopes is nil") + assert(info.Selections != nil, "types.Info.Selections is nil") + assert(info.Types != nil, "types.Info.Types is nil") + assert(info.Uses != nil, "types.Info.Uses is nil") +} From 0b3914d33537f633192b70686cf9f0e34355f4cf Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Tue, 12 Sep 2023 13:34:23 -0400 Subject: [PATCH 087/178] go/analysis/passes/defers: rename Analyzer.Name to "defers" ...to match package name and avoid a minor nuisance. Change-Id: Ifb16c6ea60cf349be36ae35eee281e95423ca24d Reviewed-on: https://go-review.googlesource.com/c/tools/+/527736 LUCI-TryBot-Result: Go LUCI Reviewed-by: Robert Findley Auto-Submit: Alan Donovan gopls-CI: kokoro --- go/analysis/passes/defers/{defer.go => defers.go} | 7 ++++--- .../passes/defers/{defer_test.go => defers_test.go} | 0 go/analysis/passes/defers/doc.go | 6 +++--- gopls/doc/analyzers.md | 4 ++-- gopls/doc/generate_test.go | 2 +- gopls/internal/lsp/source/api_json.go | 9 +++++---- 6 files changed, 15 insertions(+), 13 deletions(-) rename go/analysis/passes/defers/{defer.go => defers.go} (87%) rename go/analysis/passes/defers/{defer_test.go => defers_test.go} (100%) diff --git a/go/analysis/passes/defers/defer.go b/go/analysis/passes/defers/defers.go similarity index 87% rename from go/analysis/passes/defers/defer.go rename to go/analysis/passes/defers/defers.go index 19474bcc4e8..ed2a122f2b3 100644 --- a/go/analysis/passes/defers/defer.go +++ b/go/analysis/passes/defers/defers.go @@ -19,11 +19,12 @@ import ( //go:embed doc.go var doc string -// Analyzer is the defer analyzer. +// Analyzer is the defers analyzer. var Analyzer = &analysis.Analyzer{ - Name: "defer", + Name: "defers", Requires: []*analysis.Analyzer{inspect.Analyzer}, - Doc: analysisutil.MustExtractDoc(doc, "defer"), + URL: "https://pkg.go.dev/golang.org/x/tools/go/analysis/passes/defers", + Doc: analysisutil.MustExtractDoc(doc, "defers"), Run: run, } diff --git a/go/analysis/passes/defers/defer_test.go b/go/analysis/passes/defers/defers_test.go similarity index 100% rename from go/analysis/passes/defers/defer_test.go rename to go/analysis/passes/defers/defers_test.go diff --git a/go/analysis/passes/defers/doc.go b/go/analysis/passes/defers/doc.go index 60ad3c2cac9..bdb13516282 100644 --- a/go/analysis/passes/defers/doc.go +++ b/go/analysis/passes/defers/doc.go @@ -5,11 +5,11 @@ // Package defers defines an Analyzer that checks for common mistakes in defer // statements. // -// # Analyzer defer +// # Analyzer defers // -// defer: report common mistakes in defer statements +// defers: report common mistakes in defer statements // -// The defer analyzer reports a diagnostic when a defer statement would +// The defers analyzer reports a diagnostic when a defer statement would // result in a non-deferred call to time.Since, as experience has shown // that this is nearly always a mistake. // diff --git a/gopls/doc/analyzers.md b/gopls/doc/analyzers.md index 2ff9434d0b6..da8814f4b16 100644 --- a/gopls/doc/analyzers.md +++ b/gopls/doc/analyzers.md @@ -108,11 +108,11 @@ errors is discouraged. **Enabled by default.** -## **defer** +## **defers** report common mistakes in defer statements -The defer analyzer reports a diagnostic when a defer statement would +The defers analyzer reports a diagnostic when a defer statement would result in a non-deferred call to time.Since, as experience has shown that this is nearly always a mistake. diff --git a/gopls/doc/generate_test.go b/gopls/doc/generate_test.go index 6e1c23b94db..f92ff1fb8e1 100644 --- a/gopls/doc/generate_test.go +++ b/gopls/doc/generate_test.go @@ -23,6 +23,6 @@ func TestGenerated(t *testing.T) { t.Fatal(err) } if !ok { - t.Error("documentation needs updating. Run: cd gopls && go generate ./doc") + t.Error("documentation needs updating. Run: cd gopls && go generate") } } diff --git a/gopls/internal/lsp/source/api_json.go b/gopls/internal/lsp/source/api_json.go index a66ceeff78e..4d85ed7b5d2 100644 --- a/gopls/internal/lsp/source/api_json.go +++ b/gopls/internal/lsp/source/api_json.go @@ -269,8 +269,8 @@ var GeneratedAPIJSON = &APIJSON{ Default: "true", }, { - Name: "\"defer\"", - Doc: "report common mistakes in defer statements\n\nThe defer analyzer reports a diagnostic when a defer statement would\nresult in a non-deferred call to time.Since, as experience has shown\nthat this is nearly always a mistake.\n\nFor example:\n\n\tstart := time.Now()\n\t...\n\tdefer recordLatency(time.Since(start)) // error: call to time.Since is not deferred\n\nThe correct code is:\n\n\tdefer func() { recordLatency(time.Since(start)) }()", + Name: "\"defers\"", + Doc: "report common mistakes in defer statements\n\nThe defers analyzer reports a diagnostic when a defer statement would\nresult in a non-deferred call to time.Since, as experience has shown\nthat this is nearly always a mistake.\n\nFor example:\n\n\tstart := time.Now()\n\t...\n\tdefer recordLatency(time.Since(start)) // error: call to time.Since is not deferred\n\nThe correct code is:\n\n\tdefer func() { recordLatency(time.Since(start)) }()", Default: "true", }, { @@ -977,8 +977,9 @@ var GeneratedAPIJSON = &APIJSON{ Default: true, }, { - Name: "defer", - Doc: "report common mistakes in defer statements\n\nThe defer analyzer reports a diagnostic when a defer statement would\nresult in a non-deferred call to time.Since, as experience has shown\nthat this is nearly always a mistake.\n\nFor example:\n\n\tstart := time.Now()\n\t...\n\tdefer recordLatency(time.Since(start)) // error: call to time.Since is not deferred\n\nThe correct code is:\n\n\tdefer func() { recordLatency(time.Since(start)) }()", + Name: "defers", + Doc: "report common mistakes in defer statements\n\nThe defers analyzer reports a diagnostic when a defer statement would\nresult in a non-deferred call to time.Since, as experience has shown\nthat this is nearly always a mistake.\n\nFor example:\n\n\tstart := time.Now()\n\t...\n\tdefer recordLatency(time.Since(start)) // error: call to time.Since is not deferred\n\nThe correct code is:\n\n\tdefer func() { recordLatency(time.Since(start)) }()", + URL: "https://pkg.go.dev/golang.org/x/tools/go/analysis/passes/defers", Default: true, }, { From 559c4300daa4efe55422df9bba86d125cdf1d9ef Mon Sep 17 00:00:00 2001 From: Peter Weinbergr Date: Tue, 12 Sep 2023 10:11:42 -0400 Subject: [PATCH 088/178] tools: replace references to obsolete package ioutils ioutil defines 7 functions. 6 of these are replaced by functions in io or os with the same signature. ReadDir is deprecated, but the suggested replacement has a different signature. These changes were generated by a program, with some manual adjutments. The program replaces ReadDir with a call to a function named ioutilReadDir that has the same signature. The code for this function is added to files if necessary. The program replaces all the others with their new versions. The program removes the 'io/ioutil' import and adds, as necessary, 'os', 'io', and 'io/fs', the latter being needed for the signature of ioutilReadDir. The automatic process fails in a few ways: 1. ReadFile occurs only in a comment but the program adds an unneeded import. 2. ioutilReadDir is added to more than one file in the same package Both of these could be viewed as bugs and fixed by looking harder. After manual adjustment, two tests failed: 1. gopls/internal/lsp/regtesg/mis:TestGenerateProgress. The reason was a use of ioutil in a txtar constant. The calls were changed, but the code is not smart enough to change the import inside the string constant. (Or it's not smart enough not to change the contents of a string.) 2. gopls/internal/lsp/analysis/deprecated, which wants to see a use of ioutil These tests were adjused by hand, and all tests (-short) pass. Change-Id: If9efe40bbb0edda36173d9a88afaf71245db8e79 Reviewed-on: https://go-review.googlesource.com/c/tools/+/527675 TryBot-Result: Gopher Robot Run-TryBot: Peter Weinberger Reviewed-by: Heschi Kreinick --- cmd/auth/netrcauth/netrcauth.go | 3 +-- cmd/bundle/main.go | 3 +-- cmd/bundle/main_test.go | 5 ++-- cmd/compilebench/main.go | 11 ++++---- cmd/eg/eg.go | 3 +-- cmd/file2fuzz/main.go | 7 +++-- cmd/file2fuzz/main_test.go | 9 +++---- cmd/fiximports/main.go | 3 +-- cmd/getgo/download.go | 7 +++-- cmd/getgo/download_test.go | 3 +-- cmd/getgo/main_test.go | 9 +++---- cmd/getgo/path_test.go | 7 +++-- cmd/go-contrib-init/contrib.go | 5 ++-- cmd/godex/godex.go | 21 +++++++++++++-- cmd/godoc/godoc_test.go | 6 ++--- cmd/goimports/goimports.go | 7 +++-- cmd/gorename/gorename_test.go | 7 +++-- cmd/gotype/gotype.go | 4 +-- cmd/goyacc/yacc.go | 5 ++-- cmd/guru/guru_test.go | 5 ++-- cmd/guru/referrers.go | 2 +- cmd/guru/unit_test.go | 3 +-- cmd/present2md/main.go | 5 ++-- cmd/signature-fuzzer/fuzz-runner/runner.go | 5 ++-- cmd/stress/stress.go | 3 +-- cmd/toolstash/main.go | 27 +++++++++++++++---- copyright/copyright.go | 4 +-- go/analysis/analysistest/analysistest.go | 7 +++-- go/analysis/doc.go | 2 +- go/analysis/internal/analysisflags/flags.go | 3 +-- go/analysis/internal/checker/checker.go | 7 +++-- go/analysis/internal/checker/checker_test.go | 4 +-- go/analysis/internal/checker/fix_test.go | 7 +++-- go/analysis/internal/checker/start_test.go | 4 +-- .../passes/internal/analysisutil/util.go | 4 +-- go/buildutil/fakecontext.go | 3 +-- go/buildutil/overlay.go | 3 +-- go/buildutil/overlay_test.go | 4 +-- go/buildutil/util.go | 23 +++++++++++++--- go/buildutil/util_test.go | 3 +-- go/callgraph/cha/cha_test.go | 4 +-- go/callgraph/vta/helpers_test.go | 4 +-- go/expect/expect_test.go | 4 +-- go/gccgoexportdata/gccgoexportdata.go | 3 +-- go/internal/cgo/cgo.go | 3 +-- go/internal/gccgoimporter/importer.go | 2 +- go/loader/stdlib_test.go | 4 +-- go/packages/golist.go | 7 +++-- go/packages/overlay_test.go | 23 ++++++++-------- go/packages/packages.go | 3 +-- go/packages/packages_test.go | 27 +++++++++---------- go/packages/packagestest/expect.go | 3 +-- go/packages/packagestest/export.go | 9 +++---- go/packages/packagestest/export_test.go | 5 ++-- go/packages/packagestest/modules.go | 11 ++++---- go/ssa/source_test.go | 3 +-- godoc/server.go | 3 +-- godoc/static/gen.go | 4 +-- godoc/static/gen_test.go | 4 +-- godoc/static/makestatic.go | 3 +-- godoc/vfs/mapfs/mapfs_test.go | 4 +-- godoc/vfs/namespace.go | 2 +- godoc/vfs/os.go | 21 +++++++++++++-- godoc/vfs/vfs.go | 3 +-- godoc/vfs/zipfs/zipfs_test.go | 3 +-- gopls/doc/generate.go | 5 ++-- gopls/integration/govim/artifacts.go | 6 ++--- gopls/internal/hooks/diff.go | 5 ++-- gopls/internal/hooks/diff_test.go | 3 +-- gopls/internal/hooks/licenses_test.go | 8 +++--- gopls/internal/lsp/cmd/capabilities_test.go | 7 +++-- gopls/internal/lsp/cmd/cmd.go | 3 +-- gopls/internal/lsp/cmd/help_test.go | 6 ++--- gopls/internal/lsp/cmd/semantictokens.go | 5 ++-- gopls/internal/lsp/command/interface_test.go | 4 +-- gopls/internal/lsp/fake/sandbox.go | 5 ++-- gopls/internal/lsp/fake/workdir.go | 9 +++---- gopls/internal/lsp/fake/workdir_test.go | 3 +-- gopls/internal/regtest/bench/bench_test.go | 3 +-- gopls/internal/regtest/misc/generate_test.go | 3 +-- gopls/internal/regtest/misc/imports_test.go | 3 +-- gopls/internal/vulncheck/vulntest/db.go | 3 +-- .../vulncheck/vulntest/report_test.go | 3 +-- gopls/release/release.go | 3 +-- imports/forward.go | 4 +-- internal/apidiff/apidiff_test.go | 5 ++-- internal/diff/difftest/difftest_test.go | 5 ++-- internal/diff/lcs/old_test.go | 4 +-- internal/event/bench_test.go | 6 ++--- internal/event/export/ocagent/ocagent_test.go | 4 +-- internal/facts/facts.go | 4 +-- internal/fastwalk/fastwalk_portable.go | 21 +++++++++++++-- internal/fastwalk/fastwalk_test.go | 7 +++-- internal/gcimporter/gcimporter.go | 3 +-- internal/gcimporter/gcimporter_test.go | 27 +++++++++++++++---- internal/gcimporter/iexport_test.go | 4 +-- internal/gopathwalk/walk_test.go | 9 +++---- internal/imports/fix.go | 7 +++-- internal/imports/fix_test.go | 4 +-- internal/imports/mkindex.go | 3 +-- internal/imports/mkstdlib.go | 3 +-- internal/imports/mod.go | 23 +++++++++++++--- internal/imports/mod_test.go | 21 +++++++-------- internal/proxydir/proxydir.go | 5 ++-- internal/proxydir/proxydir_test.go | 10 +++---- internal/robustio/copyfiles.go | 4 +-- internal/robustio/robustio_flaky.go | 3 +-- internal/robustio/robustio_other.go | 3 +-- internal/testenv/testenv.go | 5 ++-- playground/socket/socket.go | 5 ++-- present/parse.go | 6 ++--- present/parse_test.go | 5 ++-- refactor/eg/eg_test.go | 3 +-- refactor/rename/mvpkg_test.go | 4 +-- refactor/rename/rename.go | 5 ++-- refactor/rename/rename_test.go | 7 +++-- txtar/archive.go | 4 +-- 117 files changed, 389 insertions(+), 346 deletions(-) diff --git a/cmd/auth/netrcauth/netrcauth.go b/cmd/auth/netrcauth/netrcauth.go index 7d29c96035e..a730e646a81 100644 --- a/cmd/auth/netrcauth/netrcauth.go +++ b/cmd/auth/netrcauth/netrcauth.go @@ -16,7 +16,6 @@ package main import ( "fmt" - "io/ioutil" "log" "net/http" "net/url" @@ -41,7 +40,7 @@ func main() { path := os.Args[1] - data, err := ioutil.ReadFile(path) + data, err := os.ReadFile(path) if err != nil { if os.IsNotExist(err) { return diff --git a/cmd/bundle/main.go b/cmd/bundle/main.go index 194797bd822..a5c426d8f88 100644 --- a/cmd/bundle/main.go +++ b/cmd/bundle/main.go @@ -79,7 +79,6 @@ import ( "go/printer" "go/token" "go/types" - "io/ioutil" "log" "os" "strconv" @@ -149,7 +148,7 @@ func main() { log.Fatal(err) } if *outputFile != "" { - err := ioutil.WriteFile(*outputFile, code, 0666) + err := os.WriteFile(*outputFile, code, 0666) if err != nil { log.Fatal(err) } diff --git a/cmd/bundle/main_test.go b/cmd/bundle/main_test.go index 10d790fa28e..4ee8521a074 100644 --- a/cmd/bundle/main_test.go +++ b/cmd/bundle/main_test.go @@ -6,7 +6,6 @@ package main import ( "bytes" - "io/ioutil" "os" "os/exec" "runtime" @@ -18,7 +17,7 @@ import ( func TestBundle(t *testing.T) { packagestest.TestAll(t, testBundle) } func testBundle(t *testing.T, x packagestest.Exporter) { load := func(name string) string { - data, err := ioutil.ReadFile(name) + data, err := os.ReadFile(name) if err != nil { t.Fatal(err) } @@ -53,7 +52,7 @@ func testBundle(t *testing.T, x packagestest.Exporter) { if got, want := string(out), load("testdata/out.golden"); got != want { t.Errorf("-- got --\n%s\n-- want --\n%s\n-- diff --", got, want) - if err := ioutil.WriteFile("testdata/out.got", out, 0644); err != nil { + if err := os.WriteFile("testdata/out.got", out, 0644); err != nil { t.Fatal(err) } t.Log(diff("testdata/out.golden", "testdata/out.got")) diff --git a/cmd/compilebench/main.go b/cmd/compilebench/main.go index 6900b2576d3..2509253791b 100644 --- a/cmd/compilebench/main.go +++ b/cmd/compilebench/main.go @@ -81,7 +81,6 @@ import ( "encoding/json" "flag" "fmt" - "io/ioutil" "log" "os" "path/filepath" @@ -388,7 +387,7 @@ func (c compile) run(name string, count int) error { opath := pkg.Dir + "/_compilebench_.o" if *flagObj { // TODO(josharian): object files are big; just read enough to find what we seek. - data, err := ioutil.ReadFile(opath) + data, err := os.ReadFile(opath) if err != nil { log.Print(err) } @@ -498,7 +497,7 @@ func runBuildCmd(name string, count int, dir, tool string, args []string) error haveAllocs, haveRSS := false, false var allocs, allocbytes, rssbytes int64 if *flagAlloc || *flagMemprofile != "" { - out, err := ioutil.ReadFile(dir + "/_compilebench_.memprof") + out, err := os.ReadFile(dir + "/_compilebench_.memprof") if err != nil { log.Print("cannot find memory profile after compilation") } @@ -531,7 +530,7 @@ func runBuildCmd(name string, count int, dir, tool string, args []string) error if *flagCount != 1 { outpath = fmt.Sprintf("%s_%d", outpath, count) } - if err := ioutil.WriteFile(outpath, out, 0666); err != nil { + if err := os.WriteFile(outpath, out, 0666); err != nil { log.Print(err) } } @@ -539,7 +538,7 @@ func runBuildCmd(name string, count int, dir, tool string, args []string) error } if *flagCpuprofile != "" { - out, err := ioutil.ReadFile(dir + "/_compilebench_.cpuprof") + out, err := os.ReadFile(dir + "/_compilebench_.cpuprof") if err != nil { log.Print(err) } @@ -547,7 +546,7 @@ func runBuildCmd(name string, count int, dir, tool string, args []string) error if *flagCount != 1 { outpath = fmt.Sprintf("%s_%d", outpath, count) } - if err := ioutil.WriteFile(outpath, out, 0666); err != nil { + if err := os.WriteFile(outpath, out, 0666); err != nil { log.Print(err) } os.Remove(dir + "/_compilebench_.cpuprof") diff --git a/cmd/eg/eg.go b/cmd/eg/eg.go index 1629b801cd4..5d21138a49e 100644 --- a/cmd/eg/eg.go +++ b/cmd/eg/eg.go @@ -15,7 +15,6 @@ import ( "go/parser" "go/token" "go/types" - "io/ioutil" "os" "path/filepath" "strings" @@ -77,7 +76,7 @@ func doMain() error { if err != nil { return err } - template, err := ioutil.ReadFile(tAbs) + template, err := os.ReadFile(tAbs) if err != nil { return err } diff --git a/cmd/file2fuzz/main.go b/cmd/file2fuzz/main.go index ed212cb9d72..c2b7ee52089 100644 --- a/cmd/file2fuzz/main.go +++ b/cmd/file2fuzz/main.go @@ -25,7 +25,6 @@ import ( "flag" "fmt" "io" - "io/ioutil" "log" "os" "path/filepath" @@ -51,7 +50,7 @@ func dirWriter(dir string) func([]byte) error { if err := os.MkdirAll(dir, 0777); err != nil { return err } - if err := ioutil.WriteFile(name, b, 0666); err != nil { + if err := os.WriteFile(name, b, 0666); err != nil { os.Remove(name) return err } @@ -98,14 +97,14 @@ func convert(inputArgs []string, outputArg string) error { output = dirWriter(outputArg) } else { output = func(b []byte) error { - return ioutil.WriteFile(outputArg, b, 0666) + return os.WriteFile(outputArg, b, 0666) } } } } for _, f := range input { - b, err := ioutil.ReadAll(f) + b, err := io.ReadAll(f) if err != nil { return fmt.Errorf("unable to read input: %s", err) } diff --git a/cmd/file2fuzz/main_test.go b/cmd/file2fuzz/main_test.go index f9f54919a33..83653d2dd77 100644 --- a/cmd/file2fuzz/main_test.go +++ b/cmd/file2fuzz/main_test.go @@ -5,7 +5,6 @@ package main import ( - "io/ioutil" "os" "os/exec" "path/filepath" @@ -118,9 +117,9 @@ func TestFile2Fuzz(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - tmp, err := ioutil.TempDir(os.TempDir(), "file2fuzz") + tmp, err := os.MkdirTemp(os.TempDir(), "file2fuzz") if err != nil { - t.Fatalf("ioutil.TempDir failed: %s", err) + t.Fatalf("os.MkdirTemp failed: %s", err) } defer os.RemoveAll(tmp) for _, f := range tc.inputFiles { @@ -129,7 +128,7 @@ func TestFile2Fuzz(t *testing.T) { t.Fatalf("failed to create test directory: %s", err) } } else { - if err := ioutil.WriteFile(filepath.Join(tmp, f.name), []byte(f.content), 0666); err != nil { + if err := os.WriteFile(filepath.Join(tmp, f.name), []byte(f.content), 0666); err != nil { t.Fatalf("failed to create test input file: %s", err) } } @@ -146,7 +145,7 @@ func TestFile2Fuzz(t *testing.T) { } for _, f := range tc.expectedFiles { - c, err := ioutil.ReadFile(filepath.Join(tmp, f.name)) + c, err := os.ReadFile(filepath.Join(tmp, f.name)) if err != nil { t.Fatalf("failed to read expected output file %q: %s", f.name, err) } diff --git a/cmd/fiximports/main.go b/cmd/fiximports/main.go index 8eeacd1eda3..0893b068756 100644 --- a/cmd/fiximports/main.go +++ b/cmd/fiximports/main.go @@ -76,7 +76,6 @@ import ( "go/parser" "go/token" "io" - "io/ioutil" "log" "os" "path" @@ -100,7 +99,7 @@ var ( // seams for testing var ( stderr io.Writer = os.Stderr - writeFile = ioutil.WriteFile + writeFile = os.WriteFile ) const usage = `fiximports: rewrite import paths to use canonical package names. diff --git a/cmd/getgo/download.go b/cmd/getgo/download.go index 86f0a2fed80..18e1aec2eef 100644 --- a/cmd/getgo/download.go +++ b/cmd/getgo/download.go @@ -15,7 +15,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "net/http" "os" "path/filepath" @@ -51,7 +50,7 @@ func downloadGoVersion(version, ops, arch, dest string) error { } defer resp.Body.Close() - tmpf, err := ioutil.TempFile("", "go") + tmpf, err := os.CreateTemp("", "go") if err != nil { return err } @@ -75,7 +74,7 @@ func downloadGoVersion(version, ops, arch, dest string) error { return fmt.Errorf("Downloading Go sha256 from %s.sha256 failed with HTTP status %s", uri, sresp.Status) } - shasum, err := ioutil.ReadAll(sresp.Body) + shasum, err := io.ReadAll(sresp.Body) if err != nil { return err } @@ -174,7 +173,7 @@ func getLatestGoVersion() (string, error) { } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - b, _ := ioutil.ReadAll(io.LimitReader(resp.Body, 1024)) + b, _ := io.ReadAll(io.LimitReader(resp.Body, 1024)) return "", fmt.Errorf("Could not get current Go release: HTTP %d: %q", resp.StatusCode, b) } var releases []struct { diff --git a/cmd/getgo/download_test.go b/cmd/getgo/download_test.go index 76cd96cbd1e..b4f2059d14e 100644 --- a/cmd/getgo/download_test.go +++ b/cmd/getgo/download_test.go @@ -8,7 +8,6 @@ package main import ( - "io/ioutil" "os" "path/filepath" "testing" @@ -19,7 +18,7 @@ func TestDownloadGoVersion(t *testing.T) { t.Skipf("Skipping download in short mode") } - tmpd, err := ioutil.TempDir("", "go") + tmpd, err := os.MkdirTemp("", "go") if err != nil { t.Fatal(err) } diff --git a/cmd/getgo/main_test.go b/cmd/getgo/main_test.go index fc28c5df904..878137dd3f4 100644 --- a/cmd/getgo/main_test.go +++ b/cmd/getgo/main_test.go @@ -10,7 +10,6 @@ package main import ( "bytes" "fmt" - "io/ioutil" "os" "os/exec" "testing" @@ -37,7 +36,7 @@ func TestMain(m *testing.M) { } func createTmpHome(t *testing.T) string { - tmpd, err := ioutil.TempDir("", "testgetgo") + tmpd, err := os.MkdirTemp("", "testgetgo") if err != nil { t.Fatalf("creating test tempdir failed: %v", err) } @@ -86,7 +85,7 @@ func TestCommandVerbose(t *testing.T) { if err != nil { t.Fatal(err) } - b, err := ioutil.ReadFile(shellConfig) + b, err := os.ReadFile(shellConfig) if err != nil { t.Fatal(err) } @@ -122,7 +121,7 @@ func TestCommandPathExists(t *testing.T) { if err != nil { t.Fatal(err) } - b, err := ioutil.ReadFile(shellConfig) + b, err := os.ReadFile(shellConfig) if err != nil { t.Fatal(err) } @@ -146,7 +145,7 @@ export PATH=$PATH:%s/go/bin t.Fatal(err) } - b, err = ioutil.ReadFile(shellConfig) + b, err = os.ReadFile(shellConfig) if err != nil { t.Fatal(err) } diff --git a/cmd/getgo/path_test.go b/cmd/getgo/path_test.go index 2249c5447b7..8195f2e68d5 100644 --- a/cmd/getgo/path_test.go +++ b/cmd/getgo/path_test.go @@ -8,7 +8,6 @@ package main import ( - "io/ioutil" "os" "path/filepath" "strings" @@ -16,7 +15,7 @@ import ( ) func TestAppendPath(t *testing.T) { - tmpd, err := ioutil.TempDir("", "go") + tmpd, err := os.MkdirTemp("", "go") if err != nil { t.Fatal(err) } @@ -35,7 +34,7 @@ func TestAppendPath(t *testing.T) { if err != nil { t.Fatal(err) } - b, err := ioutil.ReadFile(shellConfig) + b, err := os.ReadFile(shellConfig) if err != nil { t.Fatal(err) } @@ -49,7 +48,7 @@ func TestAppendPath(t *testing.T) { if err := appendToPATH(filepath.Join(GOPATH, "bin")); err != nil { t.Fatal(err) } - b, err = ioutil.ReadFile(shellConfig) + b, err = os.ReadFile(shellConfig) if err != nil { t.Fatal(err) } diff --git a/cmd/go-contrib-init/contrib.go b/cmd/go-contrib-init/contrib.go index e2bb5070c60..9b4d265025c 100644 --- a/cmd/go-contrib-init/contrib.go +++ b/cmd/go-contrib-init/contrib.go @@ -14,7 +14,6 @@ import ( "fmt" "go/build" exec "golang.org/x/sys/execabs" - "io/ioutil" "log" "os" "path/filepath" @@ -66,7 +65,7 @@ func detectrepo() string { var googleSourceRx = regexp.MustCompile(`(?m)^(go|go-review)?\.googlesource.com\b`) func checkCLA() { - slurp, err := ioutil.ReadFile(cookiesFile()) + slurp, err := os.ReadFile(cookiesFile()) if err != nil && !os.IsNotExist(err) { log.Fatal(err) } @@ -135,7 +134,7 @@ func checkGoroot() { "your GOROOT or set it to the path of your development version\n"+ "of Go.", v) } - slurp, err := ioutil.ReadFile(filepath.Join(v, "VERSION")) + slurp, err := os.ReadFile(filepath.Join(v, "VERSION")) if err == nil { slurp = bytes.TrimSpace(slurp) log.Fatalf("Your GOROOT environment variable is set to %q\n"+ diff --git a/cmd/godex/godex.go b/cmd/godex/godex.go index e1d7e2f9243..7540f58a557 100644 --- a/cmd/godex/godex.go +++ b/cmd/godex/godex.go @@ -10,7 +10,7 @@ import ( "fmt" "go/build" "go/types" - "io/ioutil" + "io/fs" "os" "path/filepath" "strings" @@ -197,7 +197,7 @@ func genPrefixes(out chan string, all bool) { } func walkDir(dirname, prefix string, out chan string) { - fiList, err := ioutil.ReadDir(dirname) + fiList, err := ioutilReadDir(dirname) if err != nil { return } @@ -209,3 +209,20 @@ func walkDir(dirname, prefix string, out chan string) { } } } + +func ioutilReadDir(dirname string) ([]fs.FileInfo, error) { + entries, err := os.ReadDir(dirname) + if err != nil { + return nil, err + } + + infos := make([]fs.FileInfo, 0, len(entries)) + for _, entry := range entries { + info, err := entry.Info() + if err != nil { + return infos, err + } + infos = append(infos, info) + } + return infos, nil +} diff --git a/cmd/godoc/godoc_test.go b/cmd/godoc/godoc_test.go index b2ebe8581a6..42582c4b228 100644 --- a/cmd/godoc/godoc_test.go +++ b/cmd/godoc/godoc_test.go @@ -9,7 +9,7 @@ import ( "context" "fmt" "go/build" - "io/ioutil" + "io" "net" "net/http" "os" @@ -111,7 +111,7 @@ func waitForServer(t *testing.T, ctx context.Context, url, match string, reverse if err != nil { continue } - body, err := ioutil.ReadAll(res.Body) + body, err := io.ReadAll(res.Body) res.Body.Close() if err != nil || res.StatusCode != http.StatusOK { continue @@ -396,7 +396,7 @@ package a; import _ "godoc.test/repo2/a"; const Name = "repo1a"`, t.Errorf("GET %s failed: %s", url, err) continue } - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) strBody := string(body) resp.Body.Close() if err != nil { diff --git a/cmd/goimports/goimports.go b/cmd/goimports/goimports.go index b354c9e8241..3b6bd72503e 100644 --- a/cmd/goimports/goimports.go +++ b/cmd/goimports/goimports.go @@ -13,7 +13,6 @@ import ( "go/scanner" exec "golang.org/x/sys/execabs" "io" - "io/ioutil" "log" "os" "path/filepath" @@ -106,7 +105,7 @@ func processFile(filename string, in io.Reader, out io.Writer, argType argumentT in = f } - src, err := ioutil.ReadAll(in) + src, err := io.ReadAll(in) if err != nil { return err } @@ -159,7 +158,7 @@ func processFile(filename string, in io.Reader, out io.Writer, argType argumentT if fi, err := os.Stat(filename); err == nil { perms = fi.Mode() & os.ModePerm } - err = ioutil.WriteFile(filename, res, perms) + err = os.WriteFile(filename, res, perms) if err != nil { return err } @@ -296,7 +295,7 @@ func gofmtMain() { } func writeTempFile(dir, prefix string, data []byte) (string, error) { - file, err := ioutil.TempFile(dir, prefix) + file, err := os.CreateTemp(dir, prefix) if err != nil { return "", err } diff --git a/cmd/gorename/gorename_test.go b/cmd/gorename/gorename_test.go index 30b87967140..f72b6f4a429 100644 --- a/cmd/gorename/gorename_test.go +++ b/cmd/gorename/gorename_test.go @@ -6,7 +6,6 @@ package main_test import ( "fmt" - "io/ioutil" "os" "os/exec" "path/filepath" @@ -315,7 +314,7 @@ func buildGorename(t *testing.T) (tmp, bin string, cleanup func()) { t.Skipf("the dependencies are not available on android") } - tmp, err := ioutil.TempDir("", "gorename-regtest-") + tmp, err := os.MkdirTemp("", "gorename-regtest-") if err != nil { t.Fatal(err) } @@ -352,7 +351,7 @@ func setUpPackages(t *testing.T, dir string, packages map[string][]string) (clea // Write the packages files for i, val := range files { file := filepath.Join(pkgDir, strconv.Itoa(i)+".go") - if err := ioutil.WriteFile(file, []byte(val), os.ModePerm); err != nil { + if err := os.WriteFile(file, []byte(val), os.ModePerm); err != nil { t.Fatal(err) } } @@ -373,7 +372,7 @@ func modifiedFiles(t *testing.T, dir string, packages map[string][]string) (resu for i, val := range files { file := filepath.Join(pkgDir, strconv.Itoa(i)+".go") // read file contents and compare to val - if contents, err := ioutil.ReadFile(file); err != nil { + if contents, err := os.ReadFile(file); err != nil { t.Fatalf("File missing: %s", err) } else if string(contents) != val { results = append(results, strings.TrimPrefix(dir, file)) diff --git a/cmd/gotype/gotype.go b/cmd/gotype/gotype.go index 08b52057f55..4a731f26233 100644 --- a/cmd/gotype/gotype.go +++ b/cmd/gotype/gotype.go @@ -97,7 +97,7 @@ import ( "go/scanner" "go/token" "go/types" - "io/ioutil" + "io" "os" "path/filepath" "sync" @@ -197,7 +197,7 @@ func parse(filename string, src interface{}) (*ast.File, error) { } func parseStdin() (*ast.File, error) { - src, err := ioutil.ReadAll(os.Stdin) + src, err := io.ReadAll(os.Stdin) if err != nil { return nil, err } diff --git a/cmd/goyacc/yacc.go b/cmd/goyacc/yacc.go index 948e5010417..5a8ede0a482 100644 --- a/cmd/goyacc/yacc.go +++ b/cmd/goyacc/yacc.go @@ -50,7 +50,6 @@ import ( "flag" "fmt" "go/format" - "io/ioutil" "math" "os" "strconv" @@ -3209,7 +3208,7 @@ func exit(status int) { } func gofmt() { - src, err := ioutil.ReadFile(oflag) + src, err := os.ReadFile(oflag) if err != nil { return } @@ -3217,7 +3216,7 @@ func gofmt() { if err != nil { return } - ioutil.WriteFile(oflag, src, 0666) + os.WriteFile(oflag, src, 0666) } var yaccpar string // will be processed version of yaccpartext: s/$$/prefix/g diff --git a/cmd/guru/guru_test.go b/cmd/guru/guru_test.go index 905a9e2cf49..d3c38e0a472 100644 --- a/cmd/guru/guru_test.go +++ b/cmd/guru/guru_test.go @@ -34,7 +34,6 @@ import ( "go/parser" "go/token" "io" - "io/ioutil" "log" "os" "os/exec" @@ -83,7 +82,7 @@ func parseRegexp(text string) (*regexp.Regexp, error) { // parseQueries parses and returns the queries in the named file. func parseQueries(t *testing.T, filename string) []*query { - filedata, err := ioutil.ReadFile(filename) + filedata, err := os.ReadFile(filename) if err != nil { t.Fatal(err) } @@ -266,7 +265,7 @@ func TestGuru(t *testing.T) { json := strings.Contains(filename, "-json/") queries := parseQueries(t, filename) golden := filename + "lden" - gotfh, err := ioutil.TempFile("", filepath.Base(filename)+"t") + gotfh, err := os.CreateTemp("", filepath.Base(filename)+"t") if err != nil { t.Fatal(err) } diff --git a/cmd/guru/referrers.go b/cmd/guru/referrers.go index d75196bf93a..70db3d1841a 100644 --- a/cmd/guru/referrers.go +++ b/cmd/guru/referrers.go @@ -765,7 +765,7 @@ func (r *referrersPackageResult) foreachRef(f func(id *ast.Ident, text string)) } } -// readFile is like ioutil.ReadFile, but +// readFile is like os.ReadFile, but // it goes through the virtualized build.Context. // If non-nil, buf must have been reset. func readFile(ctxt *build.Context, filename string, buf *bytes.Buffer) ([]byte, error) { diff --git a/cmd/guru/unit_test.go b/cmd/guru/unit_test.go index 699e6a1b10f..7c24d714f19 100644 --- a/cmd/guru/unit_test.go +++ b/cmd/guru/unit_test.go @@ -7,7 +7,6 @@ package main import ( "fmt" "go/build" - "io/ioutil" "os" "path/filepath" "runtime" @@ -27,7 +26,7 @@ func TestIssue17515(t *testing.T) { // (4) symlink & absolute: GOPATH=$HOME/src file= $HOME/go/src/test/test.go // Create a temporary home directory under /tmp - home, err := ioutil.TempDir(os.TempDir(), "home") + home, err := os.MkdirTemp(os.TempDir(), "home") if err != nil { t.Errorf("Unable to create a temporary directory in %s", os.TempDir()) } diff --git a/cmd/present2md/main.go b/cmd/present2md/main.go index 748b041e41f..a11e57ecf8b 100644 --- a/cmd/present2md/main.go +++ b/cmd/present2md/main.go @@ -25,7 +25,6 @@ import ( "flag" "fmt" "io" - "io/ioutil" "log" "net/url" "os" @@ -84,7 +83,7 @@ func main() { // If writeBack is true, the converted version is written back to file. // If writeBack is false, the converted version is printed to standard output. func convert(r io.Reader, file string, writeBack bool) error { - data, err := ioutil.ReadAll(r) + data, err := io.ReadAll(r) if err != nil { return err } @@ -184,7 +183,7 @@ func convert(r io.Reader, file string, writeBack bool) error { os.Stdout.Write(md.Bytes()) return nil } - return ioutil.WriteFile(file, md.Bytes(), 0666) + return os.WriteFile(file, md.Bytes(), 0666) } func printSectionBody(file string, depth int, w *bytes.Buffer, elems []present.Elem) { diff --git a/cmd/signature-fuzzer/fuzz-runner/runner.go b/cmd/signature-fuzzer/fuzz-runner/runner.go index b77b218f5a8..27ab975f0c8 100644 --- a/cmd/signature-fuzzer/fuzz-runner/runner.go +++ b/cmd/signature-fuzzer/fuzz-runner/runner.go @@ -12,7 +12,6 @@ package main import ( "flag" "fmt" - "io/ioutil" "log" "os" "os/exec" @@ -196,7 +195,7 @@ func (c *config) gen(singlepk int, singlefn int) { func (c *config) action(cmd []string, outfile string, emitout bool) int { st := docmdout(cmd, c.gendir, outfile) if emitout { - content, err := ioutil.ReadFile(outfile) + content, err := os.ReadFile(outfile) if err != nil { log.Fatal(err) } @@ -405,7 +404,7 @@ func main() { } verb(1, "in main, verblevel=%d", *verbflag) - tmpdir, err := ioutil.TempDir("", "fuzzrun") + tmpdir, err := os.MkdirTemp("", "fuzzrun") if err != nil { fatal("creation of tempdir failed: %v", err) } diff --git a/cmd/stress/stress.go b/cmd/stress/stress.go index f9817a1dda6..6dc563d7a87 100644 --- a/cmd/stress/stress.go +++ b/cmd/stress/stress.go @@ -20,7 +20,6 @@ package main import ( "flag" "fmt" - "io/ioutil" "os" "path/filepath" "regexp" @@ -128,7 +127,7 @@ func main() { } fails++ dir, path := filepath.Split(*flagOutput) - f, err := ioutil.TempFile(dir, path) + f, err := os.CreateTemp(dir, path) if err != nil { fmt.Printf("failed to create temp file: %v\n", err) os.Exit(1) diff --git a/cmd/toolstash/main.go b/cmd/toolstash/main.go index ddb1905ae4d..519a9428bcc 100644 --- a/cmd/toolstash/main.go +++ b/cmd/toolstash/main.go @@ -128,7 +128,7 @@ import ( "fmt" exec "golang.org/x/sys/execabs" "io" - "io/ioutil" + "io/fs" "log" "os" "path/filepath" @@ -553,7 +553,7 @@ func save() { } toolDir := filepath.Join(goroot, fmt.Sprintf("pkg/tool/%s_%s", runtime.GOOS, runtime.GOARCH)) - files, err := ioutil.ReadDir(toolDir) + files, err := ioutilReadDir(toolDir) if err != nil { log.Fatal(err) } @@ -578,7 +578,7 @@ func save() { } func restore() { - files, err := ioutil.ReadDir(stashDir) + files, err := ioutilReadDir(stashDir) if err != nil { log.Fatal(err) } @@ -626,11 +626,28 @@ func cp(src, dst string) { if *verbose { fmt.Printf("cp %s %s\n", src, dst) } - data, err := ioutil.ReadFile(src) + data, err := os.ReadFile(src) if err != nil { log.Fatal(err) } - if err := ioutil.WriteFile(dst, data, 0777); err != nil { + if err := os.WriteFile(dst, data, 0777); err != nil { log.Fatal(err) } } + +func ioutilReadDir(dirname string) ([]fs.FileInfo, error) { + entries, err := os.ReadDir(dirname) + if err != nil { + return nil, err + } + + infos := make([]fs.FileInfo, 0, len(entries)) + for _, entry := range entries { + info, err := entry.Info() + if err != nil { + return infos, err + } + infos = append(infos, info) + } + return infos, nil +} diff --git a/copyright/copyright.go b/copyright/copyright.go index db63c59922e..c084bd0cda8 100644 --- a/copyright/copyright.go +++ b/copyright/copyright.go @@ -13,7 +13,7 @@ import ( "go/parser" "go/token" "io/fs" - "io/ioutil" + "os" "path/filepath" "regexp" "strings" @@ -67,7 +67,7 @@ func checkFile(toolsDir, filename string) (bool, error) { if strings.HasSuffix(normalized, "cmd/goyacc/yacc.go") { return false, nil } - content, err := ioutil.ReadFile(filename) + content, err := os.ReadFile(filename) if err != nil { return false, err } diff --git a/go/analysis/analysistest/analysistest.go b/go/analysis/analysistest/analysistest.go index 63ca6e9eb2e..139c7587c52 100644 --- a/go/analysis/analysistest/analysistest.go +++ b/go/analysis/analysistest/analysistest.go @@ -11,7 +11,6 @@ import ( "go/format" "go/token" "go/types" - "io/ioutil" "log" "os" "path/filepath" @@ -35,7 +34,7 @@ import ( // maps file names to contents). On success it returns the name of the // directory and a cleanup function to delete it. func WriteFiles(filemap map[string]string) (dir string, cleanup func(), err error) { - gopath, err := ioutil.TempDir("", "analysistest") + gopath, err := os.MkdirTemp("", "analysistest") if err != nil { return "", nil, err } @@ -44,7 +43,7 @@ func WriteFiles(filemap map[string]string) (dir string, cleanup func(), err erro for name, content := range filemap { filename := filepath.Join(gopath, "src", name) os.MkdirAll(filepath.Dir(filename), 0777) // ignore error - if err := ioutil.WriteFile(filename, []byte(content), 0666); err != nil { + if err := os.WriteFile(filename, []byte(content), 0666); err != nil { cleanup() return "", nil, err } @@ -451,7 +450,7 @@ func check(t Testing, gopath string, pass *analysis.Pass, diagnostics []analysis // Extract 'want' comments from non-Go files. // TODO(adonovan): we may need to handle //line directives. for _, filename := range pass.OtherFiles { - data, err := ioutil.ReadFile(filename) + data, err := os.ReadFile(filename) if err != nil { t.Errorf("can't read '// want' comments from %s: %v", filename, err) continue diff --git a/go/analysis/doc.go b/go/analysis/doc.go index c5429c9e239..44867d599e4 100644 --- a/go/analysis/doc.go +++ b/go/analysis/doc.go @@ -191,7 +191,7 @@ and buildtag, inspect the raw text of Go source files or even non-Go files such as assembly. To report a diagnostic against a line of a raw text file, use the following sequence: - content, err := ioutil.ReadFile(filename) + content, err := os.ReadFile(filename) if err != nil { ... } tf := fset.AddFile(filename, -1, len(content)) tf.SetLinesForContent(content) diff --git a/go/analysis/internal/analysisflags/flags.go b/go/analysis/internal/analysisflags/flags.go index e127a42b97a..9e3fde72bb6 100644 --- a/go/analysis/internal/analysisflags/flags.go +++ b/go/analysis/internal/analysisflags/flags.go @@ -14,7 +14,6 @@ import ( "fmt" "go/token" "io" - "io/ioutil" "log" "os" "strconv" @@ -331,7 +330,7 @@ func PrintPlain(fset *token.FileSet, diag analysis.Diagnostic) { if !end.IsValid() { end = posn } - data, _ := ioutil.ReadFile(posn.Filename) + data, _ := os.ReadFile(posn.Filename) lines := strings.Split(string(data), "\n") for i := posn.Line - Context; i <= end.Line+Context; i++ { if 1 <= i && i <= len(lines) { diff --git a/go/analysis/internal/checker/checker.go b/go/analysis/internal/checker/checker.go index 2da46925de4..33ca77a06c9 100644 --- a/go/analysis/internal/checker/checker.go +++ b/go/analysis/internal/checker/checker.go @@ -17,7 +17,6 @@ import ( "go/format" "go/token" "go/types" - "io/ioutil" "log" "os" "reflect" @@ -425,7 +424,7 @@ func applyFixes(roots []*action) error { // Now we've got a set of valid edits for each file. Apply them. for path, edits := range editsByPath { - contents, err := ioutil.ReadFile(path) + contents, err := os.ReadFile(path) if err != nil { return err } @@ -440,7 +439,7 @@ func applyFixes(roots []*action) error { out = formatted } - if err := ioutil.WriteFile(path, out, 0644); err != nil { + if err := os.WriteFile(path, out, 0644); err != nil { return err } } @@ -480,7 +479,7 @@ func validateEdits(edits []diff.Edit) ([]diff.Edit, int) { // diff3Conflict returns an error describing two conflicting sets of // edits on a file at path. func diff3Conflict(path string, xlabel, ylabel string, xedits, yedits []diff.Edit) error { - contents, err := ioutil.ReadFile(path) + contents, err := os.ReadFile(path) if err != nil { return err } diff --git a/go/analysis/internal/checker/checker_test.go b/go/analysis/internal/checker/checker_test.go index 393e7f1854b..be1f1c03869 100644 --- a/go/analysis/internal/checker/checker_test.go +++ b/go/analysis/internal/checker/checker_test.go @@ -7,7 +7,7 @@ package checker_test import ( "fmt" "go/ast" - "io/ioutil" + "os" "path/filepath" "reflect" "testing" @@ -51,7 +51,7 @@ func Foo() { checker.Fix = true checker.Run([]string{"file=" + path}, []*analysis.Analyzer{analyzer}) - contents, err := ioutil.ReadFile(path) + contents, err := os.ReadFile(path) if err != nil { t.Fatal(err) } diff --git a/go/analysis/internal/checker/fix_test.go b/go/analysis/internal/checker/fix_test.go index 3ea92b38cc1..e6ac1c1f008 100644 --- a/go/analysis/internal/checker/fix_test.go +++ b/go/analysis/internal/checker/fix_test.go @@ -6,7 +6,6 @@ package checker_test import ( "flag" - "io/ioutil" "os" "os/exec" "path" @@ -151,7 +150,7 @@ func Foo() { for name, want := range fixed { path := path.Join(dir, "src", name) - contents, err := ioutil.ReadFile(path) + contents, err := os.ReadFile(path) if err != nil { t.Errorf("error reading %s: %v", path, err) } @@ -224,7 +223,7 @@ func Foo() { // No files updated for name, want := range files { path := path.Join(dir, "src", name) - contents, err := ioutil.ReadFile(path) + contents, err := os.ReadFile(path) if err != nil { t.Errorf("error reading %s: %v", path, err) } @@ -298,7 +297,7 @@ func Foo() { // No files updated for name, want := range files { path := path.Join(dir, "src", name) - contents, err := ioutil.ReadFile(path) + contents, err := os.ReadFile(path) if err != nil { t.Errorf("error reading %s: %v", path, err) } diff --git a/go/analysis/internal/checker/start_test.go b/go/analysis/internal/checker/start_test.go index ede21159bc8..6b0df3033ed 100644 --- a/go/analysis/internal/checker/start_test.go +++ b/go/analysis/internal/checker/start_test.go @@ -6,7 +6,7 @@ package checker_test import ( "go/ast" - "io/ioutil" + "os" "path/filepath" "testing" @@ -40,7 +40,7 @@ package comment checker.Fix = true checker.Run([]string{"file=" + path}, []*analysis.Analyzer{commentAnalyzer}) - contents, err := ioutil.ReadFile(path) + contents, err := os.ReadFile(path) if err != nil { t.Fatal(err) } diff --git a/go/analysis/passes/internal/analysisutil/util.go b/go/analysis/passes/internal/analysisutil/util.go index ac37e4784e1..a8d84034df1 100644 --- a/go/analysis/passes/internal/analysisutil/util.go +++ b/go/analysis/passes/internal/analysisutil/util.go @@ -12,7 +12,7 @@ import ( "go/printer" "go/token" "go/types" - "io/ioutil" + "os" ) // Format returns a string representation of the expression. @@ -69,7 +69,7 @@ func Unparen(e ast.Expr) ast.Expr { // ReadFile reads a file and adds it to the FileSet // so that we can report errors against it using lineStart. func ReadFile(fset *token.FileSet, filename string) ([]byte, *token.File, error) { - content, err := ioutil.ReadFile(filename) + content, err := os.ReadFile(filename) if err != nil { return nil, nil, err } diff --git a/go/buildutil/fakecontext.go b/go/buildutil/fakecontext.go index 15025f645f9..763d18809b4 100644 --- a/go/buildutil/fakecontext.go +++ b/go/buildutil/fakecontext.go @@ -8,7 +8,6 @@ import ( "fmt" "go/build" "io" - "io/ioutil" "os" "path" "path/filepath" @@ -76,7 +75,7 @@ func FakeContext(pkgs map[string]map[string]string) *build.Context { if !ok { return nil, fmt.Errorf("file not found: %s", filename) } - return ioutil.NopCloser(strings.NewReader(content)), nil + return io.NopCloser(strings.NewReader(content)), nil } ctxt.IsAbsPath = func(path string) bool { path = filepath.ToSlash(path) diff --git a/go/buildutil/overlay.go b/go/buildutil/overlay.go index bdbfd931478..7e371658d9e 100644 --- a/go/buildutil/overlay.go +++ b/go/buildutil/overlay.go @@ -10,7 +10,6 @@ import ( "fmt" "go/build" "io" - "io/ioutil" "path/filepath" "strconv" "strings" @@ -33,7 +32,7 @@ func OverlayContext(orig *build.Context, overlay map[string][]byte) *build.Conte // TODO(dominikh): Implement IsDir, HasSubdir and ReadDir rc := func(data []byte) (io.ReadCloser, error) { - return ioutil.NopCloser(bytes.NewBuffer(data)), nil + return io.NopCloser(bytes.NewBuffer(data)), nil } copy := *orig // make a copy diff --git a/go/buildutil/overlay_test.go b/go/buildutil/overlay_test.go index 4ee8817f422..267db3f7d63 100644 --- a/go/buildutil/overlay_test.go +++ b/go/buildutil/overlay_test.go @@ -6,7 +6,7 @@ package buildutil_test import ( "go/build" - "io/ioutil" + "io" "reflect" "strings" "testing" @@ -63,7 +63,7 @@ func TestOverlay(t *testing.T) { if err != nil { t.Errorf("unexpected error %v", err) } - b, err := ioutil.ReadAll(f) + b, err := io.ReadAll(f) if err != nil { t.Errorf("unexpected error %v", err) } diff --git a/go/buildutil/util.go b/go/buildutil/util.go index bee6390de4c..0dd8af4dd52 100644 --- a/go/buildutil/util.go +++ b/go/buildutil/util.go @@ -11,7 +11,7 @@ import ( "go/parser" "go/token" "io" - "io/ioutil" + "io/fs" "os" "path" "path/filepath" @@ -174,13 +174,13 @@ func IsDir(ctxt *build.Context, path string) bool { return err == nil && fi.IsDir() } -// ReadDir behaves like ioutil.ReadDir, +// ReadDir behaves like ioutilReadDir, // but uses the build context's file system interface, if any. func ReadDir(ctxt *build.Context, path string) ([]os.FileInfo, error) { if ctxt.ReadDir != nil { return ctxt.ReadDir(path) } - return ioutil.ReadDir(path) + return ioutilReadDir(path) } // SplitPathList behaves like filepath.SplitList, @@ -207,3 +207,20 @@ func sameFile(x, y string) bool { } return false } + +func ioutilReadDir(dirname string) ([]fs.FileInfo, error) { + entries, err := os.ReadDir(dirname) + if err != nil { + return nil, err + } + + infos := make([]fs.FileInfo, 0, len(entries)) + for _, entry := range entries { + info, err := entry.Info() + if err != nil { + return infos, err + } + infos = append(infos, info) + } + return infos, nil +} diff --git a/go/buildutil/util_test.go b/go/buildutil/util_test.go index e6761307583..6c507579a38 100644 --- a/go/buildutil/util_test.go +++ b/go/buildutil/util_test.go @@ -6,7 +6,6 @@ package buildutil_test import ( "go/build" - "io/ioutil" "os" "path/filepath" "runtime" @@ -53,7 +52,7 @@ func TestContainingPackage(t *testing.T) { if runtime.GOOS != "windows" && runtime.GOOS != "plan9" { // Make a symlink to gopath for test - tmp, err := ioutil.TempDir(os.TempDir(), "go") + tmp, err := os.MkdirTemp(os.TempDir(), "go") if err != nil { t.Errorf("Unable to create a temporary directory in %s", os.TempDir()) } diff --git a/go/callgraph/cha/cha_test.go b/go/callgraph/cha/cha_test.go index a12b3d0a348..0737a981481 100644 --- a/go/callgraph/cha/cha_test.go +++ b/go/callgraph/cha/cha_test.go @@ -16,7 +16,7 @@ import ( "go/parser" "go/token" "go/types" - "io/ioutil" + "os" "sort" "strings" "testing" @@ -98,7 +98,7 @@ func TestCHAGenerics(t *testing.T) { } func loadProgInfo(filename string, mode ssa.BuilderMode) (*ssa.Program, *ast.File, *ssa.Package, error) { - content, err := ioutil.ReadFile(filename) + content, err := os.ReadFile(filename) if err != nil { return nil, nil, nil, fmt.Errorf("couldn't read file '%s': %s", filename, err) } diff --git a/go/callgraph/vta/helpers_test.go b/go/callgraph/vta/helpers_test.go index facf6afa2ba..1a539e69b63 100644 --- a/go/callgraph/vta/helpers_test.go +++ b/go/callgraph/vta/helpers_test.go @@ -9,7 +9,7 @@ import ( "fmt" "go/ast" "go/parser" - "io/ioutil" + "os" "sort" "strings" "testing" @@ -38,7 +38,7 @@ func want(f *ast.File) []string { // `path`, assumed to define package "testdata," and the // test want result as list of strings. func testProg(path string, mode ssa.BuilderMode) (*ssa.Program, []string, error) { - content, err := ioutil.ReadFile(path) + content, err := os.ReadFile(path) if err != nil { return nil, nil, err } diff --git a/go/expect/expect_test.go b/go/expect/expect_test.go index e9ae40f7e09..da0ae984ecb 100644 --- a/go/expect/expect_test.go +++ b/go/expect/expect_test.go @@ -7,7 +7,7 @@ package expect_test import ( "bytes" "go/token" - "io/ioutil" + "os" "testing" "golang.org/x/tools/go/expect" @@ -52,7 +52,7 @@ func TestMarker(t *testing.T) { }, } { t.Run(tt.filename, func(t *testing.T) { - content, err := ioutil.ReadFile(tt.filename) + content, err := os.ReadFile(tt.filename) if err != nil { t.Fatal(err) } diff --git a/go/gccgoexportdata/gccgoexportdata.go b/go/gccgoexportdata/gccgoexportdata.go index 30ed521ea03..5df2edc13cb 100644 --- a/go/gccgoexportdata/gccgoexportdata.go +++ b/go/gccgoexportdata/gccgoexportdata.go @@ -20,7 +20,6 @@ import ( "go/token" "go/types" "io" - "io/ioutil" "strconv" "strings" @@ -46,7 +45,7 @@ func CompilerInfo(gccgo string, args ...string) (version, triple string, dirs [] // NewReader returns a reader for the export data section of an object // (.o) or archive (.a) file read from r. func NewReader(r io.Reader) (io.Reader, error) { - data, err := ioutil.ReadAll(r) + data, err := io.ReadAll(r) if err != nil { return nil, err } diff --git a/go/internal/cgo/cgo.go b/go/internal/cgo/cgo.go index 3fce4800342..38d5c6c7cd3 100644 --- a/go/internal/cgo/cgo.go +++ b/go/internal/cgo/cgo.go @@ -57,7 +57,6 @@ import ( "go/build" "go/parser" "go/token" - "io/ioutil" "log" "os" "path/filepath" @@ -70,7 +69,7 @@ import ( // ProcessFiles invokes the cgo preprocessor on bp.CgoFiles, parses // the output and returns the resulting ASTs. func ProcessFiles(bp *build.Package, fset *token.FileSet, DisplayPath func(path string) string, mode parser.Mode) ([]*ast.File, error) { - tmpdir, err := ioutil.TempDir("", strings.Replace(bp.ImportPath, "/", "_", -1)+"_C") + tmpdir, err := os.MkdirTemp("", strings.Replace(bp.ImportPath, "/", "_", -1)+"_C") if err != nil { return nil, err } diff --git a/go/internal/gccgoimporter/importer.go b/go/internal/gccgoimporter/importer.go index 1094af2c568..53f34c2fbf5 100644 --- a/go/internal/gccgoimporter/importer.go +++ b/go/internal/gccgoimporter/importer.go @@ -210,7 +210,7 @@ func GetImporter(searchpaths []string, initmap map[*types.Package]InitData) Impo // Excluded for now: Standard gccgo doesn't support this import format currently. // case goimporterMagic: // var data []byte - // data, err = ioutil.ReadAll(reader) + // data, err = io.ReadAll(reader) // if err != nil { // return // } diff --git a/go/loader/stdlib_test.go b/go/loader/stdlib_test.go index f3f3e39bf74..83d70dabdca 100644 --- a/go/loader/stdlib_test.go +++ b/go/loader/stdlib_test.go @@ -15,7 +15,7 @@ import ( "go/build" "go/token" "go/types" - "io/ioutil" + "os" "path/filepath" "runtime" "strings" @@ -190,7 +190,7 @@ func TestCgoOption(t *testing.T) { } // Load the file and check the object is declared at the right place. - b, err := ioutil.ReadFile(posn.Filename) + b, err := os.ReadFile(posn.Filename) if err != nil { t.Errorf("can't read %s: %s", posn.Filename, err) continue diff --git a/go/packages/golist.go b/go/packages/golist.go index b5de9cf9f21..1f1eade0ac8 100644 --- a/go/packages/golist.go +++ b/go/packages/golist.go @@ -9,7 +9,6 @@ import ( "context" "encoding/json" "fmt" - "io/ioutil" "log" "os" "path" @@ -1109,7 +1108,7 @@ func (state *golistState) writeOverlays() (filename string, cleanup func(), err if len(state.cfg.Overlay) == 0 { return "", func() {}, nil } - dir, err := ioutil.TempDir("", "gopackages-*") + dir, err := os.MkdirTemp("", "gopackages-*") if err != nil { return "", nil, err } @@ -1128,7 +1127,7 @@ func (state *golistState) writeOverlays() (filename string, cleanup func(), err // Create a unique filename for the overlaid files, to avoid // creating nested directories. noSeparator := strings.Join(strings.Split(filepath.ToSlash(k), "/"), "") - f, err := ioutil.TempFile(dir, fmt.Sprintf("*-%s", noSeparator)) + f, err := os.CreateTemp(dir, fmt.Sprintf("*-%s", noSeparator)) if err != nil { return "", func() {}, err } @@ -1146,7 +1145,7 @@ func (state *golistState) writeOverlays() (filename string, cleanup func(), err } // Write out the overlay file that contains the filepath mappings. filename = filepath.Join(dir, "overlay.json") - if err := ioutil.WriteFile(filename, b, 0665); err != nil { + if err := os.WriteFile(filename, b, 0665); err != nil { return "", func() {}, err } return filename, cleanup, nil diff --git a/go/packages/overlay_test.go b/go/packages/overlay_test.go index 4318739eb79..5760b7774b3 100644 --- a/go/packages/overlay_test.go +++ b/go/packages/overlay_test.go @@ -6,7 +6,6 @@ package packages_test import ( "fmt" - "io/ioutil" "log" "os" "path/filepath" @@ -499,7 +498,7 @@ func TestAdHocOverlays(t *testing.T) { // This test doesn't use packagestest because we are testing ad-hoc packages, // which are outside of $GOPATH and outside of a module. - tmp, err := ioutil.TempDir("", "testAdHocOverlays") + tmp, err := os.MkdirTemp("", "testAdHocOverlays") if err != nil { t.Fatal(err) } @@ -554,18 +553,18 @@ func TestOverlayModFileChanges(t *testing.T) { testenv.NeedsTool(t, "go") // Create two unrelated modules in a temporary directory. - tmp, err := ioutil.TempDir("", "tmp") + tmp, err := os.MkdirTemp("", "tmp") if err != nil { t.Fatal(err) } defer os.RemoveAll(tmp) // mod1 has a dependency on golang.org/x/xerrors. - mod1, err := ioutil.TempDir(tmp, "mod1") + mod1, err := os.MkdirTemp(tmp, "mod1") if err != nil { t.Fatal(err) } - if err := ioutil.WriteFile(filepath.Join(mod1, "go.mod"), []byte(`module mod1 + if err := os.WriteFile(filepath.Join(mod1, "go.mod"), []byte(`module mod1 require ( golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 @@ -575,7 +574,7 @@ func TestOverlayModFileChanges(t *testing.T) { } // mod2 does not have any dependencies. - mod2, err := ioutil.TempDir(tmp, "mod2") + mod2, err := os.MkdirTemp(tmp, "mod2") if err != nil { t.Fatal(err) } @@ -584,7 +583,7 @@ func TestOverlayModFileChanges(t *testing.T) { go 1.11 ` - if err := ioutil.WriteFile(filepath.Join(mod2, "go.mod"), []byte(want), 0775); err != nil { + if err := os.WriteFile(filepath.Join(mod2, "go.mod"), []byte(want), 0775); err != nil { t.Fatal(err) } @@ -610,7 +609,7 @@ func main() {} } // Check that mod2/go.mod has not been modified. - got, err := ioutil.ReadFile(filepath.Join(mod2, "go.mod")) + got, err := os.ReadFile(filepath.Join(mod2, "go.mod")) if err != nil { t.Fatal(err) } @@ -1045,7 +1044,7 @@ func TestOverlaysInReplace(t *testing.T) { // Create module b.com in a temporary directory. Do not add any Go files // on disk. - tmpPkgs, err := ioutil.TempDir("", "modules") + tmpPkgs, err := os.MkdirTemp("", "modules") if err != nil { t.Fatal(err) } @@ -1055,7 +1054,7 @@ func TestOverlaysInReplace(t *testing.T) { if err := os.Mkdir(dirB, 0775); err != nil { t.Fatal(err) } - if err := ioutil.WriteFile(filepath.Join(dirB, "go.mod"), []byte(fmt.Sprintf("module %s.com", dirB)), 0775); err != nil { + if err := os.WriteFile(filepath.Join(dirB, "go.mod"), []byte(fmt.Sprintf("module %s.com", dirB)), 0775); err != nil { t.Fatal(err) } if err := os.MkdirAll(filepath.Join(dirB, "inner"), 0775); err != nil { @@ -1063,7 +1062,7 @@ func TestOverlaysInReplace(t *testing.T) { } // Create a separate module that requires and replaces b.com. - tmpWorkspace, err := ioutil.TempDir("", "workspace") + tmpWorkspace, err := os.MkdirTemp("", "workspace") if err != nil { t.Fatal(err) } @@ -1078,7 +1077,7 @@ replace ( b.com => %s ) `, dirB) - if err := ioutil.WriteFile(filepath.Join(tmpWorkspace, "go.mod"), []byte(goModContent), 0775); err != nil { + if err := os.WriteFile(filepath.Join(tmpWorkspace, "go.mod"), []byte(goModContent), 0775); err != nil { t.Fatal(err) } diff --git a/go/packages/packages.go b/go/packages/packages.go index 124a6fe143b..ece0e7c603e 100644 --- a/go/packages/packages.go +++ b/go/packages/packages.go @@ -16,7 +16,6 @@ import ( "go/token" "go/types" "io" - "io/ioutil" "log" "os" "path/filepath" @@ -1127,7 +1126,7 @@ func (ld *loader) parseFile(filename string) (*ast.File, error) { var err error if src == nil { ioLimit <- true // wait - src, err = ioutil.ReadFile(filename) + src, err = os.ReadFile(filename) <-ioLimit // signal } if err != nil { diff --git a/go/packages/packages_test.go b/go/packages/packages_test.go index a89887f171c..c8a72282dbb 100644 --- a/go/packages/packages_test.go +++ b/go/packages/packages_test.go @@ -15,7 +15,6 @@ import ( "go/parser" "go/token" "go/types" - "io/ioutil" "os" "os/exec" "path/filepath" @@ -931,7 +930,7 @@ func TestAdHocPackagesBadImport(t *testing.T) { // This test doesn't use packagestest because we are testing ad-hoc packages, // which are outside of $GOPATH and outside of a module. - tmp, err := ioutil.TempDir("", "a") + tmp, err := os.MkdirTemp("", "a") if err != nil { t.Fatal(err) } @@ -942,7 +941,7 @@ func TestAdHocPackagesBadImport(t *testing.T) { import _ "badimport" const A = 1 `) - if err := ioutil.WriteFile(filename, content, 0775); err != nil { + if err := os.WriteFile(filename, content, 0775); err != nil { t.Fatal(err) } @@ -1748,7 +1747,7 @@ func testAdHocContains(t *testing.T, exporter packagestest.Exporter) { }}}) defer exported.Cleanup() - tmpfile, err := ioutil.TempFile("", "adhoc*.go") + tmpfile, err := os.CreateTemp("", "adhoc*.go") filename := tmpfile.Name() if err != nil { t.Fatal(err) @@ -1985,11 +1984,11 @@ import "C"`, } func buildFakePkgconfig(t *testing.T, env []string) string { - tmpdir, err := ioutil.TempDir("", "fakepkgconfig") + tmpdir, err := os.MkdirTemp("", "fakepkgconfig") if err != nil { t.Fatal(err) } - err = ioutil.WriteFile(filepath.Join(tmpdir, "pkg-config.go"), []byte(` + err = os.WriteFile(filepath.Join(tmpdir, "pkg-config.go"), []byte(` package main import "fmt" @@ -2452,7 +2451,7 @@ func testIssue37098(t *testing.T, exporter packagestest.Exporter) { if err != nil { t.Errorf("Failed to parse file '%s' as a Go source: %v", file, err) - contents, err := ioutil.ReadFile(file) + contents, err := os.ReadFile(file) if err != nil { t.Fatalf("Failed to read the un-parsable file '%s': %v", file, err) } @@ -2633,7 +2632,7 @@ func testExternal_NotHandled(t *testing.T, exporter packagestest.Exporter) { skipIfShort(t, "builds and links fake driver binaries") testenv.NeedsGoBuild(t) - tempdir, err := ioutil.TempDir("", "testexternal") + tempdir, err := os.MkdirTemp("", "testexternal") if err != nil { t.Fatal(err) } @@ -2652,7 +2651,7 @@ import ( ) func main() { - ioutil.ReadAll(os.Stdin) + io.ReadAll(os.Stdin) fmt.Println("{}") } `, @@ -2665,7 +2664,7 @@ import ( ) func main() { - ioutil.ReadAll(os.Stdin) + io.ReadAll(os.Stdin) fmt.Println("{\"NotHandled\": true}") } `, @@ -2755,7 +2754,7 @@ func TestEmptyEnvironment(t *testing.T) { func TestPackageLoadSingleFile(t *testing.T) { testenv.NeedsTool(t, "go") - tmp, err := ioutil.TempDir("", "a") + tmp, err := os.MkdirTemp("", "a") if err != nil { t.Fatal(err) } @@ -2763,7 +2762,7 @@ func TestPackageLoadSingleFile(t *testing.T) { filename := filepath.Join(tmp, "a.go") - if err := ioutil.WriteFile(filename, []byte(`package main; func main() { println("hello world") }`), 0775); err != nil { + if err := os.WriteFile(filename, []byte(`package main; func main() { println("hello world") }`), 0775); err != nil { t.Fatal(err) } @@ -2899,7 +2898,7 @@ func copyAll(srcPath, dstPath string) error { if info.IsDir() { return nil } - contents, err := ioutil.ReadFile(path) + contents, err := os.ReadFile(path) if err != nil { return err } @@ -2911,7 +2910,7 @@ func copyAll(srcPath, dstPath string) error { if err := os.MkdirAll(filepath.Dir(dstFilePath), 0755); err != nil { return err } - if err := ioutil.WriteFile(dstFilePath, contents, 0644); err != nil { + if err := os.WriteFile(dstFilePath, contents, 0644); err != nil { return err } return nil diff --git a/go/packages/packagestest/expect.go b/go/packages/packagestest/expect.go index 92c20a64a8d..00a30f713e2 100644 --- a/go/packages/packagestest/expect.go +++ b/go/packages/packagestest/expect.go @@ -7,7 +7,6 @@ package packagestest import ( "fmt" "go/token" - "io/ioutil" "os" "path/filepath" "reflect" @@ -226,7 +225,7 @@ func goModMarkers(e *Exported, gomod string) ([]*expect.Note, error) { } gomod = strings.TrimSuffix(gomod, ".temp") // If we are in Modules mode, copy the original contents file back into go.mod - if err := ioutil.WriteFile(gomod, content, 0644); err != nil { + if err := os.WriteFile(gomod, content, 0644); err != nil { return nil, nil } return expect.Parse(e.ExpectFileSet, gomod, content) diff --git a/go/packages/packagestest/export.go b/go/packages/packagestest/export.go index 16ded99ba6e..3558ccfdd01 100644 --- a/go/packages/packagestest/export.go +++ b/go/packages/packagestest/export.go @@ -69,7 +69,6 @@ import ( "fmt" "go/token" "io" - "io/ioutil" "log" "os" "path/filepath" @@ -198,7 +197,7 @@ func Export(t testing.TB, exporter Exporter, modules []Module) *Exported { dirname := strings.Replace(t.Name(), "/", "_", -1) dirname = strings.Replace(dirname, "#", "_", -1) // duplicate subtests get a #NNN suffix. - temp, err := ioutil.TempDir("", dirname) + temp, err := os.MkdirTemp("", dirname) if err != nil { t.Fatal(err) } @@ -254,7 +253,7 @@ func Export(t testing.TB, exporter Exporter, modules []Module) *Exported { t.Fatal(err) } case string: - if err := ioutil.WriteFile(fullpath, []byte(value), 0644); err != nil { + if err := os.WriteFile(fullpath, []byte(value), 0644); err != nil { t.Fatal(err) } default: @@ -278,7 +277,7 @@ func Export(t testing.TB, exporter Exporter, modules []Module) *Exported { // It is intended for source files that are shell scripts. func Script(contents string) Writer { return func(filename string) error { - return ioutil.WriteFile(filename, []byte(contents), 0755) + return os.WriteFile(filename, []byte(contents), 0755) } } @@ -659,7 +658,7 @@ func (e *Exported) FileContents(filename string) ([]byte, error) { if content, found := e.Config.Overlay[filename]; found { return content, nil } - content, err := ioutil.ReadFile(filename) + content, err := os.ReadFile(filename) if err != nil { return nil, err } diff --git a/go/packages/packagestest/export_test.go b/go/packages/packagestest/export_test.go index 1172f7c150a..eb13f560916 100644 --- a/go/packages/packagestest/export_test.go +++ b/go/packages/packagestest/export_test.go @@ -5,7 +5,6 @@ package packagestest_test import ( - "io/ioutil" "os" "path/filepath" "reflect" @@ -197,7 +196,7 @@ func TestMustCopyFiles(t *testing.T) { "nested/b/b.go": "package b", } - tmpDir, err := ioutil.TempDir("", t.Name()) + tmpDir, err := os.MkdirTemp("", t.Name()) if err != nil { t.Fatalf("failed to create a temporary directory: %v", err) } @@ -208,7 +207,7 @@ func TestMustCopyFiles(t *testing.T) { if err := os.MkdirAll(filepath.Dir(fullpath), 0755); err != nil { t.Fatal(err) } - if err := ioutil.WriteFile(fullpath, []byte(contents), 0644); err != nil { + if err := os.WriteFile(fullpath, []byte(contents), 0644); err != nil { t.Fatal(err) } } diff --git a/go/packages/packagestest/modules.go b/go/packages/packagestest/modules.go index 69a6c935dc7..1299c6c3c73 100644 --- a/go/packages/packagestest/modules.go +++ b/go/packages/packagestest/modules.go @@ -7,7 +7,6 @@ package packagestest import ( "context" "fmt" - "io/ioutil" "os" "path" "path/filepath" @@ -90,11 +89,11 @@ func (modules) Finalize(exported *Exported) error { // If the primary module already has a go.mod, write the contents to a temp // go.mod for now and then we will reset it when we are getting all the markers. if gomod := exported.written[exported.primary]["go.mod"]; gomod != "" { - contents, err := ioutil.ReadFile(gomod) + contents, err := os.ReadFile(gomod) if err != nil { return err } - if err := ioutil.WriteFile(gomod+".temp", contents, 0644); err != nil { + if err := os.WriteFile(gomod+".temp", contents, 0644); err != nil { return err } } @@ -115,7 +114,7 @@ func (modules) Finalize(exported *Exported) error { primaryGomod += fmt.Sprintf("\t%v %v\n", other, version) } primaryGomod += ")\n" - if err := ioutil.WriteFile(filepath.Join(primaryDir, "go.mod"), []byte(primaryGomod), 0644); err != nil { + if err := os.WriteFile(filepath.Join(primaryDir, "go.mod"), []byte(primaryGomod), 0644); err != nil { return err } @@ -136,7 +135,7 @@ func (modules) Finalize(exported *Exported) error { if v, ok := versions[module]; ok { module = v.module } - if err := ioutil.WriteFile(modfile, []byte("module "+module+"\n"), 0644); err != nil { + if err := os.WriteFile(modfile, []byte("module "+module+"\n"), 0644); err != nil { return err } files["go.mod"] = modfile @@ -193,7 +192,7 @@ func (modules) Finalize(exported *Exported) error { func writeModuleFiles(rootDir, module, ver string, filePaths map[string]string) error { fileData := make(map[string][]byte) for name, path := range filePaths { - contents, err := ioutil.ReadFile(path) + contents, err := os.ReadFile(path) if err != nil { return err } diff --git a/go/ssa/source_test.go b/go/ssa/source_test.go index 4fba8a5f5d8..9a7b30675b5 100644 --- a/go/ssa/source_test.go +++ b/go/ssa/source_test.go @@ -13,7 +13,6 @@ import ( "go/parser" "go/token" "go/types" - "io/ioutil" "os" "runtime" "strings" @@ -33,7 +32,7 @@ func TestObjValueLookup(t *testing.T) { } conf := loader.Config{ParserMode: parser.ParseComments} - src, err := ioutil.ReadFile("testdata/objlookup.go") + src, err := os.ReadFile("testdata/objlookup.go") if err != nil { t.Fatal(err) } diff --git a/godoc/server.go b/godoc/server.go index 57576e10282..a6df6d74e68 100644 --- a/godoc/server.go +++ b/godoc/server.go @@ -16,7 +16,6 @@ import ( htmlpkg "html" htmltemplate "html/template" "io" - "io/ioutil" "log" "net/http" "os" @@ -85,7 +84,7 @@ func (h *handlerServer) GetPageInfo(abspath, relpath string, mode PageInfoMode, if err != nil { return nil, err } - return ioutil.NopCloser(bytes.NewReader(data)), nil + return io.NopCloser(bytes.NewReader(data)), nil } // Make the syscall/js package always visible by default. diff --git a/godoc/static/gen.go b/godoc/static/gen.go index 7991c8208da..9fe0bd56f3c 100644 --- a/godoc/static/gen.go +++ b/godoc/static/gen.go @@ -10,7 +10,7 @@ import ( "bytes" "fmt" "go/format" - "io/ioutil" + "os" "unicode" ) @@ -71,7 +71,7 @@ func Generate() ([]byte, error) { fmt.Fprintf(buf, "%v\n\n%v\n\npackage static\n\n", license, warning) fmt.Fprintf(buf, "var Files = map[string]string{\n") for _, fn := range files { - b, err := ioutil.ReadFile(fn) + b, err := os.ReadFile(fn) if err != nil { return b, err } diff --git a/godoc/static/gen_test.go b/godoc/static/gen_test.go index 7f743290319..1f1c62e0e9c 100644 --- a/godoc/static/gen_test.go +++ b/godoc/static/gen_test.go @@ -6,7 +6,7 @@ package static import ( "bytes" - "io/ioutil" + "os" "runtime" "strconv" "testing" @@ -17,7 +17,7 @@ func TestStaticIsUpToDate(t *testing.T) { if runtime.GOOS == "android" { t.Skip("files not available on android") } - oldBuf, err := ioutil.ReadFile("static.go") + oldBuf, err := os.ReadFile("static.go") if err != nil { t.Errorf("error while reading static.go: %v\n", err) } diff --git a/godoc/static/makestatic.go b/godoc/static/makestatic.go index ef7b9042aac..a8a652f8ed5 100644 --- a/godoc/static/makestatic.go +++ b/godoc/static/makestatic.go @@ -11,7 +11,6 @@ package main import ( "fmt" - "io/ioutil" "os" "golang.org/x/tools/godoc/static" @@ -29,7 +28,7 @@ func makestatic() error { if err != nil { return fmt.Errorf("error while generating static.go: %v\n", err) } - err = ioutil.WriteFile("static.go", buf, 0666) + err = os.WriteFile("static.go", buf, 0666) if err != nil { return fmt.Errorf("error while writing static.go: %v\n", err) } diff --git a/godoc/vfs/mapfs/mapfs_test.go b/godoc/vfs/mapfs/mapfs_test.go index 6b7db290ee3..954ef7e151b 100644 --- a/godoc/vfs/mapfs/mapfs_test.go +++ b/godoc/vfs/mapfs/mapfs_test.go @@ -5,7 +5,7 @@ package mapfs import ( - "io/ioutil" + "io" "os" "reflect" "testing" @@ -36,7 +36,7 @@ func TestOpenRoot(t *testing.T) { t.Errorf("Open(%q) = %v", tt.path, err) continue } - slurp, err := ioutil.ReadAll(rsc) + slurp, err := io.ReadAll(rsc) if err != nil { t.Error(err) } diff --git a/godoc/vfs/namespace.go b/godoc/vfs/namespace.go index 23dd9794312..9326cedec5b 100644 --- a/godoc/vfs/namespace.go +++ b/godoc/vfs/namespace.go @@ -77,7 +77,7 @@ const debugNS = false // just "/src" in the final two calls. // // OS is itself an implementation of a file system: it implements -// OS(`c:\Go`).ReadDir("/src/pkg/code") as ioutil.ReadDir(`c:\Go\src\pkg\code`). +// OS(`c:\Go`).ReadDir("/src/pkg/code") as ioutilReadDir(`c:\Go\src\pkg\code`). // // Because the new path is evaluated by fs (here OS(root)), another way // to read the mount table is to mentally combine fs+new, so that this table: diff --git a/godoc/vfs/os.go b/godoc/vfs/os.go index 35d050946e6..e5f7d5e77d9 100644 --- a/godoc/vfs/os.go +++ b/godoc/vfs/os.go @@ -7,7 +7,7 @@ package vfs import ( "fmt" "go/build" - "io/ioutil" + "io/fs" "os" pathpkg "path" "path/filepath" @@ -101,5 +101,22 @@ func (root osFS) Stat(path string) (os.FileInfo, error) { } func (root osFS) ReadDir(path string) ([]os.FileInfo, error) { - return ioutil.ReadDir(root.resolve(path)) // is sorted + return ioutilReadDir(root.resolve(path)) // is sorted +} + +func ioutilReadDir(dirname string) ([]fs.FileInfo, error) { + entries, err := os.ReadDir(dirname) + if err != nil { + return nil, err + } + + infos := make([]fs.FileInfo, 0, len(entries)) + for _, entry := range entries { + info, err := entry.Info() + if err != nil { + return infos, err + } + infos = append(infos, info) + } + return infos, nil } diff --git a/godoc/vfs/vfs.go b/godoc/vfs/vfs.go index d70526d5ac9..f4ec2aa7a02 100644 --- a/godoc/vfs/vfs.go +++ b/godoc/vfs/vfs.go @@ -8,7 +8,6 @@ package vfs // import "golang.org/x/tools/godoc/vfs" import ( "io" - "io/ioutil" "os" ) @@ -54,5 +53,5 @@ func ReadFile(fs Opener, path string) ([]byte, error) { return nil, err } defer rc.Close() - return ioutil.ReadAll(rc) + return io.ReadAll(rc) } diff --git a/godoc/vfs/zipfs/zipfs_test.go b/godoc/vfs/zipfs/zipfs_test.go index 2c52a60c68c..b6f2431b0b5 100644 --- a/godoc/vfs/zipfs/zipfs_test.go +++ b/godoc/vfs/zipfs/zipfs_test.go @@ -8,7 +8,6 @@ import ( "bytes" "fmt" "io" - "io/ioutil" "os" "reflect" "testing" @@ -174,7 +173,7 @@ func TestZipFSOpenSeek(t *testing.T) { // test Seek() multiple times for i := 0; i < 3; i++ { - all, err := ioutil.ReadAll(f) + all, err := io.ReadAll(f) if err != nil { t.Error(err) return diff --git a/gopls/doc/generate.go b/gopls/doc/generate.go index 51987f6a7b0..34034fb4e58 100644 --- a/gopls/doc/generate.go +++ b/gopls/doc/generate.go @@ -18,7 +18,6 @@ import ( "go/token" "go/types" "io" - "io/ioutil" "os" "os/exec" "path/filepath" @@ -573,7 +572,7 @@ func fileForPos(pkg *packages.Package, pos token.Pos) (*ast.File, error) { } func rewriteFile(file string, api *source.APIJSON, write bool, rewrite func([]byte, *source.APIJSON) ([]byte, error)) (bool, error) { - old, err := ioutil.ReadFile(file) + old, err := os.ReadFile(file) if err != nil { return false, err } @@ -587,7 +586,7 @@ func rewriteFile(file string, api *source.APIJSON, write bool, rewrite func([]by return bytes.Equal(old, new), nil } - if err := ioutil.WriteFile(file, new, 0); err != nil { + if err := os.WriteFile(file, new, 0); err != nil { return false, err } diff --git a/gopls/integration/govim/artifacts.go b/gopls/integration/govim/artifacts.go index a069ff185aa..db375a21e41 100644 --- a/gopls/integration/govim/artifacts.go +++ b/gopls/integration/govim/artifacts.go @@ -7,7 +7,7 @@ package main import ( "flag" "fmt" - "io/ioutil" + "io" "net/http" "os" "path" @@ -56,11 +56,11 @@ func download(artifactURL string) error { if resp.StatusCode != http.StatusOK { return fmt.Errorf("got status code %d from GCS", resp.StatusCode) } - data, err := ioutil.ReadAll(resp.Body) + data, err := io.ReadAll(resp.Body) if err != nil { return fmt.Errorf("reading result: %v", err) } - if err := ioutil.WriteFile(name, data, 0644); err != nil { + if err := os.WriteFile(name, data, 0644); err != nil { return fmt.Errorf("writing artifact: %v", err) } return nil diff --git a/gopls/internal/hooks/diff.go b/gopls/internal/hooks/diff.go index a6ad65f6a26..53dc4975a36 100644 --- a/gopls/internal/hooks/diff.go +++ b/gopls/internal/hooks/diff.go @@ -7,7 +7,6 @@ package hooks import ( "encoding/json" "fmt" - "io/ioutil" "log" "os" "path/filepath" @@ -42,7 +41,7 @@ var ( // save writes a JSON record of statistics about diff requests to a temporary file. func (s *diffstat) save() { diffStatsOnce.Do(func() { - f, err := ioutil.TempFile("", "gopls-diff-stats-*") + f, err := os.CreateTemp("", "gopls-diff-stats-*") if err != nil { log.Printf("can't create diff stats temp file: %v", err) // e.g. disk full return @@ -91,7 +90,7 @@ func disaster(before, after string) string { // We use NUL as a separator: it should never appear in Go source. data := before + "\x00" + after - if err := ioutil.WriteFile(filename, []byte(data), 0600); err != nil { + if err := os.WriteFile(filename, []byte(data), 0600); err != nil { log.Printf("failed to write diff bug report: %v", err) return "" } diff --git a/gopls/internal/hooks/diff_test.go b/gopls/internal/hooks/diff_test.go index a46bf3b2d28..0a809589892 100644 --- a/gopls/internal/hooks/diff_test.go +++ b/gopls/internal/hooks/diff_test.go @@ -5,7 +5,6 @@ package hooks import ( - "io/ioutil" "os" "testing" @@ -20,7 +19,7 @@ func TestDisaster(t *testing.T) { a := "This is a string,(\u0995) just for basic\nfunctionality" b := "This is another string, (\u0996) to see if disaster will store stuff correctly" fname := disaster(a, b) - buf, err := ioutil.ReadFile(fname) + buf, err := os.ReadFile(fname) if err != nil { t.Fatal(err) } diff --git a/gopls/internal/hooks/licenses_test.go b/gopls/internal/hooks/licenses_test.go index 609f05a606c..4f3ed4f9a56 100644 --- a/gopls/internal/hooks/licenses_test.go +++ b/gopls/internal/hooks/licenses_test.go @@ -6,7 +6,7 @@ package hooks import ( "bytes" - "io/ioutil" + "os" "os/exec" "runtime" "testing" @@ -23,7 +23,7 @@ func TestLicenses(t *testing.T) { if runtime.GOOS != "linux" && runtime.GOOS != "darwin" { t.Skip("generating licenses only works on Unixes") } - tmp, err := ioutil.TempFile("", "") + tmp, err := os.CreateTemp("", "") if err != nil { t.Fatal(err) } @@ -33,11 +33,11 @@ func TestLicenses(t *testing.T) { t.Fatalf("generating licenses failed: %q, %v", out, err) } - got, err := ioutil.ReadFile(tmp.Name()) + got, err := os.ReadFile(tmp.Name()) if err != nil { t.Fatal(err) } - want, err := ioutil.ReadFile("licenses.go") + want, err := os.ReadFile("licenses.go") if err != nil { t.Fatal(err) } diff --git a/gopls/internal/lsp/cmd/capabilities_test.go b/gopls/internal/lsp/cmd/capabilities_test.go index 02ff0d950f6..e952b0dcbe7 100644 --- a/gopls/internal/lsp/cmd/capabilities_test.go +++ b/gopls/internal/lsp/cmd/capabilities_test.go @@ -7,7 +7,6 @@ package cmd import ( "context" "fmt" - "io/ioutil" "os" "path/filepath" "testing" @@ -28,15 +27,15 @@ func TestCapabilities(t *testing.T) { // Is there some missing error reporting somewhere? testenv.NeedsTool(t, "go") - tmpDir, err := ioutil.TempDir("", "fake") + tmpDir, err := os.MkdirTemp("", "fake") if err != nil { t.Fatal(err) } tmpFile := filepath.Join(tmpDir, "fake.go") - if err := ioutil.WriteFile(tmpFile, []byte(""), 0775); err != nil { + if err := os.WriteFile(tmpFile, []byte(""), 0775); err != nil { t.Fatal(err) } - if err := ioutil.WriteFile(filepath.Join(tmpDir, "go.mod"), []byte("module fake\n\ngo 1.12\n"), 0775); err != nil { + if err := os.WriteFile(filepath.Join(tmpDir, "go.mod"), []byte("module fake\n\ngo 1.12\n"), 0775); err != nil { t.Fatal(err) } defer os.RemoveAll(tmpDir) diff --git a/gopls/internal/lsp/cmd/cmd.go b/gopls/internal/lsp/cmd/cmd.go index 9a9d627880a..12976f04579 100644 --- a/gopls/internal/lsp/cmd/cmd.go +++ b/gopls/internal/lsp/cmd/cmd.go @@ -11,7 +11,6 @@ import ( "context" "flag" "fmt" - "io/ioutil" "log" "os" "reflect" @@ -703,7 +702,7 @@ func (c *cmdClient) getFile(uri span.URI) *cmdFile { c.files[uri] = file } if file.mapper == nil { - content, err := ioutil.ReadFile(uri.Filename()) + content, err := os.ReadFile(uri.Filename()) if err != nil { file.err = fmt.Errorf("getFile: %v: %v", uri, err) return file diff --git a/gopls/internal/lsp/cmd/help_test.go b/gopls/internal/lsp/cmd/help_test.go index 6bd3c8c501f..a8feb6e9001 100644 --- a/gopls/internal/lsp/cmd/help_test.go +++ b/gopls/internal/lsp/cmd/help_test.go @@ -8,7 +8,7 @@ import ( "bytes" "context" "flag" - "io/ioutil" + "os" "path/filepath" "testing" @@ -41,12 +41,12 @@ func TestHelpFiles(t *testing.T) { helpFile := filepath.Join("usage", name+".hlp") got := buf.Bytes() if *updateHelpFiles { - if err := ioutil.WriteFile(helpFile, got, 0666); err != nil { + if err := os.WriteFile(helpFile, got, 0666); err != nil { t.Errorf("Failed writing %v: %v", helpFile, err) } return } - want, err := ioutil.ReadFile(helpFile) + want, err := os.ReadFile(helpFile) if err != nil { t.Fatalf("Missing help file %q", helpFile) } diff --git a/gopls/internal/lsp/cmd/semantictokens.go b/gopls/internal/lsp/cmd/semantictokens.go index 3d2ad9c58e2..1acd83a2ac0 100644 --- a/gopls/internal/lsp/cmd/semantictokens.go +++ b/gopls/internal/lsp/cmd/semantictokens.go @@ -11,7 +11,6 @@ import ( "fmt" "go/parser" "go/token" - "io/ioutil" "log" "os" "unicode/utf8" @@ -87,7 +86,7 @@ func (c *semtok) Run(ctx context.Context, args ...string) error { return err } - buf, err := ioutil.ReadFile(args[0]) + buf, err := os.ReadFile(args[0]) if err != nil { return err } @@ -162,7 +161,7 @@ func markLine(m mark, lines [][]byte) { } func decorate(file string, result []uint32) error { - buf, err := ioutil.ReadFile(file) + buf, err := os.ReadFile(file) if err != nil { return err } diff --git a/gopls/internal/lsp/command/interface_test.go b/gopls/internal/lsp/command/interface_test.go index 2eb6f9ac819..f81a2aa22fd 100644 --- a/gopls/internal/lsp/command/interface_test.go +++ b/gopls/internal/lsp/command/interface_test.go @@ -5,7 +5,7 @@ package command_test import ( - "io/ioutil" + "os" "testing" "github.com/google/go-cmp/cmp" @@ -17,7 +17,7 @@ func TestGenerated(t *testing.T) { testenv.NeedsGoPackages(t) testenv.NeedsLocalXTools(t) - onDisk, err := ioutil.ReadFile("command_gen.go") + onDisk, err := os.ReadFile("command_gen.go") if err != nil { t.Fatal(err) } diff --git a/gopls/internal/lsp/fake/sandbox.go b/gopls/internal/lsp/fake/sandbox.go index 41188af30fe..8218bc15bc5 100644 --- a/gopls/internal/lsp/fake/sandbox.go +++ b/gopls/internal/lsp/fake/sandbox.go @@ -8,7 +8,6 @@ import ( "context" "errors" "fmt" - "io/ioutil" "os" "path/filepath" "strings" @@ -92,7 +91,7 @@ func NewSandbox(config *SandboxConfig) (_ *Sandbox, err error) { rootDir := config.RootDir if rootDir == "" { - rootDir, err = ioutil.TempDir(config.RootDir, "gopls-sandbox-") + rootDir, err = os.MkdirTemp(config.RootDir, "gopls-sandbox-") if err != nil { return nil, fmt.Errorf("creating temporary workdir: %v", err) } @@ -150,7 +149,7 @@ func NewSandbox(config *SandboxConfig) (_ *Sandbox, err error) { // is the responsibility of the caller to call os.RemoveAll on the returned // file path when it is no longer needed. func Tempdir(files map[string][]byte) (string, error) { - dir, err := ioutil.TempDir("", "gopls-tempdir-") + dir, err := os.MkdirTemp("", "gopls-tempdir-") if err != nil { return "", err } diff --git a/gopls/internal/lsp/fake/workdir.go b/gopls/internal/lsp/fake/workdir.go index d5e8eb2af22..462d54821f1 100644 --- a/gopls/internal/lsp/fake/workdir.go +++ b/gopls/internal/lsp/fake/workdir.go @@ -10,7 +10,6 @@ import ( "crypto/sha256" "fmt" "io/fs" - "io/ioutil" "os" "path/filepath" "runtime" @@ -57,7 +56,7 @@ func writeFileData(path string, content []byte, rel RelativeTo) error { } backoff := 1 * time.Millisecond for { - err := ioutil.WriteFile(fp, []byte(content), 0644) + err := os.WriteFile(fp, []byte(content), 0644) if err != nil { // This lock file violation is not handled by the robustio package, as it // indicates a real race condition that could be avoided. @@ -160,7 +159,7 @@ func toURI(fp string) protocol.DocumentURI { func (w *Workdir) ReadFile(path string) ([]byte, error) { backoff := 1 * time.Millisecond for { - b, err := ioutil.ReadFile(w.AbsPath(path)) + b, err := os.ReadFile(w.AbsPath(path)) if err != nil { if runtime.GOOS == "plan9" && strings.HasSuffix(err.Error(), " exclusive use file already open") { // Plan 9 enforces exclusive access to locked files. @@ -370,7 +369,7 @@ func (w *Workdir) pollFiles() ([]protocol.FileEvent, error) { // must read the file contents. id := fileID{mtime: info.ModTime()} if time.Since(info.ModTime()) < 2*time.Second { - data, err := ioutil.ReadFile(fp) + data, err := os.ReadFile(fp) if err != nil { return err } @@ -397,7 +396,7 @@ func (w *Workdir) pollFiles() ([]protocol.FileEvent, error) { // In this case, read the content to check whether the file actually // changed. if oldID.mtime.Equal(id.mtime) && oldID.hash != "" && id.hash == "" { - data, err := ioutil.ReadFile(fp) + data, err := os.ReadFile(fp) if err != nil { return err } diff --git a/gopls/internal/lsp/fake/workdir_test.go b/gopls/internal/lsp/fake/workdir_test.go index fe89fa72dc3..b45b5339991 100644 --- a/gopls/internal/lsp/fake/workdir_test.go +++ b/gopls/internal/lsp/fake/workdir_test.go @@ -6,7 +6,6 @@ package fake import ( "context" - "io/ioutil" "os" "sync" "testing" @@ -32,7 +31,7 @@ Hello World! func newWorkdir(t *testing.T, txt string) (*Workdir, *eventBuffer, func()) { t.Helper() - tmpdir, err := ioutil.TempDir("", "goplstest-workdir-") + tmpdir, err := os.MkdirTemp("", "goplstest-workdir-") if err != nil { t.Fatal(err) } diff --git a/gopls/internal/regtest/bench/bench_test.go b/gopls/internal/regtest/bench/bench_test.go index 0120a1a65f0..610f43fa876 100644 --- a/gopls/internal/regtest/bench/bench_test.go +++ b/gopls/internal/regtest/bench/bench_test.go @@ -11,7 +11,6 @@ import ( "flag" "fmt" "io" - "io/ioutil" "log" "os" "os/exec" @@ -77,7 +76,7 @@ func TestMain(m *testing.M) { func getTempDir() string { makeTempDirOnce.Do(func() { var err error - tempDir, err = ioutil.TempDir("", "gopls-bench") + tempDir, err = os.MkdirTemp("", "gopls-bench") if err != nil { log.Fatal(err) } diff --git a/gopls/internal/regtest/misc/generate_test.go b/gopls/internal/regtest/misc/generate_test.go index 547755fd271..0cfcab59d24 100644 --- a/gopls/internal/regtest/misc/generate_test.go +++ b/gopls/internal/regtest/misc/generate_test.go @@ -27,12 +27,11 @@ go 1.14 package main import ( - "io/ioutil" "os" ) func main() { - ioutil.WriteFile("generated.go", []byte("package " + os.Args[1] + "\n\nconst Answer = 21"), 0644) + os.WriteFile("generated.go", []byte("package " + os.Args[1] + "\n\nconst Answer = 21"), 0644) } -- lib1/lib.go -- diff --git a/gopls/internal/regtest/misc/imports_test.go b/gopls/internal/regtest/misc/imports_test.go index 82c05a6319b..1e1d303379d 100644 --- a/gopls/internal/regtest/misc/imports_test.go +++ b/gopls/internal/regtest/misc/imports_test.go @@ -5,7 +5,6 @@ package misc import ( - "io/ioutil" "os" "path/filepath" "strings" @@ -175,7 +174,7 @@ import "example.com/x" var _, _ = x.X, y.Y ` - modcache, err := ioutil.TempDir("", "TestGOMODCACHE-modcache") + modcache, err := os.MkdirTemp("", "TestGOMODCACHE-modcache") if err != nil { t.Fatal(err) } diff --git a/gopls/internal/vulncheck/vulntest/db.go b/gopls/internal/vulncheck/vulntest/db.go index 2e756e3ea33..a4ea54b95fc 100644 --- a/gopls/internal/vulncheck/vulntest/db.go +++ b/gopls/internal/vulncheck/vulntest/db.go @@ -13,7 +13,6 @@ import ( "context" "encoding/json" "fmt" - "io/ioutil" "os" "path/filepath" "sort" @@ -41,7 +40,7 @@ import ( // The returned DB's Clean method must be called to clean up the // generated database. func NewDatabase(ctx context.Context, txtarReports []byte) (*DB, error) { - disk, err := ioutil.TempDir("", "vulndb-test") + disk, err := os.MkdirTemp("", "vulndb-test") if err != nil { return nil, err } diff --git a/gopls/internal/vulncheck/vulntest/report_test.go b/gopls/internal/vulncheck/vulntest/report_test.go index c42dae805fa..31f62aba838 100644 --- a/gopls/internal/vulncheck/vulntest/report_test.go +++ b/gopls/internal/vulncheck/vulntest/report_test.go @@ -10,7 +10,6 @@ package vulntest import ( "bytes" "io" - "io/ioutil" "os" "path/filepath" "testing" @@ -19,7 +18,7 @@ import ( ) func readAll(t *testing.T, filename string) io.Reader { - d, err := ioutil.ReadFile(filename) + d, err := os.ReadFile(filename) if err != nil { t.Fatal(err) } diff --git a/gopls/release/release.go b/gopls/release/release.go index dab95822eb6..b2e0b3ca847 100644 --- a/gopls/release/release.go +++ b/gopls/release/release.go @@ -15,7 +15,6 @@ import ( "flag" "fmt" "go/types" - "io/ioutil" "log" "os" "path/filepath" @@ -109,7 +108,7 @@ func validateHardcodedVersion(version string) error { func validateGoModFile(goplsDir string) error { filename := filepath.Join(goplsDir, "go.mod") - data, err := ioutil.ReadFile(filename) + data, err := os.ReadFile(filename) if err != nil { return err } diff --git a/imports/forward.go b/imports/forward.go index d2547c74338..cb6db8893f9 100644 --- a/imports/forward.go +++ b/imports/forward.go @@ -7,8 +7,8 @@ package imports // import "golang.org/x/tools/imports" import ( - "io/ioutil" "log" + "os" "golang.org/x/tools/internal/gocommand" intimp "golang.org/x/tools/internal/imports" @@ -44,7 +44,7 @@ var LocalPrefix string func Process(filename string, src []byte, opt *Options) ([]byte, error) { var err error if src == nil { - src, err = ioutil.ReadFile(filename) + src, err = os.ReadFile(filename) if err != nil { return nil, err } diff --git a/internal/apidiff/apidiff_test.go b/internal/apidiff/apidiff_test.go index b385b7cbbab..ecf32e4a22f 100644 --- a/internal/apidiff/apidiff_test.go +++ b/internal/apidiff/apidiff_test.go @@ -8,7 +8,6 @@ import ( "bufio" "fmt" "go/types" - "io/ioutil" "os" "path/filepath" "reflect" @@ -21,7 +20,7 @@ import ( ) func TestChanges(t *testing.T) { - dir, err := ioutil.TempDir("", "apidiff_test") + dir, err := os.MkdirTemp("", "apidiff_test") if err != nil { t.Fatal(err) } @@ -66,7 +65,7 @@ func splitIntoPackages(t *testing.T, dir string) (incompatibles, compatibles []s if err := os.MkdirAll(filepath.Join(dir, "src", "apidiff"), 0700); err != nil { t.Fatal(err) } - if err := ioutil.WriteFile(filepath.Join(dir, "src", "apidiff", "go.mod"), []byte("module apidiff\n"), 0666); err != nil { + if err := os.WriteFile(filepath.Join(dir, "src", "apidiff", "go.mod"), []byte("module apidiff\n"), 0666); err != nil { t.Fatal(err) } diff --git a/internal/diff/difftest/difftest_test.go b/internal/diff/difftest/difftest_test.go index a990e522438..c64a0fa0c9f 100644 --- a/internal/diff/difftest/difftest_test.go +++ b/internal/diff/difftest/difftest_test.go @@ -9,7 +9,6 @@ package difftest_test import ( "fmt" - "io/ioutil" "os" "os/exec" "strings" @@ -41,7 +40,7 @@ func TestVerifyUnified(t *testing.T) { } func getDiffOutput(a, b string) (string, error) { - fileA, err := ioutil.TempFile("", "myers.in") + fileA, err := os.CreateTemp("", "myers.in") if err != nil { return "", err } @@ -52,7 +51,7 @@ func getDiffOutput(a, b string) (string, error) { if err := fileA.Close(); err != nil { return "", err } - fileB, err := ioutil.TempFile("", "myers.in") + fileB, err := os.CreateTemp("", "myers.in") if err != nil { return "", err } diff --git a/internal/diff/lcs/old_test.go b/internal/diff/lcs/old_test.go index 0c894316fa5..e39941ba0a5 100644 --- a/internal/diff/lcs/old_test.go +++ b/internal/diff/lcs/old_test.go @@ -6,9 +6,9 @@ package lcs import ( "fmt" - "io/ioutil" "log" "math/rand" + "os" "strings" "testing" ) @@ -218,7 +218,7 @@ func genBench(set string, n int) []struct{ before, after string } { // itself minus the last byte is faster still; I don't know why. // There is much low-hanging fruit here for further improvement. func BenchmarkLargeFileSmallDiff(b *testing.B) { - data, err := ioutil.ReadFile("old.go") // large file + data, err := os.ReadFile("old.go") // large file if err != nil { log.Fatal(err) } diff --git a/internal/event/bench_test.go b/internal/event/bench_test.go index 9ec7519b5d6..aae2a57b09f 100644 --- a/internal/event/bench_test.go +++ b/internal/event/bench_test.go @@ -6,7 +6,7 @@ package event_test import ( "context" - "io/ioutil" + "io" "log" "testing" @@ -119,7 +119,7 @@ func Benchmark(b *testing.B) { b.Run(t.name+"Noop", t.test) } - event.SetExporter(export.Spans(export.LogWriter(ioutil.Discard, false))) + event.SetExporter(export.Spans(export.LogWriter(io.Discard, false))) for _, t := range benchmarks { b.Run(t.name, t.test) } @@ -150,7 +150,7 @@ func (hooks Hooks) runBenchmark(b *testing.B) { } func init() { - log.SetOutput(ioutil.Discard) + log.SetOutput(io.Discard) } func noopExporter(ctx context.Context, ev core.Event, lm label.Map) context.Context { diff --git a/internal/event/export/ocagent/ocagent_test.go b/internal/event/export/ocagent/ocagent_test.go index 88730b10adf..38a52faede5 100644 --- a/internal/event/export/ocagent/ocagent_test.go +++ b/internal/event/export/ocagent/ocagent_test.go @@ -9,7 +9,7 @@ import ( "context" "encoding/json" "fmt" - "io/ioutil" + "io" "net/http" "sync" "testing" @@ -191,7 +191,7 @@ func (s *fakeSender) RoundTrip(req *http.Request) (*http.Response, error) { if s.data == nil { s.data = make(map[string][]byte) } - data, err := ioutil.ReadAll(req.Body) + data, err := io.ReadAll(req.Body) if err != nil { return nil, err } diff --git a/internal/facts/facts.go b/internal/facts/facts.go index ec11d5e0af1..8480ea062f7 100644 --- a/internal/facts/facts.go +++ b/internal/facts/facts.go @@ -40,7 +40,7 @@ import ( "encoding/gob" "fmt" "go/types" - "io/ioutil" + "io" "log" "reflect" "sort" @@ -356,7 +356,7 @@ func (s *Set) Encode(skipMethodSorting bool) []byte { if err := gob.NewEncoder(&buf).Encode(gobFacts); err != nil { // Fact encoding should never fail. Identify the culprit. for _, gf := range gobFacts { - if err := gob.NewEncoder(ioutil.Discard).Encode(gf); err != nil { + if err := gob.NewEncoder(io.Discard).Encode(gf); err != nil { fact := gf.Fact pkgpath := reflect.TypeOf(fact).Elem().PkgPath() log.Panicf("internal error: gob encoding of analysis fact %s failed: %v; please report a bug against fact %T in package %q", diff --git a/internal/fastwalk/fastwalk_portable.go b/internal/fastwalk/fastwalk_portable.go index 085d311600b..563c9a3c98f 100644 --- a/internal/fastwalk/fastwalk_portable.go +++ b/internal/fastwalk/fastwalk_portable.go @@ -8,7 +8,7 @@ package fastwalk import ( - "io/ioutil" + "io/fs" "os" ) @@ -17,7 +17,7 @@ import ( // If fn returns a non-nil error, readDir returns with that error // immediately. func readDir(dirName string, fn func(dirName, entName string, typ os.FileMode) error) error { - fis, err := ioutil.ReadDir(dirName) + fis, err := ioutilReadDir(dirName) if err != nil { return err } @@ -36,3 +36,20 @@ func readDir(dirName string, fn func(dirName, entName string, typ os.FileMode) e } return nil } + +func ioutilReadDir(dirname string) ([]fs.FileInfo, error) { + entries, err := os.ReadDir(dirname) + if err != nil { + return nil, err + } + + infos := make([]fs.FileInfo, 0, len(entries)) + for _, entry := range entries { + info, err := entry.Info() + if err != nil { + return infos, err + } + infos = append(infos, info) + } + return infos, nil +} diff --git a/internal/fastwalk/fastwalk_test.go b/internal/fastwalk/fastwalk_test.go index d896aebc956..b5c82bc5293 100644 --- a/internal/fastwalk/fastwalk_test.go +++ b/internal/fastwalk/fastwalk_test.go @@ -8,7 +8,6 @@ import ( "bytes" "flag" "fmt" - "io/ioutil" "os" "path/filepath" "reflect" @@ -35,7 +34,7 @@ func formatFileModes(m map[string]os.FileMode) string { } func testFastWalk(t *testing.T, files map[string]string, callback func(path string, typ os.FileMode) error, want map[string]os.FileMode) { - tempdir, err := ioutil.TempDir("", "test-fast-walk") + tempdir, err := os.MkdirTemp("", "test-fast-walk") if err != nil { t.Fatal(err) } @@ -51,7 +50,7 @@ func testFastWalk(t *testing.T, files map[string]string, callback func(path stri if strings.HasPrefix(contents, "LINK:") { symlinks[file] = filepath.FromSlash(strings.TrimPrefix(contents, "LINK:")) } else { - err = ioutil.WriteFile(file, []byte(contents), 0644) + err = os.WriteFile(file, []byte(contents), 0644) } if err != nil { t.Fatal(err) @@ -63,7 +62,7 @@ func testFastWalk(t *testing.T, files map[string]string, callback func(path stri for file, dst := range symlinks { err = os.Symlink(dst, file) if err != nil { - if writeErr := ioutil.WriteFile(file, []byte(dst), 0644); writeErr == nil { + if writeErr := os.WriteFile(file, []byte(dst), 0644); writeErr == nil { // Couldn't create symlink, but could write the file. // Probably this filesystem doesn't support symlinks. // (Perhaps we are on an older Windows and not running as administrator.) diff --git a/internal/gcimporter/gcimporter.go b/internal/gcimporter/gcimporter.go index b1223713b94..2d078ccb19c 100644 --- a/internal/gcimporter/gcimporter.go +++ b/internal/gcimporter/gcimporter.go @@ -29,7 +29,6 @@ import ( "go/token" "go/types" "io" - "io/ioutil" "os" "os/exec" "path/filepath" @@ -221,7 +220,7 @@ func Import(packages map[string]*types.Package, path, srcDir string, lookup func switch hdr { case "$$B\n": var data []byte - data, err = ioutil.ReadAll(buf) + data, err = io.ReadAll(buf) if err != nil { break } diff --git a/internal/gcimporter/gcimporter_test.go b/internal/gcimporter/gcimporter_test.go index 1407e90849e..757306ca110 100644 --- a/internal/gcimporter/gcimporter_test.go +++ b/internal/gcimporter/gcimporter_test.go @@ -17,7 +17,7 @@ import ( goparser "go/parser" "go/token" "go/types" - "io/ioutil" + "io/fs" "os" "os/exec" "path" @@ -105,7 +105,7 @@ func testPath(t *testing.T, path, srcDir string) *types.Package { } func mktmpdir(t *testing.T) string { - tmpdir, err := ioutil.TempDir("", "gcimporter_test") + tmpdir, err := os.MkdirTemp("", "gcimporter_test") if err != nil { t.Fatal("mktmpdir:", err) } @@ -286,7 +286,7 @@ func TestVersionHandling(t *testing.T) { needsCompiler(t, "gc") const dir = "./testdata/versions" - list, err := ioutil.ReadDir(dir) + list, err := ioutilReadDir(dir) if err != nil { t.Fatal(err) } @@ -321,7 +321,7 @@ func TestVersionHandling(t *testing.T) { // create file with corrupted export data // 1) read file - data, err := ioutil.ReadFile(filepath.Join(dir, name)) + data, err := os.ReadFile(filepath.Join(dir, name)) if err != nil { t.Fatal(err) } @@ -338,7 +338,7 @@ func TestVersionHandling(t *testing.T) { // 4) write the file pkgpath += "_corrupted" filename := filepath.Join(corruptdir, pkgpath) + ".a" - ioutil.WriteFile(filename, data, 0666) + os.WriteFile(filename, data, 0666) // test that importing the corrupted file results in an error _, err = Import(make(map[string]*types.Package), pkgpath, corruptdir, nil) @@ -1008,3 +1008,20 @@ func lookupObj(t *testing.T, scope *types.Scope, name string) types.Object { t.Fatalf("%s not found", name) return nil } + +func ioutilReadDir(dirname string) ([]fs.FileInfo, error) { + entries, err := os.ReadDir(dirname) + if err != nil { + return nil, err + } + + infos := make([]fs.FileInfo, 0, len(entries)) + for _, entry := range entries { + info, err := entry.Info() + if err != nil { + return infos, err + } + infos = append(infos, info) + } + return infos, nil +} diff --git a/internal/gcimporter/iexport_test.go b/internal/gcimporter/iexport_test.go index 7f77a796077..4ee79dac9d0 100644 --- a/internal/gcimporter/iexport_test.go +++ b/internal/gcimporter/iexport_test.go @@ -19,7 +19,7 @@ import ( "go/parser" "go/token" "go/types" - "io/ioutil" + "io" "math/big" "os" "reflect" @@ -55,7 +55,7 @@ func readExportFile(filename string) ([]byte, error) { return nil, fmt.Errorf("unexpected byte: %v", ch) } - return ioutil.ReadAll(buf) + return io.ReadAll(buf) } func iexport(fset *token.FileSet, version int, pkg *types.Package) ([]byte, error) { diff --git a/internal/gopathwalk/walk_test.go b/internal/gopathwalk/walk_test.go index fa4ebdc32b2..58abdcff6b3 100644 --- a/internal/gopathwalk/walk_test.go +++ b/internal/gopathwalk/walk_test.go @@ -5,7 +5,6 @@ package gopathwalk import ( - "io/ioutil" "log" "os" "path/filepath" @@ -22,7 +21,7 @@ func TestShouldTraverse(t *testing.T) { t.Skipf("skipping symlink-requiring test on %s", runtime.GOOS) } - dir, err := ioutil.TempDir("", "goimports-") + dir, err := os.MkdirTemp("", "goimports-") if err != nil { t.Fatal(err) } @@ -90,7 +89,7 @@ func TestShouldTraverse(t *testing.T) { // TestSkip tests that various goimports rules are followed in non-modules mode. func TestSkip(t *testing.T) { - dir, err := ioutil.TempDir("", "goimports-") + dir, err := os.MkdirTemp("", "goimports-") if err != nil { t.Fatal(err) } @@ -125,7 +124,7 @@ func TestSkip(t *testing.T) { // TestSkipFunction tests that scan successfully skips directories from user callback. func TestSkipFunction(t *testing.T) { - dir, err := ioutil.TempDir("", "goimports-") + dir, err := os.MkdirTemp("", "goimports-") if err != nil { t.Fatal(err) } @@ -165,7 +164,7 @@ func mapToDir(destDir string, files map[string]string) error { if strings.HasPrefix(contents, "LINK:") { err = os.Symlink(strings.TrimPrefix(contents, "LINK:"), file) } else { - err = ioutil.WriteFile(file, []byte(contents), 0644) + err = os.WriteFile(file, []byte(contents), 0644) } if err != nil { return err diff --git a/internal/imports/fix.go b/internal/imports/fix.go index d4f1b4e8a0f..c8e9aaaa636 100644 --- a/internal/imports/fix.go +++ b/internal/imports/fix.go @@ -13,7 +13,6 @@ import ( "go/build" "go/parser" "go/token" - "io/ioutil" "os" "path" "path/filepath" @@ -107,7 +106,7 @@ func parseOtherFiles(fset *token.FileSet, srcDir, filename string) []*ast.File { considerTests := strings.HasSuffix(filename, "_test.go") fileBase := filepath.Base(filename) - packageFileInfos, err := ioutil.ReadDir(srcDir) + packageFileInfos, err := ioutilReadDir(srcDir) if err != nil { return nil } @@ -974,7 +973,7 @@ func (e *ProcessEnv) buildContext() (*build.Context, error) { // HACK: setting any of the Context I/O hooks prevents Import from invoking // 'go list', regardless of GO111MODULE. This is undocumented, but it's // unlikely to change before GOPATH support is removed. - ctx.ReadDir = ioutil.ReadDir + ctx.ReadDir = ioutilReadDir return &ctx, nil } @@ -1469,7 +1468,7 @@ func VendorlessPath(ipath string) string { func loadExportsFromFiles(ctx context.Context, env *ProcessEnv, dir string, includeTest bool) (string, []string, error) { // Look for non-test, buildable .go files which could provide exports. - all, err := ioutil.ReadDir(dir) + all, err := ioutilReadDir(dir) if err != nil { return "", nil, err } diff --git a/internal/imports/fix_test.go b/internal/imports/fix_test.go index ba81affdb14..7096ff25c56 100644 --- a/internal/imports/fix_test.go +++ b/internal/imports/fix_test.go @@ -9,8 +9,8 @@ import ( "flag" "fmt" "go/build" - "io/ioutil" "log" + "os" "path/filepath" "reflect" "sort" @@ -1700,7 +1700,7 @@ func (t *goimportTest) process(module, file string, contents []byte, opts *Optio func (t *goimportTest) processNonModule(file string, contents []byte, opts *Options) ([]byte, error) { if contents == nil { var err error - contents, err = ioutil.ReadFile(file) + contents, err = os.ReadFile(file) if err != nil { return nil, err } diff --git a/internal/imports/mkindex.go b/internal/imports/mkindex.go index 36a532b0ca3..2ecc9e45e9f 100644 --- a/internal/imports/mkindex.go +++ b/internal/imports/mkindex.go @@ -19,7 +19,6 @@ import ( "go/format" "go/parser" "go/token" - "io/ioutil" "log" "os" "path" @@ -88,7 +87,7 @@ func main() { } // Write out source file. - err = ioutil.WriteFile("pkgindex.go", src, 0644) + err = os.WriteFile("pkgindex.go", src, 0644) if err != nil { log.Fatal(err) } diff --git a/internal/imports/mkstdlib.go b/internal/imports/mkstdlib.go index 470b93f1df2..3896872c234 100644 --- a/internal/imports/mkstdlib.go +++ b/internal/imports/mkstdlib.go @@ -17,7 +17,6 @@ import ( "go/format" "go/token" "io" - "io/ioutil" "log" "os" "path/filepath" @@ -101,7 +100,7 @@ func main() { if err != nil { log.Fatal(err) } - err = ioutil.WriteFile("zstdlib.go", fmtbuf, 0666) + err = os.WriteFile("zstdlib.go", fmtbuf, 0666) if err != nil { log.Fatal(err) } diff --git a/internal/imports/mod.go b/internal/imports/mod.go index 977d2389da1..a00130b9d71 100644 --- a/internal/imports/mod.go +++ b/internal/imports/mod.go @@ -9,7 +9,7 @@ import ( "context" "encoding/json" "fmt" - "io/ioutil" + "io/fs" "os" "path" "path/filepath" @@ -265,7 +265,7 @@ func (r *ModuleResolver) findPackage(importPath string) (*gocommand.ModuleJSON, } // Not cached. Read the filesystem. - pkgFiles, err := ioutil.ReadDir(pkgDir) + pkgFiles, err := ioutilReadDir(pkgDir) if err != nil { continue } @@ -370,7 +370,7 @@ func (r *ModuleResolver) dirIsNestedModule(dir string, mod *gocommand.ModuleJSON func (r *ModuleResolver) modInfo(dir string) (modDir string, modName string) { readModName := func(modFile string) string { - modBytes, err := ioutil.ReadFile(modFile) + modBytes, err := os.ReadFile(modFile) if err != nil { return "" } @@ -722,3 +722,20 @@ func modulePath(mod []byte) string { } return "" // missing module path } + +func ioutilReadDir(dirname string) ([]fs.FileInfo, error) { + entries, err := os.ReadDir(dirname) + if err != nil { + return nil, err + } + + infos := make([]fs.FileInfo, 0, len(entries)) + for _, entry := range entries { + info, err := entry.Info() + if err != nil { + return infos, err + } + infos = append(infos, info) + } + return infos, nil +} diff --git a/internal/imports/mod_test.go b/internal/imports/mod_test.go index 46831d46623..b8b103e852a 100644 --- a/internal/imports/mod_test.go +++ b/internal/imports/mod_test.go @@ -8,7 +8,6 @@ import ( "archive/zip" "context" "fmt" - "io/ioutil" "log" "os" "path/filepath" @@ -197,7 +196,7 @@ import _ "rsc.io/quote" if err := os.Chmod(filepath.Join(found.dir, "go.mod"), 0644); err != nil { t.Fatal(err) } - if err := ioutil.WriteFile(filepath.Join(found.dir, "go.mod"), []byte("module bad.com\n"), 0644); err != nil { + if err := os.WriteFile(filepath.Join(found.dir, "go.mod"), []byte("module bad.com\n"), 0644); err != nil { t.Fatal(err) } @@ -205,10 +204,10 @@ import _ "rsc.io/quote" mt.assertScanFinds("rsc.io/quote", "quote") // Rewrite the main package so that rsc.io/quote is not in scope. - if err := ioutil.WriteFile(filepath.Join(mt.env.WorkingDir, "go.mod"), []byte("module x\n"), 0644); err != nil { + if err := os.WriteFile(filepath.Join(mt.env.WorkingDir, "go.mod"), []byte("module x\n"), 0644); err != nil { t.Fatal(err) } - if err := ioutil.WriteFile(filepath.Join(mt.env.WorkingDir, "x.go"), []byte("package x\n"), 0644); err != nil { + if err := os.WriteFile(filepath.Join(mt.env.WorkingDir, "x.go"), []byte("package x\n"), 0644); err != nil { t.Fatal(err) } @@ -1000,7 +999,7 @@ func setup(t *testing.T, extraEnv map[string]string, main, wd string) *modTest { proxyOnce.Do(func() { var err error - proxyDir, err = ioutil.TempDir("", "proxy-") + proxyDir, err = os.MkdirTemp("", "proxy-") if err != nil { t.Fatal(err) } @@ -1009,7 +1008,7 @@ func setup(t *testing.T, extraEnv map[string]string, main, wd string) *modTest { } }) - dir, err := ioutil.TempDir("", t.Name()) + dir, err := os.MkdirTemp("", t.Name()) if err != nil { t.Fatal(err) } @@ -1070,7 +1069,7 @@ func writeModule(dir, ar string) error { return err } - if err := ioutil.WriteFile(fpath, f.Data, 0644); err != nil { + if err := os.WriteFile(fpath, f.Data, 0644); err != nil { return err } } @@ -1080,7 +1079,7 @@ func writeModule(dir, ar string) error { // writeProxy writes all the txtar-formatted modules in arDir to a proxy // directory in dir. func writeProxy(dir, arDir string) error { - files, err := ioutil.ReadDir(arDir) + files, err := ioutilReadDir(arDir) if err != nil { return err } @@ -1123,7 +1122,7 @@ func writeProxyModule(base, arPath string) error { z := zip.NewWriter(f) for _, f := range a.Files { if f.Name[0] == '.' { - if err := ioutil.WriteFile(filepath.Join(dir, ver+f.Name), f.Data, 0644); err != nil { + if err := os.WriteFile(filepath.Join(dir, ver+f.Name), f.Data, 0644); err != nil { return err } } else { @@ -1194,7 +1193,7 @@ import _ "rsc.io/quote" func TestInvalidModCache(t *testing.T) { testenv.NeedsTool(t, "go") - dir, err := ioutil.TempDir("", t.Name()) + dir, err := os.MkdirTemp("", t.Name()) if err != nil { t.Fatal(err) } @@ -1204,7 +1203,7 @@ func TestInvalidModCache(t *testing.T) { if err := os.MkdirAll(filepath.Join(dir, "gopath/pkg/mod/sabotage"), 0777); err != nil { t.Fatal(err) } - if err := ioutil.WriteFile(filepath.Join(dir, "gopath/pkg/mod/sabotage/x.go"), []byte("package foo\n"), 0777); err != nil { + if err := os.WriteFile(filepath.Join(dir, "gopath/pkg/mod/sabotage/x.go"), []byte("package foo\n"), 0777); err != nil { t.Fatal(err) } env := &ProcessEnv{ diff --git a/internal/proxydir/proxydir.go b/internal/proxydir/proxydir.go index 5180204064b..ffec81c264c 100644 --- a/internal/proxydir/proxydir.go +++ b/internal/proxydir/proxydir.go @@ -11,7 +11,6 @@ import ( "archive/zip" "fmt" "io" - "io/ioutil" "os" "path/filepath" "strings" @@ -44,13 +43,13 @@ func WriteModuleVersion(rootDir, module, ver string, files map[string][]byte) (r if !ok { modContents = []byte("module " + module) } - if err := ioutil.WriteFile(filepath.Join(dir, ver+".mod"), modContents, 0644); err != nil { + if err := os.WriteFile(filepath.Join(dir, ver+".mod"), modContents, 0644); err != nil { return err } // info file, just the bare bones. infoContents := []byte(fmt.Sprintf(`{"Version": "%v", "Time":"2017-12-14T13:08:43Z"}`, ver)) - if err := ioutil.WriteFile(filepath.Join(dir, ver+".info"), infoContents, 0644); err != nil { + if err := os.WriteFile(filepath.Join(dir, ver+".info"), infoContents, 0644); err != nil { return err } diff --git a/internal/proxydir/proxydir_test.go b/internal/proxydir/proxydir_test.go index 54401fb1647..c8137229b04 100644 --- a/internal/proxydir/proxydir_test.go +++ b/internal/proxydir/proxydir_test.go @@ -7,7 +7,7 @@ package proxydir import ( "archive/zip" "fmt" - "io/ioutil" + "io" "os" "path/filepath" "strings" @@ -43,7 +43,7 @@ func TestWriteModuleVersion(t *testing.T) { }, }, } - dir, err := ioutil.TempDir("", "proxydirtest-") + dir, err := os.MkdirTemp("", "proxydirtest-") if err != nil { t.Fatal(err) } @@ -54,7 +54,7 @@ func TestWriteModuleVersion(t *testing.T) { t.Fatal(err) } rootDir := filepath.Join(dir, filepath.FromSlash(test.modulePath), "@v") - gomod, err := ioutil.ReadFile(filepath.Join(rootDir, test.version+".mod")) + gomod, err := os.ReadFile(filepath.Join(rootDir, test.version+".mod")) if err != nil { t.Fatal(err) } @@ -77,7 +77,7 @@ func TestWriteModuleVersion(t *testing.T) { t.Fatal(err) } defer r.Close() - content, err := ioutil.ReadAll(r) + content, err := io.ReadAll(r) if err != nil { t.Fatal(err) } @@ -101,7 +101,7 @@ func TestWriteModuleVersion(t *testing.T) { for _, test := range lists { fp := filepath.Join(dir, filepath.FromSlash(test.modulePath), "@v", "list") - list, err := ioutil.ReadFile(fp) + list, err := os.ReadFile(fp) if err != nil { t.Fatal(err) } diff --git a/internal/robustio/copyfiles.go b/internal/robustio/copyfiles.go index 6e9f4b3875f..8c93fcd7163 100644 --- a/internal/robustio/copyfiles.go +++ b/internal/robustio/copyfiles.go @@ -53,7 +53,7 @@ func main() { content = bytes.Replace(content, []byte("windows.ERROR_SHARING_VIOLATION"), []byte("ERROR_SHARING_VIOLATION"), -1) } - // Replace os.ReadFile with ioutil.ReadFile (for 1.15 and older). We + // Replace os.ReadFile with os.ReadFile (for 1.15 and older). We // attempt to match calls (via the '('), to avoid matching mentions of // os.ReadFile in comments. // @@ -61,7 +61,7 @@ func main() { // this and break the build. if bytes.Contains(content, []byte("os.ReadFile(")) { content = bytes.Replace(content, []byte("\"os\""), []byte("\"io/ioutil\"\n\t\"os\""), 1) - content = bytes.Replace(content, []byte("os.ReadFile("), []byte("ioutil.ReadFile("), -1) + content = bytes.Replace(content, []byte("os.ReadFile("), []byte("os.ReadFile("), -1) } // Add +build constraints, for 1.16. diff --git a/internal/robustio/robustio_flaky.go b/internal/robustio/robustio_flaky.go index c6f99724468..d5c241857b4 100644 --- a/internal/robustio/robustio_flaky.go +++ b/internal/robustio/robustio_flaky.go @@ -9,7 +9,6 @@ package robustio import ( "errors" - "io/ioutil" "math/rand" "os" "syscall" @@ -75,7 +74,7 @@ func rename(oldpath, newpath string) (err error) { func readFile(filename string) ([]byte, error) { var b []byte err := retry(func() (err error, mayRetry bool) { - b, err = ioutil.ReadFile(filename) + b, err = os.ReadFile(filename) // Unlike in rename, we do not retry errFileNotFound here: it can occur // as a spurious error, but the file may also genuinely not exist, so the diff --git a/internal/robustio/robustio_other.go b/internal/robustio/robustio_other.go index c11dbf9f14b..3a20cac6cf8 100644 --- a/internal/robustio/robustio_other.go +++ b/internal/robustio/robustio_other.go @@ -8,7 +8,6 @@ package robustio import ( - "io/ioutil" "os" ) @@ -17,7 +16,7 @@ func rename(oldpath, newpath string) error { } func readFile(filename string) ([]byte, error) { - return ioutil.ReadFile(filename) + return os.ReadFile(filename) } func removeAll(path string) error { diff --git a/internal/testenv/testenv.go b/internal/testenv/testenv.go index 0fe217b3c16..4d29ebe7f72 100644 --- a/internal/testenv/testenv.go +++ b/internal/testenv/testenv.go @@ -10,7 +10,6 @@ import ( "bytes" "fmt" "go/build" - "io/ioutil" "os" "path/filepath" "runtime" @@ -67,7 +66,7 @@ func hasTool(tool string) error { switch tool { case "patch": // check that the patch tools supports the -o argument - temp, err := ioutil.TempFile("", "patch-test") + temp, err := os.CreateTemp("", "patch-test") if err != nil { return err } @@ -360,7 +359,7 @@ func WriteImportcfg(t testing.TB, dstPath string, additionalPackageFiles map[str if err != nil { t.Fatalf("preparing the importcfg failed: %s", err) } - ioutil.WriteFile(dstPath, []byte(importcfg), 0655) + os.WriteFile(dstPath, []byte(importcfg), 0655) if err != nil { t.Fatalf("writing the importcfg failed: %s", err) } diff --git a/playground/socket/socket.go b/playground/socket/socket.go index cdc665316d4..c396aac5196 100644 --- a/playground/socket/socket.go +++ b/playground/socket/socket.go @@ -22,7 +22,6 @@ import ( "go/token" exec "golang.org/x/sys/execabs" "io" - "io/ioutil" "log" "net" "net/http" @@ -356,7 +355,7 @@ func (p *process) start(body string, opt *Options) error { // (rather than the go tool process). // This makes Kill work. - path, err := ioutil.TempDir("", "present-") + path, err := os.MkdirTemp("", "present-") if err != nil { return err } @@ -376,7 +375,7 @@ func (p *process) start(body string, opt *Options) error { } hasModfile := false for _, f := range a.Files { - err = ioutil.WriteFile(filepath.Join(path, f.Name), f.Data, 0666) + err = os.WriteFile(filepath.Join(path, f.Name), f.Data, 0666) if err != nil { return err } diff --git a/present/parse.go b/present/parse.go index 4294ea5f9cc..162a382b060 100644 --- a/present/parse.go +++ b/present/parse.go @@ -11,9 +11,9 @@ import ( "fmt" "html/template" "io" - "io/ioutil" "log" "net/url" + "os" "regexp" "strings" "time" @@ -342,9 +342,9 @@ func (ctx *Context) Parse(r io.Reader, name string, mode ParseMode) (*Doc, error } // Parse parses a document from r. Parse reads assets used by the presentation -// from the file system using ioutil.ReadFile. +// from the file system using os.ReadFile. func Parse(r io.Reader, name string, mode ParseMode) (*Doc, error) { - ctx := Context{ReadFile: ioutil.ReadFile} + ctx := Context{ReadFile: os.ReadFile} return ctx.Parse(r, name, mode) } diff --git a/present/parse_test.go b/present/parse_test.go index 18d1a35080d..0e59857a3a0 100644 --- a/present/parse_test.go +++ b/present/parse_test.go @@ -7,7 +7,6 @@ package present import ( "bytes" "html/template" - "io/ioutil" "os" "os/exec" "path/filepath" @@ -33,7 +32,7 @@ func TestTestdata(t *testing.T) { continue } t.Run(name, func(t *testing.T) { - data, err := ioutil.ReadFile(file) + data, err := os.ReadFile(file) if err != nil { t.Fatalf("%s: %v", file, err) } @@ -94,7 +93,7 @@ func diff(prefix string, name1 string, b1 []byte, name2 string, b2 []byte) ([]by } func writeTempFile(prefix string, data []byte) (string, error) { - file, err := ioutil.TempFile("", prefix) + file, err := os.CreateTemp("", prefix) if err != nil { return "", err } diff --git a/refactor/eg/eg_test.go b/refactor/eg/eg_test.go index 438e6b75e47..4154e9a8f4e 100644 --- a/refactor/eg/eg_test.go +++ b/refactor/eg/eg_test.go @@ -16,7 +16,6 @@ import ( "go/parser" "go/token" "go/types" - "io/ioutil" "os" "os/exec" "path/filepath" @@ -137,7 +136,7 @@ func Test(t *testing.T) { continue } - gotf, err := ioutil.TempFile("", filepath.Base(filename)+"t") + gotf, err := os.CreateTemp("", filepath.Base(filename)+"t") if err != nil { t.Fatal(err) } diff --git a/refactor/rename/mvpkg_test.go b/refactor/rename/mvpkg_test.go index b8b4d85da4d..f201ee85aa8 100644 --- a/refactor/rename/mvpkg_test.go +++ b/refactor/rename/mvpkg_test.go @@ -8,7 +8,7 @@ import ( "fmt" "go/build" "go/token" - "io/ioutil" + "io" "path/filepath" "reflect" "regexp" @@ -387,7 +387,7 @@ var _ foo.T t.Errorf("unexpected error opening file: %s", err) return } - bytes, err := ioutil.ReadAll(f) + bytes, err := io.ReadAll(f) f.Close() if err != nil { t.Errorf("unexpected error reading file: %s", err) diff --git a/refactor/rename/rename.go b/refactor/rename/rename.go index e74e0a64024..a80381c84b1 100644 --- a/refactor/rename/rename.go +++ b/refactor/rename/rename.go @@ -19,7 +19,6 @@ import ( "go/types" exec "golang.org/x/sys/execabs" "io" - "io/ioutil" "log" "os" "path" @@ -579,12 +578,12 @@ func plural(n int) string { var writeFile = reallyWriteFile func reallyWriteFile(filename string, content []byte) error { - return ioutil.WriteFile(filename, content, 0644) + return os.WriteFile(filename, content, 0644) } func diff(filename string, content []byte) error { renamed := fmt.Sprintf("%s.%d.renamed", filename, os.Getpid()) - if err := ioutil.WriteFile(renamed, content, 0644); err != nil { + if err := os.WriteFile(renamed, content, 0644); err != nil { return err } defer os.Remove(renamed) diff --git a/refactor/rename/rename_test.go b/refactor/rename/rename_test.go index 3dfdc18967c..38c59c9d448 100644 --- a/refactor/rename/rename_test.go +++ b/refactor/rename/rename_test.go @@ -9,7 +9,6 @@ import ( "fmt" "go/build" "go/token" - "io/ioutil" "os" "os/exec" "path/filepath" @@ -1302,7 +1301,7 @@ func TestDiff(t *testing.T) { // Set up a fake GOPATH in a temporary directory, // and ensure we're in GOPATH mode. - tmpdir, err := ioutil.TempDir("", "TestDiff") + tmpdir, err := os.MkdirTemp("", "TestDiff") if err != nil { t.Fatal(err) } @@ -1329,7 +1328,7 @@ func TestDiff(t *testing.T) { go 1.15 ` - if err := ioutil.WriteFile(filepath.Join(pkgDir, "go.mod"), []byte(modFile), 0644); err != nil { + if err := os.WriteFile(filepath.Join(pkgDir, "go.mod"), []byte(modFile), 0644); err != nil { t.Fatal(err) } @@ -1339,7 +1338,7 @@ func justHereForTestingDiff() { justHereForTestingDiff() } ` - if err := ioutil.WriteFile(filepath.Join(pkgDir, "rename_test.go"), []byte(goFile), 0644); err != nil { + if err := os.WriteFile(filepath.Join(pkgDir, "rename_test.go"), []byte(goFile), 0644); err != nil { t.Fatal(err) } diff --git a/txtar/archive.go b/txtar/archive.go index 81b31454512..fd95f1e64a1 100644 --- a/txtar/archive.go +++ b/txtar/archive.go @@ -34,7 +34,7 @@ package txtar import ( "bytes" "fmt" - "io/ioutil" + "os" "strings" ) @@ -66,7 +66,7 @@ func Format(a *Archive) []byte { // ParseFile parses the named file as an archive. func ParseFile(file string) (*Archive, error) { - data, err := ioutil.ReadFile(file) + data, err := os.ReadFile(file) if err != nil { return nil, err } From d98bc1a0158f15696e2533b6a19613ddc86a2a56 Mon Sep 17 00:00:00 2001 From: Peter Aronoff Date: Wed, 13 Sep 2023 15:43:22 +0000 Subject: [PATCH 089/178] gopls: improve usage instructions for neovim 1. The previous version still used Vimscript with embedded Lua in the Custom Configuration section. I updated that to use Lua like the rest of the neovim instructions. 2. I added instructions for how to enable format on write for neovim. 3. For goimports, I effectively reverted an earlier commit and changed buf.code_action back to buf_request_sync. Because buf.code_action is async it can lead to problems (see links below for examples). 4. If buf.code_action gains an async option, this should be revisited since using it makes goimports a one-liner. Problems with buf.code_action: + https://github.com/neovim/neovim/issues/24168 + https://gist.github.com/telemachus/4d19dfb2cb32633342adeb4e5e61bb7e Change-Id: Ifdb2c9188677afe202752502147522f7d9d6b7f8 GitHub-Last-Rev: 38a1bc7e6466b1b91e1aa45e574afed59fda9cfd GitHub-Pull-Request: golang/tools#449 Reviewed-on: https://go-review.googlesource.com/c/tools/+/527759 Reviewed-by: Alan Donovan Reviewed-by: Robert Findley LUCI-TryBot-Result: Go LUCI --- gopls/doc/vim.md | 68 ++++++++++++++++++++++++++++++------------------ 1 file changed, 43 insertions(+), 25 deletions(-) diff --git a/gopls/doc/vim.md b/gopls/doc/vim.md index 8be533cfc36..ac040fb7eb3 100644 --- a/gopls/doc/vim.md +++ b/gopls/doc/vim.md @@ -140,42 +140,60 @@ cd "$dir" git clone 'https://github.com/neovim/nvim-lspconfig.git' . ``` -### Custom Configuration +### Configuration -You can add custom configuration using Lua. Here is an example of enabling the -`unusedparams` check as well as `staticcheck`: +nvim-lspconfig aims to provide reasonable defaults, so your setup can be very +brief. -```vim -lua <Imports +### Imports and Formatting Use the following configuration to have your imports organized on save using -the logic of `goimports`. Note: this requires Neovim v0.7.0 or later. +the logic of `goimports` and your code formatted. ```lua -vim.api.nvim_create_autocmd('BufWritePre', { - pattern = '*.go', +autocmd("BufWritePre", { + pattern = "*.go", callback = function() - vim.lsp.buf.code_action({ context = { only = { 'source.organizeImports' } }, apply = true }) + local params = vim.lsp.util.make_range_params() + params.context = {only = {"source.organizeImports"}} + -- buf_request_sync defaults to a 1000ms timeout. Depending on your + -- machine and codebase, you may want longer. Add an additional + -- argument after params if you find that you have to write the file + -- twice for changes to be saved. + -- E.g., vim.lsp.buf_request_sync(0, "textDocument/codeAction", params, 3000) + local result = vim.lsp.buf_request_sync(0, "textDocument/codeAction", params) + for cid, res in pairs(result or {}) do + for _, r in pairs(res.result or {}) do + if r.edit then + local enc = (vim.lsp.get_client_by_id(cid) or {}).offset_encoding or "utf-16" + vim.lsp.util.apply_workspace_edit(r.edit, enc) + end + end + end + vim.lsp.buf.format({async = false}) end }) ``` From efaab950fc6e6c5639279a3e222a50bf113a67f0 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Mon, 11 Sep 2023 08:29:54 -0400 Subject: [PATCH 090/178] internal/refactor/inline: simplify f(slice...) calls This change causes f(slice...) calls to be reduced to f(slice), and the corresponding parameter func(...T) to be replaced by func([]T), enabling parameter elimination and reduction. This is one of several baby steps towards handling the richness of variadic and spread calls. Also: - split the arrays of params and results in AnalyzeCallee; no need for Kind field. - gather parameters (incl. receiver) in the same manner as we gather arguments, and set them to nil (just like arguments) when we eliminate them. - Enable inliner logging in gopls. Curious users can read the log to find out why the inliner chose a given strategy. Also, tests. Change-Id: I2ba7dbf284c1de8c4fb0c16ef77b8df7f33398da Reviewed-on: https://go-review.googlesource.com/c/tools/+/527676 Reviewed-by: Robert Findley Auto-Submit: Alan Donovan LUCI-TryBot-Result: Go LUCI --- gopls/internal/lsp/source/inline.go | 6 +- internal/refactor/inline/callee.go | 27 ++-- internal/refactor/inline/inline.go | 175 ++++++++++++++---------- internal/refactor/inline/inline_test.go | 18 +++ 4 files changed, 139 insertions(+), 87 deletions(-) diff --git a/gopls/internal/lsp/source/inline.go b/gopls/internal/lsp/source/inline.go index a9a8e493259..64e6b82265c 100644 --- a/gopls/internal/lsp/source/inline.go +++ b/gopls/internal/lsp/source/inline.go @@ -12,6 +12,7 @@ import ( "go/ast" "go/token" "go/types" + "log" "runtime/debug" "golang.org/x/tools/go/analysis" @@ -118,8 +119,9 @@ func inlineCall(ctx context.Context, snapshot Snapshot, fh FileHandle, rng proto Content: callerPGF.Src, } - // Pass log.Printf here when debugging. - got, err := inline.Inline(nil, caller, callee) + // Users can consult the gopls log to see + // why a particular inlining strategy was chosen. + got, err := inline.Inline(log.Printf, caller, callee) if err != nil { return nil, nil, err } diff --git a/internal/refactor/inline/callee.go b/internal/refactor/inline/callee.go index 8ccfeea5e3d..10a8ca84c96 100644 --- a/internal/refactor/inline/callee.go +++ b/internal/refactor/inline/callee.go @@ -39,7 +39,8 @@ type gobCallee struct { BodyIsReturnExpr bool // function body is "return expr(s)" with trivial conversion ValidForCallStmt bool // => bodyIsReturnExpr and sole expr is f() or <-ch NumResults int // number of results (according to type, not ast.FieldList) - Params []*paramInfo // information about receiver, params, and results + Params []*paramInfo // information about parameters (incl. receiver) + Results []*paramInfo // information about result variables HasDefer bool // uses defer TotalReturns int // number of return statements TrivialReturns int // number of return statements with trivial result conversions @@ -329,6 +330,7 @@ func AnalyzeCallee(fset *token.FileSet, pkg *types.Package, info *types.Info, de return nil, err } + params, results := analyzeParams(fset, info, decl) return &Callee{gobCallee{ Content: content, PkgPath: pkg.Path(), @@ -339,7 +341,8 @@ func AnalyzeCallee(fset *token.FileSet, pkg *types.Package, info *types.Info, de BodyIsReturnExpr: bodyIsReturnExpr, ValidForCallStmt: validForCallStmt, NumResults: sig.Results().Len(), - Params: analyzeParams(fset, info, decl), + Params: params, + Results: results, HasDefer: hasDefer, TotalReturns: totalReturns, TrivialReturns: trivialReturns, @@ -362,7 +365,6 @@ func parseCompact(content []byte) (*token.FileSet, *ast.FuncDecl, error) { // A paramInfo records information about a callee receiver, parameter, or result variable. type paramInfo struct { Name string // parameter name (may be blank, or even "") - Kind string // one of {recv,param,result} Assigned bool // parameter appears on left side of an assignment statement Escapes bool // parameter has its address taken Refs []int // FuncDecl-relative byte offset of parameter ref within body @@ -372,11 +374,11 @@ type paramInfo struct { // analyzeParams computes information about parameters of function fn, // including a simple "address taken" escape analysis. // -// It returns a new array with an entry for each receiver, -// parameter, and result variable of function fn. +// It returns two new arrays, one of the receiver and parameters, and +// the other of the result variables of function fn. // // The input must be well-typed. -func analyzeParams(fset *token.FileSet, info *types.Info, decl *ast.FuncDecl) (res []*paramInfo) { +func analyzeParams(fset *token.FileSet, info *types.Info, decl *ast.FuncDecl) (params, results []*paramInfo) { fnobj, ok := info.Defs[decl.Name] if !ok { panic(fmt.Sprintf("%s: no func object for %q", @@ -386,20 +388,19 @@ func analyzeParams(fset *token.FileSet, info *types.Info, decl *ast.FuncDecl) (r paramInfos := make(map[*types.Var]*paramInfo) { sig := fnobj.Type().(*types.Signature) - newParamInfo := func(param *types.Var, kind string) *paramInfo { - info := ¶mInfo{Name: param.Name(), Kind: kind} - res = append(res, info) + newParamInfo := func(param *types.Var) *paramInfo { + info := ¶mInfo{Name: param.Name()} paramInfos[param] = info return info } if sig.Recv() != nil { - newParamInfo(sig.Recv(), "recv") + params = append(params, newParamInfo(sig.Recv())) } for i := 0; i < sig.Params().Len(); i++ { - newParamInfo(sig.Params().At(i), "param") + params = append(params, newParamInfo(sig.Params().At(i))) } for i := 0; i < sig.Results().Len(); i++ { - newParamInfo(sig.Results().At(i), "result") + results = append(results, newParamInfo(sig.Results().At(i))) } } @@ -515,7 +516,7 @@ func analyzeParams(fset *token.FileSet, info *types.Info, decl *ast.FuncDecl) (r }) } - return res + return params, results } // -- callee helpers -- diff --git a/internal/refactor/inline/inline.go b/internal/refactor/inline/inline.go index 4b9e2fad8c6..d0078cc77fd 100644 --- a/internal/refactor/inline/inline.go +++ b/internal/refactor/inline/inline.go @@ -658,8 +658,6 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu } } - sig := calleeSymbol.Type().(*types.Signature) - // Gather effective argument tuple, including receiver. // // If the receiver argument and parameter have @@ -690,6 +688,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu type argument struct { expr ast.Expr typ types.Type // may be tuple for sole non-receiver arg in spread call + spread bool // final arg is call() assigned to multiple params pure bool // expr has no effects duplicable bool // expr may be duplicated freevars map[string]bool // free names of expr @@ -714,7 +713,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // predicates (pure et al) above, as they won't // work on synthetic syntax. argIsPtr := arg.typ != deref(arg.typ) - paramIsPtr := is[*types.Pointer](sig.Recv().Type()) + paramIsPtr := is[*types.Pointer](calleeSymbol.Type().(*types.Signature).Recv().Type()) if !argIsPtr && paramIsPtr { // &recv arg.expr = &ast.UnaryExpr{Op: token.AND, X: arg.expr} @@ -743,15 +742,66 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu } } for _, arg := range caller.Call.Args { + typ := caller.Info.TypeOf(arg) args = append(args, &argument{ expr: arg, - typ: caller.Info.TypeOf(arg), + typ: typ, + spread: is[*types.Tuple](typ), // => last pure: pure(caller.Info, arg), duplicable: duplicable(caller.Info, arg), freevars: freevars(caller.Info, arg), }) } + // Gather effective parameter tuple, including the receiver if any. + type parameter struct { + obj *types.Var // parameter var from caller's signature + info *paramInfo // information from AnalyzeCallee + variadic bool // final T... parameter accumulates slice of arguments + } + var params []*parameter // including receiver; nil => parameter eliminated + { + sig := calleeSymbol.Type().(*types.Signature) + if sig.Recv() != nil { + params = append(params, ¶meter{ + obj: sig.Recv(), + info: callee.Params[0], + }) + } + for i := 0; i < sig.Params().Len(); i++ { + params = append(params, ¶meter{ + obj: sig.Params().At(i), + info: callee.Params[len(params)], + }) + } + if sig.Variadic() { + last(params).variadic = true + } + } + + // Note: computation below should be expressed in terms of + // the args and params slices, not the raw material. + + // In most calls, args and params correspond. + // + // Edge case: in a variadic call, len(args) >= len(params)-1. + // + // Edge case: in a spread call f(g()) where g is n-ary (n > 1), + // len(args) = 1 and len(params) = n, + // unless (corner case!) f is variadic, + // in which case both are again 1. + // + // If the last arg is f(slice...), the last param must be func(...T). + // We can immediately simplify both to f(slice) and func([]T), + // without changing the types of either. + if caller.Call.Ellipsis.IsValid() { + lastParamField := last(calleeDecl.Type.Params.List) + lastParamField.Type = &ast.ArrayType{ + Elt: lastParamField.Type.(*ast.Ellipsis).Elt, + } + last(params).variadic = false + } + // Parameter elimination // // Consider each parameter and its corresponding argument in turn @@ -767,50 +817,26 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // // If all conditions are met then the parameter can be eliminated // and each reference to it replaced by the argument. - var eliminatedParams []bool // (recv, params...) + // + // TODO(adonovan): support elimination of variadic parameters, + // and of spread arguments. { - // Gather effective parameter objects, - // including the receiver if any. - var paramObjs []*types.Var - if sig.Recv() != nil { - paramObjs = append(paramObjs, sig.Recv()) - } - for i := 0; i < sig.Params().Len(); i++ { - paramObjs = append(paramObjs, sig.Params().At(i)) - } - - // In most calls, args and paramObjs correspond. - // - // Edge case: in a variadic call, len(args) >= len(params)-1. - // - // Edge case: in a spread call f(g()) where g is n-ary (n > 1), - // len(args) = 1 and len(params) = n, - // unless (corner case!) f is variadic, - // in which case both are again 1. - // - // TODO(adonovan): support elimination of variadic parameters, - // and of spread arguments. - - eliminatedParams = make([]bool, len(paramObjs)) nextParam: - for i, param := range callee.Params { - if param.Kind == "result" { - break // end of parameters - } - if sig.Variadic() && i == len(paramObjs)-1 { + for i, param := range params { + if param.variadic { // final ...T parameter // TODO(adonovan): decouple the param and arg parts of this // loop so that we can handle variadic cases. logf("keeping param %q: variadic elimination not yet supported", - param.Name) + param.info.Name) continue } - if param.Escapes { - logf("keeping param %q: escapes from callee", param.Name) + if param.info.Escapes { + logf("keeping param %q: escapes from callee", param.info.Name) continue } - if param.Assigned { - logf("keeping param %q: assigned by callee", param.Name) + if param.info.Assigned { + logf("keeping param %q: assigned by callee", param.info.Name) continue // callee needs the parameter variable } @@ -821,22 +847,22 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // and thus lacking positions and types; // do it earlier (see pure/duplicable/freevars). arg := args[i] - if is[*types.Tuple](arg.typ) { + if arg.spread { // TODO(adonovan): handle elimination of spread arguments. logf("keeping param %q: argument %s is spread", - param.Name, debugFormatNode(caller.Fset, arg.expr)) + param.info.Name, debugFormatNode(caller.Fset, arg.expr)) break // spread => last argument, but not last parameter } if !arg.pure { logf("keeping param %q: argument %s is impure", - param.Name, debugFormatNode(caller.Fset, arg.expr)) + param.info.Name, debugFormatNode(caller.Fset, arg.expr)) continue // unsafe to change order or cardinality of effects } - if len(param.Refs) > 1 && !arg.duplicable { - logf("keeping param %q: argument is not duplicable", param.Name) + if len(param.info.Refs) > 1 && !arg.duplicable { + logf("keeping param %q: argument is not duplicable", param.info.Name) continue // incorrect or poor style to duplicate an expression } - if len(param.Refs) == 0 { + if len(param.info.Refs) == 0 { // Eliminating an unreferenced parameter might // remove the last reference to a caller local var. for free := range arg.freevars { @@ -846,7 +872,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // function (if any) and is indeed referenced // only by the call. logf("keeping param %q: arg contains perhaps the last reference to possible caller local %v @ %v", - param.Name, v, caller.Fset.Position(v.Pos())) + param.info.Name, v, caller.Fset.Position(v.Pos())) continue nextParam } } @@ -858,11 +884,11 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // (We don't simply wrap the argument in an explicit conversion // to the parameter type because that could increase allocation // in the number of (e.g.) string -> any conversions. - // Even when Uses = 1, the sole ref might be in a lambda that + // Even when Uses = 1, the sole ref might be in a loop or lambda that // is multiply executed.) - if len(param.Refs) > 0 && !trivialConversion(args[i].typ, paramObjs[i]) { + if len(param.info.Refs) > 0 && !trivialConversion(args[i].typ, params[i].obj) { logf("keeping param %q: argument passing converts %s to type %s", - param.Name, args[i].typ, paramObjs[i].Type()) + param.info.Name, args[i].typ, params[i].obj.Type()) continue // implicit conversion is significant } @@ -875,8 +901,8 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // because there's an intervening declaration of z // that would shadow the caller's one. for free := range arg.freevars { - if param.Shadow[free] { - logf("keeping param %q: cannot replace with argument as it has free ref to %s that is shadowed", param.Name, free) + if param.info.Shadow[free] { + logf("keeping param %q: cannot replace with argument as it has free ref to %s that is shadowed", param.info.Name, free) continue nextParam // shadowing conflict } } @@ -888,12 +914,12 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // Because arg.expr belongs to the caller, // we clone it before splicing it into the callee tree. logf("replacing parameter %q by argument %q", - param.Name, debugFormatNode(caller.Fset, arg.expr)) - for _, ref := range param.Refs { + param.info.Name, debugFormatNode(caller.Fset, arg.expr)) + for _, ref := range param.info.Refs { replaceCalleeID(ref, cloneNode(arg.expr).(ast.Expr)) } - eliminatedParams[i] = true - args[i] = nil + params[i] = nil // eliminated + args[i] = nil // eliminated } } @@ -941,7 +967,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // Emit "_, _ = args" to discard results. // Make correction for spread calls // f(g()) or x.f(g()) where g() is a tuple. - if last := args[len(args)-1]; last != nil { + if last := last(args); last != nil { if tuple, ok := last.typ.(*types.Tuple); ok { nargs += tuple.Len() - 1 } @@ -960,18 +986,13 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // Attempt to reduce parameterless calls // whose result variables do not escape. - if forall(callee.Params, func(i int, param *paramInfo) bool { - if param.Kind != "result" { // recv or param - if !eliminatedParams[i] { - logf("param %q not eliminated", param.Name) - return false - } - } else if param.Escapes { - logf("result variable %s escapes", param.Name) - return false - } - return true - }) { + allParamsEliminated := forall(params, func(i int, p *parameter) bool { + return p == nil + }) + noResultEscapes := !exists(callee.Results, func(i int, r *paramInfo) bool { + return r.Escapes + }) + if allParamsEliminated && noResultEscapes { logf("all params eliminated and no result vars escape") // Special case: parameterless call to { return exprs }. @@ -1095,9 +1116,9 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu len(ret.Results) == 1 && callee.TrivialReturns == callee.TotalReturns && !hasLabelConflict(callerPath, callee.Labels) && - forall(callee.Params, func(i int, p *paramInfo) bool { + forall(callee.Results, func(i int, p *paramInfo) bool { // all result vars are unreferenced - return p.Kind != "result" || len(p.Refs) == 0 + return len(p.Refs) == 0 }) { logf("strategy: reduce parameterless tail-call") res.old = ret @@ -1191,7 +1212,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu var names []*ast.Ident if field.Names == nil { // Unnamed parameter field (e.g. func f(int) - if !eliminatedParams[paramIdx] { + if params[paramIdx] != nil { // Give it an explicit name "_" since we will // make the receiver (if any) a regular parameter // and one cannot mix named and unnamed parameters. @@ -1203,7 +1224,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // Remove eliminated parameters in place. // If all were eliminated, delete field. for _, id := range field.Names { - if !eliminatedParams[paramIdx] { + if params[paramIdx] != nil { names = append(names, id) } paramIdx++ @@ -1232,7 +1253,8 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu Type: calleeDecl.Type, Body: calleeDecl.Body, }, - Args: remainingArgs, + Ellipsis: token.NoPos, // f(slice...) is always simplified + Args: remainingArgs, } clearPositions(newCall.Fun) res.old = caller.Call @@ -1726,3 +1748,12 @@ func exists[T any](list []T, f func(i int, x T) bool) bool { } return false } + +// last returns the last element of a slice, or zero if empty. +func last[T any](slice []T) T { + n := len(slice) + if n > 0 { + return slice[n-1] + } + return *new(T) +} diff --git a/internal/refactor/inline/inline_test.go b/internal/refactor/inline/inline_test.go index 11392e0b4ea..f99db705d70 100644 --- a/internal/refactor/inline/inline_test.go +++ b/internal/refactor/inline/inline_test.go @@ -371,6 +371,24 @@ func TestTable(t *testing.T) { `func _() I { return recover().(I).f(g()) }`, `error: can't yet inline spread call to method`, }, + { + "Variadic cancellation (basic).", + `func f(args ...any) { defer f(&args); println(args) }`, + `func _(slice []any) { f(slice...) }`, + `func _(slice []any) { func(args []any) { defer f(&args); println(args) }(slice) }`, + }, + { + "Variadic cancellation (literalization with parameter elimination).", + `func f(args ...any) { defer f(); println(args) }`, + `func _(slice []any) { f(slice...) }`, + `func _(slice []any) { func() { defer f(); println(slice) }() }`, + }, + { + "Variadic cancellation (reduction).", + `func f(args ...any) { println(args) }`, + `func _(slice []any) { f(slice...) }`, + `func _(slice []any) { println(slice) }`, + }, // TODO(adonovan): improve coverage of the cross // product of each strategy with the checklist of // concerns enumerated in the package doc comment. From 0bcc621fa0295e58a1bfb5e5b6da61f4fe24bfd8 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Tue, 12 Sep 2023 23:21:47 -0400 Subject: [PATCH 091/178] internal/refactor/inline: simplify ordinary variadics This change analyzes variadic functions more systematically. There are three call forms: ordinary, ellipsis, and spread. In the first two cases, the variadic parameter is simplified to an ordinary slice, enabling parameter elimination. (The ellipsis case was already done.) The spread case cannot be simplified. Also: - tests. - fix a bug in cloneNode: avoid Set(nil) Change-Id: I1d34022558d251bf62cbbb84aab1752c7f8f88b9 Reviewed-on: https://go-review.googlesource.com/c/tools/+/527677 Reviewed-by: Robert Findley LUCI-TryBot-Result: Go LUCI --- internal/refactor/inline/inline.go | 145 +++++++++++------- internal/refactor/inline/inline_test.go | 32 +++- .../refactor/inline/testdata/empty-body.txtar | 2 +- 3 files changed, 125 insertions(+), 54 deletions(-) diff --git a/internal/refactor/inline/inline.go b/internal/refactor/inline/inline.go index d0078cc77fd..1b474698d79 100644 --- a/internal/refactor/inline/inline.go +++ b/internal/refactor/inline/inline.go @@ -741,23 +741,24 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu } } } - for _, arg := range caller.Call.Args { - typ := caller.Info.TypeOf(arg) + for _, expr := range caller.Call.Args { + typ := caller.Info.TypeOf(expr) args = append(args, &argument{ - expr: arg, + expr: expr, typ: typ, spread: is[*types.Tuple](typ), // => last - pure: pure(caller.Info, arg), - duplicable: duplicable(caller.Info, arg), - freevars: freevars(caller.Info, arg), + pure: pure(caller.Info, expr), + duplicable: duplicable(caller.Info, expr), + freevars: freevars(caller.Info, expr), }) } // Gather effective parameter tuple, including the receiver if any. + // Simplify variadic parameters to slices (in all cases but one). type parameter struct { obj *types.Var // parameter var from caller's signature info *paramInfo // information from AnalyzeCallee - variadic bool // final T... parameter accumulates slice of arguments + variadic bool // (final) parameter is unsimplified ...T } var params []*parameter // including receiver; nil => parameter eliminated { @@ -774,34 +775,63 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu info: callee.Params[len(params)], }) } + + // Variadic function? + // + // There are three possible types of call: + // - ordinary f(a1, ..., aN) + // - ellipsis f(a1, ..., slice...) + // - spread f(recv?, g()) where g() is a tuple. + // The first two are desugared to non-variadic calls + // with an ordinary slice parameter; + // the third is tricky and cannot be reduced, and (if + // a receiver is present) cannot even be literalized. + // Fortunately it is vanishingly rare. if sig.Variadic() { - last(params).variadic = true + lastParam := last(params) + if len(args) > 0 && last(args).spread { + // spread call to variadic: tricky + lastParam.variadic = true + } else { + // ordinary/ellipsis call to variadic + + // simplify decl: func(T...) -> func([]T) + lastParamField := last(calleeDecl.Type.Params.List) + lastParamField.Type = &ast.ArrayType{ + Elt: lastParamField.Type.(*ast.Ellipsis).Elt, + } + + if caller.Call.Ellipsis.IsValid() { + // ellipsis call: f(slice...) -> f(slice) + // nop + } else { + // ordinary call: f(a1, ... aN) -> f([]T{a1, ..., aN}) + n := len(params) - 1 + ordinary, extra := args[:n], args[n:] + var elts []ast.Expr + pure := true + for _, arg := range extra { + elts = append(elts, arg.expr) + pure = pure && arg.pure + } + args = append(ordinary, &argument{ + expr: &ast.CompositeLit{ + Type: lastParamField.Type, + Elts: elts, + }, + typ: lastParam.obj.Type(), + pure: pure, + duplicable: false, + freevars: nil, // not needed + }) + } + } } } // Note: computation below should be expressed in terms of // the args and params slices, not the raw material. - // In most calls, args and params correspond. - // - // Edge case: in a variadic call, len(args) >= len(params)-1. - // - // Edge case: in a spread call f(g()) where g is n-ary (n > 1), - // len(args) = 1 and len(params) = n, - // unless (corner case!) f is variadic, - // in which case both are again 1. - // - // If the last arg is f(slice...), the last param must be func(...T). - // We can immediately simplify both to f(slice) and func([]T), - // without changing the types of either. - if caller.Call.Ellipsis.IsValid() { - lastParamField := last(calleeDecl.Type.Params.List) - lastParamField.Type = &ast.ArrayType{ - Elt: lastParamField.Type.(*ast.Ellipsis).Elt, - } - last(params).variadic = false - } - // Parameter elimination // // Consider each parameter and its corresponding argument in turn @@ -817,20 +847,23 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // // If all conditions are met then the parameter can be eliminated // and each reference to it replaced by the argument. - // - // TODO(adonovan): support elimination of variadic parameters, - // and of spread arguments. { - nextParam: + // Inv: + // in calls to variadic, len(args) >= len(params)-1 + // in spread calls to non-variadic, len(args) < len(params) + // in spread calls to variadic, len(args) <= len(params) + // (In spread calls len(args) = 1, or 2 if call has receiver.) + // Non-spread variadics have been simplified away already, + // so the args[i] lookup is safe if we stop after the spread arg. + next: for i, param := range params { - if param.variadic { - // final ...T parameter - // TODO(adonovan): decouple the param and arg parts of this - // loop so that we can handle variadic cases. - logf("keeping param %q: variadic elimination not yet supported", - param.info.Name) - continue + arg := args[i] + if arg.spread { + logf("keeping param %q and following ones: argument %s is spread", + param.info.Name, debugFormatNode(caller.Fset, arg.expr)) + break // spread => last argument, but not always last parameter } + assert(!param.variadic, "unsimplfied variadic parameter") if param.info.Escapes { logf("keeping param %q: escapes from callee", param.info.Name) continue @@ -846,13 +879,6 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // the syntax may be synthetic (not created by parser) // and thus lacking positions and types; // do it earlier (see pure/duplicable/freevars). - arg := args[i] - if arg.spread { - // TODO(adonovan): handle elimination of spread arguments. - logf("keeping param %q: argument %s is spread", - param.info.Name, debugFormatNode(caller.Fset, arg.expr)) - break // spread => last argument, but not last parameter - } if !arg.pure { logf("keeping param %q: argument %s is impure", param.info.Name, debugFormatNode(caller.Fset, arg.expr)) @@ -873,7 +899,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // only by the call. logf("keeping param %q: arg contains perhaps the last reference to possible caller local %v @ %v", param.info.Name, v, caller.Fset.Position(v.Pos())) - continue nextParam + continue next } } } @@ -903,7 +929,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu for free := range arg.freevars { if param.info.Shadow[free] { logf("keeping param %q: cannot replace with argument as it has free ref to %s that is shadowed", param.info.Name, free) - continue nextParam // shadowing conflict + continue next // shadowing conflict } } @@ -967,6 +993,15 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // Emit "_, _ = args" to discard results. // Make correction for spread calls // f(g()) or x.f(g()) where g() is a tuple. + // + // TODO(adonovan): fix: it's not valid for a + // single AssignStmt to discard a receiver and + // a spread argument; use a var decl with two specs. + // + // TODO(adonovan): if args is the []T{a1, ..., an} + // literal synthesized during variadic simplification, + // consider unwrapping it to its (pure) elements. + // Perhaps there's no harm doing this for any slice literal. if last := last(args); last != nil { if tuple, ok := last.typ.(*types.Tuple); ok { nargs += tuple.Len() - 1 @@ -1611,6 +1646,12 @@ func replaceNode(root ast.Node, from, to ast.Node) { // It omits pointers to ast.{Scope,Object} variables. func cloneNode(n ast.Node) ast.Node { var clone func(x reflect.Value) reflect.Value + set := func(dst, src reflect.Value) { + src = clone(src) + if src.IsValid() { + dst.Set(src) + } + } clone = func(x reflect.Value) reflect.Value { switch x.Kind() { case reflect.Ptr: @@ -1623,26 +1664,26 @@ func cloneNode(n ast.Node) ast.Node { return reflect.Zero(x.Type()) } y := reflect.New(x.Type().Elem()) - y.Elem().Set(clone(x.Elem())) + set(y.Elem(), x.Elem()) return y case reflect.Struct: y := reflect.New(x.Type()).Elem() for i := 0; i < x.Type().NumField(); i++ { - y.Field(i).Set(clone(x.Field(i))) + set(y.Field(i), x.Field(i)) } return y case reflect.Slice: y := reflect.MakeSlice(x.Type(), x.Len(), x.Cap()) for i := 0; i < x.Len(); i++ { - y.Index(i).Set(clone(x.Index(i))) + set(y.Index(i), x.Index(i)) } return y case reflect.Interface: y := reflect.New(x.Type()).Elem() - y.Set(clone(x.Elem())) + set(y, x.Elem()) return y case reflect.Array, reflect.Chan, reflect.Func, reflect.Map, reflect.UnsafePointer: diff --git a/internal/refactor/inline/inline_test.go b/internal/refactor/inline/inline_test.go index f99db705d70..4c246c49234 100644 --- a/internal/refactor/inline/inline_test.go +++ b/internal/refactor/inline/inline_test.go @@ -389,6 +389,36 @@ func TestTable(t *testing.T) { `func _(slice []any) { f(slice...) }`, `func _(slice []any) { println(slice) }`, }, + { + "Variadic elimination (literalization).", + `func f(x any, rest ...any) { println(x, rest) }`, + `func _() { f(1, 2, 3) }`, // NB: x int->any causes literalization, for now + `func _() { func(x any) { println(x, []any{2, 3}) }(1) }`, + }, + { + "Variadic elimination (reduction).", + `func f(x int, rest ...int) { println(x, rest) }`, + `func _() { f(1, 2, 3) }`, + `func _() { println(1, []int{2, 3}) }`, + }, + { + "Spread call to variadic (1 arg, 1 param).", + `func f(rest ...int) { println(rest) }; func g() (a, b int)`, + `func _() { f(g()) }`, + `func _() { func(rest ...int) { println(rest) }(g()) }`, + }, + { + "Spread call to variadic (1 arg, 2 params).", + `func f(x int, rest ...int) { println(x, rest) }; func g() (a, b int)`, + `func _() { f(g()) }`, + `func _() { func(x int, rest ...int) { println(x, rest) }(g()) }`, + }, + { + "Spread call to variadic (1 arg, 3 params).", + `func f(x, y int, rest ...int) { println(x, y, rest) }; func g() (a, b, c int)`, + `func _() { f(g()) }`, + `func _() { func(x, y int, rest ...int) { println(x, y, rest) }(g()) }`, + }, // TODO(adonovan): improve coverage of the cross // product of each strategy with the checklist of // concerns enumerated in the package doc comment. @@ -516,7 +546,7 @@ func TestTable(t *testing.T) { got := strings.Join(gotLines, "\n") if strings.TrimSpace(got) != strings.TrimSpace(test.want) { - t.Errorf("\nInlining this call:\t%s\nof this callee: \t%s\nproduced:\n%s\nWant:\n\n%s", + t.Fatalf("\nInlining this call:\t%s\nof this callee: \t%s\nproduced:\n%s\nWant:\n\n%s", test.caller, test.callee, got, diff --git a/internal/refactor/inline/testdata/empty-body.txtar b/internal/refactor/inline/testdata/empty-body.txtar index 77311c33702..8983fda8c6e 100644 --- a/internal/refactor/inline/testdata/empty-body.txtar +++ b/internal/refactor/inline/testdata/empty-body.txtar @@ -59,7 +59,7 @@ func g() int package a func _(ch chan int) { - _, _, _, _, _ = -1, ch, len(""), g(), <-ch //@ inline(re"empty", empty2) + _ = []any{-1, ch, len(""), g(), <-ch} //@ inline(re"empty", empty2) } func g() int From 715a45271b44f634229f45a32b9de4e23661d9ab Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Wed, 13 Sep 2023 09:54:30 -0400 Subject: [PATCH 092/178] internal/refactor/inline: doc: optimizing compiler analogy Change-Id: Id4ca3d2650a270a654b9de9b0ba2adc7324e40c8 Reviewed-on: https://go-review.googlesource.com/c/tools/+/527995 Reviewed-by: Robert Findley LUCI-TryBot-Result: Go LUCI --- internal/refactor/inline/inline.go | 35 +++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/internal/refactor/inline/inline.go b/internal/refactor/inline/inline.go index 1b474698d79..733bc7c5859 100644 --- a/internal/refactor/inline/inline.go +++ b/internal/refactor/inline/inline.go @@ -45,12 +45,27 @@ // reflection over the call stack, but this exception to the rule is // explicitly allowed.) // -// In a number of special cases it is possible to entirely replace -// ("reduce") the call by a copy of the function's body in which -// parameters have been replaced by arguments. The inliner supports a -// small number of reduction strategies, and we expect this set to -// grow. Nonetheless, sound reduction is surprisingly tricky. The -// following section lists some of the challenges. +// In many cases it is possible to entirely replace ("reduce") the +// call by a copy of the function's body in which parameters have been +// replaced by arguments. The inliner supports a number of reduction +// strategies, and we expect this set to grow. Nonetheless, sound +// reduction is surprisingly tricky. +// +// The inliner is in some ways like an optimizing compiler. A compiler +// is considered correct if it doesn't change the meaning of the +// program in translation from source language to target language. An +// optimizing compiler exploits the particulars of the input to +// generate better code, where "better" usually means more efficient. +// When a case is found in which it emits suboptimal code, the +// compiler is improved to recognize more cases, or more rules, and +// more exceptions to rules; this process has no end. Inlining is +// similar except that "better" code means tidier code. The baseline +// translation (literalization) is correct, but there are endless +// rules--and exceptions to rules--by which the output can be +// improved. +// +// The following section lists some of the challenges, and ways in +// which they can be addressed. // // - All effects of the call argument expressions must be preserved, // both in their number (they must not be eliminated or repeated), @@ -137,8 +152,12 @@ // - If the function body uses 'defer' and the inlined call is not a // tail-call, inlining may delay the deferred effects. // -// - Each control label that is used by both caller and callee must -// be α-renamed. +// - Because the scope of a control label is the entire function, a +// call cannot be reduced if the caller and callee have intersecting +// sets of control labels. (It is possible to α-rename any +// conflicting ones, but our colleagues building C++ refactoring +// tools report that, when tools must choose new identifiers, they +// generally do a poor job.) // // - Given // From eaf809ad3a197cce3032408b3bcc14c817de0473 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Thu, 14 Sep 2023 10:31:14 -0400 Subject: [PATCH 093/178] internal/refactor/inline: 2 fixes in AnalyzeCallee recursion 1. We forgot to descend into CompositeLit.Type. 2. We forgot to recursively visit subtrees. These bugs were unearthed by an "everything" test, which inlines every function call and type-checks the result, to follow. Change-Id: I17d79134cb8c4b088b017e6022508bc233ad2454 Reviewed-on: https://go-review.googlesource.com/c/tools/+/528396 Reviewed-by: Robert Findley LUCI-TryBot-Result: Go LUCI --- internal/refactor/inline/callee.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/internal/refactor/inline/callee.go b/internal/refactor/inline/callee.go index 10a8ca84c96..370cb8ae38f 100644 --- a/internal/refactor/inline/callee.go +++ b/internal/refactor/inline/callee.go @@ -109,8 +109,9 @@ func AnalyzeCallee(fset *token.FileSet, pkg *types.Package, info *types.Info, de freeRefs []freeRef // free refs that may need renaming unexported []string // free refs to unexported objects, for later error checks ) - var visit func(n ast.Node) bool - visit = func(n ast.Node) bool { + var f func(n ast.Node) bool + visit := func(n ast.Node) { ast.Inspect(n, f) } + f = func(n ast.Node) bool { switch n := n.(type) { case *ast.SelectorExpr: // Check selections of free fields/methods. @@ -130,6 +131,9 @@ func AnalyzeCallee(fset *token.FileSet, pkg *types.Package, info *types.Info, de // whether keyed or unkeyed. (Logic assumes well-typedness.) litType := deref(info.TypeOf(n)) if s, ok := typeparams.CoreType(litType).(*types.Struct); ok { + if n.Type != nil { + visit(n.Type) + } for i, elt := range n.Elts { var field *types.Var var value ast.Expr @@ -195,7 +199,7 @@ func AnalyzeCallee(fset *token.FileSet, pkg *types.Package, info *types.Info, de } return true } - ast.Inspect(decl, visit) + visit(decl) // Analyze callee body for "return results" form, where // results is one or more expressions or an n-ary call, From 940ffda00916bcd8df839d5b895727dac2c69a12 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Wed, 13 Sep 2023 14:33:40 -0400 Subject: [PATCH 094/178] internal/refactor/inline: introduce "binding decls" A binding declaration is a var decl that declares a set of parameter variables and assigns them to their argument values. The vars are divided into specs of the same type in the same way that parameters are grouped into fields. Types are explicit, so that the correct assignment conversions occur. A binding decl allows many reduction strategies to work even when some parameters were not eliminated by substitution, or when some result vars escape, because it keeps the cardinality of variables unchanged. For example, the call f(a, b, c) where: func f(x, y T, z U) { stmts } can be reduced to: { var ( x, y T = a, b z U = c ) stmts } It is not always possible to create a binding decl. For example, if U has "x" among its free names, then the declaration of x above shadows the correct x needed by U. Strategies have been updated to insert a binding decl where appropriate, and tests updated. Change-Id: I1c7837df6f2e26a4426758fa07b066d378428221 Reviewed-on: https://go-review.googlesource.com/c/tools/+/528175 LUCI-TryBot-Result: Go LUCI Reviewed-by: Robert Findley --- internal/refactor/inline/inline.go | 577 ++++++++++++------ internal/refactor/inline/inline_test.go | 38 +- .../inline/testdata/basic-literal.txtar | 12 +- .../refactor/inline/testdata/method.txtar | 25 +- .../inline/testdata/multistmt-body.txtar | 12 +- 5 files changed, 451 insertions(+), 213 deletions(-) diff --git a/internal/refactor/inline/inline.go b/internal/refactor/inline/inline.go index 733bc7c5859..9f000e49489 100644 --- a/internal/refactor/inline/inline.go +++ b/internal/refactor/inline/inline.go @@ -99,7 +99,7 @@ // "return expr", in some contexts it may be syntactically // impossible to reduce the call. Consider: // -// } else if x := f(); cond { ... } +// if x := f(); cond { ... } // // Go has no equivalent to Lisp's progn or Rust's blocks, // nor ML's let expressions (let param = arg in body); @@ -107,6 +107,11 @@ // Reduction strategies must therefore consider the syntactic // context of the call. // +// In such situations we could work harder to extract a statement +// context for the call, by transforming it to: +// +// { x := f(); if cond { ... } } +// // - Similarly, without the equivalent of Rust-style blocks and // first-class tuples, there is no general way to reduce a call // to a function such as @@ -260,7 +265,8 @@ // But note that the existing algorithm makes widespread assumptions // that the callee is a package-level function or method. // -// - Eliminate parens inserted conservatively when they are redundant. +// - Eliminate parens and braces inserted conservatively when they +// are redundant. // // - Allow non-'go' build systems such as Bazel/Blaze a chance to // decide whether an import is accessible using logic other than @@ -722,7 +728,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu typ: caller.Info.TypeOf(sel.X), pure: pure(caller.Info, sel.X), duplicable: duplicable(caller.Info, sel.X), - freevars: freevars(caller.Info, sel.X), + freevars: freeVars(caller.Info, sel.X), } args = append(args, arg) @@ -768,7 +774,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu spread: is[*types.Tuple](typ), // => last pure: pure(caller.Info, expr), duplicable: duplicable(caller.Info, expr), - freevars: freevars(caller.Info, expr), + freevars: freeVars(caller.Info, expr), }) } @@ -853,6 +859,11 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // Parameter elimination // + // TODO(adonovan): use the term "parameter substitution" + // instead because a "binding decl" (see below) is an + // alternative way that parameters can be eliminated even when + // they can't be substituted. + // // Consider each parameter and its corresponding argument in turn // and evaluate these conditions: // @@ -882,7 +893,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu param.info.Name, debugFormatNode(caller.Fset, arg.expr)) break // spread => last argument, but not always last parameter } - assert(!param.variadic, "unsimplfied variadic parameter") + assert(!param.variadic, "unsimplified variadic parameter") if param.info.Escapes { logf("keeping param %q: escapes from callee", param.info.Name) continue @@ -975,20 +986,176 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu } } - // TODO(adonovan): eliminate all remaining parameters - // by replacing a call f(a1, a2) - // to func f(x T1, y T2) {body} by - // { var x T1 = a1 - // var y T2 = a2 - // body } - // if x ∉ freevars(a2) or freevars(T2), and so on, - // plus the usual checks for return conversions (if any), - // complex control, etc. + // Modify callee's FuncDecl.Type.Params to remove eliminated + // parameters and move the receiver (if any) to the head of + // the ordinary parameters. + // + // The logic is fiddly because of the three forms of ast.Field: + // func(int), func(x int), func(x, y int) + // + // Also, ensure that all remaining parameters are named + // to avoid a mix of named/unnamed when joining (recv, params...). + // func (T) f(int, bool) -> (_ T, _ int, _ bool) + // (Strictly, we need do this only for methods and only when + // the namednesses of Recv and Params differ; that might be tidier.) + { + paramIdx := 0 // index in original parameter list (incl. receiver) + var newParams []*ast.Field + filterParams := func(field *ast.Field) { + var names []*ast.Ident + if field.Names == nil { + // Unnamed parameter field (e.g. func f(int) + if params[paramIdx] != nil { + // Give it an explicit name "_" since we will + // make the receiver (if any) a regular parameter + // and one cannot mix named and unnamed parameters. + names = append(names, makeIdent("_")) + } + paramIdx++ + } else { + // Named parameter field e.g. func f(x, y int) + // Remove eliminated parameters in place. + // If all were eliminated, delete field. + for _, id := range field.Names { + if pinfo := params[paramIdx]; pinfo != nil { + // Rename unreferenced parameters with "_". + // This is crucial for binding decls, since + // unlike parameters, they are subject to + // "unreferenced var" checks. + if len(pinfo.info.Refs) == 0 { + id = makeIdent("_") + } + names = append(names, id) + } + paramIdx++ + } + } + if names != nil { + newParams = append(newParams, &ast.Field{ + Names: names, + Type: field.Type, + }) + } + } + if calleeDecl.Recv != nil { + filterParams(calleeDecl.Recv.List[0]) + calleeDecl.Recv = nil + } + for _, field := range calleeDecl.Type.Params.List { + filterParams(field) + } + calleeDecl.Type.Params.List = newParams + } + + // Construct a "binding decl" that implements parameter assignment. + // + // If we succeed, the declaration may be used by reduction + // strategies to relax the requirement that all parameters + // have been eliminated. + // + // For example, a call: + // f(a0, a1, a2) + // where: + // func f(p0, p1 T0, p2 T1) { body } + // reduces to: + // { + // var ( + // p0, p1 T0 = a0, a1 + // p2 T1 = a2 + // ) + // body + // } // - // If viable, use this with the reduction strategies below - // that produce a block (not a value). + // so long as p0, p1 ∉ freevars(T1) or freevars(a2), and so on, + // because each spec is statically resolved in sequence and + // dynamically assigned in sequence. By contrast, all + // parameters are resolved simultaneously and assigned + // simultaneously. + // + // The pX names should already be blank ("_") if the parameter + // is unreferenced; this avoids "unreferenced local var" checks. + // + // Strategies may impose additional checks on return + // conversions, labels, defer, etc. + bindingDeclStmt := func() ast.Stmt { + // Spread calls are tricky as they may not align with the + // parameters' field groupings nor types. + // For example, given + // func g() (int, string) + // the call + // f(g()) + // is legal with these decls of f: + // func f(int, string) + // func f(x, y any) + // func f(x, y ...any) + // TODO(adonovan): support binding decls for spread calls by + // splitting parameter groupings as needed. + if lastArg := last(args); lastArg != nil && lastArg.spread { + logf("binding decls not yet supported for spread calls") + return nil + } + + var ( + values = remainingArgs + specs []ast.Spec + shadowed = make(map[string]bool) // names defined by previous specs + ) + for _, field := range calleeDecl.Type.Params.List { + // Each field (param group) becomes a ValueSpec. + spec := &ast.ValueSpec{ + Names: field.Names, + Type: field.Type, + Values: values[:len(field.Names)], + } + values = values[len(field.Names):] + + // Compute union of free names of type and values + // and detect shadowing. Values is the arguments + // (caller syntax), so we can use type info. + // But Type is the untyped callee syntax, + // so we have to use a syntax-only algorithm. + free := make(map[string]bool) + for _, value := range spec.Values { + for name := range freeVars(caller.Info, value) { + free[name] = true + } + } + freeishNames(free, field.Type) + for name := range free { + if shadowed[name] { + logf("binding decl would shadow free name %q", name) + return nil + } + } + for _, id := range spec.Names { + if id.Name != "_" { + shadowed[id.Name] = true + } + } + + specs = append(specs, spec) + } + decl := &ast.DeclStmt{ + Decl: &ast.GenDecl{ + Tok: token.VAR, + Specs: specs, + }, + } + logf("binding decl: %s", debugFormatNode(caller.Fset, decl)) + return decl + }() // -- let the inlining strategies begin -- + // + // When we commit to a strategy, we log a message of the form: + // + // "strategy: reduce expr-context call to { return expr }" + // + // This is a terse way of saying: + // + // we plan to reduce a call + // that appears in expression context + // to a function whose body is of the form { return expr } // TODO(adonovan): split this huge function into a sequence of // function calls with an error sentinel that means "try the @@ -1027,7 +1194,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu } } res.new = &ast.AssignStmt{ - Lhs: blanks[ast.Expr](nargs), + Lhs: blanks(nargs), Tok: token.ASSIGN, Rhs: remainingArgs, } @@ -1046,52 +1213,83 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu noResultEscapes := !exists(callee.Results, func(i int, r *paramInfo) bool { return r.Escapes }) - if allParamsEliminated && noResultEscapes { - logf("all params eliminated and no result vars escape") - - // Special case: parameterless call to { return exprs }. - // - // => reduce to: exprs (if legal) - // or: _, _ = expr (otherwise) - // - // If: - // - the body is just "return expr" with trivial implicit conversions, - // - all parameters have been eliminated, and - // - no result var escapes, - // then the call expression can be replaced by the - // callee's body expression, suitably substituted. - if callee.BodyIsReturnExpr { - logf("strategy: reduce parameterless call to { return expr }") - - results := calleeDecl.Body.List[0].(*ast.ReturnStmt).Results + // Special case: call to { return exprs }. + // + // Reduces to: + // { var (bindings); _, _ = exprs } + // or _, _ = exprs + // or expr + // + // If: + // - the body is just "return expr" with trivial implicit conversions, + // - all parameters have been eliminated or + // can be replaced by a binding decl, and + // - no result var escapes, + // then the call expression can be replaced by the + // callee's body expression, suitably substituted. + if callee.BodyIsReturnExpr { + results := calleeDecl.Body.List[0].(*ast.ReturnStmt).Results + + context := callContext(callerPath) + + // statement context + if stmt, ok := context.(*ast.ExprStmt); ok && + (allParamsEliminated && noResultEscapes || bindingDeclStmt != nil) { + logf("strategy: reduce stmt-context call to { return exprs }") clearPositions(calleeDecl.Body) - context := callContext(callerPath) - if stmt, ok := context.(*ast.ExprStmt); ok { - logf("call in statement context") - - if callee.ValidForCallStmt { - logf("callee body is valid as statement") - // Replace the statement with the callee expr. + if callee.ValidForCallStmt { + logf("callee body is valid as statement") + // Inv: len(results) == 1 + if allParamsEliminated && noResultEscapes { + // Reduces to: expr res.old = caller.Call - res.new = results[0] // Inv: len(results) == 1 + res.new = results[0] } else { - logf("callee body is not valid as statement") - // The call is a standalone statement, but the - // callee body is not suitable as a standalone statement - // (f() or <-ch), explicitly discard the results: - // _, _ = expr + // Reduces to: { var (bindings); expr } res.old = stmt - res.new = &ast.AssignStmt{ - Lhs: blanks[ast.Expr](callee.NumResults), - Tok: token.ASSIGN, - Rhs: results, + res.new = &ast.BlockStmt{ + List: []ast.Stmt{ + bindingDeclStmt, + &ast.ExprStmt{X: results[0]}, + }, + } + } + } else { + logf("callee body is not valid as statement") + // The call is a standalone statement, but the + // callee body is not suitable as a standalone statement + // (f() or <-ch), explicitly discard the results: + // Reduces to: _, _ = exprs + discard := &ast.AssignStmt{ + Lhs: blanks(callee.NumResults), + Tok: token.ASSIGN, + Rhs: results, + } + res.old = stmt + if allParamsEliminated && noResultEscapes { + // Reduces to: _, _ = exprs + res.new = discard + } else { + // Reduces to: { var (bindings); _, _ = exprs } + res.new = &ast.BlockStmt{ + List: []ast.Stmt{ + bindingDeclStmt, + discard, + }, } } + } + return res, nil + } - } else if callee.NumResults == 1 { - logf("call in expression context") + // expression context + if allParamsEliminated && noResultEscapes { + clearPositions(calleeDecl.Body) + + if callee.NumResults == 1 { + logf("strategy: reduce expr-context call to { return expr }") // A single return operand inlined to a unary // expression context may need parens. Otherwise: @@ -1106,9 +1304,8 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // but the y and z subtrees are safe. res.old = caller.Call res.new = &ast.ParenExpr{X: results[0]} - } else { - logf("call in spread context") + logf("strategy: reduce spread-context call to { return expr }") // The call returns multiple results but is // not a standalone call statement. It must @@ -1141,96 +1338,105 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu } return res, nil } + } - // Special case: parameterless tail-call. - // - // Inlining: - // return f(args) - // where: - // func f(params) (results) { body } - // reduces to: - // { body } - // so long as: - // - all parameters are eliminated; - // - call is a tail-call; - // - all returns in body have trivial result conversions; - // - there is no label conflict; - // - no result variable is referenced by name. - // - // The body may use defer, arbitrary control flow, and - // multiple returns. - // - // TODO(adonovan): omit the braces if the sets of - // names in the two blocks are disjoint. - // - // TODO(adonovan): add a strategy for a 'void tail - // call', i.e. a call statement prior to an (explicit - // or implicit) return. - if ret, ok := callContext(callerPath).(*ast.ReturnStmt); ok && - len(ret.Results) == 1 && - callee.TrivialReturns == callee.TotalReturns && - !hasLabelConflict(callerPath, callee.Labels) && - forall(callee.Results, func(i int, p *paramInfo) bool { - // all result vars are unreferenced - return len(p.Refs) == 0 - }) { - logf("strategy: reduce parameterless tail-call") - res.old = ret - res.new = calleeDecl.Body - clearPositions(calleeDecl.Body) - return res, nil + // Special case: tail-call. + // + // Inlining: + // return f(args) + // where: + // func f(params) (results) { body } + // reduces to: + // { var (bindings); body } + // { body } + // so long as: + // - all parameters have been eliminated or + // can be replaced by a binding declaration; + // - call is a tail-call; + // - all returns in body have trivial result conversions; + // - there is no label conflict; + // - no result variable is referenced by name. + // + // The body may use defer, arbitrary control flow, and + // multiple returns. + // + // TODO(adonovan): omit the braces if the sets of + // names in the two blocks are disjoint. + // + // TODO(adonovan): add a strategy for a 'void tail + // call', i.e. a call statement prior to an (explicit + // or implicit) return. + if ret, ok := callContext(callerPath).(*ast.ReturnStmt); ok && + len(ret.Results) == 1 && + callee.TrivialReturns == callee.TotalReturns && + (allParamsEliminated && noResultEscapes || bindingDeclStmt != nil) && + !hasLabelConflict(callerPath, callee.Labels) && + forall(callee.Results, func(i int, p *paramInfo) bool { + // all result vars are unreferenced + return len(p.Refs) == 0 + }) { + logf("strategy: reduce tail-call") + body := calleeDecl.Body + clearPositions(body) + if !(allParamsEliminated && noResultEscapes) { + body.List = prepend(bindingDeclStmt, body.List...) } + res.old = ret + res.new = body + return res, nil + } - // Special case: parameterless call to void function - // - // Inlining: - // Special case: parameterless call to void function - // - // Inlining: - // f(args) - // where: - // func f(params) { stmts } - // reduces to: - // { stmts } - // so long as: - // - callee is a void function (no returns) - // - callee does not use defer - // - there is no label conflict between caller and callee - // - all parameters have been eliminated. - // - // If there is only a single statement, the braces are omitted. - if stmt := callStmt(callerPath); stmt != nil && - !callee.HasDefer && - !hasLabelConflict(callerPath, callee.Labels) && - callee.TotalReturns == 0 { - logf("strategy: reduce parameterless call to { stmt } from a call stmt") - - body := calleeDecl.Body - var repl ast.Stmt = body - if len(body.List) == 1 { - repl = body.List[0] // singleton: omit braces - } - clearPositions(repl) - res.old = stmt - res.new = repl - return res, nil + // Special case: call to void function + // + // Inlining: + // f(args) + // where: + // func f(params) { stmts } + // reduces to: + // { var (bindings); stmts } + // { stmts } + // so long as: + // - callee is a void function (no returns) + // - callee does not use defer + // - there is no label conflict between caller and callee + // - all parameters have been eliminated or + // can be replaced by a binding decl. + // + // If there is only a single statement, the braces are omitted. + if stmt := callStmt(callerPath); stmt != nil && + (allParamsEliminated && noResultEscapes || bindingDeclStmt != nil) && + !callee.HasDefer && + !hasLabelConflict(callerPath, callee.Labels) && + callee.TotalReturns == 0 { + logf("strategy: reduce stmt-context call to { stmts }") + body := calleeDecl.Body + var repl ast.Stmt = body + clearPositions(repl) + if !(allParamsEliminated && noResultEscapes) { + body.List = prepend(bindingDeclStmt, body.List...) } - - // TODO(adonovan): parameterless call to { stmt; return expr } - // from one of these contexts: - // x, y = f() - // x, y := f() - // var x, y = f() - // => - // var (x T1, y T2); { stmts; x, y = expr } - // - // Because the params are no longer declared simultaneously - // we need to check that (for example) x ∉ freevars(T2), - // in addition to the usual checks for arg/result conversions, - // complex control, etc. - // Also test cases where expr is an n-ary call (spread returns). + if len(body.List) == 1 { + repl = body.List[0] // singleton: omit braces + } + res.old = stmt + res.new = repl + return res, nil } + // TODO(adonovan): parameterless call to { stmt; return expr } + // from one of these contexts: + // x, y = f() + // x, y := f() + // var x, y = f() + // => + // var (x T1, y T2); { stmts; x, y = expr } + // + // Because the params are no longer declared simultaneously + // we need to check that (for example) x ∉ freevars(T2), + // in addition to the usual checks for arg/result conversions, + // complex control, etc. + // Also test cases where expr is an n-ary call (spread returns). + // Literalization isn't quite infallible. // Consider a spread call to a method in which // no parameters are eliminated, e.g. @@ -1249,57 +1455,6 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // Infallible general case: literalization. logf("strategy: literalization") - // Modify callee's FuncDecl.Type.Params to remove eliminated - // parameters and move the receiver (if any) to the head of - // the ordinary parameters. - // - // The logic is fiddly because of the three forms of ast.Field: - // func(int), func(x int), func(x, y int) - // - // Also, ensure that all remaining parameters are named - // to avoid a mix of named/unnamed when joining (recv, params...). - // func (T) f(int, bool) -> (_ T, _ int, _ bool) - { - paramIdx := 0 // index in original parameter list (incl. receiver) - var newParams []*ast.Field - filterParams := func(field *ast.Field) { - var names []*ast.Ident - if field.Names == nil { - // Unnamed parameter field (e.g. func f(int) - if params[paramIdx] != nil { - // Give it an explicit name "_" since we will - // make the receiver (if any) a regular parameter - // and one cannot mix named and unnamed parameters. - names = blanks[*ast.Ident](1) - } - paramIdx++ - } else { - // Named parameter field e.g. func f(x, y int) - // Remove eliminated parameters in place. - // If all were eliminated, delete field. - for _, id := range field.Names { - if params[paramIdx] != nil { - names = append(names, id) - } - paramIdx++ - } - } - if names != nil { - newParams = append(newParams, &ast.Field{ - Names: names, - Type: field.Type, - }) - } - } - if calleeDecl.Recv != nil { - filterParams(calleeDecl.Recv.List[0]) - calleeDecl.Recv = nil - } - for _, field := range calleeDecl.Type.Params.List { - filterParams(field) - } - calleeDecl.Type.Params.List = newParams - } // Emit a new call to a function literal in place of // the callee name, with appropriate replacements. newCall := &ast.CallExpr{ @@ -1318,10 +1473,10 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // -- predicates over expressions -- -// freevars returns the names of all free identifiers of e: +// freeVars returns the names of all free identifiers of e: // those lexically referenced by it but not defined within it. // (Fields and methods are not included.) -func freevars(info *types.Info, e ast.Expr) map[string]bool { +func freeVars(info *types.Info, e ast.Expr) map[string]bool { free := make(map[string]bool) ast.Inspect(e, func(n ast.Node) bool { if id, ok := n.(*ast.Ident); ok { @@ -1335,6 +1490,36 @@ func freevars(info *types.Info, e ast.Expr) map[string]bool { return free } +// freeishNames computes an over-approximation to the free names +// of the type syntax t, inserting values into the map. +// +// Because we don't have go/types annotations, we can't give an exact +// result in all cases. In particular, an array type [n]T might have a +// size such as unsafe.Sizeof(func() int{stmts...}()) and now the +// precise answer depends upon all the statement syntax too. But that +// never happens in practice. +func freeishNames(free map[string]bool, t ast.Expr) { + var visit func(n ast.Node) bool + visit = func(n ast.Node) bool { + switch n := n.(type) { + case *ast.Ident: + free[n.Name] = true + + case *ast.SelectorExpr: + ast.Inspect(n.X, visit) + return false // don't visit .Sel + + case *ast.Field: + ast.Inspect(n.Type, visit) + // Don't visit .Names: + // FuncType parameters, interface methods, struct fields + return false + } + return true + } + ast.Inspect(t, visit) +} + // pure reports whether the expression is pure, that is, // has no side effects nor potential to panic. // @@ -1465,13 +1650,13 @@ func assert(cond bool, msg string) { } // blanks returns a slice of n > 0 blank identifiers. -func blanks[E ast.Expr](n int) []E { +func blanks(n int) []ast.Expr { if n == 0 { panic("blanks(0)") } - res := make([]E, n) + res := make([]ast.Expr, n) for i := range res { - res[i] = any(makeIdent("_")).(E) // ugh + res[i] = makeIdent("_") } return res } diff --git a/internal/refactor/inline/inline_test.go b/internal/refactor/inline/inline_test.go index 4c246c49234..b3ca6e596f8 100644 --- a/internal/refactor/inline/inline_test.go +++ b/internal/refactor/inline/inline_test.go @@ -391,9 +391,9 @@ func TestTable(t *testing.T) { }, { "Variadic elimination (literalization).", - `func f(x any, rest ...any) { println(x, rest) }`, - `func _() { f(1, 2, 3) }`, // NB: x int->any causes literalization, for now - `func _() { func(x any) { println(x, []any{2, 3}) }(1) }`, + `func f(x any, rest ...any) { defer println(x, rest) }`, // defer => literalization + `func _() { f(1, 2, 3) }`, + `func _() { func(x any) { defer println(x, []any{2, 3}) }(1) }`, }, { "Variadic elimination (reduction).", @@ -419,6 +419,38 @@ func TestTable(t *testing.T) { `func _() { f(g()) }`, `func _() { func(x, y int, rest ...int) { println(x, y, rest) }(g()) }`, }, + { + "Binding declaration (x eliminated).", + `func f(w, x, y any, z int) { println(w, y, z) }; func g(int) int`, + `func _() { f(g(0), g(1), g(2), g(3)) }`, + `func _() { + { + var ( + w, _, y any = g(0), g(1), g(2) + z int = g(3) + ) + println(w, y, z) + } +}`, + }, + { + "Binding decl in reduction of stmt-context call to { return exprs }", + `func f(ch chan int) int { return <-ch }; func g() chan int`, + `func _() { f(g()) }`, + `func _() { + { + var ch chan int = g() + <-ch + } +}`, + }, + { + "No binding decl due to shadowing of int", + `func f(int, y any, z int) { defer println(int, y, z) }; func g() int`, + `func _() { f(g(), g(), g()) }`, + `func _() { func(int, y any, z int) { defer println(int, y, z) }(g(), g(), g()) } +`, + }, // TODO(adonovan): improve coverage of the cross // product of each strategy with the checklist of // concerns enumerated in the package doc comment. diff --git a/internal/refactor/inline/testdata/basic-literal.txtar b/internal/refactor/inline/testdata/basic-literal.txtar index c5e95960e20..a74fbda42de 100644 --- a/internal/refactor/inline/testdata/basic-literal.txtar +++ b/internal/refactor/inline/testdata/basic-literal.txtar @@ -1,8 +1,10 @@ Basic tests of inlining by literalization. +The use of defer forces literalization. + recover() is an example of a function with effects, -so it (currently) defeats reduction. But note that the -other parameter of 'add' is eliminated. +defeating elimination of parameter x; but parameter +y is eliminated by substitution. -- go.mod -- module testdata @@ -15,13 +17,13 @@ func _() { add(recover().(int), 2) //@ inline(re"add", add1) } -func add(x, y int) int { return x + y } +func add(x, y int) int { defer print(); return x + y } -- add1 -- package a func _() { - func(x int) int { return x + 2 }(recover().(int)) //@ inline(re"add", add1) + func(x int) int { defer print(); return x + 2 }(recover().(int)) //@ inline(re"add", add1) } -func add(x, y int) int { return x + y } +func add(x, y int) int { defer print(); return x + y } diff --git a/internal/refactor/inline/testdata/method.txtar b/internal/refactor/inline/testdata/method.txtar index 511a90e4703..61c082eefda 100644 --- a/internal/refactor/inline/testdata/method.txtar +++ b/internal/refactor/inline/testdata/method.txtar @@ -29,7 +29,10 @@ type T int func (T) f0() { println() } func _(x T) { - func(_ T) { println() }(x) //@ inline(re"f0", f0) + { + var _ T = x + println() + } //@ inline(re"f0", f0) } -- a/g0.go -- @@ -47,7 +50,10 @@ package a func (recv *T) g0() { println() } func _(x T) { - func(recv *T) { println() }(&x) //@ inline(re"g0", g0) + { + var _ *T = &x + println() + } //@ inline(re"g0", g0) } -- a/f1.go -- @@ -65,7 +71,10 @@ package a func (T) f1(int, int) { println() } func _(x T) { - func(_ T) { println() }(x) //@ inline(re"f1", f1) + { + var _ T = x + println() + } //@ inline(re"f1", f1) } -- a/g1.go -- @@ -83,7 +92,10 @@ package a func (recv *T) g1(int, int) { println() } func _(x T) { - func(recv *T) { println() }(&x) //@ inline(re"g1", g1) + { + var _ *T = &x + println() + } //@ inline(re"g1", g1) } -- a/h.go -- @@ -103,7 +115,10 @@ func (T) h() int { return 1 } func _() { var ptr *T - func(_ T) int { return 1 }(*ptr) //@ inline(re"h", h) + { + var _ T = *ptr + _ = 1 + } //@ inline(re"h", h) } -- a/i.go -- diff --git a/internal/refactor/inline/testdata/multistmt-body.txtar b/internal/refactor/inline/testdata/multistmt-body.txtar index b0e82f21715..1dc39c5c3b9 100644 --- a/internal/refactor/inline/testdata/multistmt-body.txtar +++ b/internal/refactor/inline/testdata/multistmt-body.txtar @@ -1,9 +1,9 @@ Tests of reduction of calls to multi-statement bodies. -a1: literalized, because replacing parameter x with -argument z would cause a shadowing conflict. +a1: reduced to a block with a parameter binding decl. + (Parameter x can't be substituted by z without a shadowing conflict.) -a2: reduced (no shadowing). +a2: reduced with parameter substitution (no shadowing). a3: literalized, because of the return statement. @@ -29,7 +29,11 @@ package a func _() { z := 1 - func(x int) { z := 1; print(x + 2 + z) }(z) //@ inline(re"f", out1) + { + var x int = z + z := 1 + print(x + 2 + z) + } //@ inline(re"f", out1) } func f(x, y int) { From 8421a35b9ef4b19c2cb517671c48c9154ee51841 Mon Sep 17 00:00:00 2001 From: "Hana (Hyang-Ah) Kim" Date: Wed, 13 Sep 2023 12:15:37 -0400 Subject: [PATCH 095/178] gopls/lsp/command: add gopls.add_telemetry_counters Gopls clients can ask gopls to increment the specified counters using this custom command. Other than communication errors or obvious violation of protocol (e.g. unmatched names/values length, type errors), this will never return an error even when some counters cannot be updated. golang.org/x/telemetry/counter APIs does not report errors. Change-Id: I21dbbba461f04819df8a7b2190fbe77547375d99 Reviewed-on: https://go-review.googlesource.com/c/tools/+/527821 Run-TryBot: Hyang-Ah Hana Kim TryBot-Result: Gopher Robot Reviewed-by: Robert Findley --- gopls/doc/commands.md | 16 ++++++++++++++ gopls/internal/lsp/command.go | 11 ++++++++++ gopls/internal/lsp/command/command_gen.go | 20 +++++++++++++++++ gopls/internal/lsp/command/interface.go | 14 ++++++++++++ gopls/internal/lsp/source/api_json.go | 6 ++++++ gopls/internal/telemetry/telemetry.go | 12 +++++++++++ gopls/internal/telemetry/telemetry_go118.go | 3 +++ gopls/internal/telemetry/telemetry_test.go | 24 ++++++++++++++++++++- 8 files changed, 105 insertions(+), 1 deletion(-) diff --git a/gopls/doc/commands.md b/gopls/doc/commands.md index 1a8c4dc99f8..a2ffda56a6e 100644 --- a/gopls/doc/commands.md +++ b/gopls/doc/commands.md @@ -41,6 +41,22 @@ Args: } ``` +### **update the given telemetry counters.** +Identifier: `gopls.add_telemetry_counters` + +Gopls will prepend "fwd/" to all the counters updated using this command +to avoid conflicts with other counters gopls collects. + +Args: + +``` +{ + // Names and Values must have the same length. + "Names": []string, + "Values": []int64, +} +``` + ### **Apply a fix** Identifier: `gopls.apply_fix` diff --git a/gopls/internal/lsp/command.go b/gopls/internal/lsp/command.go index 99b0b384ffe..a64007b988a 100644 --- a/gopls/internal/lsp/command.go +++ b/gopls/internal/lsp/command.go @@ -28,6 +28,7 @@ import ( "golang.org/x/tools/gopls/internal/lsp/protocol" "golang.org/x/tools/gopls/internal/lsp/source" "golang.org/x/tools/gopls/internal/span" + "golang.org/x/tools/gopls/internal/telemetry" "golang.org/x/tools/gopls/internal/vulncheck" "golang.org/x/tools/gopls/internal/vulncheck/scan" "golang.org/x/tools/internal/event" @@ -63,6 +64,16 @@ type commandHandler struct { params *protocol.ExecuteCommandParams } +// AddTelemetryCounters implements command.Interface. +func (*commandHandler) AddTelemetryCounters(_ context.Context, args command.AddTelemetryCountersArgs) error { + if len(args.Names) != len(args.Values) { + return fmt.Errorf("Names and Values must have the same length") + } + // invalid counter update requests will be silently dropped. (no audience) + telemetry.AddForwardedCounters(args.Names, args.Values) + return nil +} + // commandConfig configures common command set-up and execution. type commandConfig struct { async bool // whether to run the command asynchronously. Async commands can only return errors. diff --git a/gopls/internal/lsp/command/command_gen.go b/gopls/internal/lsp/command/command_gen.go index 00b76579601..40eda278190 100644 --- a/gopls/internal/lsp/command/command_gen.go +++ b/gopls/internal/lsp/command/command_gen.go @@ -21,6 +21,7 @@ import ( const ( AddDependency Command = "add_dependency" AddImport Command = "add_import" + AddTelemetryCounters Command = "add_telemetry_counters" ApplyFix Command = "apply_fix" CheckUpgrades Command = "check_upgrades" EditGoDirective Command = "edit_go_directive" @@ -52,6 +53,7 @@ const ( var Commands = []Command{ AddDependency, AddImport, + AddTelemetryCounters, ApplyFix, CheckUpgrades, EditGoDirective, @@ -94,6 +96,12 @@ func Dispatch(ctx context.Context, params *protocol.ExecuteCommandParams, s Inte return nil, err } return nil, s.AddImport(ctx, a0) + case "gopls.add_telemetry_counters": + var a0 AddTelemetryCountersArgs + if err := UnmarshalArgs(params.Arguments, &a0); err != nil { + return nil, err + } + return nil, s.AddTelemetryCounters(ctx, a0) case "gopls.apply_fix": var a0 ApplyFixArgs if err := UnmarshalArgs(params.Arguments, &a0); err != nil { @@ -272,6 +280,18 @@ func NewAddImportCommand(title string, a0 AddImportArgs) (protocol.Command, erro }, nil } +func NewAddTelemetryCountersCommand(title string, a0 AddTelemetryCountersArgs) (protocol.Command, error) { + args, err := MarshalArgs(a0) + if err != nil { + return protocol.Command{}, err + } + return protocol.Command{ + Title: title, + Command: "gopls.add_telemetry_counters", + Arguments: args, + }, nil +} + func NewApplyFixCommand(title string, a0 ApplyFixArgs) (protocol.Command, error) { args, err := MarshalArgs(a0) if err != nil { diff --git a/gopls/internal/lsp/command/interface.go b/gopls/internal/lsp/command/interface.go index 261776d5ad1..2ae50fb0e87 100644 --- a/gopls/internal/lsp/command/interface.go +++ b/gopls/internal/lsp/command/interface.go @@ -189,6 +189,12 @@ type Interface interface { // RunGoWorkCommand: run `go work [args...]`, and apply the resulting go.work // edits to the current go.work file. RunGoWorkCommand(context.Context, RunGoWorkArgs) error + + // AddTelemetryCounters: update the given telemetry counters. + // + // Gopls will prepend "fwd/" to all the counters updated using this command + // to avoid conflicts with other counters gopls collects. + AddTelemetryCounters(context.Context, AddTelemetryCountersArgs) error } type RunTestsArgs struct { @@ -499,3 +505,11 @@ type RunGoWorkArgs struct { InitFirst bool // Whether to run `go work init` first Args []string // Args to pass to `go work` } + +// AddTelemetryCountersArgs holds the arguments to the AddCounters command +// that updates the telemetry counters. +type AddTelemetryCountersArgs struct { + // Names and Values must have the same length. + Names []string // Name of counters. + Values []int64 // Values added to the corresponding counters. Must be non-negative. +} diff --git a/gopls/internal/lsp/source/api_json.go b/gopls/internal/lsp/source/api_json.go index 4d85ed7b5d2..d0fdb2008ce 100644 --- a/gopls/internal/lsp/source/api_json.go +++ b/gopls/internal/lsp/source/api_json.go @@ -709,6 +709,12 @@ var GeneratedAPIJSON = &APIJSON{ Doc: "Ask the server to add an import path to a given Go file. The method will\ncall applyEdit on the client so that clients don't have to apply the edit\nthemselves.", ArgDoc: "{\n\t// ImportPath is the target import path that should\n\t// be added to the URI file\n\t\"ImportPath\": string,\n\t// URI is the file that the ImportPath should be\n\t// added to\n\t\"URI\": string,\n}", }, + { + Command: "gopls.add_telemetry_counters", + Title: "update the given telemetry counters.", + Doc: "Gopls will prepend \"fwd/\" to all the counters updated using this command\nto avoid conflicts with other counters gopls collects.", + ArgDoc: "{\n\t// Names and Values must have the same length.\n\t\"Names\": []string,\n\t\"Values\": []int64,\n}", + }, { Command: "gopls.apply_fix", Title: "Apply a fix", diff --git a/gopls/internal/telemetry/telemetry.go b/gopls/internal/telemetry/telemetry.go index db75e1a7fbf..840896d516d 100644 --- a/gopls/internal/telemetry/telemetry.go +++ b/gopls/internal/telemetry/telemetry.go @@ -68,3 +68,15 @@ func RecordViewGoVersion(x int) { name := fmt.Sprintf("gopls/goversion:1.%d", x) counter.Inc(name) } + +// AddForwardedCounters adds the given counters on behalf of clients. +// Names and values must have the same length. +func AddForwardedCounters(names []string, values []int64) { + for i, n := range names { + v := values[i] + if n == "" || v < 0 { + continue // Should we report an error? Who is the audience? + } + counter.Add("fwd/"+n, v) + } +} diff --git a/gopls/internal/telemetry/telemetry_go118.go b/gopls/internal/telemetry/telemetry_go118.go index b0c1197cb77..2d1d6040490 100644 --- a/gopls/internal/telemetry/telemetry_go118.go +++ b/gopls/internal/telemetry/telemetry_go118.go @@ -17,3 +17,6 @@ func RecordClientInfo(params *protocol.ParamInitialize) { func RecordViewGoVersion(x int) { } + +func AddForwardedCounters(names []string, values []int64) { +} diff --git a/gopls/internal/telemetry/telemetry_test.go b/gopls/internal/telemetry/telemetry_test.go index 93751bff1d8..57fff1c4798 100644 --- a/gopls/internal/telemetry/telemetry_test.go +++ b/gopls/internal/telemetry/telemetry_test.go @@ -18,6 +18,8 @@ import ( "golang.org/x/telemetry/counter/countertest" // requires go1.21+ "golang.org/x/tools/gopls/internal/bug" "golang.org/x/tools/gopls/internal/hooks" + "golang.org/x/tools/gopls/internal/lsp/command" + "golang.org/x/tools/gopls/internal/lsp/protocol" . "golang.org/x/tools/gopls/internal/lsp/regtest" ) @@ -43,8 +45,9 @@ func TestTelemetry(t *testing.T) { Modes(Default), // must be in-process to receive the bug report below Settings{"showBugReports": true}, ClientName("Visual Studio Code"), - ).Run(t, "", func(t *testing.T, env *Env) { + ).Run(t, "", func(_ *testing.T, env *Env) { goversion = strconv.Itoa(env.GoVersion()) + addForwardedCounters(env, []string{"vscode/linter:a"}, []int64{1}) const desc = "got a bug" bug.Report(desc) // want a stack counter with the trace starting from here. env.Await(ShownMessage(desc)) @@ -52,9 +55,11 @@ func TestTelemetry(t *testing.T) { // gopls/editor:client // gopls/goversion:1.x + // fwd/vscode/linter:a for _, c := range []*counter.Counter{ counter.New("gopls/client:" + editor), counter.New("gopls/goversion:1." + goversion), + counter.New("fwd/vscode/linter:a"), } { count, err := countertest.ReadCounter(c) if err != nil || count != 1 { @@ -75,6 +80,23 @@ func TestTelemetry(t *testing.T) { } } +func addForwardedCounters(env *Env, names []string, values []int64) { + args, err := command.MarshalArgs(command.AddTelemetryCountersArgs{ + Names: names, Values: values, + }) + if err != nil { + env.T.Fatal(err) + } + var res error + env.ExecuteCommand(&protocol.ExecuteCommandParams{ + Command: command.AddTelemetryCounters.ID(), + Arguments: args, + }, res) + if res != nil { + env.T.Errorf("%v failed - %v", command.AddTelemetryCounters.ID(), res) + } +} + func hasEntry(counts map[string]uint64, pattern string, want uint64) bool { for k, v := range counts { if strings.Contains(k, pattern) && v == want { From e2393aba883af17118d4142960a1d41b72f2c7b9 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Thu, 14 Sep 2023 17:58:12 -0400 Subject: [PATCH 096/178] gopls/internal/lsp/source: inliner: don't spam log This change makes the inliner use the gopls event log so that we don't pollute stderr during testing. Change-Id: I3ede9963c4be5babed8cc2b3f5b089923fdaee65 Reviewed-on: https://go-review.googlesource.com/c/tools/+/528557 LUCI-TryBot-Result: Go LUCI Reviewed-by: Robert Findley --- gopls/internal/lsp/source/inline.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/gopls/internal/lsp/source/inline.go b/gopls/internal/lsp/source/inline.go index 64e6b82265c..23389e1163d 100644 --- a/gopls/internal/lsp/source/inline.go +++ b/gopls/internal/lsp/source/inline.go @@ -12,7 +12,6 @@ import ( "go/ast" "go/token" "go/types" - "log" "runtime/debug" "golang.org/x/tools/go/analysis" @@ -23,6 +22,7 @@ import ( "golang.org/x/tools/gopls/internal/lsp/safetoken" "golang.org/x/tools/gopls/internal/span" "golang.org/x/tools/internal/diff" + "golang.org/x/tools/internal/event" "golang.org/x/tools/internal/refactor/inline" ) @@ -119,9 +119,12 @@ func inlineCall(ctx context.Context, snapshot Snapshot, fh FileHandle, rng proto Content: callerPGF.Src, } - // Users can consult the gopls log to see + // Users can consult the gopls event log to see // why a particular inlining strategy was chosen. - got, err := inline.Inline(log.Printf, caller, callee) + logf := func(format string, args ...any) { + event.Log(ctx, "inliner: "+fmt.Sprintf(format, args...)) + } + got, err := inline.Inline(logf, caller, callee) if err != nil { return nil, nil, err } From dca7c8240ba7568cb887bccab1c8b0edbfdafc1e Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Wed, 13 Sep 2023 16:52:13 -0400 Subject: [PATCH 097/178] gopls/internal/regtest: support full features of old completion markers Refactor the marker tests to support binding arbitrary data to identifiers. This generalizes @loc, and allows us to support the @item completion item annotations of the old marker tests. It turns out that the position of @item notes in the old marker framework doesn't really matter, so perhaps there is a clearer way to implement them. Nevertheless, for now the priority is to migrate the tests, and the easiest way to do that is to require very little modification when translating them to the new framework. Furthermore, the filtering of builtins is ported to the new marker framework, but in a much simpler way: just lookup in types.Universe and the token package. I think this change makes sense philosophically, and simplifies the test framework. For example, @loc is no longer a special case, and the markerFunc implementation is simpler. However, it's still a rather lot of machinery. We can further simplify later, but for now the priority is to eliminate the old framework. Notably, as part of this change it became problematic for the @rename markers to accept an Identifier (because Identifiers now refer to either Golden content or bound values). I think it makes sense to change the argument to rename to a string anyway, since the user input is a string. For golang/go#54845 Change-Id: I728d96b0d8a4b29cdabb60cdabe59500aa4ad577 Reviewed-on: https://go-review.googlesource.com/c/tools/+/528335 Reviewed-by: Alan Donovan LUCI-TryBot-Result: Go LUCI --- gopls/internal/lsp/regtest/marker.go | 498 ++++++++++-------- gopls/internal/lsp/testdata/bad/bad0_go120.go | 24 - gopls/internal/lsp/testdata/bad/bad0_go121.go | 26 - gopls/internal/lsp/testdata/bad/bad1.go | 34 -- .../internal/lsp/testdata/summary.txt.golden | 4 +- .../lsp/testdata/summary_go1.21.txt.golden | 4 +- .../marker/testdata/completion/bad.txt | 68 +++ .../marker/testdata/completion/issue59096.txt | 4 +- .../marker/testdata/completion/issue60545.txt | 6 +- .../testdata/diagnostics/useinternal.txt | 21 + .../regtest/marker/testdata/rename/basic.txt | 28 +- .../marker/testdata/rename/conflict.txt | 8 +- .../regtest/marker/testdata/rename/embed.txt | 12 +- .../marker/testdata/rename/generics.txt | 92 ++-- .../marker/testdata/rename/issue60789.txt | 8 +- .../marker/testdata/rename/issue61294.txt | 4 +- .../marker/testdata/rename/issue61640.txt | 4 +- .../marker/testdata/rename/issue61813.txt | 4 +- .../marker/testdata/rename/methods.txt | 8 +- .../marker/testdata/rename/typeswitch.txt | 12 +- .../marker/testdata/rename/unexported.txt | 2 +- 21 files changed, 484 insertions(+), 387 deletions(-) delete mode 100644 gopls/internal/lsp/testdata/bad/bad0_go120.go delete mode 100644 gopls/internal/lsp/testdata/bad/bad0_go121.go delete mode 100644 gopls/internal/lsp/testdata/bad/bad1.go create mode 100644 gopls/internal/regtest/marker/testdata/completion/bad.txt create mode 100644 gopls/internal/regtest/marker/testdata/diagnostics/useinternal.txt diff --git a/gopls/internal/lsp/regtest/marker.go b/gopls/internal/lsp/regtest/marker.go index 422acc75c3c..ca4c9a8f83d 100644 --- a/gopls/internal/lsp/regtest/marker.go +++ b/gopls/internal/lsp/regtest/marker.go @@ -11,6 +11,7 @@ import ( "flag" "fmt" "go/token" + "go/types" "io/fs" "log" "os" @@ -156,8 +157,8 @@ var update = flag.Bool("update", false, "if set, update test data during marker // current document, with results compared to the @codelens annotations in // the current document. // -// - complete(location, ...labels): specifies expected completion results at -// the given location. +// - complete(location, ...items): specifies expected completion results at +// the given location. Must be used in conjunction with @item. // // - diag(location, regexp): specifies an expected diagnostic matching the // given regexp at the given location. The test runner requires @@ -196,6 +197,13 @@ var update = flag.Bool("update", false, "if set, update test data during marker // textDocument/implementation query at the src location and // checks that the resulting set of locations matches want. // +// - item(label, details, kind): defines a completion item with the provided +// fields. This information is not positional, and therefore @item markers +// may occur anywhere in the source. Used in conjunction with @complete. +// +// TODO(rfindley): rethink whether floating @item annotations are the best +// way to specify completion results. +// // - loc(name, location): specifies the name for a location in the source. These // locations may be referenced by other markers. // @@ -397,7 +405,7 @@ func RunMarkerTests(t *testing.T, dir string) { } if _, ok := config.Settings["diagnosticsDelay"]; !ok { if config.Settings == nil { - config.Settings = make(map[string]interface{}) + config.Settings = make(map[string]any) } config.Settings["diagnosticsDelay"] = "10ms" } @@ -407,7 +415,7 @@ func RunMarkerTests(t *testing.T, dir string) { test: test, env: newEnv(t, cache, test.files, test.proxyFiles, test.writeGoSum, config), settings: config.Settings, - locations: make(map[expect.Identifier]protocol.Location), + data: make(map[expect.Identifier]any), diags: make(map[protocol.Location][]protocol.Diagnostic), extraNotes: make(map[protocol.DocumentURI]map[string][]*expect.Note), } @@ -423,29 +431,6 @@ func RunMarkerTests(t *testing.T, dir string) { for file := range test.files { run.env.OpenFile(file) } - - // Pre-process locations. - var markers []marker - for _, note := range test.notes { - fn, ok := markerFuncs[note.Name] - if !ok { - // TODO(rfindley): simplify these deeply nested APIs. - uri := run.env.Sandbox.Workdir.URI(run.test.fset.File(note.Pos).Name()) - if run.extraNotes[uri] == nil { - run.extraNotes[uri] = make(map[string][]*expect.Note) - } - run.extraNotes[uri][note.Name] = append(run.extraNotes[uri][note.Name], note) - continue - } - mark := marker{run: run, note: note, fn: fn} - switch note.Name { - case "loc": // as a special case, locations are collected before other markers - mark.execute() - default: - markers = append(markers, mark) - } - } - // Wait for the didOpen notifications to be processed, then collect // diagnostics. var diags map[string]*protocol.PublishDiagnosticsParams @@ -464,9 +449,25 @@ func RunMarkerTests(t *testing.T, dir string) { } } + var markers []marker + for _, note := range test.notes { + mark := marker{run: run, note: note} + if fn, ok := dataFuncs[note.Name]; ok { + fn(mark) + } else if _, ok := actionFuncs[note.Name]; ok { + markers = append(markers, mark) // save for later + } else { + uri := mark.uri() + if run.extraNotes[uri] == nil { + run.extraNotes[uri] = make(map[string][]*expect.Note) + } + run.extraNotes[uri][note.Name] = append(run.extraNotes[uri][note.Name], note) + } + } + // Invoke each remaining marker in the test. for _, mark := range markers { - mark.execute() + actionFuncs[mark.note.Name](mark) } // Any remaining (un-eliminated) diagnostics are an error. @@ -521,7 +522,6 @@ func RunMarkerTests(t *testing.T, dir string) { type marker struct { run *markerTestRun note *expect.Note - fn markerFunc } // server returns the LSP server for the marker test run. @@ -532,7 +532,7 @@ func (m marker) server() protocol.Server { // errorf reports an error with a prefix indicating the position of the marker note. // // It formats the error message using mark.sprintf. -func (mark marker) errorf(format string, args ...interface{}) { +func (mark marker) errorf(format string, args ...any) { msg := mark.sprintf(format, args...) // TODO(adonovan): consider using fmt.Fprintf(os.Stderr)+t.Fail instead of // t.Errorf to avoid reporting uninteresting positions in the Go source of @@ -541,72 +541,162 @@ func (mark marker) errorf(format string, args ...interface{}) { mark.run.env.T.Errorf("%s: %s", mark.run.fmtPos(mark.note.Pos), msg) } -// execute invokes the marker's function with the arguments from note. -func (mark marker) execute() { - // The first converter corresponds to the *Env argument. - // All others must be converted from the marker syntax. - args := []reflect.Value{reflect.ValueOf(mark)} - var convert converter - for i, in := range mark.note.Args { - if i < len(mark.fn.converters) { - convert = mark.fn.converters[i] - } else if !mark.fn.variadic { - goto arity // too many args - } +// A dataFunc is a func that binds an identifier to a value. The first argument +// to the provided func must be of type `marker`, and the func must return +// exactly one result. dataFuncs run before markerFuncs. +// +// When associated with a note, the first argument of the note must be an +// identifier, and remaining arguments are converted and passed to the provided +// function along with the mark context. +// +// For example, given a data func with signature +// +// func(mark marker, label, details, kind string) CompletionItem +// +// The resulting dataFunc can associated with @item notes, and invoked as follows: +// +// //@item(FooCompletion, "Foo", "func() int", "func") +// +// In this example, the name 'FooCompletion' is bound to the completion item +// produced with the given arguments, and may be referenced by other marker +// funcs in the test, by passing the FooCompletion identifier. +// +// dataFuncs should not mutate the test environment. +func dataFunc(fn any) func(marker) { + ftype := reflect.TypeOf(fn) + if ftype.NumIn() == 0 || ftype.In(0) != markerType { + panic(fmt.Sprintf("data function %#v must accept marker as its first argument", ftype)) + } + if ftype.NumOut() != 1 { + panic(fmt.Sprintf("data function %#v must have exactly 1 result", ftype)) + } - // Special handling for the blank identifier: treat it as the zero value. - if ident, ok := in.(expect.Identifier); ok && ident == "_" { - zero := reflect.Zero(mark.fn.paramTypes[i]) - args = append(args, zero) - continue + return func(mark marker) { + if len(mark.note.Args) == 0 || !is[expect.Identifier](mark.note.Args[0]) { + mark.errorf("first argument to a data func must be an identifier") + return + } + id := mark.note.Args[0].(expect.Identifier) + if alt, ok := mark.run.data[id]; ok { + mark.errorf("%s already declared as %T", id, alt) + return + } + args := append([]any{mark}, mark.note.Args[1:]...) + argValues, err := convertArgs(mark, ftype, args) + if err != nil { + mark.errorf("converting args: %v", err) + return } + results := reflect.ValueOf(fn).Call(argValues) + mark.run.data[id] = results[0].Interface() + } +} + +// A markerFunc is a func that executes after all dataFuncs, performs some +// operation, and verifies an assertion. +// +// The first argument of the provided function must be of type `marker`, and +// the function must not return any results. +// +// When associated with a note, the notes arguments are converted and passed to +// the provided function along with the mark context. +// +// markerFuncs should not mutate the test environment. +func markerFunc(fn any) func(marker) { + ftype := reflect.TypeOf(fn) + if ftype.NumIn() == 0 || ftype.In(0) != markerType { + panic(fmt.Sprintf("marker function %#v must accept marker as its first argument", ftype)) + } + if ftype.NumOut() != 0 { + panic(fmt.Sprintf("action function %#v cannot have results", ftype)) + } - out, err := convert(mark, in) + return func(mark marker) { + args := append([]any{mark}, mark.note.Args...) + argValues, err := convertArgs(mark, ftype, args) if err != nil { - mark.errorf("converting argument #%d of %s (%v): %v", i, mark.note.Name, in, err) + mark.errorf("converting args: %v", err) return } - args = append(args, reflect.ValueOf(out)) + reflect.ValueOf(fn).Call(argValues) } - if len(args) < len(mark.fn.converters) { - goto arity // too few args +} + +func convertArgs(mark marker, ftype reflect.Type, args []any) ([]reflect.Value, error) { + var ( + argValues []reflect.Value + pnext int // next param index + p reflect.Type // current param + ) + for i, arg := range args { + if i < ftype.NumIn() { + p = ftype.In(pnext) + pnext++ + } else if p == nil || !ftype.IsVariadic() { + // The actual number of arguments expected by the mark varies, depending + // on whether this is a data func or an action func. + // + // Since this error indicates a bug, probably OK to have an imprecise + // error message here. + return nil, fmt.Errorf("too many arguments to %s", mark.note.Name) + } + elemType := p + if ftype.IsVariadic() && pnext == ftype.NumIn() { + elemType = p.Elem() + } + var v reflect.Value + if id, ok := arg.(expect.Identifier); ok && id == "_" { + v = reflect.Zero(elemType) + } else { + a, err := convert(mark, arg, elemType) + if err != nil { + return nil, err + } + v = reflect.ValueOf(a) + } + argValues = append(argValues, v) + } + // Check that we have sufficient arguments. If the function is variadic, we + // do not need arguments for the final parameter. + if pnext < ftype.NumIn()-1 || pnext == ftype.NumIn()-1 && !ftype.IsVariadic() { + // Same comment as above: OK to be vague here. + return nil, fmt.Errorf("not enough arguments to %s", mark.note.Name) } + return argValues, nil +} - mark.fn.fn.Call(args) - return +// is reports whether arg is a T. +func is[T any](arg any) bool { + _, ok := arg.(T) + return ok +} -arity: - mark.errorf("got %d arguments to %s, want %d", - len(mark.note.Args), mark.note.Name, len(mark.fn.converters)) +// Supported data functions. See [dataFunc] for more details. +var dataFuncs = map[string]func(marker){ + "loc": dataFunc(locMarker), + "item": dataFunc(completionItemMarker), } -// Supported marker functions. -// -// Each marker function must accept a marker as its first argument, with -// subsequent arguments converted from the marker arguments. -// -// Marker funcs should not mutate the test environment (e.g. via opening files -// or applying edits in the editor). -var markerFuncs = map[string]markerFunc{ - "acceptcompletion": makeMarkerFunc(acceptCompletionMarker), - "codeaction": makeMarkerFunc(codeActionMarker), - "codeactionerr": makeMarkerFunc(codeActionErrMarker), - "codelenses": makeMarkerFunc(codeLensesMarker), - "complete": makeMarkerFunc(completeMarker), - "def": makeMarkerFunc(defMarker), - "diag": makeMarkerFunc(diagMarker), - "foldingrange": makeMarkerFunc(foldingRangeMarker), - "format": makeMarkerFunc(formatMarker), - "highlight": makeMarkerFunc(highlightMarker), - "hover": makeMarkerFunc(hoverMarker), - "implementation": makeMarkerFunc(implementationMarker), - "loc": makeMarkerFunc(locMarker), - "rename": makeMarkerFunc(renameMarker), - "renameerr": makeMarkerFunc(renameErrMarker), - "suggestedfix": makeMarkerFunc(suggestedfixMarker), - "symbol": makeMarkerFunc(symbolMarker), - "refs": makeMarkerFunc(refsMarker), - "workspacesymbol": makeMarkerFunc(workspaceSymbolMarker), +// Supported marker functions. See [markerFunc] for more details. +var actionFuncs = map[string]func(marker){ + "acceptcompletion": markerFunc(acceptCompletionMarker), + "codeaction": markerFunc(codeActionMarker), + "codeactionerr": markerFunc(codeActionErrMarker), + "codelenses": markerFunc(codeLensesMarker), + "complete": markerFunc(completeMarker), + "def": markerFunc(defMarker), + "diag": markerFunc(diagMarker), + "foldingrange": markerFunc(foldingRangeMarker), + "format": markerFunc(formatMarker), + "highlight": markerFunc(highlightMarker), + "hover": markerFunc(hoverMarker), + "implementation": markerFunc(implementationMarker), + "rename": markerFunc(renameMarker), + "renameerr": markerFunc(renameErrMarker), + "suggestedfix": markerFunc(suggestedfixMarker), + "symbol": markerFunc(symbolMarker), + "refs": markerFunc(refsMarker), + "workspacesymbol": markerFunc(workspaceSymbolMarker), } // markerTest holds all the test data extracted from a test txtar archive. @@ -614,17 +704,17 @@ var markerFuncs = map[string]markerFunc{ // See the documentation for RunMarkerTests for more information on the archive // format. type markerTest struct { - name string // relative path to the txtar file in the testdata dir - fset *token.FileSet // fileset used for parsing notes - content []byte // raw test content - archive *txtar.Archive // original test archive - settings map[string]interface{} // gopls settings - capabilities []byte // content of capabilities.json file - env map[string]string // editor environment - proxyFiles map[string][]byte // proxy content - files map[string][]byte // data files from the archive (excluding special files) - notes []*expect.Note // extracted notes from data files - golden map[string]*Golden // extracted golden content, by identifier name + name string // relative path to the txtar file in the testdata dir + fset *token.FileSet // fileset used for parsing notes + content []byte // raw test content + archive *txtar.Archive // original test archive + settings map[string]any // gopls settings + capabilities []byte // content of capabilities.json file + env map[string]string // editor environment + proxyFiles map[string][]byte // proxy content + files map[string][]byte // data files from the archive (excluding special files) + notes []*expect.Note // extracted notes from data files + golden map[expect.Identifier]*Golden // extracted golden content, by identifier name skipReason string // the skip reason extracted from the "skip" archive file flags []string // flags extracted from the special "flags" archive file. @@ -663,7 +753,7 @@ func (l stringListValue) String() string { return strings.Join([]string(l), ",") } -func (t *markerTest) getGolden(id string) *Golden { +func (t *markerTest) getGolden(id expect.Identifier) *Golden { golden, ok := t.golden[id] // If there was no golden content for this identifier, we must create one // to handle the case where -update is set: we need a place to store @@ -685,7 +775,7 @@ func (t *markerTest) getGolden(id string) *Golden { // When -update is set, golden captures the updated golden contents for later // writing. type Golden struct { - id string + id expect.Identifier data map[string][]byte // key "" => @id itself updated map[string][]byte } @@ -764,7 +854,7 @@ func loadMarkerTest(name string, content []byte) (*markerTest, error) { content: content, archive: archive, files: make(map[string][]byte), - golden: make(map[string]*Golden), + golden: make(map[expect.Identifier]*Golden), } for _, file := range archive.Files { switch { @@ -799,7 +889,8 @@ func loadMarkerTest(name string, content []byte) (*markerTest, error) { } case strings.HasPrefix(file.Name, "@"): // golden content - id, name, _ := strings.Cut(file.Name[len("@"):], "/") + idstring, name, _ := strings.Cut(file.Name[len("@"):], "/") + id := expect.Identifier(idstring) // Note that a file.Name of just "@id" gives (id, name) = ("id", ""). if _, ok := test.golden[id]; !ok { test.golden[id] = &Golden{ @@ -853,7 +944,7 @@ func formatTest(test *markerTest) ([]byte, error) { updatedGolden := make(map[string][]byte) for id, g := range test.golden { for name, data := range g.updated { - filename := "@" + path.Join(id, name) // name may be "" + filename := "@" + path.Join(string(id), name) // name may be "" updatedGolden[filename] = data } } @@ -938,24 +1029,16 @@ func newEnv(t *testing.T, cache *cache.Cache, files, proxyFiles map[string][]byt } } -// A markerFunc is a reflectively callable @mark implementation function. -type markerFunc struct { - fn reflect.Value // the func to invoke - paramTypes []reflect.Type // parameter types, for zero values - converters []converter // to convert non-blank arguments - variadic bool -} - // A markerTestRun holds the state of one run of a marker test archive. type markerTestRun struct { test *markerTest env *Env - settings map[string]interface{} + settings map[string]any // Collected information. // Each @diag/@suggestedfix marker eliminates an entry from diags. - locations map[expect.Identifier]protocol.Location - diags map[protocol.Location][]protocol.Diagnostic // diagnostics by position; location end == start + data map[expect.Identifier]any + diags map[protocol.Location][]protocol.Diagnostic // diagnostics by position; location end == start // Notes that weren't associated with a top-level marker func. They may be // consumed by another marker (e.g. @codelenses collects @codelens markers). @@ -967,11 +1050,11 @@ type markerTestRun struct { // arguments of the following types: // - token.Pos: formatted using (*markerTestRun).fmtPos // - protocol.Location: formatted using (*markerTestRun).fmtLoc -func (c *marker) sprintf(format string, args ...interface{}) string { +func (c *marker) sprintf(format string, args ...any) string { if false { _ = fmt.Sprintf(format, args...) // enable vet printf checker } - var args2 []interface{} + var args2 []any for _, arg := range args { switch arg := arg.(type) { case token.Pos: @@ -1080,36 +1163,11 @@ func (run *markerTestRun) fmtLocDetails(loc protocol.Location, includeTxtPos boo } } -// makeMarkerFunc uses reflection to create a markerFunc for the given func value. -func makeMarkerFunc(fn interface{}) markerFunc { - mi := markerFunc{ - fn: reflect.ValueOf(fn), - } - mtyp := mi.fn.Type() - mi.variadic = mtyp.IsVariadic() - if mtyp.NumIn() == 0 || mtyp.In(0) != markerType { - panic(fmt.Sprintf("marker function %#v must accept marker as its first argument", mi.fn)) - } - if mtyp.NumOut() != 0 { - panic(fmt.Sprintf("marker function %#v must not have results", mi.fn)) - } - for a := 1; a < mtyp.NumIn(); a++ { - in := mtyp.In(a) - if mi.variadic && a == mtyp.NumIn()-1 { - in = in.Elem() // for ...T, convert to T - } - mi.paramTypes = append(mi.paramTypes, in) - c := makeConverter(in) - mi.converters = append(mi.converters, c) - } - return mi -} - // ---- converters ---- // converter is the signature of argument converters. // A converter should return an error rather than calling marker.errorf(). -type converter func(marker, interface{}) (interface{}, error) +type converter func(marker, any) (any, error) // Types with special conversions. var ( @@ -1120,28 +1178,37 @@ var ( wantErrorType = reflect.TypeOf(wantError{}) ) -func makeConverter(paramType reflect.Type) converter { +func convert(mark marker, arg any, paramType reflect.Type) (any, error) { + if paramType == goldenType { + id, ok := arg.(expect.Identifier) + if !ok { + return nil, fmt.Errorf("invalid input type %T: golden key must be an identifier", arg) + } + return mark.run.test.getGolden(id), nil + } + if id, ok := arg.(expect.Identifier); ok { + if arg, ok := mark.run.data[id]; ok { + return arg, nil + } + } + argType := reflect.TypeOf(arg) + if argType.AssignableTo(paramType) { + return arg, nil // no conversion required + } switch paramType { - case goldenType: - return goldenConverter case locationType: - return locationConverter + return convertLocation(mark, arg) case wantErrorType: - return wantErrorConverter + return convertWantError(mark, arg) default: - return func(_ marker, arg interface{}) (interface{}, error) { - if argType := reflect.TypeOf(arg); argType != paramType { - return nil, fmt.Errorf("cannot convert type %s to %s", argType, paramType) - } - return arg, nil - } + return nil, fmt.Errorf("cannot convert type %s to %s", argType, paramType) } } -// locationConverter converts a string argument into the protocol location -// corresponding to the first position of the string in the line preceding the -// note. -func locationConverter(mark marker, arg interface{}) (interface{}, error) { +// convertLocation converts a string or regexp argument into the protocol +// location corresponding to the first position of the string (or first match +// of the regexp) in the line preceding the note. +func convertLocation(mark marker, arg any) (protocol.Location, error) { switch arg := arg.(type) { case string: startOff, preceding, m, err := linePreceding(mark.run, mark.note.Pos) @@ -1150,20 +1217,14 @@ func locationConverter(mark marker, arg interface{}) (interface{}, error) { } idx := bytes.Index(preceding, []byte(arg)) if idx < 0 { - return nil, fmt.Errorf("substring %q not found in %q", arg, preceding) + return protocol.Location{}, fmt.Errorf("substring %q not found in %q", arg, preceding) } off := startOff + idx return m.OffsetLocation(off, off+len(arg)) case *regexp.Regexp: return findRegexpInLine(mark.run, mark.note.Pos, arg) - case expect.Identifier: - loc, ok := mark.run.locations[arg] - if !ok { - return nil, fmt.Errorf("no location named %q", arg) - } - return loc, nil default: - return nil, fmt.Errorf("cannot convert argument type %T to location (must be a string to match the preceding line)", arg) + return protocol.Location{}, fmt.Errorf("cannot convert argument type %T to location (must be a string to match the preceding line)", arg) } } @@ -1211,22 +1272,22 @@ func linePreceding(run *markerTestRun, pos token.Pos) (int, []byte, *protocol.Ma return startOff, m.Content[startOff:endOff], m, nil } -// wantErrorConverter converts a string, regexp, or identifier +// convertWantError converts a string, regexp, or identifier // argument into a wantError. The string is a substring of the // expected error, the regexp is a pattern than matches the expected // error, and the identifier is a golden file containing the expected // error. -func wantErrorConverter(mark marker, arg interface{}) (interface{}, error) { +func convertWantError(mark marker, arg any) (wantError, error) { switch arg := arg.(type) { case string: return wantError{substr: arg}, nil case *regexp.Regexp: return wantError{pattern: arg}, nil case expect.Identifier: - golden := mark.run.test.getGolden(string(arg)) + golden := mark.run.test.getGolden(arg) return wantError{golden: golden}, nil default: - return nil, fmt.Errorf("cannot convert %T to wantError (want: string, regexp, or identifier)", arg) + return wantError{}, fmt.Errorf("cannot convert %T to wantError (want: string, regexp, or identifier)", arg) } } @@ -1286,17 +1347,6 @@ func (we wantError) check(mark marker, err error) { } } -// goldenConverter converts an identifier into the Golden directory of content -// prefixed by @ in the test archive file. -func goldenConverter(mark marker, arg interface{}) (interface{}, error) { - switch arg := arg.(type) { - case expect.Identifier: - return mark.run.test.getGolden(string(arg)), nil - default: - return nil, fmt.Errorf("invalid input type %T: golden key must be an identifier", arg) - } -} - // checkChangedFiles compares the files changed by an operation with their expected (golden) state. func checkChangedFiles(mark marker, changed map[string][]byte, golden *Golden) { // Check changed files match expectations. @@ -1324,26 +1374,70 @@ func checkChangedFiles(mark marker, changed map[string][]byte, golden *Golden) { // ---- marker functions ---- +// completionItem is a simplified summary of a completion item. +type completionItem struct { + Label, Detail, Kind string +} + +func completionItemMarker(mark marker, label, detail, kind string) completionItem { + return completionItem{ + Label: label, + Detail: detail, + Kind: kind, + // TODO(rfindley): add variadic documentation? It is supported by the old + // marker tests but almost no test cases use it. + } +} + // completeMarker implements the @complete marker, running // textDocument/completion at the given src location and asserting that the // results match the expected results. -// -// TODO(rfindley): for now, this is just a quick check against the expected -// completion labels. We could do more by assembling richer completion items, -// as is done in the old marker tests. Does that add value? If so, perhaps we -// should support a variant form of the argument, labelOrItem, which allows the -// string form or item form. -func completeMarker(mark marker, src protocol.Location, want ...string) { +func completeMarker(mark marker, src protocol.Location, want ...completionItem) { list := mark.run.env.Completion(src) - var got []string - for _, item := range list.Items { - got = append(got, item.Label) + items := filterBuiltinsAndKeywords(list.Items) + var got []completionItem + for i, item := range items { + simplified := completionItem{ + Label: item.Label, + Detail: item.Detail, + Kind: fmt.Sprint(item.Kind), + } + // Support short-hand notation: if Detail or Kind are omitted from the + // item, don't match them. + if i < len(want) { + if want[i].Detail == "" { + simplified.Detail = "" + } + if want[i].Kind == "" { + simplified.Kind = "" + } + } + if i < len(want) && want[i].Detail == "" { + simplified.Detail = "" + } + got = append(got, simplified) } + if diff := cmp.Diff(want, got); diff != "" { mark.errorf("Completion(...) returned unexpect results (-want +got):\n%s", diff) } } +// filterBuiltinsAndKeywords filters out builtins and keywords from completion +// results. +// +// It over-approximates, and does not detect if builtins are shadowed. +func filterBuiltinsAndKeywords(items []protocol.CompletionItem) []protocol.CompletionItem { + keep := 0 + for _, item := range items { + if types.Universe.Lookup(item.Label) == nil && token.Lookup(item.Label) == token.IDENT { + items[keep] = item + keep++ + } + } + return items[:keep] +} + // acceptCompletionMarker implements the @acceptCompletion marker, running // textDocument/completion at the given src location and accepting the // candidate with the given label. The resulting source must match the provided @@ -1525,14 +1619,7 @@ func hoverMarker(mark marker, src, dst protocol.Location, golden *Golden) { // locMarker implements the @loc marker. It is executed before other // markers, so that locations are available. -func locMarker(mark marker, name expect.Identifier, loc protocol.Location) { - if prev, dup := mark.run.locations[name]; dup { - mark.errorf("location %q already declared at %s", - name, mark.run.fmtLoc(prev)) - return - } - mark.run.locations[name] = loc -} +func locMarker(mark marker, loc protocol.Location) protocol.Location { return loc } // diagMarker implements the @diag marker. It eliminates diagnostics from // the observed set in mark.test. @@ -1561,8 +1648,8 @@ func removeDiagnostic(mark marker, loc protocol.Location, re *regexp.Regexp) (pr } // renameMarker implements the @rename(location, new, golden) marker. -func renameMarker(mark marker, loc protocol.Location, newName expect.Identifier, golden *Golden) { - changed, err := rename(mark.run.env, loc, string(newName)) +func renameMarker(mark marker, loc protocol.Location, newName string, golden *Golden) { + changed, err := rename(mark.run.env, loc, newName) if err != nil { mark.errorf("rename failed: %v. (Use @renameerr for expected errors.)", err) return @@ -1571,8 +1658,8 @@ func renameMarker(mark marker, loc protocol.Location, newName expect.Identifier, } // renameErrMarker implements the @renamererr(location, new, error) marker. -func renameErrMarker(mark marker, loc protocol.Location, newName expect.Identifier, wantErr wantError) { - _, err := rename(mark.run.env, loc, string(newName)) +func renameErrMarker(mark marker, loc protocol.Location, newName string, wantErr wantError) { + _, err := rename(mark.run.env, loc, newName) wantErr.check(mark, err) } @@ -1684,7 +1771,7 @@ func codeLensesMarker(mark marker) { } var want []codeLens - mark.collectExtraNotes("codelens", makeMarkerFunc(func(mark marker, loc protocol.Location, title string) { + mark.consumeExtraNotes("codelens", markerFunc(func(mark marker, loc protocol.Location, title string) { want = append(want, codeLens{loc.Range, title}) })) @@ -1703,16 +1790,15 @@ func codeLensesMarker(mark marker) { } } -// collectExtraNotes runs the provided markerFunc for each extra note with the -// given name, and marks all matching notes as used. -func (mark marker) collectExtraNotes(name string, f markerFunc) { +// consumeExtraNotes runs the provided func for each extra note with the given +// name, and deletes all matching notes. +func (mark marker) consumeExtraNotes(name string, f func(marker)) { uri := mark.uri() notes := mark.run.extraNotes[uri][name] delete(mark.run.extraNotes[uri], name) for _, note := range notes { - mark := marker{run: mark.run, note: note, fn: f} - mark.execute() + f(marker{run: mark.run, note: note}) } } @@ -1902,7 +1988,7 @@ func symbolMarker(mark marker, golden *Golden) { if err != nil { mark.run.env.T.Fatal(err) } - if _, ok := symbol.(map[string]interface{})["location"]; ok { + if _, ok := symbol.(map[string]any)["location"]; ok { // This case is not reached because Editor initialization // enables HierarchicalDocumentSymbolSupport. // TODO(adonovan): test this too. diff --git a/gopls/internal/lsp/testdata/bad/bad0_go120.go b/gopls/internal/lsp/testdata/bad/bad0_go120.go deleted file mode 100644 index 78ddb0b4081..00000000000 --- a/gopls/internal/lsp/testdata/bad/bad0_go120.go +++ /dev/null @@ -1,24 +0,0 @@ -//go:build go1.11 && !go1.21 -// +build go1.11,!go1.21 - -package bad - -import _ "golang.org/lsptests/assign/internal/secret" //@diag("\"golang.org/lsptests/assign/internal/secret\"", "compiler", "could not import golang.org/lsptests/assign/internal/secret \\(invalid use of internal package \"golang.org/lsptests/assign/internal/secret\"\\)", "error") - -func stuff() { //@item(stuff, "stuff", "func()", "func") - x := "heeeeyyyy" - random2(x) //@diag("x", "compiler", "cannot use x \\(variable of type string\\) as int value in argument to random2", "error") - random2(1) //@complete("dom", random, random2, random3) - y := 3 //@diag("y", "compiler", "y declared (and|but) not used", "error") -} - -type bob struct { //@item(bob, "bob", "struct{...}", "struct") - x int -} - -func _() { - var q int - _ = &bob{ - f: q, //@diag("f: q", "compiler", "unknown field f in struct literal", "error") - } -} diff --git a/gopls/internal/lsp/testdata/bad/bad0_go121.go b/gopls/internal/lsp/testdata/bad/bad0_go121.go deleted file mode 100644 index c4f4ecc6383..00000000000 --- a/gopls/internal/lsp/testdata/bad/bad0_go121.go +++ /dev/null @@ -1,26 +0,0 @@ -//go:build go1.21 -// +build go1.21 - -package bad - -// TODO(matloob): uncomment this and remove the space between the // and the @diag -// once the changes that produce the new go list error are submitted. -import _ "golang.org/lsptests/assign/internal/secret" //@diag("\"golang.org/lsptests/assign/internal/secret\"", "compiler", "could not import golang.org/lsptests/assign/internal/secret \\(invalid use of internal package \"golang.org/lsptests/assign/internal/secret\"\\)", "error"),diag("_", "go list", "use of internal package golang.org/lsptests/assign/internal/secret not allowed", "error") - -func stuff() { //@item(stuff, "stuff", "func()", "func") - x := "heeeeyyyy" - random2(x) //@diag("x", "compiler", "cannot use x \\(variable of type string\\) as int value in argument to random2", "error") - random2(1) //@complete("dom", random, random2, random3) - y := 3 //@diag("y", "compiler", "y declared (and|but) not used", "error") -} - -type bob struct { //@item(bob, "bob", "struct{...}", "struct") - x int -} - -func _() { - var q int - _ = &bob{ - f: q, //@diag("f: q", "compiler", "unknown field f in struct literal", "error") - } -} diff --git a/gopls/internal/lsp/testdata/bad/bad1.go b/gopls/internal/lsp/testdata/bad/bad1.go deleted file mode 100644 index 13b3d0af61c..00000000000 --- a/gopls/internal/lsp/testdata/bad/bad1.go +++ /dev/null @@ -1,34 +0,0 @@ -//go:build go1.11 -// +build go1.11 - -package bad - -// See #36637 -type stateFunc func() stateFunc //@item(stateFunc, "stateFunc", "func() stateFunc", "type") - -var a unknown //@item(global_a, "a", "unknown", "var"),diag("unknown", "compiler", "(undeclared name|undefined): unknown", "error") - -func random() int { //@item(random, "random", "func() int", "func") - //@complete("", global_a, bob, random, random2, random3, stateFunc, stuff) - return 0 -} - -func random2(y int) int { //@item(random2, "random2", "func(y int) int", "func"),item(bad_y_param, "y", "int", "var") - x := 6 //@item(x, "x", "int", "var"),diag("x", "compiler", "x declared (and|but) not used", "error") - var q blah //@item(q, "q", "blah", "var"),diag("q", "compiler", "q declared (and|but) not used", "error"),diag("blah", "compiler", "(undeclared name|undefined): blah", "error") - var t **blob //@item(t, "t", "**blob", "var"),diag("t", "compiler", "t declared (and|but) not used", "error"),diag("blob", "compiler", "(undeclared name|undefined): blob", "error") - //@complete("", q, t, x, bad_y_param, global_a, bob, random, random2, random3, stateFunc, stuff) - - return y -} - -func random3(y ...int) { //@item(random3, "random3", "func(y ...int)", "func"),item(y_variadic_param, "y", "[]int", "var") - //@complete("", y_variadic_param, global_a, bob, random, random2, random3, stateFunc, stuff) - - var ch chan (favType1) //@item(ch, "ch", "chan (favType1)", "var"),diag("ch", "compiler", "ch declared (and|but) not used", "error"),diag("favType1", "compiler", "(undeclared name|undefined): favType1", "error") - var m map[keyType]int //@item(m, "m", "map[keyType]int", "var"),diag("m", "compiler", "m declared (and|but) not used", "error"),diag("keyType", "compiler", "(undeclared name|undefined): keyType", "error") - var arr []favType2 //@item(arr, "arr", "[]favType2", "var"),diag("arr", "compiler", "arr declared (and|but) not used", "error"),diag("favType2", "compiler", "(undeclared name|undefined): favType2", "error") - var fn1 func() badResult //@item(fn1, "fn1", "func() badResult", "var"),diag("fn1", "compiler", "fn1 declared (and|but) not used", "error"),diag("badResult", "compiler", "(undeclared name|undefined): badResult", "error") - var fn2 func(badParam) //@item(fn2, "fn2", "func(badParam)", "var"),diag("fn2", "compiler", "fn2 declared (and|but) not used", "error"),diag("badParam", "compiler", "(undeclared name|undefined): badParam", "error") - //@complete("", arr, ch, fn1, fn2, m, y_variadic_param, global_a, bob, random, random2, random3, stateFunc, stuff) -} diff --git a/gopls/internal/lsp/testdata/summary.txt.golden b/gopls/internal/lsp/testdata/summary.txt.golden index c814ebb9322..29d58216de1 100644 --- a/gopls/internal/lsp/testdata/summary.txt.golden +++ b/gopls/internal/lsp/testdata/summary.txt.golden @@ -1,13 +1,13 @@ -- summary -- CallHierarchyCount = 2 -CompletionsCount = 264 +CompletionsCount = 259 CompletionSnippetCount = 115 UnimportedCompletionsCount = 5 DeepCompletionsCount = 5 FuzzyCompletionsCount = 8 RankedCompletionsCount = 174 CaseSensitiveCompletionsCount = 4 -DiagnosticsCount = 23 +DiagnosticsCount = 3 SemanticTokenCount = 3 SuggestedFixCount = 80 MethodExtractionCount = 8 diff --git a/gopls/internal/lsp/testdata/summary_go1.21.txt.golden b/gopls/internal/lsp/testdata/summary_go1.21.txt.golden index 2e78a46913a..4da0693b855 100644 --- a/gopls/internal/lsp/testdata/summary_go1.21.txt.golden +++ b/gopls/internal/lsp/testdata/summary_go1.21.txt.golden @@ -1,13 +1,13 @@ -- summary -- CallHierarchyCount = 2 -CompletionsCount = 263 +CompletionsCount = 258 CompletionSnippetCount = 115 UnimportedCompletionsCount = 5 DeepCompletionsCount = 5 FuzzyCompletionsCount = 8 RankedCompletionsCount = 174 CaseSensitiveCompletionsCount = 4 -DiagnosticsCount = 24 +DiagnosticsCount = 3 SemanticTokenCount = 3 SuggestedFixCount = 80 MethodExtractionCount = 8 diff --git a/gopls/internal/regtest/marker/testdata/completion/bad.txt b/gopls/internal/regtest/marker/testdata/completion/bad.txt new file mode 100644 index 00000000000..4da021ae322 --- /dev/null +++ b/gopls/internal/regtest/marker/testdata/completion/bad.txt @@ -0,0 +1,68 @@ +This test exercises completion in the presence of type errors. + +Note: this test was ported from the old marker tests, which did not enable +unimported completion. Enabling it causes matches in e.g. crypto/rand. + +-- settings.json -- +{ + "completeUnimported": false +} + +-- go.mod -- +module bad.test + +go 1.18 + +-- bad/bad0.go -- +package bad + +func stuff() { //@item(stuff, "stuff", "func()", "func") + x := "heeeeyyyy" + random2(x) //@diag("x", re"cannot use x \\(variable of type string\\) as int value in argument to random2") + random2(1) //@complete("dom", random, random2, random3) + y := 3 //@diag("y", re"y declared (and|but) not used") +} + +type bob struct { //@item(bob, "bob", "struct{...}", "struct") + x int +} + +func _() { + var q int + _ = &bob{ + f: q, //@diag("f: q", re"unknown field f in struct literal") + } +} + +-- bad/bad1.go -- +package bad + +// See #36637 +type stateFunc func() stateFunc //@item(stateFunc, "stateFunc", "func() stateFunc", "type") + +var a unknown //@item(global_a, "a", "unknown", "var"),diag("unknown", re"(undeclared name|undefined): unknown") + +func random() int { //@item(random, "random", "func() int", "func") + //@complete("", global_a, bob, random, random2, random3, stateFunc, stuff) + return 0 +} + +func random2(y int) int { //@item(random2, "random2", "func(y int) int", "func"),item(bad_y_param, "y", "int", "var") + x := 6 //@item(x, "x", "int", "var"),diag("x", re"x declared (and|but) not used") + var q blah //@item(q, "q", "blah", "var"),diag("q", re"q declared (and|but) not used"),diag("blah", re"(undeclared name|undefined): blah") + var t **blob //@item(t, "t", "**blob", "var"),diag("t", re"t declared (and|but) not used"),diag("blob", re"(undeclared name|undefined): blob") + //@complete("", q, t, x, bad_y_param, global_a, bob, random, random2, random3, stateFunc, stuff) + + return y +} + +func random3(y ...int) { //@item(random3, "random3", "func(y ...int)", "func"),item(y_variadic_param, "y", "[]int", "var") + //@complete("", y_variadic_param, global_a, bob, random, random2, random3, stateFunc, stuff) + + var ch chan (favType1) //@item(ch, "ch", "chan (favType1)", "var"),diag("ch", re"ch declared (and|but) not used"),diag("favType1", re"(undeclared name|undefined): favType1") + var m map[keyType]int //@item(m, "m", "map[keyType]int", "var"),diag("m", re"m declared (and|but) not used"),diag("keyType", re"(undeclared name|undefined): keyType") + var arr []favType2 //@item(arr, "arr", "[]favType2", "var"),diag("arr", re"arr declared (and|but) not used"),diag("favType2", re"(undeclared name|undefined): favType2") + var fn1 func() badResult //@item(fn1, "fn1", "func() badResult", "var"),diag("fn1", re"fn1 declared (and|but) not used"),diag("badResult", re"(undeclared name|undefined): badResult") + var fn2 func(badParam) //@item(fn2, "fn2", "func(badParam)", "var"),diag("fn2", re"fn2 declared (and|but) not used"),diag("badParam", re"(undeclared name|undefined): badParam") + //@complete("", arr, ch, fn1, fn2, m, y_variadic_param, global_a, bob, random, random2, random3, stateFunc, stuff) +} diff --git a/gopls/internal/regtest/marker/testdata/completion/issue59096.txt b/gopls/internal/regtest/marker/testdata/completion/issue59096.txt index 44bb10ae91d..23d82c4dc9c 100644 --- a/gopls/internal/regtest/marker/testdata/completion/issue59096.txt +++ b/gopls/internal/regtest/marker/testdata/completion/issue59096.txt @@ -9,9 +9,11 @@ module example.com package a func _() { - b.(foo) //@complete(re"b.()", "B"), diag("b", re"(undefined|undeclared name): b") + b.(foo) //@complete(re"b.()", B), diag("b", re"(undefined|undeclared name): b") } +//@item(B, "B", "const (from \"example.com/b\")", "const") + -- b/b.go -- package b diff --git a/gopls/internal/regtest/marker/testdata/completion/issue60545.txt b/gopls/internal/regtest/marker/testdata/completion/issue60545.txt index 67221a67563..4d204979b6a 100644 --- a/gopls/internal/regtest/marker/testdata/completion/issue60545.txt +++ b/gopls/internal/regtest/marker/testdata/completion/issue60545.txt @@ -8,8 +8,12 @@ go 1.18 -- main.go -- package main +//@item(Print, "Print", "func (from \"fmt\")", "func") +//@item(Printf, "Printf", "func (from \"fmt\")", "func") +//@item(Println, "Println", "func (from \"fmt\")", "func") + func main() { - fmt.p //@complete(re"p()","Print", "Printf", "Println"), diag("fmt", re"(undefined|undeclared)") + fmt.p //@complete(re"fmt.p()", Print, Printf, Println), diag("fmt", re"(undefined|undeclared)") } -- other.go -- diff --git a/gopls/internal/regtest/marker/testdata/diagnostics/useinternal.txt b/gopls/internal/regtest/marker/testdata/diagnostics/useinternal.txt new file mode 100644 index 00000000000..11a3cc9a0c0 --- /dev/null +++ b/gopls/internal/regtest/marker/testdata/diagnostics/useinternal.txt @@ -0,0 +1,21 @@ +This test checks a diagnostic for invalid use of internal packages. + +This list error changed in Go 1.21. + +-- flags -- +-min_go=go1.21 + +-- go.mod -- +module bad.test + +go 1.18 + +-- assign/internal/secret/secret.go -- +package secret + +func Hello() {} + +-- bad/bad.go -- +package bad + +import _ "bad.test/assign/internal/secret" //@diag("\"bad.test/assign/internal/secret\"", re"could not import bad.test/assign/internal/secret \\(invalid use of internal package \"bad.test/assign/internal/secret\"\\)"),diag("_", re"use of internal package bad.test/assign/internal/secret not allowed") diff --git a/gopls/internal/regtest/marker/testdata/rename/basic.txt b/gopls/internal/regtest/marker/testdata/rename/basic.txt index 28de07c3482..8a1d42d23ec 100644 --- a/gopls/internal/regtest/marker/testdata/rename/basic.txt +++ b/gopls/internal/regtest/marker/testdata/rename/basic.txt @@ -3,36 +3,36 @@ This test performs basic coverage of 'rename' within a single package. -- basic.go -- package p -func f(x int) { println(x) } //@rename("x", y, xToy) +func f(x int) { println(x) } //@rename("x", "y", xToy) -- @xToy/basic.go -- package p -func f(y int) { println(y) } //@rename("x", y, xToy) +func f(y int) { println(y) } //@rename("x", "y", xToy) -- alias.go -- package p // from golang/go#61625 type LongNameHere struct{} -type A = LongNameHere //@rename("A", B, AToB) +type A = LongNameHere //@rename("A", "B", AToB) func Foo() A +-- errors.go -- +package p + +func _(x []int) { //@renameerr("_", "blank", `can't rename "_"`) + x = append(x, 1) //@renameerr("append", "blank", "built in and cannot be renamed") + x = nil //@renameerr("nil", "blank", "built in and cannot be renamed") + x = nil //@renameerr("x", "x", "old and new names are the same: x") + _ = 1 //@renameerr("1", "x", "no identifier found") +} + -- @AToB/alias.go -- package p // from golang/go#61625 type LongNameHere struct{} -type B = LongNameHere //@rename("A", B, AToB) +type B = LongNameHere //@rename("A", "B", AToB) func Foo() B --- errors.go -- -package p - -func _(x []int) { //@renameerr("_", blank, `can't rename "_"`) - x = append(x, 1) //@renameerr("append", blank, "built in and cannot be renamed") - x = nil //@renameerr("nil", blank, "built in and cannot be renamed") - x = nil //@renameerr("x", x, "old and new names are the same: x") - _ = 1 //@renameerr("1", x, "no identifier found") -} - diff --git a/gopls/internal/regtest/marker/testdata/rename/conflict.txt b/gopls/internal/regtest/marker/testdata/rename/conflict.txt index 18438c8a801..3d7d21cb3e4 100644 --- a/gopls/internal/regtest/marker/testdata/rename/conflict.txt +++ b/gopls/internal/regtest/marker/testdata/rename/conflict.txt @@ -12,7 +12,7 @@ var x int func f(y int) { println(x) - println(y) //@renameerr("y", x, errSuperBlockConflict) + println(y) //@renameerr("y", "x", errSuperBlockConflict) } -- @errSuperBlockConflict -- @@ -25,7 +25,7 @@ package sub var a int func f2(b int) { - println(a) //@renameerr("a", b, errSubBlockConflict) + println(a) //@renameerr("a", "b", errSubBlockConflict) println(b) } @@ -36,7 +36,7 @@ sub/p.go:5:9: by this intervening var definition -- pkgname/p.go -- package pkgname -import e1 "errors" //@renameerr("e1", errors, errImportConflict) +import e1 "errors" //@renameerr("e1", "errors", errImportConflict) import "errors" var _ = errors.New @@ -51,7 +51,7 @@ var x int -- pkgname2/p2.go -- package pkgname2 -import "errors" //@renameerr("errors", x, errImportConflict2) +import "errors" //@renameerr("errors", "x", errImportConflict2) var _ = errors.New -- @errImportConflict2 -- diff --git a/gopls/internal/regtest/marker/testdata/rename/embed.txt b/gopls/internal/regtest/marker/testdata/rename/embed.txt index 68cf771bc0f..c0b0301fac6 100644 --- a/gopls/internal/regtest/marker/testdata/rename/embed.txt +++ b/gopls/internal/regtest/marker/testdata/rename/embed.txt @@ -7,30 +7,30 @@ go 1.12 -- a/a.go -- package a -type A int //@rename("A", A2, type) +type A int //@rename("A", "A2", type) -- b/b.go -- package b import "example.com/a" -type B struct { a.A } //@renameerr("A", A3, errAnonField) +type B struct { a.A } //@renameerr("A", "A3", errAnonField) -var _ = new(B).A //@renameerr("A", A4, errAnonField) +var _ = new(B).A //@renameerr("A", "A4", errAnonField) -- @errAnonField -- can't rename embedded fields: rename the type directly or name the field -- @type/a/a.go -- package a -type A2 int //@rename("A", A2, type) +type A2 int //@rename("A", "A2", type) -- @type/b/b.go -- package b import "example.com/a" -type B struct { a.A2 } //@renameerr("A", A3, errAnonField) +type B struct { a.A2 } //@renameerr("A", "A3", errAnonField) -var _ = new(B).A2 //@renameerr("A", A4, errAnonField) +var _ = new(B).A2 //@renameerr("A", "A4", errAnonField) diff --git a/gopls/internal/regtest/marker/testdata/rename/generics.txt b/gopls/internal/regtest/marker/testdata/rename/generics.txt index 9f015ee2d08..db64bb4fbf9 100644 --- a/gopls/internal/regtest/marker/testdata/rename/generics.txt +++ b/gopls/internal/regtest/marker/testdata/rename/generics.txt @@ -21,7 +21,7 @@ package a type I int -func (I) m() {} //@rename("m", M, mToM) +func (I) m() {} //@rename("m", "M", mToM) func _[P ~[]int]() { _ = P{} @@ -32,7 +32,7 @@ package a type I int -func (I) M() {} //@rename("m", M, mToM) +func (I) M() {} //@rename("m", "M", mToM) func _[P ~[]int]() { _ = P{} @@ -41,43 +41,43 @@ func _[P ~[]int]() { -- g.go -- package a -type S[P any] struct { //@rename("P", Q, PToQ) +type S[P any] struct { //@rename("P", "Q", PToQ) P P F func(P) P } func F[R any](r R) { - var _ R //@rename("R", S, RToS) + var _ R //@rename("R", "S", RToS) } -- @PToQ/g.go -- package a -type S[Q any] struct { //@rename("P", Q, PToQ) +type S[Q any] struct { //@rename("P", "Q", PToQ) P Q F func(Q) Q } func F[R any](r R) { - var _ R //@rename("R", S, RToS) + var _ R //@rename("R", "S", RToS) } -- @RToS/g.go -- package a -type S[P any] struct { //@rename("P", Q, PToQ) +type S[P any] struct { //@rename("P", "Q", PToQ) P P F func(P) P } func F[S any](r S) { - var _ S //@rename("R", S, RToS) + var _ S //@rename("R", "S", RToS) } -- issue61635/p.go -- package issue61635 -type builder[S ~[]F, F ~string] struct { //@rename("S", T, SToT) +type builder[S ~[]F, F ~string] struct { //@rename("S", "T", SToT) name string elements S elemData map[F][]ElemData[F] @@ -101,7 +101,7 @@ var _ issue61635.ElemData[string] -- @SToT/issue61635/p.go -- package issue61635 -type builder[T ~[]F, F ~string] struct { //@rename("S", T, SToT) +type builder[T ~[]F, F ~string] struct { //@rename("S", "T", SToT) name string elements T elemData map[F][]ElemData[F] @@ -118,123 +118,123 @@ type BuilderImpl[S ~[]F, F ~string] struct{ builder[S, F] } -- instances/type.go -- package instances -type R[P any] struct { //@rename("R", u, Rtou) - Next *R[P] //@rename("R", s, RTos) +type R[P any] struct { //@rename("R", "u", Rtou) + Next *R[P] //@rename("R", "s", RTos) } -func (rv R[P]) Do(R[P]) R[P] { //@rename("Do", Do1, DoToDo1) +func (rv R[P]) Do(R[P]) R[P] { //@rename("Do", "Do1", DoToDo1) var x R[P] - return rv.Do(x) //@rename("Do", Do2, DoToDo2) + return rv.Do(x) //@rename("Do", "Do2", DoToDo2) } func _() { - var x R[int] //@rename("R", r, RTor) + var x R[int] //@rename("R", "r", RTor) x = x.Do(x) } -- @RTos/instances/type.go -- package instances -type s[P any] struct { //@rename("R", u, Rtou) - Next *s[P] //@rename("R", s, RTos) +type s[P any] struct { //@rename("R", "u", Rtou) + Next *s[P] //@rename("R", "s", RTos) } -func (rv s[P]) Do(s[P]) s[P] { //@rename("Do", Do1, DoToDo1) +func (rv s[P]) Do(s[P]) s[P] { //@rename("Do", "Do1", DoToDo1) var x s[P] - return rv.Do(x) //@rename("Do", Do2, DoToDo2) + return rv.Do(x) //@rename("Do", "Do2", DoToDo2) } func _() { - var x s[int] //@rename("R", r, RTor) + var x s[int] //@rename("R", "r", RTor) x = x.Do(x) } -- @Rtou/instances/type.go -- package instances -type u[P any] struct { //@rename("R", u, Rtou) - Next *u[P] //@rename("R", s, RTos) +type u[P any] struct { //@rename("R", "u", Rtou) + Next *u[P] //@rename("R", "s", RTos) } -func (rv u[P]) Do(u[P]) u[P] { //@rename("Do", Do1, DoToDo1) +func (rv u[P]) Do(u[P]) u[P] { //@rename("Do", "Do1", DoToDo1) var x u[P] - return rv.Do(x) //@rename("Do", Do2, DoToDo2) + return rv.Do(x) //@rename("Do", "Do2", DoToDo2) } func _() { - var x u[int] //@rename("R", r, RTor) + var x u[int] //@rename("R", "r", RTor) x = x.Do(x) } -- @DoToDo1/instances/type.go -- package instances -type R[P any] struct { //@rename("R", u, Rtou) - Next *R[P] //@rename("R", s, RTos) +type R[P any] struct { //@rename("R", "u", Rtou) + Next *R[P] //@rename("R", "s", RTos) } -func (rv R[P]) Do1(R[P]) R[P] { //@rename("Do", Do1, DoToDo1) +func (rv R[P]) Do1(R[P]) R[P] { //@rename("Do", "Do1", DoToDo1) var x R[P] - return rv.Do1(x) //@rename("Do", Do2, DoToDo2) + return rv.Do1(x) //@rename("Do", "Do2", DoToDo2) } func _() { - var x R[int] //@rename("R", r, RTor) + var x R[int] //@rename("R", "r", RTor) x = x.Do1(x) } -- @DoToDo2/instances/type.go -- package instances -type R[P any] struct { //@rename("R", u, Rtou) - Next *R[P] //@rename("R", s, RTos) +type R[P any] struct { //@rename("R", "u", Rtou) + Next *R[P] //@rename("R", "s", RTos) } -func (rv R[P]) Do2(R[P]) R[P] { //@rename("Do", Do1, DoToDo1) +func (rv R[P]) Do2(R[P]) R[P] { //@rename("Do", "Do1", DoToDo1) var x R[P] - return rv.Do2(x) //@rename("Do", Do2, DoToDo2) + return rv.Do2(x) //@rename("Do", "Do2", DoToDo2) } func _() { - var x R[int] //@rename("R", r, RTor) + var x R[int] //@rename("R", "r", RTor) x = x.Do2(x) } -- instances/func.go -- package instances -func Foo[P any](p P) { //@rename("Foo", Bar, FooToBar) - Foo(p) //@rename("Foo", Baz, FooToBaz) +func Foo[P any](p P) { //@rename("Foo", "Bar", FooToBar) + Foo(p) //@rename("Foo", "Baz", FooToBaz) } -- @FooToBar/instances/func.go -- package instances -func Bar[P any](p P) { //@rename("Foo", Bar, FooToBar) - Bar(p) //@rename("Foo", Baz, FooToBaz) +func Bar[P any](p P) { //@rename("Foo", "Bar", FooToBar) + Bar(p) //@rename("Foo", "Baz", FooToBaz) } -- @FooToBaz/instances/func.go -- package instances -func Baz[P any](p P) { //@rename("Foo", Bar, FooToBar) - Baz(p) //@rename("Foo", Baz, FooToBaz) +func Baz[P any](p P) { //@rename("Foo", "Bar", FooToBar) + Baz(p) //@rename("Foo", "Baz", FooToBaz) } -- @RTor/instances/type.go -- package instances -type r[P any] struct { //@rename("R", u, Rtou) - Next *r[P] //@rename("R", s, RTos) +type r[P any] struct { //@rename("R", "u", Rtou) + Next *r[P] //@rename("R", "s", RTos) } -func (rv r[P]) Do(r[P]) r[P] { //@rename("Do", Do1, DoToDo1) +func (rv r[P]) Do(r[P]) r[P] { //@rename("Do", "Do1", DoToDo1) var x r[P] - return rv.Do(x) //@rename("Do", Do2, DoToDo2) + return rv.Do(x) //@rename("Do", "Do2", DoToDo2) } func _() { - var x r[int] //@rename("R", r, RTor) + var x r[int] //@rename("R", "r", RTor) x = x.Do(x) } diff --git a/gopls/internal/regtest/marker/testdata/rename/issue60789.txt b/gopls/internal/regtest/marker/testdata/rename/issue60789.txt index ee2a084581b..40173320c74 100644 --- a/gopls/internal/regtest/marker/testdata/rename/issue60789.txt +++ b/gopls/internal/regtest/marker/testdata/rename/issue60789.txt @@ -13,8 +13,8 @@ go 1.12 -- a/a.go -- package a -type unexported int -func (unexported) F() {} //@rename("F", G, fToG) +type unexported int +func (unexported) F() {} //@rename("F", "G", fToG) var _ = unexported(0).F @@ -29,8 +29,8 @@ import _ "example.com/a" -- @fToG/a/a.go -- package a -type unexported int -func (unexported) G() {} //@rename("F", G, fToG) +type unexported int +func (unexported) G() {} //@rename("F", "G", fToG) var _ = unexported(0).G diff --git a/gopls/internal/regtest/marker/testdata/rename/issue61294.txt b/gopls/internal/regtest/marker/testdata/rename/issue61294.txt index 83d68582883..3ce1dbc7670 100644 --- a/gopls/internal/regtest/marker/testdata/rename/issue61294.txt +++ b/gopls/internal/regtest/marker/testdata/rename/issue61294.txt @@ -13,7 +13,7 @@ package a func One() -func Two(One int) //@rename("One", Three, OneToThree) +func Two(One int) //@rename("One", "Three", OneToThree) -- b/b.go -- package b @@ -25,5 +25,5 @@ package a func One() -func Two(Three int) //@rename("One", Three, OneToThree) +func Two(Three int) //@rename("One", "Three", OneToThree) diff --git a/gopls/internal/regtest/marker/testdata/rename/issue61640.txt b/gopls/internal/regtest/marker/testdata/rename/issue61640.txt index 91c2b76933d..70a6123ab32 100644 --- a/gopls/internal/regtest/marker/testdata/rename/issue61640.txt +++ b/gopls/internal/regtest/marker/testdata/rename/issue61640.txt @@ -9,7 +9,7 @@ package a // This file is adapted from the example in the issue. type builder[S ~[]int] struct { - elements S //@rename("elements", elements2, OneToTwo) + elements S //@rename("elements", "elements2", OneToTwo) } type BuilderImpl[S ~[]int] struct{ builder[S] } @@ -30,7 +30,7 @@ package a // This file is adapted from the example in the issue. type builder[S ~[]int] struct { - elements2 S //@rename("elements", elements2, OneToTwo) + elements2 S //@rename("elements", "elements2", OneToTwo) } type BuilderImpl[S ~[]int] struct{ builder[S] } diff --git a/gopls/internal/regtest/marker/testdata/rename/issue61813.txt b/gopls/internal/regtest/marker/testdata/rename/issue61813.txt index ae5162b84a4..52813f869a4 100644 --- a/gopls/internal/regtest/marker/testdata/rename/issue61813.txt +++ b/gopls/internal/regtest/marker/testdata/rename/issue61813.txt @@ -5,7 +5,7 @@ package p type P struct{} -func (P) M() {} //@rename("M", N, MToN) +func (P) M() {} //@rename("M", "N", MToN) var x = []*P{{}} -- @MToN/p.go -- @@ -13,6 +13,6 @@ package p type P struct{} -func (P) N() {} //@rename("M", N, MToN) +func (P) N() {} //@rename("M", "N", MToN) var x = []*P{{}} diff --git a/gopls/internal/regtest/marker/testdata/rename/methods.txt b/gopls/internal/regtest/marker/testdata/rename/methods.txt index 1bd985bcf57..05a5cd8697b 100644 --- a/gopls/internal/regtest/marker/testdata/rename/methods.txt +++ b/gopls/internal/regtest/marker/testdata/rename/methods.txt @@ -12,7 +12,7 @@ package a type A int -func (A) F() {} //@renameerr("F", G, errAfToG) +func (A) F() {} //@renameerr("F", "G", errAfToG) -- b/b.go -- package b @@ -20,7 +20,7 @@ package b import "example.com/a" import "example.com/c" -type B interface { F() } //@rename("F", G, BfToG) +type B interface { F() } //@rename("F", "G", BfToG) var _ B = a.A(0) var _ B = c.C(0) @@ -30,7 +30,7 @@ package c type C int -func (C) F() {} //@renameerr("F", G, errCfToG) +func (C) F() {} //@renameerr("F", "G", errCfToG) -- d/d.go -- package d @@ -49,7 +49,7 @@ package b import "example.com/a" import "example.com/c" -type B interface { G() } //@rename("F", G, BfToG) +type B interface { G() } //@rename("F", "G", BfToG) var _ B = a.A(0) var _ B = c.C(0) diff --git a/gopls/internal/regtest/marker/testdata/rename/typeswitch.txt b/gopls/internal/regtest/marker/testdata/rename/typeswitch.txt index 6743b99ef29..252c8db7af6 100644 --- a/gopls/internal/regtest/marker/testdata/rename/typeswitch.txt +++ b/gopls/internal/regtest/marker/testdata/rename/typeswitch.txt @@ -4,11 +4,11 @@ This test covers the special case of renaming a type switch var. package p func _(x interface{}) { - switch y := x.(type) { //@rename("y", z, yToZ) + switch y := x.(type) { //@rename("y", "z", yToZ) case string: - print(y) //@rename("y", z, yToZ) + print(y) //@rename("y", "z", yToZ) default: - print(y) //@rename("y", z, yToZ) + print(y) //@rename("y", "z", yToZ) } } @@ -16,11 +16,11 @@ func _(x interface{}) { package p func _(x interface{}) { - switch z := x.(type) { //@rename("y", z, yToZ) + switch z := x.(type) { //@rename("y", "z", yToZ) case string: - print(z) //@rename("y", z, yToZ) + print(z) //@rename("y", "z", yToZ) default: - print(z) //@rename("y", z, yToZ) + print(z) //@rename("y", "z", yToZ) } } diff --git a/gopls/internal/regtest/marker/testdata/rename/unexported.txt b/gopls/internal/regtest/marker/testdata/rename/unexported.txt index e5631fa4907..ed60f666d4b 100644 --- a/gopls/internal/regtest/marker/testdata/rename/unexported.txt +++ b/gopls/internal/regtest/marker/testdata/rename/unexported.txt @@ -11,7 +11,7 @@ go 1.12 -- a/a.go -- package a -var S struct{ X int } //@renameerr("X", x, oops) +var S struct{ X int } //@renameerr("X", "x", oops) -- a/a_test.go -- package a_test From afa68c987ff77effb35084b038436997a4954435 Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Thu, 14 Sep 2023 17:08:07 -0400 Subject: [PATCH 098/178] gopls/internal: move builtin completion test to a regtest This test is more naturally expressed as a regtest. By moving it, we eliminate the last place where the old marker test summary differs at a go version. For golang/go#54845 Change-Id: I7a99e2fba3bbec01ddf1aa93fb69cfb1c56e9882 Reviewed-on: https://go-review.googlesource.com/c/tools/+/528735 LUCI-TryBot-Result: Go LUCI Reviewed-by: Alan Donovan --- .../lsp/testdata/builtins/builtin_go117.go | 8 ---- .../lsp/testdata/builtins/builtin_go118.go | 8 ---- .../lsp/testdata/builtins/builtin_go121.go | 8 ---- .../lsp/testdata/builtins/builtin_go122.go | 8 ---- .../lsp/testdata/builtins/builtins.go | 39 +-------------- .../internal/lsp/testdata/summary.txt.golden | 2 +- .../lsp/testdata/summary_go1.21.txt.golden | 22 --------- gopls/internal/lsp/tests/tests.go | 9 +--- .../regtest/completion/completion_test.go | 47 +++++++++++++++++++ 9 files changed, 50 insertions(+), 101 deletions(-) delete mode 100644 gopls/internal/lsp/testdata/builtins/builtin_go117.go delete mode 100644 gopls/internal/lsp/testdata/builtins/builtin_go118.go delete mode 100644 gopls/internal/lsp/testdata/builtins/builtin_go121.go delete mode 100644 gopls/internal/lsp/testdata/builtins/builtin_go122.go delete mode 100644 gopls/internal/lsp/testdata/summary_go1.21.txt.golden diff --git a/gopls/internal/lsp/testdata/builtins/builtin_go117.go b/gopls/internal/lsp/testdata/builtins/builtin_go117.go deleted file mode 100644 index 57abcde1517..00000000000 --- a/gopls/internal/lsp/testdata/builtins/builtin_go117.go +++ /dev/null @@ -1,8 +0,0 @@ -//go:build !go1.18 -// +build !go1.18 - -package builtins - -func _() { - //@complete("", append, bool, byte, cap, close, complex, complex128, complex64, copy, delete, error, _false, float32, float64, imag, int, int16, int32, int64, int8, len, make, new, panic, print, println, real, recover, rune, string, _true, uint, uint16, uint32, uint64, uint8, uintptr, _nil) -} diff --git a/gopls/internal/lsp/testdata/builtins/builtin_go118.go b/gopls/internal/lsp/testdata/builtins/builtin_go118.go deleted file mode 100644 index dabffcc679c..00000000000 --- a/gopls/internal/lsp/testdata/builtins/builtin_go118.go +++ /dev/null @@ -1,8 +0,0 @@ -//go:build go1.18 && !go1.21 -// +build go1.18,!go1.21 - -package builtins - -func _() { - //@complete("", any, append, bool, byte, cap, close, comparable, complex, complex128, complex64, copy, delete, error, _false, float32, float64, imag, int, int16, int32, int64, int8, len, make, new, panic, print, println, real, recover, rune, string, _true, uint, uint16, uint32, uint64, uint8, uintptr, _nil) -} diff --git a/gopls/internal/lsp/testdata/builtins/builtin_go121.go b/gopls/internal/lsp/testdata/builtins/builtin_go121.go deleted file mode 100644 index 14f59def9ac..00000000000 --- a/gopls/internal/lsp/testdata/builtins/builtin_go121.go +++ /dev/null @@ -1,8 +0,0 @@ -//go:build go1.21 && !go1.22 && ignore -// +build go1.21,!go1.22,ignore - -package builtins - -func _() { - //@complete("", any, append, bool, byte, cap, clear, close, comparable, complex, complex128, complex64, copy, delete, error, _false, float32, float64, imag, int, int16, int32, int64, int8, len, make, max, min, new, panic, print, println, real, recover, rune, string, _true, uint, uint16, uint32, uint64, uint8, uintptr, _nil) -} diff --git a/gopls/internal/lsp/testdata/builtins/builtin_go122.go b/gopls/internal/lsp/testdata/builtins/builtin_go122.go deleted file mode 100644 index f799c1225a1..00000000000 --- a/gopls/internal/lsp/testdata/builtins/builtin_go122.go +++ /dev/null @@ -1,8 +0,0 @@ -//go:build go1.22 && ignore -// +build go1.22,ignore - -package builtins - -func _() { - //@complete("", any, append, bool, byte, cap, clear, close, comparable, complex, complex128, complex64, copy, delete, error, _false, float32, float64, imag, int, int16, int32, int64, int8, len, make, max, min, new, panic, print, println, real, recover, rune, string, _true, uint, uint16, uint32, uint64, uint8, uintptr, zero, _nil) -} diff --git a/gopls/internal/lsp/testdata/builtins/builtins.go b/gopls/internal/lsp/testdata/builtins/builtins.go index a6450362a78..bd47477d831 100644 --- a/gopls/internal/lsp/testdata/builtins/builtins.go +++ b/gopls/internal/lsp/testdata/builtins/builtins.go @@ -1,50 +1,13 @@ package builtins -// Definitions of builtin completion items. +// Definitions of builtin completion items that are still used in tests. -/* any */ //@item(any, "any", "", "interface") -/* Create markers for builtin types. Only for use by this test. -/* append(slice []Type, elems ...Type) []Type */ //@item(append, "append", "func(slice []Type, elems ...Type) []Type", "func") /* bool */ //@item(bool, "bool", "", "type") -/* byte */ //@item(byte, "byte", "", "type") -/* cap(v Type) int */ //@item(cap, "cap", "func(v Type) int", "func") -/* clear[T interface{ ~[]Type | ~map[Type]Type1 }](t T) */ //@item(clear, "clear", "func(t T)", "func") -/* close(c chan<- Type) */ //@item(close, "close", "func(c chan<- Type)", "func") -/* comparable */ //@item(comparable, "comparable", "", "interface") /* complex(r float64, i float64) */ //@item(complex, "complex", "func(r float64, i float64) complex128", "func") -/* complex128 */ //@item(complex128, "complex128", "", "type") -/* complex64 */ //@item(complex64, "complex64", "", "type") -/* copy(dst []Type, src []Type) int */ //@item(copy, "copy", "func(dst []Type, src []Type) int", "func") -/* delete(m map[Type]Type1, key Type) */ //@item(delete, "delete", "func(m map[Type]Type1, key Type)", "func") -/* error */ //@item(error, "error", "", "interface") -/* false */ //@item(_false, "false", "", "const") /* float32 */ //@item(float32, "float32", "", "type") /* float64 */ //@item(float64, "float64", "", "type") /* imag(c complex128) float64 */ //@item(imag, "imag", "func(c complex128) float64", "func") /* int */ //@item(int, "int", "", "type") -/* int16 */ //@item(int16, "int16", "", "type") -/* int32 */ //@item(int32, "int32", "", "type") -/* int64 */ //@item(int64, "int64", "", "type") -/* int8 */ //@item(int8, "int8", "", "type") /* iota */ //@item(iota, "iota", "", "const") -/* len(v Type) int */ //@item(len, "len", "func(v Type) int", "func") -/* max(x T, y ...T) T */ //@item(max, "max", "func(x T, y ...T) T", "func") -/* min(y T, y ...T) T */ //@item(min, "min", "func(x T, y ...T) T", "func") -/* make(t Type, size ...int) Type */ //@item(make, "make", "func(t Type, size ...int) Type", "func") -/* new(Type) *Type */ //@item(new, "new", "func(Type) *Type", "func") -/* nil */ //@item(_nil, "nil", "", "var") -/* panic(v interface{}) */ //@item(panic, "panic", "func(v interface{})", "func") -/* print(args ...Type) */ //@item(print, "print", "func(args ...Type)", "func") -/* println(args ...Type) */ //@item(println, "println", "func(args ...Type)", "func") -/* real(c complex128) float64 */ //@item(real, "real", "func(c complex128) float64", "func") -/* recover() interface{} */ //@item(recover, "recover", "func() interface{}", "func") -/* rune */ //@item(rune, "rune", "", "type") /* string */ //@item(string, "string", "", "type") /* true */ //@item(_true, "true", "", "const") -/* uint */ //@item(uint, "uint", "", "type") -/* uint16 */ //@item(uint16, "uint16", "", "type") -/* uint32 */ //@item(uint32, "uint32", "", "type") -/* uint64 */ //@item(uint64, "uint64", "", "type") -/* uint8 */ //@item(uint8, "uint8", "", "type") -/* uintptr */ //@item(uintptr, "uintptr", "", "type") -/* zero */ //@item(zero, "zero", "", "var") diff --git a/gopls/internal/lsp/testdata/summary.txt.golden b/gopls/internal/lsp/testdata/summary.txt.golden index 29d58216de1..4da0693b855 100644 --- a/gopls/internal/lsp/testdata/summary.txt.golden +++ b/gopls/internal/lsp/testdata/summary.txt.golden @@ -1,6 +1,6 @@ -- summary -- CallHierarchyCount = 2 -CompletionsCount = 259 +CompletionsCount = 258 CompletionSnippetCount = 115 UnimportedCompletionsCount = 5 DeepCompletionsCount = 5 diff --git a/gopls/internal/lsp/testdata/summary_go1.21.txt.golden b/gopls/internal/lsp/testdata/summary_go1.21.txt.golden deleted file mode 100644 index 4da0693b855..00000000000 --- a/gopls/internal/lsp/testdata/summary_go1.21.txt.golden +++ /dev/null @@ -1,22 +0,0 @@ --- summary -- -CallHierarchyCount = 2 -CompletionsCount = 258 -CompletionSnippetCount = 115 -UnimportedCompletionsCount = 5 -DeepCompletionsCount = 5 -FuzzyCompletionsCount = 8 -RankedCompletionsCount = 174 -CaseSensitiveCompletionsCount = 4 -DiagnosticsCount = 3 -SemanticTokenCount = 3 -SuggestedFixCount = 80 -MethodExtractionCount = 8 -DefinitionsCount = 46 -TypeDefinitionsCount = 18 -InlayHintsCount = 5 -RenamesCount = 48 -PrepareRenamesCount = 7 -SignaturesCount = 33 -LinksCount = 7 -SelectionRangesCount = 3 - diff --git a/gopls/internal/lsp/tests/tests.go b/gopls/internal/lsp/tests/tests.go index 87d2a2257bb..2a0bda3623d 100644 --- a/gopls/internal/lsp/tests/tests.go +++ b/gopls/internal/lsp/tests/tests.go @@ -41,6 +41,7 @@ const ( overlayFileSuffix = ".overlay" goldenFileSuffix = ".golden" inFileSuffix = ".in" + summaryFile = "summary.txt" // The module path containing the testdata packages. // @@ -49,14 +50,6 @@ const ( testModule = "golang.org/lsptests" ) -var summaryFile = "summary.txt" - -func init() { - if testenv.Go1Point() >= 21 { - summaryFile = "summary_go1.21.txt" - } -} - var UpdateGolden = flag.Bool("golden", false, "Update golden files") // These type names apparently avoid the need to repeat the diff --git a/gopls/internal/regtest/completion/completion_test.go b/gopls/internal/regtest/completion/completion_test.go index 117e940e012..4294b92c2db 100644 --- a/gopls/internal/regtest/completion/completion_test.go +++ b/gopls/internal/regtest/completion/completion_test.go @@ -6,6 +6,7 @@ package completion import ( "fmt" + "sort" "strings" "testing" "time" @@ -868,3 +869,49 @@ use ./dir/foobar/ } }) } + +func TestBuiltinCompletion(t *testing.T) { + const files = ` +-- go.mod -- +module mod.com + +go 1.18 +-- a.go -- +package a + +func _() { + // here +} +` + + Run(t, files, func(t *testing.T, env *Env) { + env.OpenFile("a.go") + result := env.Completion(env.RegexpSearch("a.go", `// here`)) + builtins := []string{ + "any", "append", "bool", "byte", "cap", "close", + "comparable", "complex", "complex128", "complex64", "copy", "delete", + "error", "false", "float32", "float64", "imag", "int", "int16", "int32", + "int64", "int8", "len", "make", "new", "panic", "print", "println", "real", + "recover", "rune", "string", "true", "uint", "uint16", "uint32", "uint64", + "uint8", "uintptr", "nil", + } + if testenv.Go1Point() >= 21 { + builtins = append(builtins, "clear", "max", "min") + } + sort.Strings(builtins) + var got []string + + for _, item := range result.Items { + // TODO(rfindley): for flexibility, ignore zero while it is being + // implemented. Remove this if/when zero lands. + if item.Label != "zero" { + got = append(got, item.Label) + } + } + sort.Strings(got) + + if diff := cmp.Diff(builtins, got); diff != "" { + t.Errorf("Completion: unexpected mismatch:\n%s", diff) + } + }) +} From 866a6b0ff32adf32534b859fc6499a7e95f09054 Mon Sep 17 00:00:00 2001 From: "Hana (Hyang-Ah) Kim" Date: Wed, 13 Sep 2023 13:56:14 -0400 Subject: [PATCH 099/178] gopls: update x/telemetry to the latest This picks up changes upload logic and x/telemetry api. Change-Id: I3946fcb3e4416cae7abed49d428617b1b8db167f Reviewed-on: https://go-review.googlesource.com/c/tools/+/528397 Run-TryBot: Hyang-Ah Hana Kim Reviewed-by: Robert Findley TryBot-Result: Gopher Robot --- gopls/go.mod | 2 +- gopls/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/gopls/go.mod b/gopls/go.mod index f17636979f7..542619f4a2a 100644 --- a/gopls/go.mod +++ b/gopls/go.mod @@ -10,7 +10,7 @@ require ( golang.org/x/mod v0.12.0 golang.org/x/sync v0.3.0 golang.org/x/sys v0.12.0 - golang.org/x/telemetry v0.0.0-20230822160736-17171dbf1d76 + golang.org/x/telemetry v0.0.0-20230914213300-46be8a514e94 golang.org/x/text v0.13.0 golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 golang.org/x/vuln v1.0.1 diff --git a/gopls/go.sum b/gopls/go.sum index 0b3aa549a7e..347d4e06dac 100644 --- a/gopls/go.sum +++ b/gopls/go.sum @@ -44,8 +44,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/telemetry v0.0.0-20230822160736-17171dbf1d76 h1:Lv25uIMpljmSMN0+GCC+xgiC/4ikIdKMkQfw/EVq2Nk= -golang.org/x/telemetry v0.0.0-20230822160736-17171dbf1d76/go.mod h1:kO7uNSGGmqCHII6C0TYfaLwSBIfcyhj53//nu0+Fy4A= +golang.org/x/telemetry v0.0.0-20230914213300-46be8a514e94 h1:wwhvP22pWmWAdBy6nMmHj0rgDU99Br9H9YVV+rRj7E4= +golang.org/x/telemetry v0.0.0-20230914213300-46be8a514e94/go.mod h1:ppZ76JTkRgJC2GQEgtVY3fiuJR+N8FU2MAlp+gfN1E4= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From 38f5195e40da4a96aecdc6b82f803629ef664f3f Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Thu, 14 Sep 2023 15:56:35 -0400 Subject: [PATCH 100/178] internal/refactor/inline: treat self-ref as free ref Another bug unearthed by the everything test. Also, a regression test. Change-Id: I4b45e83c1fee7fd461f366fc25ef69512300aa74 Reviewed-on: https://go-review.googlesource.com/c/tools/+/528616 LUCI-TryBot-Result: Go LUCI Reviewed-by: Robert Findley --- internal/refactor/inline/callee.go | 4 +-- .../inline/testdata/crosspkg-selfref.txtar | 32 +++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 internal/refactor/inline/testdata/crosspkg-selfref.txtar diff --git a/internal/refactor/inline/callee.go b/internal/refactor/inline/callee.go index 370cb8ae38f..696a4ea05d5 100644 --- a/internal/refactor/inline/callee.go +++ b/internal/refactor/inline/callee.go @@ -170,8 +170,8 @@ func AnalyzeCallee(fset *token.FileSet, pkg *types.Package, info *types.Info, de unexported = append(unexported, n.Name) } - // Record free reference. - if !within(obj.Pos(), decl) { + // Record free reference (incl. self-reference). + if obj == fn || !within(obj.Pos(), decl) { objidx, ok := freeObjIndex[obj] if !ok { objidx = len(freeObjIndex) diff --git a/internal/refactor/inline/testdata/crosspkg-selfref.txtar b/internal/refactor/inline/testdata/crosspkg-selfref.txtar new file mode 100644 index 00000000000..0c45be87d92 --- /dev/null +++ b/internal/refactor/inline/testdata/crosspkg-selfref.txtar @@ -0,0 +1,32 @@ +A self-reference counts as a free reference, +so that it gets properly package-qualified as needed. +(Regression test for a bug.) + +-- go.mod -- +module testdata +go 1.12 + +-- a/a.go -- +package a + +import "testdata/b" + +func _() { + b.F(1) //@ inline(re"F", output) +} + +-- b/b.go -- +package b + +func F(x int) { + F(x + 2) +} + +-- output -- +package a + +import "testdata/b" + +func _() { + b.F(1 + 2) //@ inline(re"F", output) +} From 2c15796c1971ee0059600b98beeeabe35fa7c3af Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Fri, 15 Sep 2023 12:20:08 -0400 Subject: [PATCH 101/178] internal/diff/lcs: increase search depth to 100 This greatly improves the diffs obtained from typical inliner refactorings, which involve one change to the imports and one change to a function call, with a large gap in between. How does the cost relate to this parameter? Do I hear any advance on 100? Change-Id: Ia006ebf3374f9894d4c561d61853e1adaa47dca6 Reviewed-on: https://go-review.googlesource.com/c/tools/+/528359 LUCI-TryBot-Result: Go LUCI Reviewed-by: Robert Findley --- internal/diff/lcs/old.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/diff/lcs/old.go b/internal/diff/lcs/old.go index 7af11fc896c..a14ae9119ca 100644 --- a/internal/diff/lcs/old.go +++ b/internal/diff/lcs/old.go @@ -29,7 +29,7 @@ func DiffRunes(a, b []rune) []Diff { return diff(runesSeqs{a, b}) } func diff(seqs sequences) []Diff { // A limit on how deeply the LCS algorithm should search. The value is just a guess. - const maxDiffs = 30 + const maxDiffs = 100 diff, _ := compute(seqs, twosided, maxDiffs/2) return diff } From c4f811e371d59b92d1aabb51328cd4b99bd0c7b5 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Fri, 15 Sep 2023 11:21:15 -0400 Subject: [PATCH 102/178] internal/refactor/inline: reject generic methods for now Plus a test. Change-Id: Icdee9e7a4f095074f9c5b9e93d4b53950a824fd6 Reviewed-on: https://go-review.googlesource.com/c/tools/+/528358 LUCI-TryBot-Result: Go LUCI Reviewed-by: Robert Findley --- internal/refactor/inline/callee.go | 2 +- internal/refactor/inline/inline_test.go | 12 ++++++++++++ internal/refactor/inline/util.go | 16 ++++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/internal/refactor/inline/callee.go b/internal/refactor/inline/callee.go index 696a4ea05d5..3e9f397fdca 100644 --- a/internal/refactor/inline/callee.go +++ b/internal/refactor/inline/callee.go @@ -97,7 +97,7 @@ func AnalyzeCallee(fset *token.FileSet, pkg *types.Package, info *types.Info, de // ident or qualified ident to prevent "if x == struct{}" // parsing ambiguity, or "T(x)" where T = "*int" or "func()" // from misparsing. - if decl.Type.TypeParams != nil { + if funcHasTypeParams(decl) { return nil, fmt.Errorf("cannot inline generic function %s: type parameters are not yet supported", name) } diff --git a/internal/refactor/inline/inline_test.go b/internal/refactor/inline/inline_test.go index b3ca6e596f8..82d5185fc1e 100644 --- a/internal/refactor/inline/inline_test.go +++ b/internal/refactor/inline/inline_test.go @@ -324,6 +324,18 @@ func TestTable(t *testing.T) { callee, caller string // Go source files (sans package decl) of caller, callee want string // expected new portion of caller file, or "error: regexp" }{ + { + "Generic functions are not yet supported.", + `func f[T any](x T) T { return x }`, + `var _ = f(0)`, + `error: type parameters are not yet supported`, + }, + { + "Methods on generic types are not yet supported.", + `type G[T any] struct{}; func (G[T]) f(x T) T { return x }`, + `var _ = G[int]{}.f(0)`, + `error: type parameters are not yet supported`, + }, { "Basic", `func f(x int) int { return x }`, diff --git a/internal/refactor/inline/util.go b/internal/refactor/inline/util.go index 87b0f7b0228..7ca5b084eec 100644 --- a/internal/refactor/inline/util.go +++ b/internal/refactor/inline/util.go @@ -53,3 +53,19 @@ func checkInfoFields(info *types.Info) { assert(info.Types != nil, "types.Info.Types is nil") assert(info.Uses != nil, "types.Info.Uses is nil") } + +func funcHasTypeParams(decl *ast.FuncDecl) bool { + // generic function? + if decl.Type.TypeParams != nil { + return true + } + // method on generic type? + if decl.Recv != nil { + t := decl.Recv.List[0].Type + if u, ok := t.(*ast.StarExpr); ok { + t = u.X + } + return is[*ast.IndexExpr](t) || is[*ast.IndexListExpr](t) + } + return false +} From 673f26324e9f93d8c2bd94455b430308ce693a73 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Thu, 14 Sep 2023 16:37:53 -0400 Subject: [PATCH 103/178] internal/refactor/inline: update docs I forgot to do this in the CL that implemented binding decls. Change-Id: Idf3fcb5aac8e253e63ae599bc38f1eef38db2beb Reviewed-on: https://go-review.googlesource.com/c/tools/+/528675 Reviewed-by: Robert Findley LUCI-TryBot-Result: Go LUCI --- internal/refactor/inline/inline.go | 64 ++++++++++++------------------ 1 file changed, 26 insertions(+), 38 deletions(-) diff --git a/internal/refactor/inline/inline.go b/internal/refactor/inline/inline.go index 9f000e49489..67d8775cbc9 100644 --- a/internal/refactor/inline/inline.go +++ b/internal/refactor/inline/inline.go @@ -82,6 +82,31 @@ // with effects, but with further analysis of the sequence of // strict effects within the callee we could relax this constraint. // +// - When not all parameters can be substituted by their arguments +// (e.g. due to possible effects), if the call appears in a +// statement context, the inliner may introduce a var declaration +// that declares the parameter variables (with the correct types) +// and assigns them to their corresponding argument values. +// The rest of the function body may then follow. +// For example, the call +// +// f(1, 2) +// +// to the function +// +// func f(x, y int32) { stmts } +// +// may be reduced to +// +// { var x, y int32 = 1, 2; stmts }. +// +// There are many reasons why this is not always possible. For +// example, true parameters are statically resolved in the same +// scope, and are dynamically assigned their arguments in +// parallel; but each spec in a var declaration is statically +// resolved in sequence and dynamically executed in sequence, so +// earlier parameters may shadow references in later ones. +// // - Even an argument expression as simple as ptr.x may not be // referentially transparent, because another argument may have the // effect of changing the value of ptr. @@ -221,47 +246,10 @@ // panics? Can we avoid evaluating an argument x.f // or a[i] when the corresponding parameter is unused? // -// - When caller syntax permits a block, replace argument-to-parameter -// assignment by a set of local var decls, e.g. f(1, 2) would -// become { var x, y = 1, 2; body... }. -// -// But even this is complicated: a single var decl initializer -// cannot declare all the parameters and initialize them to their -// arguments in one go if they have varied types. Instead, -// one must use multiple specs such as: -// -// { var x int = 1; var y int32 = 2; body ...} -// -// but this means that the initializer expression for y is -// within the scope of x, so it may require α-renaming. -// -// It is tempting to use a short var decl { x, y := 1, 2; body ...} -// as it permits simultaneous declaration and initialization -// of many variables of varied type. However, one must take care -// to convert each argument expression to the correct parameter -// variable type, perhaps explicitly. (Consider "x := 1 << 64".) -// -// Also, as a matter of style, having all parameter declarations -// and argument expressions in a single statement is potentially -// unwieldy. -// // - Support inlining of generic functions, replacing type parameters // by their instantiations. // -// - Support inlining of calls to function literals such as: -// -// f := func(...) { ... } -// -// f() -// -// including recursive ones: -// -// var f func(...) -// -// f = func(...) { ...f...} -// -// f() -// +// - Support inlining of calls to function literals ("closures"). // But note that the existing algorithm makes widespread assumptions // that the callee is a package-level function or method. // From c1a2c238e0aefe3a25818de5e7cd53e0dc1723f6 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Thu, 14 Sep 2023 14:08:28 -0400 Subject: [PATCH 104/178] internal/refactor/inline: handle implicit field selections A call x.f(args) may implicitly select embedded fields x.A.B.C.f(args). Similarly, T.f(x, args) may call C.f(x.A.B.C, args). In both cases there may be implicit pointer traversals. This change updates the receiver logic to handle these cases correctly and make the selections explicit. This may fail if any of A, B, C is unexported from a different package. (The bugs were unearthed by the "everything test", to follow in CL 528495.) Also, tests. Change-Id: I843b45af85a31ae5b3a8d0ab029d328f4323de05 Reviewed-on: https://go-review.googlesource.com/c/tools/+/528555 Auto-Submit: Alan Donovan LUCI-TryBot-Result: Go LUCI Reviewed-by: Robert Findley --- internal/refactor/inline/inline.go | 141 +++++++++++------- internal/refactor/inline/inline_test.go | 39 +++++ internal/refactor/inline/testdata/embed.txtar | 28 ++++ 3 files changed, 155 insertions(+), 53 deletions(-) create mode 100644 internal/refactor/inline/testdata/embed.txtar diff --git a/internal/refactor/inline/inline.go b/internal/refactor/inline/inline.go index 67d8775cbc9..72073cc57c9 100644 --- a/internal/refactor/inline/inline.go +++ b/internal/refactor/inline/inline.go @@ -293,6 +293,7 @@ import ( "golang.org/x/tools/go/ast/astutil" "golang.org/x/tools/go/types/typeutil" "golang.org/x/tools/imports" + "golang.org/x/tools/internal/typeparams" ) // A Caller describes the function call and its enclosing context. @@ -707,63 +708,97 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu freevars map[string]bool // free names of expr } var args []*argument // effective arguments; nil => eliminated - if calleeDecl.Recv != nil { - sel := astutil.Unparen(caller.Call.Fun).(*ast.SelectorExpr) - if caller.Info.Selections[sel].Kind() == types.MethodVal { - // Move receiver argument recv.f(args) to argument list f(&recv, args). - arg := &argument{ - expr: sel.X, - typ: caller.Info.TypeOf(sel.X), - pure: pure(caller.Info, sel.X), - duplicable: duplicable(caller.Info, sel.X), - freevars: freeVars(caller.Info, sel.X), + { + // TODO(adonovan): extract to a function (in a separate CL). + callArgs := caller.Call.Args + if calleeDecl.Recv != nil { + sel := astutil.Unparen(caller.Call.Fun).(*ast.SelectorExpr) + seln := caller.Info.Selections[sel] + var recvArg ast.Expr + switch seln.Kind() { + case types.MethodVal: // recv.f(callArgs) + recvArg = sel.X + case types.MethodExpr: // T.f(recv, callArgs) + recvArg = callArgs[0] + callArgs = callArgs[1:] } - args = append(args, arg) + if recvArg != nil { + // Compute all the type-based predicates now, + // before we start meddling with the syntax; + // the meddling will update them. + arg := &argument{ + expr: recvArg, + typ: caller.Info.TypeOf(recvArg), + pure: pure(caller.Info, recvArg), + duplicable: duplicable(caller.Info, recvArg), + freevars: freeVars(caller.Info, recvArg), + } + recvArg = nil // prevent accidental use + + // Move receiver argument recv.f(args) to argument list f(&recv, args). + args = append(args, arg) + + // Make field selections explicit (recv.f -> recv.y.f), + // updating arg.{expr,typ}. + indices := seln.Index() + for _, index := range indices[:len(indices)-1] { + t := deref(arg.typ) + fld := typeparams.CoreType(t).(*types.Struct).Field(index) + if fld.Pkg() != caller.Types && !fld.Exported() { + return nil, fmt.Errorf("in %s, implicit reference to unexported field .%s cannot be made explicit", + debugFormatNode(caller.Fset, caller.Call.Fun), + fld.Name()) + } + arg.expr = &ast.SelectorExpr{ + X: arg.expr, + Sel: makeIdent(fld.Name()), + } + arg.typ = fld.Type() + } - // Make * or & explicit. - // - // We do this after we've computed the type-based - // predicates (pure et al) above, as they won't - // work on synthetic syntax. - argIsPtr := arg.typ != deref(arg.typ) - paramIsPtr := is[*types.Pointer](calleeSymbol.Type().(*types.Signature).Recv().Type()) - if !argIsPtr && paramIsPtr { - // &recv - arg.expr = &ast.UnaryExpr{Op: token.AND, X: arg.expr} - arg.typ = types.NewPointer(arg.typ) - } else if argIsPtr && !paramIsPtr { - // *recv - arg.expr = &ast.StarExpr{X: arg.expr} - arg.typ = deref(arg.typ) - - // Technically *recv is non-pure and - // non-duplicable, as side effects - // could change the pointer between - // multiple reads. But unfortunately - // this really degrades many of our tests. - // - // TODO(adonovan): improve the precision - // purity and duplicability. - // For example, *new(T) is actually pure. - // And *ptr, where ptr doesn't escape and - // has no assignments other than its decl, - // is also pure; this is very common. - // - // arg.pure = false - // arg.duplicable = false + // Make * or & explicit. + argIsPtr := arg.typ != deref(arg.typ) + paramIsPtr := is[*types.Pointer](seln.Obj().Type().(*types.Signature).Recv().Type()) + if !argIsPtr && paramIsPtr { + // &recv + arg.expr = &ast.UnaryExpr{Op: token.AND, X: arg.expr} + arg.typ = types.NewPointer(arg.typ) + } else if argIsPtr && !paramIsPtr { + // *recv + arg.expr = &ast.StarExpr{X: arg.expr} + arg.typ = deref(arg.typ) + + // Technically *recv is non-pure and + // non-duplicable, as side effects + // could change the pointer between + // multiple reads. But unfortunately + // this really degrades many of our tests. + // (The indices loop above should similarly + // update these flags when traversing pointers.) + // + // TODO(adonovan): improve the precision of + // purity and duplicability. + // For example, *new(T) is actually pure. + // And *ptr, where ptr doesn't escape and + // has no assignments other than its decl, + // is also pure; this is very common. + // + // arg.pure = false + // arg.duplicable = false + } } } - } - for _, expr := range caller.Call.Args { - typ := caller.Info.TypeOf(expr) - args = append(args, &argument{ - expr: expr, - typ: typ, - spread: is[*types.Tuple](typ), // => last - pure: pure(caller.Info, expr), - duplicable: duplicable(caller.Info, expr), - freevars: freeVars(caller.Info, expr), - }) + for _, expr := range callArgs { + typ := caller.Info.TypeOf(expr) + args = append(args, &argument{ + expr: expr, + typ: typ, + spread: is[*types.Tuple](typ), // => last + pure: pure(caller.Info, expr), + duplicable: duplicable(caller.Info, expr), + freevars: freeVars(caller.Info, expr), + }) + } } // Gather effective parameter tuple, including the receiver if any. diff --git a/internal/refactor/inline/inline_test.go b/internal/refactor/inline/inline_test.go index 82d5185fc1e..430ee49760d 100644 --- a/internal/refactor/inline/inline_test.go +++ b/internal/refactor/inline/inline_test.go @@ -463,6 +463,45 @@ func TestTable(t *testing.T) { `func _() { func(int, y any, z int) { defer println(int, y, z) }(g(), g(), g()) } `, }, + // Embedded fields: + { + "Embedded fields in x.f method selection (direct).", + `type T int; func (t T) f() { print(t) }; type U struct{ T }`, + `func _(u U) { u.f() }`, + `func _(u U) { print(u.T) }`, + }, + { + "Embedded fields in x.f method selection (implicit *).", + `type ( T int; U struct{*T}; V struct {U} ); func (t T) f() { print(t) }`, + `func _(v V) { v.f() }`, + `func _(v V) { print(*v.U.T) }`, + }, + { + "Embedded fields in x.f method selection (implicit &).", + `type ( T int; U struct{T}; V struct {U} ); func (t *T) f() { print(t) }`, + `func _(v V) { v.f() }`, + `func _(v V) { print(&v.U.T) }`, + }, + // Now the same tests again with T.f(recv). + { + "Embedded fields in T.f method selection.", + `type T int; func (t T) f() { print(t) }; type U struct{ T }`, + `func _(u U) { U.f(u) }`, + `func _(u U) { print(u.T) }`, + }, + { + "Embedded fields in T.f method selection (implicit *).", + `type ( T int; U struct{*T}; V struct {U} ); func (t T) f() { print(t) }`, + `func _(v V) { V.f(v) }`, + `func _(v V) { print(*v.U.T) }`, + }, + { + "Embedded fields in (*T).f method selection.", + `type ( T int; U struct{T}; V struct {U} ); func (t *T) f() { print(t) }`, + `func _(v V) { (*V).f(&v) }`, + `func _(v V) { print(&(&v).U.T) }`, + }, + // TODO(adonovan): improve coverage of the cross // product of each strategy with the checklist of // concerns enumerated in the package doc comment. diff --git a/internal/refactor/inline/testdata/embed.txtar b/internal/refactor/inline/testdata/embed.txtar new file mode 100644 index 00000000000..ab52f5a5a00 --- /dev/null +++ b/internal/refactor/inline/testdata/embed.txtar @@ -0,0 +1,28 @@ +Test of implicit field selections in method calls. + +The two level wrapping T -> unexported -> U is required +to exercise the implicit selections exportedness check; +with only a single level, the receiver declaration in +"func (unexported) F()" would fail the earlier +unexportedness check. + +-- go.mod -- +module testdata +go 1.12 + +-- a/a.go -- +package a + +import "testdata/b" + +func _(x b.T) { + x.F() //@ inline(re"F", re"in x.F, implicit reference to unexported field .unexported cannot be made explicit") +} + +-- b/b.go -- +package b + +type T struct { unexported } +type unexported struct { U } +type U struct{} +func (U) F() {} From 55383756d1cf9dd034e986bd34ec36eb48260f13 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Thu, 14 Sep 2023 17:15:15 -0400 Subject: [PATCH 105/178] internal/refactor/inline: fix import shadowing bug Plus a test. Change-Id: I30b96c231dcfce5cb20972eb6d1822f83cb45faf Reviewed-on: https://go-review.googlesource.com/c/tools/+/528556 Reviewed-by: Robert Findley Auto-Submit: Alan Donovan LUCI-TryBot-Result: Go LUCI --- internal/refactor/inline/inline.go | 73 ++++++++------- .../inline/testdata/import-shadow.txtar | 89 +++++++++++++++---- 2 files changed, 114 insertions(+), 48 deletions(-) diff --git a/internal/refactor/inline/inline.go b/internal/refactor/inline/inline.go index 72073cc57c9..5294550e9e3 100644 --- a/internal/refactor/inline/inline.go +++ b/internal/refactor/inline/inline.go @@ -530,47 +530,58 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // qualified identifier (QI) in the callee is always // represented by a QI in the caller, allowing us to treat a // QI like a selection on a package name. - importMap := make(map[string]string) // maps package path to local name + importMap := make(map[string][]string) // maps package path to local name(s) for _, imp := range caller.File.Imports { - if pkgname, ok := importedPkgName(caller.Info, imp); ok && pkgname.Name() != "." { - importMap[pkgname.Imported().Path()] = pkgname.Name() + if pkgname, ok := importedPkgName(caller.Info, imp); ok && + pkgname.Name() != "." && + pkgname.Name() != "_" { + path := pkgname.Imported().Path() + importMap[path] = append(importMap[path], pkgname.Name()) } } // localImportName returns the local name for a given imported package path. var newImports []*ast.ImportSpec localImportName := func(path string) string { - name, ok := importMap[path] - if !ok { - // import added by callee - // - // Choose local PkgName based on last segment of - // package path plus, if needed, a numeric suffix to - // ensure uniqueness. - // - // TODO(adonovan): preserve the PkgName used - // in the original source, or, for a dot import, - // use the package's declared name. - base := pathpkg.Base(path) - name = base - for n := 0; callerLookup(name) != nil; n++ { - name = fmt.Sprintf("%s%d", base, n) + // Does an import exist? + for _, name := range importMap[path] { + // Check that either the import preexisted, + // or that it was newly added (no PkgName) but is not shadowed. + found := callerLookup(name) + if is[*types.PkgName](found) || found == nil { + return name } + } - // TODO(adonovan): don't use a renaming import - // unless the local name differs from either - // the package name or the last segment of path. - // This requires that we tabulate (path, declared name, local name) - // triples for each package referenced by the callee. - newImports = append(newImports, &ast.ImportSpec{ - Name: makeIdent(name), - Path: &ast.BasicLit{ - Kind: token.STRING, - Value: strconv.Quote(path), - }, - }) - importMap[path] = name + // import added by callee + // + // Choose local PkgName based on last segment of + // package path plus, if needed, a numeric suffix to + // ensure uniqueness. + // + // TODO(adonovan): preserve the PkgName used + // in the original source, or, for a dot import, + // use the package's declared name. + base := pathpkg.Base(path) + name := base + for n := 0; callerLookup(name) != nil; n++ { + name = fmt.Sprintf("%s%d", base, n) } + + // TODO(adonovan): don't use a renaming import + // unless the local name differs from either + // the package name or the last segment of path. + // This requires that we tabulate (path, declared name, local name) + // triples for each package referenced by the callee. + logf("adding import %s %q", name, path) + newImports = append(newImports, &ast.ImportSpec{ + Name: makeIdent(name), + Path: &ast.BasicLit{ + Kind: token.STRING, + Value: strconv.Quote(path), + }, + }) + importMap[path] = append(importMap[path], name) return name } diff --git a/internal/refactor/inline/testdata/import-shadow.txtar b/internal/refactor/inline/testdata/import-shadow.txtar index e7ce5a26e07..c311ee69425 100644 --- a/internal/refactor/inline/testdata/import-shadow.txtar +++ b/internal/refactor/inline/testdata/import-shadow.txtar @@ -1,5 +1,14 @@ -Test of heuristic for generating a fresh import PkgName. -The names c and c0 are taken, so it uses c1. +Just because a package (e.g. log) is imported by the caller, +and the name log is in scope, doesn't mean the name in scope +refers to the package: it could be locally shadowed. + +Two scenarios below: + +1. a second (renaming) import is added because the first import is + locally shadowed. + +2. a new import is added with a fresh name because the default + name is locally shadowed. -- go.mod -- module testdata @@ -9,35 +18,81 @@ go 1.12 package a import "testdata/b" +import "log" func A() { - const c = 1 - type c0 int - b.B() //@ inline(re"B", result) + const log = "shadow" + b.B() //@ inline(re"B", bresult) } +var _ log.Logger + -- b/b.go -- package b -import "testdata/c" - -func B() { c.C() } +import "log" --- c/c.go -- -package c - -func C() {} +func B() { + log.Printf("") +} --- result -- +-- bresult -- package a import ( - c1 "testdata/c" + "log" + log0 "log" ) func A() { - const c = 1 - type c0 int - c1.C() //@ inline(re"B", result) + const log = "shadow" + log0.Printf("") //@ inline(re"B", bresult) +} + +var _ log.Logger + +-- go.mod -- +module testdata +go 1.12 + +-- a/a.go -- +package a + +import "testdata/b" + +var x b.T + +func A(b int) { + x.F() //@ inline(re"F", fresult) +} + +-- b/b.go -- +package b + +type T struct{} + +func (T) F() { + One() + Two() } +func One() {} +func Two() {} + +-- fresult -- +package a + +import ( + "testdata/b" + b0 "testdata/b" +) + +var x b.T + +func A(b int) { + { + var _ b0.T = x + b0.One() + b0.Two() + } //@ inline(re"F", fresult) +} From 365db56d19afc7187d4873f5e633a85629c0ebe9 Mon Sep 17 00:00:00 2001 From: Peter Weinberger Date: Wed, 13 Sep 2023 12:37:49 -0400 Subject: [PATCH 106/178] tools: clean up after removing all references to ioutil Although ioutil.ReadDir is deprecated, the suggested replacement has a different signature, returning []fs.DirEntry rather than []fs.FileEntry. Some of the places where this occurs are better left referring to ioutil, as build.Config wants the old behavior. (go/buildutil/util.go and godoc/vfs/os.go) When someday build gets updated, these can be easily found, and changed. Some of the place use Mode().IsRegular() (cmd/toolstash/mail.go and internal/fastwalk/fastwal_portable.go) and the code needs a minor adjustment. And, happily, in all the other places one can use os.ReadDir directly as only Name() is called. There are no remaining instances of the generated ioutilReadDir(). Change-Id: I165ca27eafe2fe37fdf14390543b21f7e198281e Reviewed-on: https://go-review.googlesource.com/c/tools/+/528135 TryBot-Result: Gopher Robot Run-TryBot: Peter Weinberger Reviewed-by: Heschi Kreinick --- cmd/godex/godex.go | 20 +------------- cmd/toolstash/main.go | 37 ++++++++++---------------- go/buildutil/util.go | 23 +++------------- godoc/vfs/namespace.go | 2 +- godoc/vfs/os.go | 21 ++------------- internal/fastwalk/fastwalk_portable.go | 28 +++++-------------- internal/gcimporter/gcimporter_test.go | 20 +------------- internal/imports/fix.go | 10 ++++--- internal/imports/mod.go | 20 +------------- internal/imports/mod_test.go | 2 +- 10 files changed, 37 insertions(+), 146 deletions(-) diff --git a/cmd/godex/godex.go b/cmd/godex/godex.go index 7540f58a557..e91dbfcea5f 100644 --- a/cmd/godex/godex.go +++ b/cmd/godex/godex.go @@ -10,7 +10,6 @@ import ( "fmt" "go/build" "go/types" - "io/fs" "os" "path/filepath" "strings" @@ -197,7 +196,7 @@ func genPrefixes(out chan string, all bool) { } func walkDir(dirname, prefix string, out chan string) { - fiList, err := ioutilReadDir(dirname) + fiList, err := os.ReadDir(dirname) if err != nil { return } @@ -209,20 +208,3 @@ func walkDir(dirname, prefix string, out chan string) { } } } - -func ioutilReadDir(dirname string) ([]fs.FileInfo, error) { - entries, err := os.ReadDir(dirname) - if err != nil { - return nil, err - } - - infos := make([]fs.FileInfo, 0, len(entries)) - for _, entry := range entries { - info, err := entry.Info() - if err != nil { - return infos, err - } - infos = append(infos, info) - } - return infos, nil -} diff --git a/cmd/toolstash/main.go b/cmd/toolstash/main.go index 519a9428bcc..7f38524dfb1 100644 --- a/cmd/toolstash/main.go +++ b/cmd/toolstash/main.go @@ -126,15 +126,15 @@ import ( "bufio" "flag" "fmt" - exec "golang.org/x/sys/execabs" "io" - "io/fs" "log" "os" "path/filepath" "runtime" "strings" "time" + + exec "golang.org/x/sys/execabs" ) var usageMessage = `usage: toolstash [-n] [-v] [-cmp] command line @@ -553,13 +553,17 @@ func save() { } toolDir := filepath.Join(goroot, fmt.Sprintf("pkg/tool/%s_%s", runtime.GOOS, runtime.GOARCH)) - files, err := ioutilReadDir(toolDir) + files, err := os.ReadDir(toolDir) if err != nil { log.Fatal(err) } for _, file := range files { - if shouldSave(file.Name()) && file.Mode().IsRegular() { + info, err := file.Info() + if err != nil { + log.Fatal(err) + } + if shouldSave(file.Name()) && info.Mode().IsRegular() { cp(filepath.Join(toolDir, file.Name()), filepath.Join(stashDir, file.Name())) } } @@ -578,13 +582,17 @@ func save() { } func restore() { - files, err := ioutilReadDir(stashDir) + files, err := os.ReadDir(stashDir) if err != nil { log.Fatal(err) } for _, file := range files { - if shouldSave(file.Name()) && file.Mode().IsRegular() { + info, err := file.Info() + if err != nil { + log.Fatal(err) + } + if shouldSave(file.Name()) && info.Mode().IsRegular() { targ := toolDir if isBinTool(file.Name()) { targ = binDir @@ -634,20 +642,3 @@ func cp(src, dst string) { log.Fatal(err) } } - -func ioutilReadDir(dirname string) ([]fs.FileInfo, error) { - entries, err := os.ReadDir(dirname) - if err != nil { - return nil, err - } - - infos := make([]fs.FileInfo, 0, len(entries)) - for _, entry := range entries { - info, err := entry.Info() - if err != nil { - return infos, err - } - infos = append(infos, info) - } - return infos, nil -} diff --git a/go/buildutil/util.go b/go/buildutil/util.go index 0dd8af4dd52..bee6390de4c 100644 --- a/go/buildutil/util.go +++ b/go/buildutil/util.go @@ -11,7 +11,7 @@ import ( "go/parser" "go/token" "io" - "io/fs" + "io/ioutil" "os" "path" "path/filepath" @@ -174,13 +174,13 @@ func IsDir(ctxt *build.Context, path string) bool { return err == nil && fi.IsDir() } -// ReadDir behaves like ioutilReadDir, +// ReadDir behaves like ioutil.ReadDir, // but uses the build context's file system interface, if any. func ReadDir(ctxt *build.Context, path string) ([]os.FileInfo, error) { if ctxt.ReadDir != nil { return ctxt.ReadDir(path) } - return ioutilReadDir(path) + return ioutil.ReadDir(path) } // SplitPathList behaves like filepath.SplitList, @@ -207,20 +207,3 @@ func sameFile(x, y string) bool { } return false } - -func ioutilReadDir(dirname string) ([]fs.FileInfo, error) { - entries, err := os.ReadDir(dirname) - if err != nil { - return nil, err - } - - infos := make([]fs.FileInfo, 0, len(entries)) - for _, entry := range entries { - info, err := entry.Info() - if err != nil { - return infos, err - } - infos = append(infos, info) - } - return infos, nil -} diff --git a/godoc/vfs/namespace.go b/godoc/vfs/namespace.go index 9326cedec5b..23dd9794312 100644 --- a/godoc/vfs/namespace.go +++ b/godoc/vfs/namespace.go @@ -77,7 +77,7 @@ const debugNS = false // just "/src" in the final two calls. // // OS is itself an implementation of a file system: it implements -// OS(`c:\Go`).ReadDir("/src/pkg/code") as ioutilReadDir(`c:\Go\src\pkg\code`). +// OS(`c:\Go`).ReadDir("/src/pkg/code") as ioutil.ReadDir(`c:\Go\src\pkg\code`). // // Because the new path is evaluated by fs (here OS(root)), another way // to read the mount table is to mentally combine fs+new, so that this table: diff --git a/godoc/vfs/os.go b/godoc/vfs/os.go index e5f7d5e77d9..35d050946e6 100644 --- a/godoc/vfs/os.go +++ b/godoc/vfs/os.go @@ -7,7 +7,7 @@ package vfs import ( "fmt" "go/build" - "io/fs" + "io/ioutil" "os" pathpkg "path" "path/filepath" @@ -101,22 +101,5 @@ func (root osFS) Stat(path string) (os.FileInfo, error) { } func (root osFS) ReadDir(path string) ([]os.FileInfo, error) { - return ioutilReadDir(root.resolve(path)) // is sorted -} - -func ioutilReadDir(dirname string) ([]fs.FileInfo, error) { - entries, err := os.ReadDir(dirname) - if err != nil { - return nil, err - } - - infos := make([]fs.FileInfo, 0, len(entries)) - for _, entry := range entries { - info, err := entry.Info() - if err != nil { - return infos, err - } - infos = append(infos, info) - } - return infos, nil + return ioutil.ReadDir(root.resolve(path)) // is sorted } diff --git a/internal/fastwalk/fastwalk_portable.go b/internal/fastwalk/fastwalk_portable.go index 563c9a3c98f..27e860243e1 100644 --- a/internal/fastwalk/fastwalk_portable.go +++ b/internal/fastwalk/fastwalk_portable.go @@ -8,7 +8,6 @@ package fastwalk import ( - "io/fs" "os" ) @@ -17,16 +16,20 @@ import ( // If fn returns a non-nil error, readDir returns with that error // immediately. func readDir(dirName string, fn func(dirName, entName string, typ os.FileMode) error) error { - fis, err := ioutilReadDir(dirName) + fis, err := os.ReadDir(dirName) if err != nil { return err } skipFiles := false for _, fi := range fis { - if fi.Mode().IsRegular() && skipFiles { + info, err := fi.Info() + if err != nil { + return err + } + if info.Mode().IsRegular() && skipFiles { continue } - if err := fn(dirName, fi.Name(), fi.Mode()&os.ModeType); err != nil { + if err := fn(dirName, fi.Name(), info.Mode()&os.ModeType); err != nil { if err == ErrSkipFiles { skipFiles = true continue @@ -36,20 +39,3 @@ func readDir(dirName string, fn func(dirName, entName string, typ os.FileMode) e } return nil } - -func ioutilReadDir(dirname string) ([]fs.FileInfo, error) { - entries, err := os.ReadDir(dirname) - if err != nil { - return nil, err - } - - infos := make([]fs.FileInfo, 0, len(entries)) - for _, entry := range entries { - info, err := entry.Info() - if err != nil { - return infos, err - } - infos = append(infos, info) - } - return infos, nil -} diff --git a/internal/gcimporter/gcimporter_test.go b/internal/gcimporter/gcimporter_test.go index 757306ca110..3af088b23d8 100644 --- a/internal/gcimporter/gcimporter_test.go +++ b/internal/gcimporter/gcimporter_test.go @@ -17,7 +17,6 @@ import ( goparser "go/parser" "go/token" "go/types" - "io/fs" "os" "os/exec" "path" @@ -286,7 +285,7 @@ func TestVersionHandling(t *testing.T) { needsCompiler(t, "gc") const dir = "./testdata/versions" - list, err := ioutilReadDir(dir) + list, err := os.ReadDir(dir) if err != nil { t.Fatal(err) } @@ -1008,20 +1007,3 @@ func lookupObj(t *testing.T, scope *types.Scope, name string) types.Object { t.Fatalf("%s not found", name) return nil } - -func ioutilReadDir(dirname string) ([]fs.FileInfo, error) { - entries, err := os.ReadDir(dirname) - if err != nil { - return nil, err - } - - infos := make([]fs.FileInfo, 0, len(entries)) - for _, entry := range entries { - info, err := entry.Info() - if err != nil { - return infos, err - } - infos = append(infos, info) - } - return infos, nil -} diff --git a/internal/imports/fix.go b/internal/imports/fix.go index c8e9aaaa636..01e8ba5fa2d 100644 --- a/internal/imports/fix.go +++ b/internal/imports/fix.go @@ -13,6 +13,8 @@ import ( "go/build" "go/parser" "go/token" + "io/fs" + "io/ioutil" "os" "path" "path/filepath" @@ -106,7 +108,7 @@ func parseOtherFiles(fset *token.FileSet, srcDir, filename string) []*ast.File { considerTests := strings.HasSuffix(filename, "_test.go") fileBase := filepath.Base(filename) - packageFileInfos, err := ioutilReadDir(srcDir) + packageFileInfos, err := os.ReadDir(srcDir) if err != nil { return nil } @@ -973,7 +975,7 @@ func (e *ProcessEnv) buildContext() (*build.Context, error) { // HACK: setting any of the Context I/O hooks prevents Import from invoking // 'go list', regardless of GO111MODULE. This is undocumented, but it's // unlikely to change before GOPATH support is removed. - ctx.ReadDir = ioutilReadDir + ctx.ReadDir = ioutil.ReadDir return &ctx, nil } @@ -1468,11 +1470,11 @@ func VendorlessPath(ipath string) string { func loadExportsFromFiles(ctx context.Context, env *ProcessEnv, dir string, includeTest bool) (string, []string, error) { // Look for non-test, buildable .go files which could provide exports. - all, err := ioutilReadDir(dir) + all, err := os.ReadDir(dir) if err != nil { return "", nil, err } - var files []os.FileInfo + var files []fs.DirEntry for _, fi := range all { name := fi.Name() if !strings.HasSuffix(name, ".go") || (!includeTest && strings.HasSuffix(name, "_test.go")) { diff --git a/internal/imports/mod.go b/internal/imports/mod.go index a00130b9d71..5f4d435d3cc 100644 --- a/internal/imports/mod.go +++ b/internal/imports/mod.go @@ -9,7 +9,6 @@ import ( "context" "encoding/json" "fmt" - "io/fs" "os" "path" "path/filepath" @@ -265,7 +264,7 @@ func (r *ModuleResolver) findPackage(importPath string) (*gocommand.ModuleJSON, } // Not cached. Read the filesystem. - pkgFiles, err := ioutilReadDir(pkgDir) + pkgFiles, err := os.ReadDir(pkgDir) if err != nil { continue } @@ -722,20 +721,3 @@ func modulePath(mod []byte) string { } return "" // missing module path } - -func ioutilReadDir(dirname string) ([]fs.FileInfo, error) { - entries, err := os.ReadDir(dirname) - if err != nil { - return nil, err - } - - infos := make([]fs.FileInfo, 0, len(entries)) - for _, entry := range entries { - info, err := entry.Info() - if err != nil { - return infos, err - } - infos = append(infos, info) - } - return infos, nil -} diff --git a/internal/imports/mod_test.go b/internal/imports/mod_test.go index b8b103e852a..26dac639062 100644 --- a/internal/imports/mod_test.go +++ b/internal/imports/mod_test.go @@ -1079,7 +1079,7 @@ func writeModule(dir, ar string) error { // writeProxy writes all the txtar-formatted modules in arDir to a proxy // directory in dir. func writeProxy(dir, arDir string) error { - files, err := ioutilReadDir(arDir) + files, err := os.ReadDir(arDir) if err != nil { return err } From 28990ac7a35f0ac1cb093ba39259d92061a5a24c Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Fri, 15 Sep 2023 15:32:47 -0400 Subject: [PATCH 107/178] internal/refactor/inline: fix bug in /internal/ check for std std contains packages named internal/foo, which are a corner case in the internal package check because (a) they have no leading slash and (b) the parent directory "" is a prefix of everything. Plus, a test. Change-Id: I89cd9c8aab38115c5e8f4862c56139599e43d50c Reviewed-on: https://go-review.googlesource.com/c/tools/+/528360 Reviewed-by: Robert Findley LUCI-TryBot-Result: Go LUCI Auto-Submit: Alan Donovan --- internal/refactor/inline/inline.go | 35 ++++++++++++++----- .../inline/testdata/std-internal.txtar | 15 ++++++++ 2 files changed, 42 insertions(+), 8 deletions(-) create mode 100644 internal/refactor/inline/testdata/std-internal.txtar diff --git a/internal/refactor/inline/inline.go b/internal/refactor/inline/inline.go index 5294550e9e3..02ab43b51af 100644 --- a/internal/refactor/inline/inline.go +++ b/internal/refactor/inline/inline.go @@ -378,15 +378,10 @@ func Inline(logf func(string, ...any), caller *Caller, callee *Callee) ([]byte, f.Decls = prepend[ast.Decl](importDecl, f.Decls...) } for _, spec := range res.newImports { - // Check that all imports (in particular, the new ones) are accessible. - // TODO(adonovan): allow customization of the accessibility relation - // (e.g. for Bazel). + // Check that the new imports are accessible. path, _ := strconv.Unquote(spec.Path.Value) - // TODO(adonovan): better segment hygiene. - if i := strings.Index(path, "/internal/"); i >= 0 { - if !strings.HasPrefix(caller.Types.Path(), path[:i]) { - return nil, fmt.Errorf("can't inline function %v as its body refers to inaccessible package %q", callee, path) - } + if !canImport(caller.Types.Path(), path) { + return nil, fmt.Errorf("can't inline function %v as its body refers to inaccessible package %q", callee, path) } importDecl.Specs = append(importDecl.Specs, spec) } @@ -2036,3 +2031,27 @@ func last[T any](slice []T) T { } return *new(T) } + +// canImport reports whether one package is allowed to import another. +// +// TODO(adonovan): allow customization of the accessibility relation +// (e.g. for Bazel). +func canImport(from, to string) bool { + // TODO(adonovan): better segment hygiene. + if strings.HasPrefix(to, "internal/") { + // Special case: only std packages may import internal/... + // We can't reliably know whether we're in std, so we + // use a heuristic on the first segment. + first, _, _ := strings.Cut(from, "/") + if strings.Contains(first, ".") { + return false // example.com/foo ∉ std + } + if first == "testdata" { + return false // testdata/foo ∉ std + } + } + if i := strings.LastIndex(to, "/internal/"); i >= 0 { + return strings.HasPrefix(from, to[:i]) + } + return true +} diff --git a/internal/refactor/inline/testdata/std-internal.txtar b/internal/refactor/inline/testdata/std-internal.txtar new file mode 100644 index 00000000000..460cdacd604 --- /dev/null +++ b/internal/refactor/inline/testdata/std-internal.txtar @@ -0,0 +1,15 @@ + +std packages are a special case of the internal package check. + +This test assumes that strings.Index refers to internal/bytealg. + +-- go.mod -- +module testdata +go 1.12 + +-- a/a.go -- +package a + +import "strings" + +var _ = strings.Index("", "") //@ inline(re"Index", re`inaccessible package "internal/bytealg"`) From 6d90c13c7adeddfe171a42742d380b6c9095f7ca Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Thu, 14 Sep 2023 17:54:10 -0400 Subject: [PATCH 108/178] internal/refactor/inline: ignore line directives Line directives should be ignored (at both caller and callee) as they would otherwise cause the inliner to make a mess of the caller file. This change ignores them consistently in the tests, and adds a heuristic assertion to reject inputs that seem to be affected by line directives. Plus: a test. Change-Id: I1a9bffd935fd7b288c47647304a2a6529779434b Reviewed-on: https://go-review.googlesource.com/c/tools/+/528298 Reviewed-by: Robert Findley LUCI-TryBot-Result: Go LUCI --- internal/refactor/inline/callee.go | 10 +++--- internal/refactor/inline/inline.go | 33 ++++++++++++++--- internal/refactor/inline/inline_test.go | 2 +- .../inline/testdata/line-directives.txtar | 35 +++++++++++++++++++ 4 files changed, 70 insertions(+), 10 deletions(-) create mode 100644 internal/refactor/inline/testdata/line-directives.txtar diff --git a/internal/refactor/inline/callee.go b/internal/refactor/inline/callee.go index 3e9f397fdca..9fff8493bc3 100644 --- a/internal/refactor/inline/callee.go +++ b/internal/refactor/inline/callee.go @@ -70,6 +70,10 @@ type object struct { // This design allows separate analysis of callers and callees in the // golang.org/x/tools/go/analysis framework: the inlining information // about a callee can be recorded as a "fact". +// +// The content should be the actual input to the compiler, not the +// apparent source file according to any //line directives that +// may be present within it. func AnalyzeCallee(fset *token.FileSet, pkg *types.Package, info *types.Info, decl *ast.FuncDecl, content []byte) (*Callee, error) { checkInfoFields(info) @@ -322,11 +326,9 @@ func AnalyzeCallee(fset *token.FileSet, pkg *types.Package, info *types.Info, de // callee file content; all we need is "package _; func f() { ... }". // This reduces the size of analysis facts. // - // (For ease of debugging we could insert a //line directive after - // the package decl but it seems more trouble than it's worth.) - // // Offsets in the callee information are "relocatable" // since they are all relative to the FuncDecl. + content = append([]byte("package _\n"), content[offsetOf(fset, decl.Pos()):offsetOf(fset, decl.End())]...) // Sanity check: re-parse the compacted content. @@ -386,7 +388,7 @@ func analyzeParams(fset *token.FileSet, info *types.Info, decl *ast.FuncDecl) (p fnobj, ok := info.Defs[decl.Name] if !ok { panic(fmt.Sprintf("%s: no func object for %q", - fset.Position(decl.Name.Pos()), decl.Name)) // ill-typed? + fset.PositionFor(decl.Name.Pos(), false), decl.Name)) // ill-typed? } paramInfos := make(map[*types.Var]*paramInfo) diff --git a/internal/refactor/inline/inline.go b/internal/refactor/inline/inline.go index 02ab43b51af..e2f1a2af2a4 100644 --- a/internal/refactor/inline/inline.go +++ b/internal/refactor/inline/inline.go @@ -305,7 +305,7 @@ type Caller struct { Info *types.Info File *ast.File Call *ast.CallExpr - Content []byte + Content []byte // source of file containing } // Inline inlines the called function (callee) into the function call (caller) @@ -325,7 +325,11 @@ func Inline(logf func(string, ...any), caller *Caller, callee *Callee) ([]byte, } logf("inline %s @ %v", debugFormatNode(caller.Fset, caller.Call), - caller.Fset.Position(caller.Call.Lparen)) + caller.Fset.PositionFor(caller.Call.Lparen, false)) + + if !consistentOffsets(caller) { + return nil, fmt.Errorf("internal error: caller syntax positions are inconsistent with file content (did you forget to use FileSet.PositionFor when computing the file name?)") + } res, err := inline(logf, caller, &callee.impl) if err != nil { @@ -611,7 +615,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu if found.Pos().IsValid() { return nil, fmt.Errorf("cannot inline because built-in %q is shadowed in caller by a %s (line %d)", obj.Name, objectKind(found), - caller.Fset.Position(found.Pos()).Line) + caller.Fset.PositionFor(found.Pos(), false).Line) } } else { @@ -627,7 +631,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu if !isPkgLevel(found) { return nil, fmt.Errorf("cannot inline because %q is shadowed in caller by a %s (line %d)", obj.Name, objectKind(found), - caller.Fset.Position(found.Pos()).Line) + caller.Fset.PositionFor(found.Pos(), false).Line) } } else { // Cross-package reference. @@ -957,7 +961,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // function (if any) and is indeed referenced // only by the call. logf("keeping param %q: arg contains perhaps the last reference to possible caller local %v @ %v", - param.info.Name, v, caller.Fset.Position(v.Pos())) + param.info.Name, v, caller.Fset.PositionFor(v.Pos(), false)) continue next } } @@ -2055,3 +2059,22 @@ func canImport(from, to string) bool { } return true } + +// consistentOffsets reports whether the portion of caller.Content +// that corresponds to caller.Call can be parsed as a call expression. +// If not, the client has provided inconsistent information, possibly +// because they forgot to ignore line directives when computing the +// filename enclosing the call. +// This is just a heuristic. +func consistentOffsets(caller *Caller) bool { + start := offsetOf(caller.Fset, caller.Call.Pos()) + end := offsetOf(caller.Fset, caller.Call.End()) + if !(0 < start && start < end && end <= len(caller.Content)) { + return false + } + expr, err := parser.ParseExpr(string(caller.Content[start:end])) + if err != nil { + return false + } + return is[*ast.CallExpr](expr) +} diff --git a/internal/refactor/inline/inline_test.go b/internal/refactor/inline/inline_test.go index 430ee49760d..cb52f3182ac 100644 --- a/internal/refactor/inline/inline_test.go +++ b/internal/refactor/inline/inline_test.go @@ -93,7 +93,7 @@ func TestData(t *testing.T) { continue } for _, note := range notes { - posn := pkg.Fset.Position(note.Pos) + posn := pkg.Fset.PositionFor(note.Pos, false) if note.Name != "inline" { t.Errorf("%s: invalid marker @%s", posn, note.Name) continue diff --git a/internal/refactor/inline/testdata/line-directives.txtar b/internal/refactor/inline/testdata/line-directives.txtar new file mode 100644 index 00000000000..66ae9ede335 --- /dev/null +++ b/internal/refactor/inline/testdata/line-directives.txtar @@ -0,0 +1,35 @@ +Test of line directives in caller and caller. +Neither should have any effect on inlining. + +-- go.mod -- +module testdata +go 1.12 + +-- a/a.go -- +package a + +import "testdata/b" + +func A() { +//line b2.go:3:3 + b.F() //@ inline(re"F", result) +} + +-- b/b.go -- +package b + +//line b2.go:1:1 +func F() { println("hi") } + +-- b/b2.go -- +package b + +func NotWhatYouWereLookingFor() {} + +-- result -- +package a + +func A() { +//line b2.go:3:3 + println("hi") //@ inline(re"F", result) +} From a9b2439b862eccc29e5ed35012352ed4e888b6f5 Mon Sep 17 00:00:00 2001 From: Peter Weinbergr Date: Mon, 18 Sep 2023 16:18:52 -0400 Subject: [PATCH 109/178] go/packages: remove use of ioutil in test The ioutil cleanup missed this case where ioutil was used in a string creating a test package. It changed ReadAll to io.ReadAll but did not change the corresponding import, and this was not caught because (probably) I used go test -short. Fixes: golang.go/go#62703 Change-Id: Ia12796f4232a01aed989d783aece98307a707bcd Reviewed-on: https://go-review.googlesource.com/c/tools/+/529298 TryBot-Result: Gopher Robot Reviewed-by: Heschi Kreinick Run-TryBot: Peter Weinberger --- go/packages/packages_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/packages/packages_test.go b/go/packages/packages_test.go index c8a72282dbb..60fdf9fbadd 100644 --- a/go/packages/packages_test.go +++ b/go/packages/packages_test.go @@ -2646,7 +2646,7 @@ func testExternal_NotHandled(t *testing.T, exporter packagestest.Exporter) { import ( "fmt" - "io/ioutil" + "io" "os" ) @@ -2659,7 +2659,7 @@ func main() { import ( "fmt" - "io/ioutil" + "io" "os" ) From 91fde38c0e0b43e4c335acec12c02202fbafe7bf Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Fri, 15 Sep 2023 12:13:15 -0400 Subject: [PATCH 110/178] gopls/internal/lsp/tests: eliminate several old marker types This CL is the unraveling of a single thread, which ended up spanning many different tests and markers. The goal was to eliminate the three remaining diagnostic markers in the old test framework, but due to the tightly bound nature of the old marker tests, that led to the elimination of Diagnostics, Definitions, TypeDefinitions, and UnimportedCompletions, along with the definition of several new markers in the new marker tests. The new tests are perhaps cleaner in some areas, but still a tangle: my goal is to delete the old marker tests ASAP, and it's easier to port them more-or-less faithfully rather than read through and fix bugs / broken assertions, verbose APIs, unexpected diagnostics, etc. To this end, the -ignore_extra_diags flag is introduced as some of the old tests had too many diagnostics for me to annotate. At least this looseness is contained only to the tests in which the flag is set. Also, the @rank and @snippet markers didn't make a ton of sense to me (they refer to completion items but only care about labels), but I didn't want to fix that. At a later time, I may go back through and change the way completion items are represented: the current style is rather verbose. Some completion markers in the old framework were changing settings for each mark, sometimes applying _multiple_ settings (see signatureHelp). The new marker framework does not support this (rightly so, in my opinion), and so where appropriate certain tests such as unimported completion had to be moved into a different test file so that they can use different settings. As for the tests themselves: they are _mostly_ the same as before, though I had to tweak them in various small ways to make them work. For example annotating diagnostics, redefining an import, changing the marker signature, etc. I don't think I meaningfully changed the test assertions, but I don't expect this to be carefully verified during review. I can only hope that the new state is slightly more maintainable than the old; and provides us a platform to eventually make the tests significantly more maintainable. For golang/go#54845 Change-Id: I701bfcbfecf32cd780caba9e324a134e2d9bd048 Reviewed-on: https://go-review.googlesource.com/c/tools/+/528736 Reviewed-by: Alan Donovan LUCI-TryBot-Result: Go LUCI --- gopls/internal/lsp/completion_test.go | 9 - gopls/internal/lsp/fake/editor.go | 11 +- gopls/internal/lsp/lsp_test.go | 87 --- gopls/internal/lsp/regtest/marker.go | 172 +++++- gopls/internal/lsp/regtest/wrappers.go | 13 +- .../lsp/testdata/arraytype/array_type.go.in | 50 -- .../lsp/testdata/badstmt/badstmt.go.in | 28 - .../lsp/testdata/badstmt/badstmt_2.go.in | 9 - .../lsp/testdata/badstmt/badstmt_3.go.in | 9 - .../lsp/testdata/badstmt/badstmt_4.go.in | 11 - gopls/internal/lsp/testdata/bar/bar.go.in | 47 -- gopls/internal/lsp/testdata/baz/baz.go.in | 33 -- gopls/internal/lsp/testdata/cgo/declarecgo.go | 27 - .../lsp/testdata/cgo/declarecgo.go.golden | 30 - .../lsp/testdata/cgo/declarecgo_nocgo.go | 6 - .../lsp/testdata/cgoimport/usecgo.go.golden | 30 - .../lsp/testdata/cgoimport/usecgo.go.in | 9 - .../danglingstmt/dangling_selector_2.go | 8 +- gopls/internal/lsp/testdata/foo/foo.go | 30 - .../internal/lsp/testdata/godef/a/a_x_test.go | 9 - .../lsp/testdata/godef/a/a_x_test.go.golden | 26 - gopls/internal/lsp/testdata/godef/a/d.go | 69 --- .../internal/lsp/testdata/godef/a/d.go.golden | 191 ------- gopls/internal/lsp/testdata/godef/a/f.go | 16 - .../internal/lsp/testdata/godef/a/f.go.golden | 34 -- gopls/internal/lsp/testdata/godef/a/h.go | 147 ----- .../internal/lsp/testdata/godef/a/h.go.golden | 161 ------ gopls/internal/lsp/testdata/godef/b/e.go | 31 - .../internal/lsp/testdata/godef/b/e.go.golden | 156 ----- .../godef/broken/unclosedIf.go.golden | 31 - .../testdata/godef/broken/unclosedIf.go.in | 9 - .../importedcomplit/imported_complit.go.in | 28 +- gopls/internal/lsp/testdata/nodisk/empty | 1 - .../lsp/testdata/nodisk/nodisk.overlay.go | 9 - .../lsp/testdata/selector/selector.go.in | 66 --- .../testdata/snippets/literal_snippets.go.in | 233 -------- .../snippets/literal_snippets118.go.in | 14 - .../lsp/testdata/snippets/snippets.go.in | 3 + .../internal/lsp/testdata/summary.txt.golden | 12 +- gopls/internal/lsp/testdata/testy/testy.go | 5 - .../internal/lsp/testdata/testy/testy_test.go | 18 - .../lsp/testdata/testy/testy_test.go.golden | 3 - gopls/internal/lsp/testdata/typdef/typdef.go | 65 --- .../lsp/testdata/unimported/export_test.go | 3 - .../lsp/testdata/unimported/unimported.go.in | 23 - .../unimported/unimported_cand_type.go | 16 - .../lsp/testdata/unimported/x_test.go | 9 - gopls/internal/lsp/tests/tests.go | 138 ----- gopls/internal/lsp/tests/util.go | 71 --- .../regtest/completion/completion_test.go | 39 +- .../marker/testdata/completion/foobarbaz.txt | 541 ++++++++++++++++++ .../marker/testdata/completion/lit.txt | 49 ++ .../marker/testdata/completion/testy.txt | 57 ++ .../marker/testdata/completion/unimported.txt | 88 +++ .../marker/testdata/definition/cgo.txt | 62 ++ .../regtest/marker/testdata/hover/godef.txt | 406 +++++++++++++ .../marker/testdata/typedef/typedef.txt | 68 +++ .../internal/regtest/misc/definition_test.go | 7 +- .../internal/regtest/misc/references_test.go | 8 +- .../internal/regtest/modfile/modfile_test.go | 2 +- 60 files changed, 1506 insertions(+), 2037 deletions(-) delete mode 100644 gopls/internal/lsp/testdata/arraytype/array_type.go.in delete mode 100644 gopls/internal/lsp/testdata/badstmt/badstmt.go.in delete mode 100644 gopls/internal/lsp/testdata/badstmt/badstmt_2.go.in delete mode 100644 gopls/internal/lsp/testdata/badstmt/badstmt_3.go.in delete mode 100644 gopls/internal/lsp/testdata/badstmt/badstmt_4.go.in delete mode 100644 gopls/internal/lsp/testdata/bar/bar.go.in delete mode 100644 gopls/internal/lsp/testdata/baz/baz.go.in delete mode 100644 gopls/internal/lsp/testdata/cgo/declarecgo.go delete mode 100644 gopls/internal/lsp/testdata/cgo/declarecgo.go.golden delete mode 100644 gopls/internal/lsp/testdata/cgo/declarecgo_nocgo.go delete mode 100644 gopls/internal/lsp/testdata/cgoimport/usecgo.go.golden delete mode 100644 gopls/internal/lsp/testdata/cgoimport/usecgo.go.in delete mode 100644 gopls/internal/lsp/testdata/foo/foo.go delete mode 100644 gopls/internal/lsp/testdata/godef/a/a_x_test.go delete mode 100644 gopls/internal/lsp/testdata/godef/a/a_x_test.go.golden delete mode 100644 gopls/internal/lsp/testdata/godef/a/d.go delete mode 100644 gopls/internal/lsp/testdata/godef/a/d.go.golden delete mode 100644 gopls/internal/lsp/testdata/godef/a/f.go delete mode 100644 gopls/internal/lsp/testdata/godef/a/f.go.golden delete mode 100644 gopls/internal/lsp/testdata/godef/a/h.go delete mode 100644 gopls/internal/lsp/testdata/godef/a/h.go.golden delete mode 100644 gopls/internal/lsp/testdata/godef/b/e.go delete mode 100644 gopls/internal/lsp/testdata/godef/b/e.go.golden delete mode 100644 gopls/internal/lsp/testdata/godef/broken/unclosedIf.go.golden delete mode 100644 gopls/internal/lsp/testdata/godef/broken/unclosedIf.go.in delete mode 100644 gopls/internal/lsp/testdata/nodisk/empty delete mode 100644 gopls/internal/lsp/testdata/nodisk/nodisk.overlay.go delete mode 100644 gopls/internal/lsp/testdata/selector/selector.go.in delete mode 100644 gopls/internal/lsp/testdata/snippets/literal_snippets.go.in delete mode 100644 gopls/internal/lsp/testdata/snippets/literal_snippets118.go.in delete mode 100644 gopls/internal/lsp/testdata/testy/testy.go delete mode 100644 gopls/internal/lsp/testdata/testy/testy_test.go delete mode 100644 gopls/internal/lsp/testdata/testy/testy_test.go.golden delete mode 100644 gopls/internal/lsp/testdata/typdef/typdef.go delete mode 100644 gopls/internal/lsp/testdata/unimported/export_test.go delete mode 100644 gopls/internal/lsp/testdata/unimported/unimported.go.in delete mode 100644 gopls/internal/lsp/testdata/unimported/unimported_cand_type.go delete mode 100644 gopls/internal/lsp/testdata/unimported/x_test.go create mode 100644 gopls/internal/regtest/marker/testdata/completion/foobarbaz.txt create mode 100644 gopls/internal/regtest/marker/testdata/completion/lit.txt create mode 100644 gopls/internal/regtest/marker/testdata/completion/testy.txt create mode 100644 gopls/internal/regtest/marker/testdata/completion/unimported.txt create mode 100644 gopls/internal/regtest/marker/testdata/definition/cgo.txt create mode 100644 gopls/internal/regtest/marker/testdata/hover/godef.txt create mode 100644 gopls/internal/regtest/marker/testdata/typedef/typedef.txt diff --git a/gopls/internal/lsp/completion_test.go b/gopls/internal/lsp/completion_test.go index 48ec9ea094e..06a6a09aa1a 100644 --- a/gopls/internal/lsp/completion_test.go +++ b/gopls/internal/lsp/completion_test.go @@ -49,15 +49,6 @@ func (r *runner) CompletionSnippet(t *testing.T, src span.Span, expected tests.C } } -func (r *runner) UnimportedCompletion(t *testing.T, src span.Span, test tests.Completion, items tests.CompletionItems) { - got := r.callCompletion(t, src, func(opts *source.Options) {}) - got = tests.FilterBuiltins(src, got) - want := expected(t, test, items) - if diff := tests.CheckCompletionOrder(want, got, false); diff != "" { - t.Errorf("%s", diff) - } -} - func (r *runner) DeepCompletion(t *testing.T, src span.Span, test tests.Completion, items tests.CompletionItems) { got := r.callCompletion(t, src, func(opts *source.Options) { opts.DeepCompletion = true diff --git a/gopls/internal/lsp/fake/editor.go b/gopls/internal/lsp/fake/editor.go index 78db2cca68b..2cf77d19ff3 100644 --- a/gopls/internal/lsp/fake/editor.go +++ b/gopls/internal/lsp/fake/editor.go @@ -794,10 +794,7 @@ func (e *Editor) setBufferContentLocked(ctx context.Context, path string, dirty // GoToDefinition jumps to the definition of the symbol at the given position // in an open buffer. It returns the location of the resulting jump. -// -// TODO(rfindley): rename to "Definition", to be consistent with LSP -// terminology. -func (e *Editor) GoToDefinition(ctx context.Context, loc protocol.Location) (protocol.Location, error) { +func (e *Editor) Definition(ctx context.Context, loc protocol.Location) (protocol.Location, error) { if err := e.checkBufferLocation(loc); err != nil { return protocol.Location{}, err } @@ -812,9 +809,9 @@ func (e *Editor) GoToDefinition(ctx context.Context, loc protocol.Location) (pro return e.extractFirstLocation(ctx, resp) } -// GoToTypeDefinition jumps to the type definition of the symbol at the given location -// in an open buffer. -func (e *Editor) GoToTypeDefinition(ctx context.Context, loc protocol.Location) (protocol.Location, error) { +// TypeDefinition jumps to the type definition of the symbol at the given +// location in an open buffer. +func (e *Editor) TypeDefinition(ctx context.Context, loc protocol.Location) (protocol.Location, error) { if err := e.checkBufferLocation(loc); err != nil { return protocol.Location{}, err } diff --git a/gopls/internal/lsp/lsp_test.go b/gopls/internal/lsp/lsp_test.go index 9934fee2d37..0bda4e6a6d0 100644 --- a/gopls/internal/lsp/lsp_test.go +++ b/gopls/internal/lsp/lsp_test.go @@ -14,7 +14,6 @@ import ( "strings" "testing" - "github.com/google/go-cmp/cmp" "golang.org/x/tools/gopls/internal/bug" "golang.org/x/tools/gopls/internal/lsp/cache" "golang.org/x/tools/gopls/internal/lsp/command" @@ -210,13 +209,6 @@ func (r *runner) CallHierarchy(t *testing.T, spn span.Span, expectedCalls *tests } } -func (r *runner) Diagnostics(t *testing.T, uri span.URI, want []*source.Diagnostic) { - // Get the diagnostics for this view if we have not done it before. - v := r.server.session.ViewByName(r.data.Config.Dir) - r.collectDiagnostics(v) - tests.CompareDiagnostics(t, uri, want, r.diagnostics[uri]) -} - func (r *runner) SemanticTokens(t *testing.T, spn span.Span) { uri := spn.URI() filename := uri.Filename() @@ -400,85 +392,6 @@ func (r *runner) MethodExtraction(t *testing.T, start span.Span, end span.Span) } } -// TODO(rfindley): This handler needs more work. The output is still a bit hard -// to read (range diffs do not format nicely), and it is too entangled with hover. -func (r *runner) Definition(t *testing.T, _ span.Span, d tests.Definition) { - sm, err := r.data.Mapper(d.Src.URI()) - if err != nil { - t.Fatal(err) - } - loc, err := sm.SpanLocation(d.Src) - if err != nil { - t.Fatalf("failed for %v: %v", d.Src, err) - } - tdpp := protocol.LocationTextDocumentPositionParams(loc) - var got []protocol.Location - var hover *protocol.Hover - if d.IsType { - params := &protocol.TypeDefinitionParams{ - TextDocumentPositionParams: tdpp, - } - got, err = r.server.TypeDefinition(r.ctx, params) - } else { - params := &protocol.DefinitionParams{ - TextDocumentPositionParams: tdpp, - } - got, err = r.server.Definition(r.ctx, params) - if err != nil { - t.Fatalf("failed for %v: %+v", d.Src, err) - } - v := &protocol.HoverParams{ - TextDocumentPositionParams: tdpp, - } - hover, err = r.server.Hover(r.ctx, v) - } - if err != nil { - t.Fatalf("failed for %v: %v", d.Src, err) - } - dm, err := r.data.Mapper(d.Def.URI()) - if err != nil { - t.Fatal(err) - } - def, err := dm.SpanLocation(d.Def) - if err != nil { - t.Fatal(err) - } - if !d.OnlyHover { - want := []protocol.Location{def} - if diff := cmp.Diff(want, got); diff != "" { - t.Fatalf("Definition(%s) mismatch (-want +got):\n%s", d.Src, diff) - } - } - didSomething := false - if hover != nil { - didSomething = true - tag := fmt.Sprintf("%s-hoverdef", d.Name) - want := string(r.data.Golden(t, tag, d.Src.URI().Filename(), func() ([]byte, error) { - return []byte(hover.Contents.Value), nil - })) - got := hover.Contents.Value - if diff := tests.DiffMarkdown(want, got); diff != "" { - t.Errorf("%s: markdown mismatch:\n%s", d.Src, diff) - } - } - if !d.OnlyHover { - didSomething = true - locURI := got[0].URI.SpanURI() - lm, err := r.data.Mapper(locURI) - if err != nil { - t.Fatal(err) - } - if def, err := lm.LocationSpan(got[0]); err != nil { - t.Fatalf("failed for %v: %v", got[0], err) - } else if def != d.Def { - t.Errorf("for %v got %v want %v", d.Src, def, d.Def) - } - } - if !didSomething { - t.Errorf("no tests ran for %s", d.Src.URI()) - } -} - func (r *runner) InlayHints(t *testing.T, spn span.Span) { uri := spn.URI() filename := uri.Filename() diff --git a/gopls/internal/lsp/regtest/marker.go b/gopls/internal/lsp/regtest/marker.go index ca4c9a8f83d..17d210c5df8 100644 --- a/gopls/internal/lsp/regtest/marker.go +++ b/gopls/internal/lsp/regtest/marker.go @@ -111,6 +111,7 @@ var update = flag.Bool("update", false, "if set, update test data during marker // in these directories before running the test. // -skip_goos=a,b,c instructs the test runner to skip the test for the // listed GOOS values. +// -ignore_extra_diags suppresses errors for unmatched diagnostics // TODO(rfindley): using build constraint expressions for -skip_goos would // be clearer. // TODO(rfindley): support flag values containing whitespace. @@ -199,7 +200,8 @@ var update = flag.Bool("update", false, "if set, update test data during marker // // - item(label, details, kind): defines a completion item with the provided // fields. This information is not positional, and therefore @item markers -// may occur anywhere in the source. Used in conjunction with @complete. +// may occur anywhere in the source. Used in conjunction with @complete, +// snippet, or rank. // // TODO(rfindley): rethink whether floating @item annotations are the best // way to specify completion results. @@ -220,9 +222,23 @@ var update = flag.Bool("update", false, "if set, update test data during marker // This action is executed for its editing effects on the source files. // Like rename, the golden directory contains the expected transformed files. // -// - refs(location, want ...location): executes a 'references' query at the -// first location and asserts that the result is the set of 'want' locations. -// The first want location must be the declaration (assumedly unique). +// - rank(location, ...completionItem): executes a textDocument/completion +// request at the given location, and verifies that each expected +// completion item occurs in the results, in the expected order. Other +// unexpected completion items may occur in the results. +// TODO(rfindley): this should accept a slice of labels, rather than +// completion items. +// +// - refs(location, want ...location): executes a textDocument/references +// request at the first location and asserts that the result is the set of +// 'want' locations. The first want location must be the declaration +// (assumedly unique). +// +// - snippet(location, completionItem, snippet): executes a +// textDocument/completion request at the location, and searches for a +// result with label matching that of the provided completion item +// (TODO(rfindley): accept a label rather than a completion item). Check +// the the result snippet matches the provided snippet. // // - symbol(golden): makes a textDocument/documentSymbol request // for the enclosing file, formats the response with one symbol @@ -342,7 +358,6 @@ var update = flag.Bool("update", false, "if set, update test data during marker // - CompletionItems // - Completions // - CompletionSnippets -// - UnimportedCompletions // - DeepCompletions // - FuzzyCompletions // - CaseSensitiveCompletions @@ -471,9 +486,11 @@ func RunMarkerTests(t *testing.T, dir string) { } // Any remaining (un-eliminated) diagnostics are an error. - for loc, diags := range run.diags { - for _, diag := range diags { - t.Errorf("%s: unexpected diagnostic: %q", run.fmtLoc(loc), diag.Message) + if !test.ignoreExtraDiags { + for loc, diags := range run.diags { + for _, diag := range diags { + t.Errorf("%s: unexpected diagnostic: %q", run.fmtLoc(loc), diag.Message) + } } } @@ -691,11 +708,15 @@ var actionFuncs = map[string]func(marker){ "highlight": markerFunc(highlightMarker), "hover": markerFunc(hoverMarker), "implementation": markerFunc(implementationMarker), + "rank": markerFunc(rankMarker), + "refs": markerFunc(refsMarker), "rename": markerFunc(renameMarker), "renameerr": markerFunc(renameErrMarker), + "signature": markerFunc(signatureMarker), + "snippet": markerFunc(snippetMarker), "suggestedfix": markerFunc(suggestedfixMarker), "symbol": markerFunc(symbolMarker), - "refs": markerFunc(refsMarker), + "typedef": markerFunc(typedefMarker), "workspacesymbol": markerFunc(workspaceSymbolMarker), } @@ -720,10 +741,11 @@ type markerTest struct { flags []string // flags extracted from the special "flags" archive file. // Parsed flags values. - minGoVersion string - cgo bool - writeGoSum []string // comma separated dirs to write go sum for - skipGOOS []string // comma separated GOOS values to skip + minGoVersion string + cgo bool + writeGoSum []string // comma separated dirs to write go sum for + skipGOOS []string // comma separated GOOS values to skip + ignoreExtraDiags bool } // flagSet returns the flagset used for parsing the special "flags" file in the @@ -734,6 +756,7 @@ func (t *markerTest) flagSet() *flag.FlagSet { flags.BoolVar(&t.cgo, "cgo", false, "if set, requires cgo (both the cgo tool and CGO_ENABLED=1)") flags.Var((*stringListValue)(&t.writeGoSum), "write_sumfile", "if set, write the sumfile for these directories") flags.Var((*stringListValue)(&t.skipGOOS), "skip_goos", "if set, skip this test on these GOOS values") + flags.BoolVar(&t.ignoreExtraDiags, "ignore_extra_diags", false, "if set, suppress errors for unmatched diagnostics") return flags } @@ -1188,11 +1211,13 @@ func convert(mark marker, arg any, paramType reflect.Type) (any, error) { } if id, ok := arg.(expect.Identifier); ok { if arg, ok := mark.run.data[id]; ok { + if !reflect.TypeOf(arg).AssignableTo(paramType) { + return nil, fmt.Errorf("cannot convert %v to %s", arg, paramType) + } return arg, nil } } - argType := reflect.TypeOf(arg) - if argType.AssignableTo(paramType) { + if reflect.TypeOf(arg).AssignableTo(paramType) { return arg, nil // no conversion required } switch paramType { @@ -1201,7 +1226,7 @@ func convert(mark marker, arg any, paramType reflect.Type) (any, error) { case wantErrorType: return convertWantError(mark, arg) default: - return nil, fmt.Errorf("cannot convert type %s to %s", argType, paramType) + return nil, fmt.Errorf("cannot convert %v to %s", arg, paramType) } } @@ -1374,18 +1399,78 @@ func checkChangedFiles(mark marker, changed map[string][]byte, golden *Golden) { // ---- marker functions ---- +// TODO(rfindley): consolidate documentation of these markers. They are already +// documented above, so much of the documentation here is redundant. + // completionItem is a simplified summary of a completion item. type completionItem struct { - Label, Detail, Kind string + Label, Detail, Kind, Documentation string } -func completionItemMarker(mark marker, label, detail, kind string) completionItem { - return completionItem{ - Label: label, - Detail: detail, - Kind: kind, - // TODO(rfindley): add variadic documentation? It is supported by the old - // marker tests but almost no test cases use it. +func completionItemMarker(mark marker, label string, other ...string) completionItem { + if len(other) > 3 { + mark.errorf("too many arguments to @item: expect at most 4") + } + item := completionItem{ + Label: label, + } + if len(other) > 0 { + item.Detail = other[0] + } + if len(other) > 1 { + item.Kind = other[1] + } + if len(other) > 2 { + item.Documentation = other[2] + } + return item +} + +func rankMarker(mark marker, src protocol.Location, items ...completionItem) { + list := mark.run.env.Completion(src) + var got []string + // Collect results that are present in items, preserving their order. + for _, g := range list.Items { + for _, w := range items { + if g.Label == w.Label { + got = append(got, g.Label) + break + } + } + } + var want []string + for _, w := range items { + want = append(want, w.Label) + } + if diff := cmp.Diff(want, got); diff != "" { + mark.errorf("completion rankings do not match (-want +got):\n%s", diff) + } +} + +func snippetMarker(mark marker, src protocol.Location, item completionItem, want string) { + list := mark.run.env.Completion(src) + var ( + found bool + got string + all []string // for errors + ) + items := filterBuiltinsAndKeywords(list.Items) + for _, i := range items { + all = append(all, i.Label) + if i.Label == item.Label { + found = true + if i.TextEdit != nil { + got = i.TextEdit.NewText + } + break + } + } + if !found { + mark.errorf("no completion item found matching %s (got: %v)", item.Label, all) + return + } + if got != want { + mark.errorf("snippets do not match: got %q, want %q", got, want) } } @@ -1402,7 +1487,15 @@ func completeMarker(mark marker, src protocol.Location, want ...completionItem) Detail: item.Detail, Kind: fmt.Sprint(item.Kind), } - // Support short-hand notation: if Detail or Kind are omitted from the + if item.Documentation != nil { + switch v := item.Documentation.Value.(type) { + case string: + simplified.Documentation = v + case protocol.MarkupContent: + simplified.Documentation = strings.TrimSpace(v.Value) // trim newlines + } + } + // Support short-hand notation: if Detail, Kind, or Documentation are omitted from the // item, don't match them. if i < len(want) { if want[i].Detail == "" { @@ -1411,13 +1504,15 @@ func completeMarker(mark marker, src protocol.Location, want ...completionItem) if want[i].Kind == "" { simplified.Kind = "" } - } - if i < len(want) && want[i].Detail == "" { - simplified.Detail = "" + if want[i].Documentation == "" { + simplified.Documentation = "" + } } got = append(got, simplified) } - + if len(want) == 0 { + want = nil // got is nil if empty + } if diff := cmp.Diff(want, got); diff != "" { mark.errorf("Completion(...) returned unexpect results (-want +got):\n%s", diff) } @@ -1488,6 +1583,14 @@ func defMarker(mark marker, src, dst protocol.Location) { } } +func typedefMarker(mark marker, src, dst protocol.Location) { + got := mark.run.env.TypeDefinition(src) + if got != dst { + mark.errorf("type definition location does not match:\n\tgot: %s\n\twant %s", + mark.run.fmtLoc(got), mark.run.fmtLoc(dst)) + } +} + func foldingRangeMarker(mark marker, g *Golden) { env := mark.run.env ranges, err := mark.server().FoldingRange(env.Ctx, &protocol.FoldingRangeParams{ @@ -1663,6 +1766,17 @@ func renameErrMarker(mark marker, loc protocol.Location, newName string, wantErr wantErr.check(mark, err) } +func signatureMarker(mark marker, src protocol.Location, want string) { + got := mark.run.env.SignatureHelp(src) + if got == nil || len(got.Signatures) != 1 { + mark.errorf("signatureHelp = %v, want exactly 1 signature", got) + return + } + if got := got.Signatures[0].Label; got != want { + mark.errorf("signatureHelp: got %q, want %q", got, want) + } +} + // rename returns the new contents of the files that would be modified // by renaming the identifier at loc to newName. func rename(env *Env, loc protocol.Location, newName string) (map[string][]byte, error) { diff --git a/gopls/internal/lsp/regtest/wrappers.go b/gopls/internal/lsp/regtest/wrappers.go index d0df0869718..d56d0c00335 100644 --- a/gopls/internal/lsp/regtest/wrappers.go +++ b/gopls/internal/lsp/regtest/wrappers.go @@ -155,9 +155,20 @@ func (e *Env) SaveBufferWithoutActions(name string) { // GoToDefinition goes to definition in the editor, calling t.Fatal on any // error. It returns the path and position of the resulting jump. +// +// TODO(rfindley): rename this to just 'Definition'. func (e *Env) GoToDefinition(loc protocol.Location) protocol.Location { e.T.Helper() - loc, err := e.Editor.GoToDefinition(e.Ctx, loc) + loc, err := e.Editor.Definition(e.Ctx, loc) + if err != nil { + e.T.Fatal(err) + } + return loc +} + +func (e *Env) TypeDefinition(loc protocol.Location) protocol.Location { + e.T.Helper() + loc, err := e.Editor.TypeDefinition(e.Ctx, loc) if err != nil { e.T.Fatal(err) } diff --git a/gopls/internal/lsp/testdata/arraytype/array_type.go.in b/gopls/internal/lsp/testdata/arraytype/array_type.go.in deleted file mode 100644 index ac1a3e78297..00000000000 --- a/gopls/internal/lsp/testdata/arraytype/array_type.go.in +++ /dev/null @@ -1,50 +0,0 @@ -package arraytype - -import ( - "golang.org/lsptests/foo" -) - -func _() { - var ( - val string //@item(atVal, "val", "string", "var") - ) - - // disabled - see issue #54822 - [] // complete(" //", PackageFoo) - - []val //@complete(" //") - - []foo.StructFoo //@complete(" //", StructFoo) - - []foo.StructFoo(nil) //@complete("(", StructFoo) - - []*foo.StructFoo //@complete(" //", StructFoo) - - [...]foo.StructFoo //@complete(" //", StructFoo) - - [2][][4]foo.StructFoo //@complete(" //", StructFoo) - - []struct { f []foo.StructFoo } //@complete(" }", StructFoo) -} - -func _() { - type myInt int //@item(atMyInt, "myInt", "int", "type") - - var mark []myInt //@item(atMark, "mark", "[]myInt", "var") - - var s []myInt //@item(atS, "s", "[]myInt", "var") - s = []m //@complete(" //", atMyInt) - // disabled - see issue #54822 - s = [] // complete(" //", atMyInt, PackageFoo) - - var a [1]myInt - a = [1]m //@complete(" //", atMyInt) - - var ds [][]myInt - ds = [][]m //@complete(" //", atMyInt) -} - -func _() { - var b [0]byte //@item(atByte, "b", "[0]byte", "var") - var _ []byte = b //@snippet(" //", atByte, "b[:]", "b[:]") -} diff --git a/gopls/internal/lsp/testdata/badstmt/badstmt.go.in b/gopls/internal/lsp/testdata/badstmt/badstmt.go.in deleted file mode 100644 index 3b8f9e06b39..00000000000 --- a/gopls/internal/lsp/testdata/badstmt/badstmt.go.in +++ /dev/null @@ -1,28 +0,0 @@ -package badstmt - -import ( - "golang.org/lsptests/foo" -) - -// (The syntax error causes suppression of diagnostics for type errors. -// See issue #59888.) - -func _(x int) { - defer foo.F //@complete(" //", Foo),diag(" //", "syntax", "function must be invoked in defer statement|expression in defer must be function call", "error") - defer foo.F //@complete(" //", Foo) -} - -func _() { - switch true { - case true: - go foo.F //@complete(" //", Foo) - } -} - -func _() { - defer func() { - foo.F //@complete(" //", Foo),snippet(" //", Foo, "Foo()", "Foo()") - - foo. //@rank(" //", Foo) - } -} diff --git a/gopls/internal/lsp/testdata/badstmt/badstmt_2.go.in b/gopls/internal/lsp/testdata/badstmt/badstmt_2.go.in deleted file mode 100644 index 6af9c35e3cf..00000000000 --- a/gopls/internal/lsp/testdata/badstmt/badstmt_2.go.in +++ /dev/null @@ -1,9 +0,0 @@ -package badstmt - -import ( - "golang.org/lsptests/foo" -) - -func _() { - defer func() { foo. } //@rank(" }", Foo) -} diff --git a/gopls/internal/lsp/testdata/badstmt/badstmt_3.go.in b/gopls/internal/lsp/testdata/badstmt/badstmt_3.go.in deleted file mode 100644 index d135e201505..00000000000 --- a/gopls/internal/lsp/testdata/badstmt/badstmt_3.go.in +++ /dev/null @@ -1,9 +0,0 @@ -package badstmt - -import ( - "golang.org/lsptests/foo" -) - -func _() { - go foo. //@rank(" //", Foo, IntFoo),snippet(" //", Foo, "Foo()", "Foo()") -} diff --git a/gopls/internal/lsp/testdata/badstmt/badstmt_4.go.in b/gopls/internal/lsp/testdata/badstmt/badstmt_4.go.in deleted file mode 100644 index 6afd635ec2d..00000000000 --- a/gopls/internal/lsp/testdata/badstmt/badstmt_4.go.in +++ /dev/null @@ -1,11 +0,0 @@ -package badstmt - -import ( - "golang.org/lsptests/foo" -) - -func _() { - go func() { - defer foo. //@rank(" //", Foo, IntFoo) - } -} diff --git a/gopls/internal/lsp/testdata/bar/bar.go.in b/gopls/internal/lsp/testdata/bar/bar.go.in deleted file mode 100644 index 502bdf74060..00000000000 --- a/gopls/internal/lsp/testdata/bar/bar.go.in +++ /dev/null @@ -1,47 +0,0 @@ -// +build go1.11 - -package bar - -import ( - "golang.org/lsptests/foo" //@item(foo, "foo", "\"golang.org/lsptests/foo\"", "package") -) - -func helper(i foo.IntFoo) {} //@item(helper, "helper", "func(i foo.IntFoo)", "func") - -func _() { - help //@complete("l", helper) - _ = foo.StructFoo{} //@complete("S", IntFoo, StructFoo) -} - -// Bar is a function. -func Bar() { //@item(Bar, "Bar", "func()", "func", "Bar is a function.") - foo.Foo() //@complete("F", Foo, IntFoo, StructFoo) - var _ foo.IntFoo //@complete("I", IntFoo, StructFoo) - foo.() //@complete("(", Foo, IntFoo, StructFoo) -} - -func _() { - var Valentine int //@item(Valentine, "Valentine", "int", "var") - - _ = foo.StructFoo{ - Valu //@complete(" //", Value) - } - _ = foo.StructFoo{ - Va //@complete("a", Value, Valentine) - } - _ = foo.StructFoo{ - Value: 5, //@complete("a", Value) - } - _ = foo.StructFoo{ - //@complete("", Value, Valentine, foo, helper, Bar) - } - _ = foo.StructFoo{ - Value: Valen //@complete("le", Valentine) - } - _ = foo.StructFoo{ - Value: //@complete(" //", Valentine, foo, helper, Bar) - } - _ = foo.StructFoo{ - Value: //@complete(" ", Valentine, foo, helper, Bar) - } -} diff --git a/gopls/internal/lsp/testdata/baz/baz.go.in b/gopls/internal/lsp/testdata/baz/baz.go.in deleted file mode 100644 index 94952e1267b..00000000000 --- a/gopls/internal/lsp/testdata/baz/baz.go.in +++ /dev/null @@ -1,33 +0,0 @@ -// +build go1.11 - -package baz - -import ( - "golang.org/lsptests/bar" - - f "golang.org/lsptests/foo" -) - -var FooStruct f.StructFoo - -func Baz() { - defer bar.Bar() //@complete("B", Bar) - // TODO(rstambler): Test completion here. - defer bar.B - var x f.IntFoo //@complete("n", IntFoo),typdef("x", IntFoo) - bar.Bar() //@complete("B", Bar) -} - -func _() { - bob := f.StructFoo{Value: 5} - if x := bob. //@complete(" //", Value) - switch true == false { - case true: - if x := bob. //@complete(" //", Value) - case false: - } - if x := bob.Va //@complete("a", Value) - switch true == true { - default: - } -} diff --git a/gopls/internal/lsp/testdata/cgo/declarecgo.go b/gopls/internal/lsp/testdata/cgo/declarecgo.go deleted file mode 100644 index c283cdfb2b7..00000000000 --- a/gopls/internal/lsp/testdata/cgo/declarecgo.go +++ /dev/null @@ -1,27 +0,0 @@ -package cgo - -/* -#include -#include - -void myprint(char* s) { - printf("%s\n", s); -} -*/ -import "C" - -import ( - "fmt" - "unsafe" -) - -func Example() { //@mark(funccgoexample, "Example"),item(funccgoexample, "Example", "func()", "func") - fmt.Println() - cs := C.CString("Hello from stdio\n") - C.myprint(cs) - C.free(unsafe.Pointer(cs)) -} - -func _() { - Example() //@godef("ample", funccgoexample),complete("ample", funccgoexample) -} diff --git a/gopls/internal/lsp/testdata/cgo/declarecgo.go.golden b/gopls/internal/lsp/testdata/cgo/declarecgo.go.golden deleted file mode 100644 index 0d6fbb0fff6..00000000000 --- a/gopls/internal/lsp/testdata/cgo/declarecgo.go.golden +++ /dev/null @@ -1,30 +0,0 @@ --- funccgoexample-definition -- -cgo/declarecgo.go:18:6-13: defined here as ```go -func Example() -``` - -[`cgo.Example` on pkg.go.dev](https://pkg.go.dev/golang.org/lsptests/cgo#Example) --- funccgoexample-definition-json -- -{ - "span": { - "uri": "file://cgo/declarecgo.go", - "start": { - "line": 18, - "column": 6, - "offset": 151 - }, - "end": { - "line": 18, - "column": 13, - "offset": 158 - } - }, - "description": "```go\nfunc Example()\n```\n\n[`cgo.Example` on pkg.go.dev](https://pkg.go.dev/golang.org/lsptests/cgo#Example)" -} - --- funccgoexample-hoverdef -- -```go -func Example() -``` - -[`cgo.Example` on pkg.go.dev](https://pkg.go.dev/golang.org/lsptests/cgo#Example) diff --git a/gopls/internal/lsp/testdata/cgo/declarecgo_nocgo.go b/gopls/internal/lsp/testdata/cgo/declarecgo_nocgo.go deleted file mode 100644 index a05c01257d0..00000000000 --- a/gopls/internal/lsp/testdata/cgo/declarecgo_nocgo.go +++ /dev/null @@ -1,6 +0,0 @@ -//+build !cgo - -package cgo - -// Set a dummy marker to keep the test framework happy. The tests should be skipped. -var _ = "Example" //@mark(funccgoexample, "Example"),godef("ample", funccgoexample),complete("ample", funccgoexample) diff --git a/gopls/internal/lsp/testdata/cgoimport/usecgo.go.golden b/gopls/internal/lsp/testdata/cgoimport/usecgo.go.golden deleted file mode 100644 index 03fc22468ca..00000000000 --- a/gopls/internal/lsp/testdata/cgoimport/usecgo.go.golden +++ /dev/null @@ -1,30 +0,0 @@ --- funccgoexample-definition -- -cgo/declarecgo.go:18:6-13: defined here as ```go -func cgo.Example() -``` - -[`cgo.Example` on pkg.go.dev](https://pkg.go.dev/golang.org/lsptests/cgo#Example) --- funccgoexample-definition-json -- -{ - "span": { - "uri": "file://cgo/declarecgo.go", - "start": { - "line": 18, - "column": 6, - "offset": 151 - }, - "end": { - "line": 18, - "column": 13, - "offset": 158 - } - }, - "description": "```go\nfunc cgo.Example()\n```\n\n[`cgo.Example` on pkg.go.dev](https://pkg.go.dev/golang.org/lsptests/cgo#Example)" -} - --- funccgoexample-hoverdef -- -```go -func cgo.Example() -``` - -[`cgo.Example` on pkg.go.dev](https://pkg.go.dev/golang.org/lsptests/cgo#Example) diff --git a/gopls/internal/lsp/testdata/cgoimport/usecgo.go.in b/gopls/internal/lsp/testdata/cgoimport/usecgo.go.in deleted file mode 100644 index 414a739da99..00000000000 --- a/gopls/internal/lsp/testdata/cgoimport/usecgo.go.in +++ /dev/null @@ -1,9 +0,0 @@ -package cgoimport - -import ( - "golang.org/lsptests/cgo" -) - -func _() { - cgo.Example() //@godef("ample", funccgoexample),complete("ample", funccgoexample) -} diff --git a/gopls/internal/lsp/testdata/danglingstmt/dangling_selector_2.go b/gopls/internal/lsp/testdata/danglingstmt/dangling_selector_2.go index 8d4b15bff6a..e25b00ce6c3 100644 --- a/gopls/internal/lsp/testdata/danglingstmt/dangling_selector_2.go +++ b/gopls/internal/lsp/testdata/danglingstmt/dangling_selector_2.go @@ -1,8 +1,10 @@ package danglingstmt -import "golang.org/lsptests/foo" +// TODO: re-enable this test, which was broken when the foo package was removed. +// (we can replicate the relevant definitions in the new marker test) +// import "golang.org/lsptests/foo" func _() { - foo. //@rank(" //", Foo) - var _ = []string{foo.} //@rank("}", Foo) + foo. // rank(" //", Foo) + var _ = []string{foo.} // rank("}", Foo) } diff --git a/gopls/internal/lsp/testdata/foo/foo.go b/gopls/internal/lsp/testdata/foo/foo.go deleted file mode 100644 index 490ff2e657c..00000000000 --- a/gopls/internal/lsp/testdata/foo/foo.go +++ /dev/null @@ -1,30 +0,0 @@ -package foo //@mark(PackageFoo, "foo"),item(PackageFoo, "foo", "\"golang.org/lsptests/foo\"", "package") - -type StructFoo struct { //@item(StructFoo, "StructFoo", "struct{...}", "struct") - Value int //@item(Value, "Value", "int", "field") -} - -// Pre-set this marker, as we don't have a "source" for it in this package. -/* Error() */ //@item(Error, "Error", "func() string", "method") - -func Foo() { //@item(Foo, "Foo", "func()", "func") - var err error - err.Error() //@complete("E", Error) -} - -func _() { - var sFoo StructFoo //@complete("t", StructFoo) - if x := sFoo; x.Value == 1 { //@complete("V", Value),typdef("sFoo", StructFoo) - return - } -} - -func _() { - shadowed := 123 - { - shadowed := "hi" //@item(shadowed, "shadowed", "string", "var") - sha //@complete("a", shadowed) - } -} - -type IntFoo int //@item(IntFoo, "IntFoo", "int", "type") diff --git a/gopls/internal/lsp/testdata/godef/a/a_x_test.go b/gopls/internal/lsp/testdata/godef/a/a_x_test.go deleted file mode 100644 index f166f055084..00000000000 --- a/gopls/internal/lsp/testdata/godef/a/a_x_test.go +++ /dev/null @@ -1,9 +0,0 @@ -package a_test - -import ( - "testing" -) - -func TestA2(t *testing.T) { //@TestA2,godef(TestA2, TestA2) - Nonexistant() //@diag("Nonexistant", "compiler", "(undeclared name|undefined): Nonexistant", "error") -} diff --git a/gopls/internal/lsp/testdata/godef/a/a_x_test.go.golden b/gopls/internal/lsp/testdata/godef/a/a_x_test.go.golden deleted file mode 100644 index 2e3064794f2..00000000000 --- a/gopls/internal/lsp/testdata/godef/a/a_x_test.go.golden +++ /dev/null @@ -1,26 +0,0 @@ --- TestA2-definition -- -godef/a/a_x_test.go:7:6-12: defined here as ```go -func TestA2(t *testing.T) -``` --- TestA2-definition-json -- -{ - "span": { - "uri": "file://godef/a/a_x_test.go", - "start": { - "line": 7, - "column": 6, - "offset": 44 - }, - "end": { - "line": 7, - "column": 12, - "offset": 50 - } - }, - "description": "```go\nfunc TestA2(t *testing.T)\n```" -} - --- TestA2-hoverdef -- -```go -func TestA2(t *testing.T) -``` diff --git a/gopls/internal/lsp/testdata/godef/a/d.go b/gopls/internal/lsp/testdata/godef/a/d.go deleted file mode 100644 index a1d17ad0da3..00000000000 --- a/gopls/internal/lsp/testdata/godef/a/d.go +++ /dev/null @@ -1,69 +0,0 @@ -package a //@mark(a, "a "),hoverdef("a ", a) - -import "fmt" - -type Thing struct { //@Thing - Member string //@Member -} - -var Other Thing //@Other - -func Things(val []string) []Thing { //@Things - return nil -} - -func (t Thing) Method(i int) string { //@Method - return t.Member -} - -func (t Thing) Method3() { -} - -func (t *Thing) Method2(i int, j int) (error, string) { - return nil, t.Member -} - -func (t *Thing) private() { -} - -func useThings() { - t := Thing{ //@mark(aStructType, "ing") - Member: "string", //@mark(fMember, "ember") - } - fmt.Print(t.Member) //@mark(aMember, "ember") - fmt.Print(Other) //@mark(aVar, "ther") - Things() //@mark(aFunc, "ings") - t.Method() //@mark(aMethod, "eth") -} - -type NextThing struct { //@NextThing - Thing - Value int -} - -func (n NextThing) another() string { - return n.Member -} - -// Shadows Thing.Method3 -func (n *NextThing) Method3() int { - return n.Value -} - -var nextThing NextThing //@hoverdef("NextThing", NextThing) - -/*@ -godef(aStructType, Thing) -godef(aMember, Member) -godef(aVar, Other) -godef(aFunc, Things) -godef(aMethod, Method) -godef(fMember, Member) -godef(Member, Member) - -//param -//package name -//const -//anon field - -*/ diff --git a/gopls/internal/lsp/testdata/godef/a/d.go.golden b/gopls/internal/lsp/testdata/godef/a/d.go.golden deleted file mode 100644 index ee687750c3e..00000000000 --- a/gopls/internal/lsp/testdata/godef/a/d.go.golden +++ /dev/null @@ -1,191 +0,0 @@ --- Member-definition -- -godef/a/d.go:6:2-8: defined here as ```go -field Member string -``` - -@Member - - -[`(a.Thing).Member` on pkg.go.dev](https://pkg.go.dev/golang.org/lsptests/godef/a#Thing.Member) --- Member-definition-json -- -{ - "span": { - "uri": "file://godef/a/d.go", - "start": { - "line": 6, - "column": 2, - "offset": 90 - }, - "end": { - "line": 6, - "column": 8, - "offset": 96 - } - }, - "description": "```go\nfield Member string\n```\n\n@Member\n\n\n[`(a.Thing).Member` on pkg.go.dev](https://pkg.go.dev/golang.org/lsptests/godef/a#Thing.Member)" -} - --- Member-hoverdef -- -```go -field Member string -``` - -@Member - - -[`(a.Thing).Member` on pkg.go.dev](https://pkg.go.dev/golang.org/lsptests/godef/a#Thing.Member) --- Method-definition -- -godef/a/d.go:15:16-22: defined here as ```go -func (Thing).Method(i int) string -``` - -[`(a.Thing).Method` on pkg.go.dev](https://pkg.go.dev/golang.org/lsptests/godef/a#Thing.Method) --- Method-definition-json -- -{ - "span": { - "uri": "file://godef/a/d.go", - "start": { - "line": 15, - "column": 16, - "offset": 219 - }, - "end": { - "line": 15, - "column": 22, - "offset": 225 - } - }, - "description": "```go\nfunc (Thing).Method(i int) string\n```\n\n[`(a.Thing).Method` on pkg.go.dev](https://pkg.go.dev/golang.org/lsptests/godef/a#Thing.Method)" -} - --- Method-hoverdef -- -```go -func (Thing).Method(i int) string -``` - -[`(a.Thing).Method` on pkg.go.dev](https://pkg.go.dev/golang.org/lsptests/godef/a#Thing.Method) --- NextThing-hoverdef -- -```go -type NextThing struct { - Thing - Value int -} - -func (*NextThing).Method3() int -func (NextThing).another() string -``` - -[`a.NextThing` on pkg.go.dev](https://pkg.go.dev/golang.org/lsptests/godef/a#NextThing) --- Other-definition -- -godef/a/d.go:9:5-10: defined here as ```go -var Other Thing -``` - -@Other - - -[`a.Other` on pkg.go.dev](https://pkg.go.dev/golang.org/lsptests/godef/a#Other) --- Other-definition-json -- -{ - "span": { - "uri": "file://godef/a/d.go", - "start": { - "line": 9, - "column": 5, - "offset": 121 - }, - "end": { - "line": 9, - "column": 10, - "offset": 126 - } - }, - "description": "```go\nvar Other Thing\n```\n\n@Other\n\n\n[`a.Other` on pkg.go.dev](https://pkg.go.dev/golang.org/lsptests/godef/a#Other)" -} - --- Other-hoverdef -- -```go -var Other Thing -``` - -@Other - - -[`a.Other` on pkg.go.dev](https://pkg.go.dev/golang.org/lsptests/godef/a#Other) --- Thing-definition -- -godef/a/d.go:5:6-11: defined here as ```go -type Thing struct { - Member string //@Member -} - -func (Thing).Method(i int) string -func (*Thing).Method2(i int, j int) (error, string) -func (Thing).Method3() -func (*Thing).private() -``` - -[`a.Thing` on pkg.go.dev](https://pkg.go.dev/golang.org/lsptests/godef/a#Thing) --- Thing-definition-json -- -{ - "span": { - "uri": "file://godef/a/d.go", - "start": { - "line": 5, - "column": 6, - "offset": 65 - }, - "end": { - "line": 5, - "column": 11, - "offset": 70 - } - }, - "description": "```go\ntype Thing struct {\n\tMember string //@Member\n}\n\nfunc (Thing).Method(i int) string\nfunc (*Thing).Method2(i int, j int) (error, string)\nfunc (Thing).Method3()\nfunc (*Thing).private()\n```\n\n[`a.Thing` on pkg.go.dev](https://pkg.go.dev/golang.org/lsptests/godef/a#Thing)" -} - --- Thing-hoverdef -- -```go -type Thing struct { - Member string //@Member -} - -func (Thing).Method(i int) string -func (*Thing).Method2(i int, j int) (error, string) -func (Thing).Method3() -func (*Thing).private() -``` - -[`a.Thing` on pkg.go.dev](https://pkg.go.dev/golang.org/lsptests/godef/a#Thing) --- Things-definition -- -godef/a/d.go:11:6-12: defined here as ```go -func Things(val []string) []Thing -``` - -[`a.Things` on pkg.go.dev](https://pkg.go.dev/golang.org/lsptests/godef/a#Things) --- Things-definition-json -- -{ - "span": { - "uri": "file://godef/a/d.go", - "start": { - "line": 11, - "column": 6, - "offset": 148 - }, - "end": { - "line": 11, - "column": 12, - "offset": 154 - } - }, - "description": "```go\nfunc Things(val []string) []Thing\n```\n\n[`a.Things` on pkg.go.dev](https://pkg.go.dev/golang.org/lsptests/godef/a#Things)" -} - --- Things-hoverdef -- -```go -func Things(val []string) []Thing -``` - -[`a.Things` on pkg.go.dev](https://pkg.go.dev/golang.org/lsptests/godef/a#Things) --- a-hoverdef -- -Package a is a package for testing go to definition. - diff --git a/gopls/internal/lsp/testdata/godef/a/f.go b/gopls/internal/lsp/testdata/godef/a/f.go deleted file mode 100644 index 10f88262a81..00000000000 --- a/gopls/internal/lsp/testdata/godef/a/f.go +++ /dev/null @@ -1,16 +0,0 @@ -// Package a is a package for testing go to definition. -package a - -import "fmt" - -func TypeStuff() { //@Stuff - var x string - - switch y := interface{}(x).(type) { //@mark(switchY, "y"),godef("y", switchY) - case int: //@mark(intY, "int") - fmt.Printf("%v", y) //@hoverdef("y", intY) - case string: //@mark(stringY, "string") - fmt.Printf("%v", y) //@hoverdef("y", stringY) - } - -} diff --git a/gopls/internal/lsp/testdata/godef/a/f.go.golden b/gopls/internal/lsp/testdata/godef/a/f.go.golden deleted file mode 100644 index a084356c06b..00000000000 --- a/gopls/internal/lsp/testdata/godef/a/f.go.golden +++ /dev/null @@ -1,34 +0,0 @@ --- intY-hoverdef -- -```go -var y int -``` --- stringY-hoverdef -- -```go -var y string -``` --- switchY-definition -- -godef/a/f.go:8:9-10: defined here as ```go -var y interface{} -``` --- switchY-definition-json -- -{ - "span": { - "uri": "file://godef/a/f.go", - "start": { - "line": 8, - "column": 9, - "offset": 76 - }, - "end": { - "line": 8, - "column": 10, - "offset": 77 - } - }, - "description": "```go\nvar y interface{}\n```" -} - --- switchY-hoverdef -- -```go -var y interface{} -``` diff --git a/gopls/internal/lsp/testdata/godef/a/h.go b/gopls/internal/lsp/testdata/godef/a/h.go deleted file mode 100644 index 5a5dcc6784d..00000000000 --- a/gopls/internal/lsp/testdata/godef/a/h.go +++ /dev/null @@ -1,147 +0,0 @@ -package a - -func _() { - type s struct { - nested struct { - // nested number - number int64 //@mark(nestedNumber, "number") - } - nested2 []struct { - // nested string - str string //@mark(nestedString, "str") - } - x struct { - x struct { - x struct { - x struct { - x struct { - // nested map - m map[string]float64 //@mark(nestedMap, "m") - } - } - } - } - } - } - - var t s - _ = t.nested.number //@hoverdef("number", nestedNumber) - _ = t.nested2[0].str //@hoverdef("str", nestedString) - _ = t.x.x.x.x.x.m //@hoverdef("m", nestedMap) -} - -func _() { - var s struct { - // a field - a int //@mark(structA, "a") - // b nested struct - b struct { //@mark(structB, "b") - // c field of nested struct - c int //@mark(structC, "c") - } - } - _ = s.a //@hoverdef("a", structA) - _ = s.b //@hoverdef("b", structB) - _ = s.b.c //@hoverdef("c", structC) - - var arr []struct { - // d field - d int //@mark(arrD, "d") - // e nested struct - e struct { //@mark(arrE, "e") - // f field of nested struct - f int //@mark(arrF, "f") - } - } - _ = arr[0].d //@hoverdef("d", arrD) - _ = arr[0].e //@hoverdef("e", arrE) - _ = arr[0].e.f //@hoverdef("f", arrF) - - var complex []struct { - c <-chan map[string][]struct { - // h field - h int //@mark(complexH, "h") - // i nested struct - i struct { //@mark(complexI, "i") - // j field of nested struct - j int //@mark(complexJ, "j") - } - } - } - _ = (<-complex[0].c)["0"][0].h //@hoverdef("h", complexH) - _ = (<-complex[0].c)["0"][0].i //@hoverdef("i", complexI) - _ = (<-complex[0].c)["0"][0].i.j //@hoverdef("j", complexJ) - - var mapWithStructKey map[struct { - // X key field - x []string //@mark(mapStructKeyX, "x") - }]int - for k := range mapWithStructKey { - _ = k.x //@hoverdef("x", mapStructKeyX) - } - - var mapWithStructKeyAndValue map[struct { - // Y key field - y string //@mark(mapStructKeyY, "y") - }]struct { - // X value field - x string //@mark(mapStructValueX, "x") - } - for k, v := range mapWithStructKeyAndValue { - // TODO: we don't show docs for y field because both map key and value - // are structs. And in this case, we parse only map value - _ = k.y //@hoverdef("y", mapStructKeyY) - _ = v.x //@hoverdef("x", mapStructValueX) - } - - var i []map[string]interface { - // open method comment - open() error //@mark(openMethod, "open") - } - i[0]["1"].open() //@hoverdef("open", openMethod) -} - -func _() { - test := struct { - // test description - desc string //@mark(testDescription, "desc") - }{} - _ = test.desc //@hoverdef("desc", testDescription) - - for _, tt := range []struct { - // test input - in map[string][]struct { //@mark(testInput, "in") - // test key - key string //@mark(testInputKey, "key") - // test value - value interface{} //@mark(testInputValue, "value") - } - result struct { - v <-chan struct { - // expected test value - value int //@mark(testResultValue, "value") - } - } - }{} { - _ = tt.in //@hoverdef("in", testInput) - _ = tt.in["0"][0].key //@hoverdef("key", testInputKey) - _ = tt.in["0"][0].value //@hoverdef("value", testInputValue) - - _ = (<-tt.result.v).value //@hoverdef("value", testResultValue) - } -} - -func _() { - getPoints := func() []struct { - // X coord - x int //@mark(returnX, "x") - // Y coord - y int //@mark(returnY, "y") - } { - return nil - } - - r := getPoints() - r[0].x //@hoverdef("x", returnX) - r[0].y //@hoverdef("y", returnY) -} diff --git a/gopls/internal/lsp/testdata/godef/a/h.go.golden b/gopls/internal/lsp/testdata/godef/a/h.go.golden deleted file mode 100644 index 7cef9ee967a..00000000000 --- a/gopls/internal/lsp/testdata/godef/a/h.go.golden +++ /dev/null @@ -1,161 +0,0 @@ --- arrD-hoverdef -- -```go -field d int -``` - -d field - --- arrE-hoverdef -- -```go -field e struct{f int} -``` - -e nested struct - --- arrF-hoverdef -- -```go -field f int -``` - -f field of nested struct - --- complexH-hoverdef -- -```go -field h int -``` - -h field - --- complexI-hoverdef -- -```go -field i struct{j int} -``` - -i nested struct - --- complexJ-hoverdef -- -```go -field j int -``` - -j field of nested struct - --- mapStructKeyX-hoverdef -- -```go -field x []string -``` - -X key field - --- mapStructKeyY-hoverdef -- -```go -field y string -``` - -Y key field - --- mapStructValueX-hoverdef -- -```go -field x string -``` - -X value field - --- nestedMap-hoverdef -- -```go -field m map[string]float64 -``` - -nested map - --- nestedNumber-hoverdef -- -```go -field number int64 -``` - -nested number - --- nestedString-hoverdef -- -```go -field str string -``` - -nested string - --- openMethod-hoverdef -- -```go -func (interface).open() error -``` - -open method comment - --- returnX-hoverdef -- -```go -field x int -``` - -X coord - --- returnY-hoverdef -- -```go -field y int -``` - -Y coord - --- structA-hoverdef -- -```go -field a int -``` - -a field - --- structB-hoverdef -- -```go -field b struct{c int} -``` - -b nested struct - --- structC-hoverdef -- -```go -field c int -``` - -c field of nested struct - --- testDescription-hoverdef -- -```go -field desc string -``` - -test description - --- testInput-hoverdef -- -```go -field in map[string][]struct{key string; value interface{}} -``` - -test input - --- testInputKey-hoverdef -- -```go -field key string -``` - -test key - --- testInputValue-hoverdef -- -```go -field value interface{} -``` - -test value - --- testResultValue-hoverdef -- -```go -field value int -``` - -expected test value - diff --git a/gopls/internal/lsp/testdata/godef/b/e.go b/gopls/internal/lsp/testdata/godef/b/e.go deleted file mode 100644 index 9c81cad3171..00000000000 --- a/gopls/internal/lsp/testdata/godef/b/e.go +++ /dev/null @@ -1,31 +0,0 @@ -package b - -import ( - "fmt" - - "golang.org/lsptests/godef/a" -) - -func useThings() { - t := a.Thing{} //@mark(bStructType, "ing") - fmt.Print(t.Member) //@mark(bMember, "ember") - fmt.Print(a.Other) //@mark(bVar, "ther") - a.Things() //@mark(bFunc, "ings") -} - -/*@ -godef(bStructType, Thing) -godef(bMember, Member) -godef(bVar, Other) -godef(bFunc, Things) -*/ - -func _() { - var x interface{} //@mark(eInterface, "interface{}") - switch x := x.(type) { //@hoverdef("x", eInterface) - case string: //@mark(eString, "string") - fmt.Println(x) //@hoverdef("x", eString) - case int: //@mark(eInt, "int") - fmt.Println(x) //@hoverdef("x", eInt) - } -} diff --git a/gopls/internal/lsp/testdata/godef/b/e.go.golden b/gopls/internal/lsp/testdata/godef/b/e.go.golden deleted file mode 100644 index 3d7d8979771..00000000000 --- a/gopls/internal/lsp/testdata/godef/b/e.go.golden +++ /dev/null @@ -1,156 +0,0 @@ --- Member-definition -- -godef/a/d.go:6:2-8: defined here as ```go -field Member string -``` - -@Member - - -[`(a.Thing).Member` on pkg.go.dev](https://pkg.go.dev/golang.org/lsptests/godef/a#Thing.Member) --- Member-definition-json -- -{ - "span": { - "uri": "file://godef/a/d.go", - "start": { - "line": 6, - "column": 2, - "offset": 90 - }, - "end": { - "line": 6, - "column": 8, - "offset": 96 - } - }, - "description": "```go\nfield Member string\n```\n\n@Member\n\n\n[`(a.Thing).Member` on pkg.go.dev](https://pkg.go.dev/golang.org/lsptests/godef/a#Thing.Member)" -} - --- Member-hoverdef -- -```go -field Member string -``` - -@Member - - -[`(a.Thing).Member` on pkg.go.dev](https://pkg.go.dev/golang.org/lsptests/godef/a#Thing.Member) --- Other-definition -- -godef/a/d.go:9:5-10: defined here as ```go -var a.Other a.Thing -``` - -@Other - - -[`a.Other` on pkg.go.dev](https://pkg.go.dev/golang.org/lsptests/godef/a#Other) --- Other-definition-json -- -{ - "span": { - "uri": "file://godef/a/d.go", - "start": { - "line": 9, - "column": 5, - "offset": 121 - }, - "end": { - "line": 9, - "column": 10, - "offset": 126 - } - }, - "description": "```go\nvar a.Other a.Thing\n```\n\n@Other\n\n\n[`a.Other` on pkg.go.dev](https://pkg.go.dev/golang.org/lsptests/godef/a#Other)" -} - --- Other-hoverdef -- -```go -var a.Other a.Thing -``` - -@Other - - -[`a.Other` on pkg.go.dev](https://pkg.go.dev/golang.org/lsptests/godef/a#Other) --- Thing-definition -- -godef/a/d.go:5:6-11: defined here as ```go -type Thing struct { - Member string //@Member -} - -func (a.Thing).Method(i int) string -func (*a.Thing).Method2(i int, j int) (error, string) -func (a.Thing).Method3() -``` - -[`a.Thing` on pkg.go.dev](https://pkg.go.dev/golang.org/lsptests/godef/a#Thing) --- Thing-definition-json -- -{ - "span": { - "uri": "file://godef/a/d.go", - "start": { - "line": 5, - "column": 6, - "offset": 65 - }, - "end": { - "line": 5, - "column": 11, - "offset": 70 - } - }, - "description": "```go\ntype Thing struct {\n\tMember string //@Member\n}\n\nfunc (a.Thing).Method(i int) string\nfunc (*a.Thing).Method2(i int, j int) (error, string)\nfunc (a.Thing).Method3()\n```\n\n[`a.Thing` on pkg.go.dev](https://pkg.go.dev/golang.org/lsptests/godef/a#Thing)" -} - --- Thing-hoverdef -- -```go -type Thing struct { - Member string //@Member -} - -func (a.Thing).Method(i int) string -func (*a.Thing).Method2(i int, j int) (error, string) -func (a.Thing).Method3() -``` - -[`a.Thing` on pkg.go.dev](https://pkg.go.dev/golang.org/lsptests/godef/a#Thing) --- Things-definition -- -godef/a/d.go:11:6-12: defined here as ```go -func a.Things(val []string) []a.Thing -``` - -[`a.Things` on pkg.go.dev](https://pkg.go.dev/golang.org/lsptests/godef/a#Things) --- Things-definition-json -- -{ - "span": { - "uri": "file://godef/a/d.go", - "start": { - "line": 11, - "column": 6, - "offset": 148 - }, - "end": { - "line": 11, - "column": 12, - "offset": 154 - } - }, - "description": "```go\nfunc a.Things(val []string) []a.Thing\n```\n\n[`a.Things` on pkg.go.dev](https://pkg.go.dev/golang.org/lsptests/godef/a#Things)" -} - --- Things-hoverdef -- -```go -func a.Things(val []string) []a.Thing -``` - -[`a.Things` on pkg.go.dev](https://pkg.go.dev/golang.org/lsptests/godef/a#Things) --- eInt-hoverdef -- -```go -var x int -``` --- eInterface-hoverdef -- -```go -var x interface{} -``` --- eString-hoverdef -- -```go -var x string -``` diff --git a/gopls/internal/lsp/testdata/godef/broken/unclosedIf.go.golden b/gopls/internal/lsp/testdata/godef/broken/unclosedIf.go.golden deleted file mode 100644 index 9ce869848cb..00000000000 --- a/gopls/internal/lsp/testdata/godef/broken/unclosedIf.go.golden +++ /dev/null @@ -1,31 +0,0 @@ --- myUnclosedIf-definition -- -godef/broken/unclosedIf.go:7:7-19: defined here as ```go -var myUnclosedIf string -``` - -@myUnclosedIf --- myUnclosedIf-definition-json -- -{ - "span": { - "uri": "file://godef/broken/unclosedIf.go", - "start": { - "line": 7, - "column": 7, - "offset": 68 - }, - "end": { - "line": 7, - "column": 19, - "offset": 80 - } - }, - "description": "```go\nvar myUnclosedIf string\n```\n\n@myUnclosedIf" -} - --- myUnclosedIf-hoverdef -- -```go -var myUnclosedIf string -``` - -@myUnclosedIf - diff --git a/gopls/internal/lsp/testdata/godef/broken/unclosedIf.go.in b/gopls/internal/lsp/testdata/godef/broken/unclosedIf.go.in deleted file mode 100644 index 0f2cf1b1e5d..00000000000 --- a/gopls/internal/lsp/testdata/godef/broken/unclosedIf.go.in +++ /dev/null @@ -1,9 +0,0 @@ -package broken - -import "fmt" - -func unclosedIf() { - if false { - var myUnclosedIf string //@myUnclosedIf - fmt.Printf("s = %v\n", myUnclosedIf) //@godef("my", myUnclosedIf) -} diff --git a/gopls/internal/lsp/testdata/importedcomplit/imported_complit.go.in b/gopls/internal/lsp/testdata/importedcomplit/imported_complit.go.in index 2f4cbada141..05ba54006a5 100644 --- a/gopls/internal/lsp/testdata/importedcomplit/imported_complit.go.in +++ b/gopls/internal/lsp/testdata/importedcomplit/imported_complit.go.in @@ -1,22 +1,32 @@ package importedcomplit import ( - "golang.org/lsptests/foo" + // TODO(rfindley): re-enable after moving to new framework + // "golang.org/lsptests/foo" + + // import completions (separate blocks to avoid comment alignment) + "crypto/elli" //@complete("\" //", cryptoImport) - // import completions "fm" //@complete("\" //", fmtImport) + "go/pars" //@complete("\" //", parserImport) - "golang.org/lsptests/signa" //@complete("na\" //", signatureImport) + + namedParser "go/pars" //@complete("\" //", parserImport) + "golang.org/lspte" //@complete("\" //", lsptestsImport) - "crypto/elli" //@complete("\" //", cryptoImport) + "golang.org/lsptests/sign" //@complete("\" //", signatureImport) + "golang.org/lsptests/sign" //@complete("ests", lsptestsImport) - namedParser "go/pars" //@complete("\" //", parserImport) + + "golang.org/lsptests/signa" //@complete("na\" //", signatureImport) ) func _() { var V int //@item(icVVar, "V", "int", "var") - _ = foo.StructFoo{V} //@complete("}", Value, icVVar) + + // TODO(rfindley): re-enable after moving to new framework + // _ = foo.StructFoo{V} // complete("}", Value, icVVar) } func _() { @@ -25,14 +35,16 @@ func _() { ab int //@item(icABVar, "ab", "int", "var") ) - _ = foo.StructFoo{a} //@complete("}", abVar, aaVar) + // TODO(rfindley): re-enable after moving to new framework + // _ = foo.StructFoo{a} // complete("}", abVar, aaVar) var s struct { AA string //@item(icFieldAA, "AA", "string", "field") AB int //@item(icFieldAB, "AB", "int", "field") } - _ = foo.StructFoo{s.} //@complete("}", icFieldAB, icFieldAA) + // TODO(rfindley): re-enable after moving to new framework + //_ = foo.StructFoo{s.} // complete("}", icFieldAB, icFieldAA) } /* "fmt" */ //@item(fmtImport, "fmt", "\"fmt\"", "package") diff --git a/gopls/internal/lsp/testdata/nodisk/empty b/gopls/internal/lsp/testdata/nodisk/empty deleted file mode 100644 index 0c10a42f942..00000000000 --- a/gopls/internal/lsp/testdata/nodisk/empty +++ /dev/null @@ -1 +0,0 @@ -an empty file so that this directory exists \ No newline at end of file diff --git a/gopls/internal/lsp/testdata/nodisk/nodisk.overlay.go b/gopls/internal/lsp/testdata/nodisk/nodisk.overlay.go deleted file mode 100644 index 08aebd12f7b..00000000000 --- a/gopls/internal/lsp/testdata/nodisk/nodisk.overlay.go +++ /dev/null @@ -1,9 +0,0 @@ -package nodisk - -import ( - "golang.org/lsptests/foo" -) - -func _() { - foo.Foo() //@complete("F", Foo, IntFoo, StructFoo) -} diff --git a/gopls/internal/lsp/testdata/selector/selector.go.in b/gopls/internal/lsp/testdata/selector/selector.go.in deleted file mode 100644 index b1498a08c77..00000000000 --- a/gopls/internal/lsp/testdata/selector/selector.go.in +++ /dev/null @@ -1,66 +0,0 @@ -// +build go1.11 - -package selector - -import ( - "golang.org/lsptests/bar" -) - -type S struct { - B, A, C int //@item(Bf, "B", "int", "field"),item(Af, "A", "int", "field"),item(Cf, "C", "int", "field") -} - -func _() { - _ = S{}.; //@complete(";", Af, Bf, Cf) -} - -type bob struct { a int } //@item(a, "a", "int", "field") -type george struct { b int } -type jack struct { c int } //@item(c, "c", "int", "field") -type jill struct { d int } - -func (b *bob) george() *george {} //@item(george, "george", "func() *george", "method") -func (g *george) jack() *jack {} -func (j *jack) jill() *jill {} //@item(jill, "jill", "func() *jill", "method") - -func _() { - b := &bob{} - y := b.george(). - jack(); - y.; //@complete(";", c, jill) -} - -func _() { - bar. //@complete(" /", Bar) - x := 5 - - var b *bob - b. //@complete(" /", a, george) - y, z := 5, 6 - - b. //@complete(" /", a, george) - y, z, a, b, c := 5, 6 -} - -func _() { - bar. //@complete(" /", Bar) - bar.Bar() - - bar. //@complete(" /", Bar) - go f() -} - -func _() { - var b *bob - if y != b. //@complete(" /", a, george) - z := 5 - - if z + y + 1 + b. //@complete(" /", a, george) - r, s, t := 4, 5 - - if y != b. //@complete(" /", a, george) - z = 5 - - if z + y + 1 + b. //@complete(" /", a, george) - r = 4 -} diff --git a/gopls/internal/lsp/testdata/snippets/literal_snippets.go.in b/gopls/internal/lsp/testdata/snippets/literal_snippets.go.in deleted file mode 100644 index c6e6c0fbd60..00000000000 --- a/gopls/internal/lsp/testdata/snippets/literal_snippets.go.in +++ /dev/null @@ -1,233 +0,0 @@ -package snippets - -import ( - "bytes" - "context" - "go/ast" - "net/http" - "sort" - - "golang.org/lsptests/foo" -) - -func _() { - []int{} //@item(litIntSlice, "[]int{}", "", "var") - &[]int{} //@item(litIntSliceAddr, "&[]int{}", "", "var") - make([]int, 0) //@item(makeIntSlice, "make([]int, 0)", "", "func") - - var _ *[]int = in //@snippet(" //", litIntSliceAddr, "&[]int{$0\\}", "&[]int{$0\\}") - var _ **[]int = in //@complete(" //") - - var slice []int - slice = i //@snippet(" //", litIntSlice, "[]int{$0\\}", "[]int{$0\\}") - slice = m //@snippet(" //", makeIntSlice, "make([]int, ${1:})", "make([]int, ${1:0})") -} - -func _() { - type namedInt []int - - namedInt{} //@item(litNamedSlice, "namedInt{}", "", "var") - make(namedInt, 0) //@item(makeNamedSlice, "make(namedInt, 0)", "", "func") - - var namedSlice namedInt - namedSlice = n //@snippet(" //", litNamedSlice, "namedInt{$0\\}", "namedInt{$0\\}") - namedSlice = m //@snippet(" //", makeNamedSlice, "make(namedInt, ${1:})", "make(namedInt, ${1:0})") -} - -func _() { - make(chan int) //@item(makeChan, "make(chan int)", "", "func") - - var ch chan int - ch = m //@snippet(" //", makeChan, "make(chan int)", "make(chan int)") -} - -func _() { - map[string]struct{}{} //@item(litMap, "map[string]struct{}{}", "", "var") - make(map[string]struct{}) //@item(makeMap, "make(map[string]struct{})", "", "func") - - var m map[string]struct{} - m = m //@snippet(" //", litMap, "map[string]struct{\\}{$0\\}", "map[string]struct{\\}{$0\\}") - m = m //@snippet(" //", makeMap, "make(map[string]struct{\\})", "make(map[string]struct{\\})") - - struct{}{} //@item(litEmptyStruct, "struct{}{}", "", "var") - - m["hi"] = s //@snippet(" //", litEmptyStruct, "struct{\\}{\\}", "struct{\\}{\\}") -} - -func _() { - type myStruct struct{ i int } //@item(myStructType, "myStruct", "struct{...}", "struct") - - myStruct{} //@item(litStruct, "myStruct{}", "", "var") - &myStruct{} //@item(litStructPtr, "&myStruct{}", "", "var") - - var ms myStruct - ms = m //@snippet(" //", litStruct, "myStruct{$0\\}", "myStruct{$0\\}") - - var msPtr *myStruct - msPtr = m //@snippet(" //", litStructPtr, "&myStruct{$0\\}", "&myStruct{$0\\}") - - msPtr = &m //@snippet(" //", litStruct, "myStruct{$0\\}", "myStruct{$0\\}") - - type myStructCopy struct { i int } //@item(myStructCopyType, "myStructCopy", "struct{...}", "struct") - - // Don't offer literal completion for convertible structs. - ms = myStruct //@complete(" //", litStruct, myStructType, myStructCopyType) -} - -type myImpl struct{} - -func (myImpl) foo() {} - -func (*myImpl) bar() {} - -type myBasicImpl string - -func (myBasicImpl) foo() {} - -func _() { - type myIntf interface { - foo() - } - - myImpl{} //@item(litImpl, "myImpl{}", "", "var") - - var mi myIntf - mi = m //@snippet(" //", litImpl, "myImpl{\\}", "myImpl{\\}") - - myBasicImpl() //@item(litBasicImpl, "myBasicImpl()", "string", "var") - - mi = m //@snippet(" //", litBasicImpl, "myBasicImpl($0)", "myBasicImpl($0)") - - // only satisfied by pointer to myImpl - type myPtrIntf interface { - bar() - } - - &myImpl{} //@item(litImplPtr, "&myImpl{}", "", "var") - - var mpi myPtrIntf - mpi = m //@snippet(" //", litImplPtr, "&myImpl{\\}", "&myImpl{\\}") -} - -func _() { - var s struct{ i []int } //@item(litSliceField, "i", "[]int", "field") - var foo []int - // no literal completions after selector - foo = s.i //@complete(" //", litSliceField) -} - -func _() { - type myStruct struct{ i int } //@item(litMyStructType, "myStruct", "struct{...}", "struct") - myStruct{} //@item(litMyStruct, "myStruct{}", "", "var") - - foo := func(s string, args ...myStruct) {} - // Don't give literal slice candidate for variadic arg. - // Do give literal candidates for variadic element. - foo("", myStruct) //@complete(")", litMyStruct, litMyStructType) -} - -func _() { - Buffer{} //@item(litBuffer, "Buffer{}", "", "var") - - var b *bytes.Buffer - b = bytes.Bu //@snippet(" //", litBuffer, "Buffer{\\}", "Buffer{\\}") -} - -func _() { - _ = "func(...) {}" //@item(litFunc, "func(...) {}", "", "var") - - sort.Slice(nil, fun) //@complete(")", litFunc),snippet(")", litFunc, "func(i, j int) bool {$0\\}", "func(i, j int) bool {$0\\}") - - http.HandleFunc("", f) //@snippet(")", litFunc, "func(w http.ResponseWriter, r *http.Request) {$0\\}", "func(${1:w} http.ResponseWriter, ${2:r} *http.Request) {$0\\}") - - // no literal "func" completions - http.Handle("", fun) //@complete(")") - - http.HandlerFunc() //@item(handlerFunc, "http.HandlerFunc()", "", "var") - http.Handle("", h) //@snippet(")", handlerFunc, "http.HandlerFunc($0)", "http.HandlerFunc($0)") - http.Handle("", http.HandlerFunc()) //@snippet("))", litFunc, "func(w http.ResponseWriter, r *http.Request) {$0\\}", "func(${1:w} http.ResponseWriter, ${2:r} *http.Request) {$0\\}") - - var namedReturn func(s string) (b bool) - namedReturn = f //@snippet(" //", litFunc, "func(s string) (b bool) {$0\\}", "func(s string) (b bool) {$0\\}") - - var multiReturn func() (bool, int) - multiReturn = f //@snippet(" //", litFunc, "func() (bool, int) {$0\\}", "func() (bool, int) {$0\\}") - - var multiNamedReturn func() (b bool, i int) - multiNamedReturn = f //@snippet(" //", litFunc, "func() (b bool, i int) {$0\\}", "func() (b bool, i int) {$0\\}") - - var duplicateParams func(myImpl, int, myImpl) - duplicateParams = f //@snippet(" //", litFunc, "func(mi1 myImpl, i int, mi2 myImpl) {$0\\}", "func(${1:mi1} myImpl, ${2:i} int, ${3:mi2} myImpl) {$0\\}") - - type aliasImpl = myImpl - var aliasParams func(aliasImpl) aliasImpl - aliasParams = f //@snippet(" //", litFunc, "func(ai aliasImpl) aliasImpl {$0\\}", "func(${1:ai} aliasImpl) aliasImpl {$0\\}") - - const two = 2 - var builtinTypes func([]int, [two]bool, map[string]string, struct{ i int }, interface{ foo() }, <-chan int) - builtinTypes = f //@snippet(" //", litFunc, "func(i1 []int, b [two]bool, m map[string]string, s struct{ i int \\}, i2 interface{ foo() \\}, c <-chan int) {$0\\}", "func(${1:i1} []int, ${2:b} [two]bool, ${3:m} map[string]string, ${4:s} struct{ i int \\}, ${5:i2} interface{ foo() \\}, ${6:c} <-chan int) {$0\\}") - - var _ func(ast.Node) = f //@snippet(" //", litFunc, "func(n ast.Node) {$0\\}", "func(${1:n} ast.Node) {$0\\}") - var _ func(error) = f //@snippet(" //", litFunc, "func(err error) {$0\\}", "func(${1:err} error) {$0\\}") - var _ func(context.Context) = f //@snippet(" //", litFunc, "func(ctx context.Context) {$0\\}", "func(${1:ctx} context.Context) {$0\\}") - - type context struct {} - var _ func(context) = f //@snippet(" //", litFunc, "func(ctx context) {$0\\}", "func(${1:ctx} context) {$0\\}") -} - -func _() { - StructFoo{} //@item(litStructFoo, "StructFoo{}", "struct{...}", "struct") - - var sfp *foo.StructFoo - // Don't insert the "&" before "StructFoo{}". - sfp = foo.Str //@snippet(" //", litStructFoo, "StructFoo{$0\\}", "StructFoo{$0\\}") - - var sf foo.StructFoo - sf = foo.Str //@snippet(" //", litStructFoo, "StructFoo{$0\\}", "StructFoo{$0\\}") - sf = foo. //@snippet(" //", litStructFoo, "StructFoo{$0\\}", "StructFoo{$0\\}") -} - -func _() { - float64() //@item(litFloat64, "float64()", "float64", "var") - - // don't complete to "&float64()" - var _ *float64 = float64 //@complete(" //") - - var f float64 - f = fl //@complete(" //", litFloat64),snippet(" //", litFloat64, "float64($0)", "float64($0)") - - type myInt int - myInt() //@item(litMyInt, "myInt()", "", "var") - - var mi myInt - mi = my //@snippet(" //", litMyInt, "myInt($0)", "myInt($0)") -} - -func _() { - type ptrStruct struct { - p *ptrStruct - } - - ptrStruct{} //@item(litPtrStruct, "ptrStruct{}", "", "var") - - ptrStruct{ - p: &ptrSt, //@rank(",", litPtrStruct) - } - - &ptrStruct{} //@item(litPtrStructPtr, "&ptrStruct{}", "", "var") - - &ptrStruct{ - p: ptrSt, //@rank(",", litPtrStructPtr) - } -} - -func _() { - f := func(...[]int) {} - f() //@snippet(")", litIntSlice, "[]int{$0\\}", "[]int{$0\\}") -} - - -func _() { - // don't complete to "untyped int()" - []int{}[untyped] //@complete("] //") -} diff --git a/gopls/internal/lsp/testdata/snippets/literal_snippets118.go.in b/gopls/internal/lsp/testdata/snippets/literal_snippets118.go.in deleted file mode 100644 index 8251a6384a3..00000000000 --- a/gopls/internal/lsp/testdata/snippets/literal_snippets118.go.in +++ /dev/null @@ -1,14 +0,0 @@ -// +build go1.18 -//go:build go1.18 - -package snippets - -type Tree[T any] struct{} - -func (tree Tree[T]) Do(f func(s T)) {} - -func _() { - _ = "func(...) {}" //@item(litFunc, "func(...) {}", "", "var") - var t Tree[string] - t.Do(fun) //@complete(")", litFunc),snippet(")", litFunc, "func(s string) {$0\\}", "func(s string) {$0\\}") -} diff --git a/gopls/internal/lsp/testdata/snippets/snippets.go.in b/gopls/internal/lsp/testdata/snippets/snippets.go.in index 58150c644ca..79bff334233 100644 --- a/gopls/internal/lsp/testdata/snippets/snippets.go.in +++ b/gopls/internal/lsp/testdata/snippets/snippets.go.in @@ -1,5 +1,8 @@ package snippets +// Pre-set this marker, as we don't have a "source" for it in this package. +/* Error() */ //@item(Error, "Error", "func() string", "method") + type AliasType = int //@item(sigAliasType, "AliasType", "AliasType", "type") func foo(i int, b bool) {} //@item(snipFoo, "foo", "func(i int, b bool)", "func") diff --git a/gopls/internal/lsp/testdata/summary.txt.golden b/gopls/internal/lsp/testdata/summary.txt.golden index 4da0693b855..7059a381d19 100644 --- a/gopls/internal/lsp/testdata/summary.txt.golden +++ b/gopls/internal/lsp/testdata/summary.txt.golden @@ -1,22 +1,18 @@ -- summary -- CallHierarchyCount = 2 -CompletionsCount = 258 -CompletionSnippetCount = 115 -UnimportedCompletionsCount = 5 +CompletionsCount = 194 +CompletionSnippetCount = 74 DeepCompletionsCount = 5 FuzzyCompletionsCount = 8 -RankedCompletionsCount = 174 +RankedCompletionsCount = 166 CaseSensitiveCompletionsCount = 4 -DiagnosticsCount = 3 SemanticTokenCount = 3 SuggestedFixCount = 80 MethodExtractionCount = 8 -DefinitionsCount = 46 -TypeDefinitionsCount = 18 InlayHintsCount = 5 RenamesCount = 48 PrepareRenamesCount = 7 -SignaturesCount = 33 +SignaturesCount = 32 LinksCount = 7 SelectionRangesCount = 3 diff --git a/gopls/internal/lsp/testdata/testy/testy.go b/gopls/internal/lsp/testdata/testy/testy.go deleted file mode 100644 index 9f74091af87..00000000000 --- a/gopls/internal/lsp/testdata/testy/testy.go +++ /dev/null @@ -1,5 +0,0 @@ -package testy - -func a() { //@item(funcA, "a", "func()", "func") - //@complete("", funcA) -} diff --git a/gopls/internal/lsp/testdata/testy/testy_test.go b/gopls/internal/lsp/testdata/testy/testy_test.go deleted file mode 100644 index 793aacfd825..00000000000 --- a/gopls/internal/lsp/testdata/testy/testy_test.go +++ /dev/null @@ -1,18 +0,0 @@ -package testy - -import ( - "testing" - - sig "golang.org/lsptests/signature" - "golang.org/lsptests/snippets" -) - -func TestSomething(t *testing.T) { //@item(TestSomething, "TestSomething(t *testing.T)", "", "func") - var x int //@mark(testyX, "x"),diag("x", "compiler", "x declared (and|but) not used", "error") - a() //@mark(testyA, "a") -} - -func _() { - _ = snippets.X(nil) //@signature("nil", "X(_ map[sig.Alias]types.CoolAlias) map[sig.Alias]types.CoolAlias", 0) - var _ sig.Alias -} diff --git a/gopls/internal/lsp/testdata/testy/testy_test.go.golden b/gopls/internal/lsp/testdata/testy/testy_test.go.golden deleted file mode 100644 index cafc380d065..00000000000 --- a/gopls/internal/lsp/testdata/testy/testy_test.go.golden +++ /dev/null @@ -1,3 +0,0 @@ --- X(_ map[sig.Alias]types.CoolAlias) map[sig.Alias]types.CoolAlias-signature -- -X(_ map[sig.Alias]types.CoolAlias) map[sig.Alias]types.CoolAlias - diff --git a/gopls/internal/lsp/testdata/typdef/typdef.go b/gopls/internal/lsp/testdata/typdef/typdef.go deleted file mode 100644 index bd2ea4b00c7..00000000000 --- a/gopls/internal/lsp/testdata/typdef/typdef.go +++ /dev/null @@ -1,65 +0,0 @@ -package typdef - -type Struct struct { //@item(Struct, "Struct", "struct{...}", "struct") - Field string -} - -type Int int //@item(Int, "Int", "int", "type") - -func _() { - var ( - value Struct - point *Struct - ) - _ = value //@typdef("value", Struct) - _ = point //@typdef("point", Struct) - - var ( - array [3]Struct - slice []Struct - ch chan Struct - complex [3]chan *[5][]Int - ) - _ = array //@typdef("array", Struct) - _ = slice //@typdef("slice", Struct) - _ = ch //@typdef("ch", Struct) - _ = complex //@typdef("complex", Int) - - var s struct { - x struct { - xx struct { - field1 []Struct - field2 []Int - } - } - } - s.x.xx.field1 //@typdef("field1", Struct) - s.x.xx.field2 //@typdef("field2", Int) -} - -func F1() Int { return 0 } -func F2() (Int, float64) { return 0, 0 } -func F3() (Struct, int, bool, error) { return Struct{}, 0, false, nil } -func F4() (**int, Int, bool, *error) { return nil, Struct{}, false, nil } -func F5() (int, float64, error, Struct) { return 0, 0, nil, Struct{} } -func F6() (int, float64, ***Struct, error) { return 0, 0, nil, nil } - -func _() { - F1() //@typdef("F1", Int) - F2() //@typdef("F2", Int) - F3() //@typdef("F3", Struct) - F4() //@typdef("F4", Int) - F5() //@typdef("F5", Struct) - F6() //@typdef("F6", Struct) - - f := func() Int { return 0 } - f() //@typdef("f", Int) -} - -// https://github.com/golang/go/issues/38589#issuecomment-620350922 -func _() { - type myFunc func(int) Int //@item(myFunc, "myFunc", "func", "type") - - var foo myFunc - bar := foo() //@typdef("foo", myFunc) -} diff --git a/gopls/internal/lsp/testdata/unimported/export_test.go b/gopls/internal/lsp/testdata/unimported/export_test.go deleted file mode 100644 index 707768e1da2..00000000000 --- a/gopls/internal/lsp/testdata/unimported/export_test.go +++ /dev/null @@ -1,3 +0,0 @@ -package unimported - -var TestExport int //@item(testexport, "TestExport", "var (from \"golang.org/lsptests/unimported\")", "var") diff --git a/gopls/internal/lsp/testdata/unimported/unimported.go.in b/gopls/internal/lsp/testdata/unimported/unimported.go.in deleted file mode 100644 index 74d51ffe82a..00000000000 --- a/gopls/internal/lsp/testdata/unimported/unimported.go.in +++ /dev/null @@ -1,23 +0,0 @@ -package unimported - -func _() { - http //@unimported("p", nethttp) - // container/ring is extremely unlikely to be imported by anything, so shouldn't have type information. - ring.Ring //@unimported("Ring", ringring) - signature.Foo //@unimported("Foo", signaturefoo) - - context.Bac //@unimported(" //", contextBackground) -} - -// Create markers for unimported std lib packages. Only for use by this test. -/* http */ //@item(nethttp, "http", "\"net/http\"", "package") - -/* ring.Ring */ //@item(ringring, "Ring", "(from \"container/ring\")", "var") - -/* signature.Foo */ //@item(signaturefoo, "Foo", "func (from \"golang.org/lsptests/signature\")", "func") - -/* context.Background */ //@item(contextBackground, "Background", "func (from \"context\")", "func") - -// Now that we no longer type-check imported completions, -// we don't expect the context.Background().Err method (see golang/go#58663). -/* context.Background().Err */ //@item(contextBackgroundErr, "Background().Err", "func (from \"context\")", "method") diff --git a/gopls/internal/lsp/testdata/unimported/unimported_cand_type.go b/gopls/internal/lsp/testdata/unimported/unimported_cand_type.go deleted file mode 100644 index 554c426a998..00000000000 --- a/gopls/internal/lsp/testdata/unimported/unimported_cand_type.go +++ /dev/null @@ -1,16 +0,0 @@ -package unimported - -import ( - _ "context" - - "golang.org/lsptests/baz" - _ "golang.org/lsptests/signature" // provide type information for unimported completions in the other file -) - -func _() { - foo.StructFoo{} //@item(litFooStructFoo, "foo.StructFoo{}", "struct{...}", "struct") - - // We get the literal completion for "foo.StructFoo{}" even though we haven't - // imported "foo" yet. - baz.FooStruct = f //@snippet(" //", litFooStructFoo, "foo.StructFoo{$0\\}", "foo.StructFoo{$0\\}") -} diff --git a/gopls/internal/lsp/testdata/unimported/x_test.go b/gopls/internal/lsp/testdata/unimported/x_test.go deleted file mode 100644 index 681dcb2536d..00000000000 --- a/gopls/internal/lsp/testdata/unimported/x_test.go +++ /dev/null @@ -1,9 +0,0 @@ -package unimported_test - -import ( - "testing" -) - -func TestSomething(t *testing.T) { - _ = unimported.TestExport //@unimported("TestExport", testexport) -} diff --git a/gopls/internal/lsp/tests/tests.go b/gopls/internal/lsp/tests/tests.go index 2a0bda3623d..5b7074fe6fa 100644 --- a/gopls/internal/lsp/tests/tests.go +++ b/gopls/internal/lsp/tests/tests.go @@ -55,11 +55,9 @@ var UpdateGolden = flag.Bool("golden", false, "Update golden files") // These type names apparently avoid the need to repeat the // type in the field name and the make() expression. type CallHierarchy = map[span.Span]*CallHierarchyResult -type Diagnostics = map[span.URI][]*source.Diagnostic type CompletionItems = map[token.Pos]*completion.CompletionItem type Completions = map[span.Span][]Completion type CompletionSnippets = map[span.Span][]CompletionSnippet -type UnimportedCompletions = map[span.Span][]Completion type DeepCompletions = map[span.Span][]Completion type FuzzyCompletions = map[span.Span][]Completion type CaseSensitiveCompletions = map[span.Span][]Completion @@ -67,7 +65,6 @@ type RankCompletions = map[span.Span][]Completion type SemanticTokens = []span.Span type SuggestedFixes = map[span.Span][]SuggestedFix type MethodExtractions = map[span.Span]span.Span -type Definitions = map[span.Span]Definition type Renames = map[span.Span]string type PrepareRenames = map[span.Span]*source.PrepareItem type InlayHints = []span.Span @@ -80,11 +77,9 @@ type Data struct { Config packages.Config Exported *packagestest.Exported CallHierarchy CallHierarchy - Diagnostics Diagnostics CompletionItems CompletionItems Completions Completions CompletionSnippets CompletionSnippets - UnimportedCompletions UnimportedCompletions DeepCompletions DeepCompletions FuzzyCompletions FuzzyCompletions CaseSensitiveCompletions CaseSensitiveCompletions @@ -92,7 +87,6 @@ type Data struct { SemanticTokens SemanticTokens SuggestedFixes SuggestedFixes MethodExtractions MethodExtractions - Definitions Definitions Renames Renames InlayHints InlayHints PrepareRenames PrepareRenames @@ -120,10 +114,8 @@ type Data struct { // we can abolish the interface now. type Tests interface { CallHierarchy(*testing.T, span.Span, *CallHierarchyResult) - Diagnostics(*testing.T, span.URI, []*source.Diagnostic) Completion(*testing.T, span.Span, Completion, CompletionItems) CompletionSnippet(*testing.T, span.Span, CompletionSnippet, bool, CompletionItems) - UnimportedCompletion(*testing.T, span.Span, Completion, CompletionItems) DeepCompletion(*testing.T, span.Span, Completion, CompletionItems) FuzzyCompletion(*testing.T, span.Span, Completion, CompletionItems) CaseSensitiveCompletion(*testing.T, span.Span, Completion, CompletionItems) @@ -131,7 +123,6 @@ type Tests interface { SemanticTokens(*testing.T, span.Span) SuggestedFix(*testing.T, span.Span, []SuggestedFix, int) MethodExtraction(*testing.T, span.Span, span.Span) - Definition(*testing.T, span.Span, Definition) InlayHints(*testing.T, span.Span) Rename(*testing.T, span.Span, string) PrepareRename(*testing.T, span.Span, *source.PrepareItem) @@ -141,22 +132,12 @@ type Tests interface { SelectionRanges(*testing.T, span.Span) } -type Definition struct { - Name string - IsType bool - OnlyHover bool - Src, Def span.Span -} - type CompletionTestType int const ( // Default runs the standard completion tests. CompletionDefault = CompletionTestType(iota) - // Unimported tests the autocompletion of unimported packages. - CompletionUnimported - // Deep tests deep completion. CompletionDeep @@ -221,7 +202,6 @@ func DefaultOptions(o *source.Options) { source.Work: {}, source.Tmpl: {}, } - o.HoverKind = source.SynopsisDocumentation o.InsertTextFormat = protocol.SnippetTextFormat o.CompletionBudget = time.Minute o.HierarchicalDocumentSymbolSupport = true @@ -255,16 +235,13 @@ func RunTests(t *testing.T, dataDir string, includeMultiModule bool, f func(*tes func load(t testing.TB, mode string, dir string) *Data { datum := &Data{ CallHierarchy: make(CallHierarchy), - Diagnostics: make(Diagnostics), CompletionItems: make(CompletionItems), Completions: make(Completions), CompletionSnippets: make(CompletionSnippets), - UnimportedCompletions: make(UnimportedCompletions), DeepCompletions: make(DeepCompletions), FuzzyCompletions: make(FuzzyCompletions), RankCompletions: make(RankCompletions), CaseSensitiveCompletions: make(CaseSensitiveCompletions), - Definitions: make(Definitions), Renames: make(Renames), PrepareRenames: make(PrepareRenames), SuggestedFixes: make(SuggestedFixes), @@ -404,19 +381,14 @@ func load(t testing.TB, mode string, dir string) *Data { // Collect any data that needs to be used by subsequent tests. if err := datum.Exported.Expect(map[string]interface{}{ - "diag": datum.collectDiagnostics, "item": datum.collectCompletionItems, "complete": datum.collectCompletions(CompletionDefault), - "unimported": datum.collectCompletions(CompletionUnimported), "deep": datum.collectCompletions(CompletionDeep), "fuzzy": datum.collectCompletions(CompletionFuzzy), "casesensitive": datum.collectCompletions(CompletionCaseSensitive), "rank": datum.collectCompletions(CompletionRank), "snippet": datum.collectCompletionSnippets, "semantic": datum.collectSemanticTokens, - "godef": datum.collectDefinitions, - "typdef": datum.collectTypeDefinitions, - "hoverdef": datum.collectHoverDefinitions, "inlayHint": datum.collectInlayHints, "rename": datum.collectRenames, "prepare": datum.collectPrepareRenames, @@ -432,13 +404,6 @@ func load(t testing.TB, mode string, dir string) *Data { t.Fatal(err) } - // Collect names for the entries that require golden files. - if err := datum.Exported.Expect(map[string]interface{}{ - "godef": datum.collectDefinitionNames, - "hoverdef": datum.collectDefinitionNames, - }); err != nil { - t.Fatal(err) - } if mode == "MultiModule" { if err := moveFile(filepath.Join(datum.Config.Dir, "go.mod"), filepath.Join(datum.Config.Dir, "testmodule/go.mod")); err != nil { t.Fatal(err) @@ -540,11 +505,6 @@ func Run(t *testing.T, tests Tests, data *Data) { } }) - t.Run("UnimportedCompletion", func(t *testing.T) { - t.Helper() - eachCompletion(t, data.UnimportedCompletions, tests.UnimportedCompletion) - }) - t.Run("DeepCompletion", func(t *testing.T) { t.Helper() eachCompletion(t, data.DeepCompletions, tests.DeepCompletion) @@ -565,20 +525,6 @@ func Run(t *testing.T, tests Tests, data *Data) { eachCompletion(t, data.RankCompletions, tests.RankCompletion) }) - t.Run("Diagnostics", func(t *testing.T) { - t.Helper() - for uri, want := range data.Diagnostics { - // Check if we should skip this URI if the -modfile flag is not available. - if shouldSkip(data, uri) { - continue - } - t.Run(uriName(uri), func(t *testing.T) { - t.Helper() - tests.Diagnostics(t, uri, want) - }) - } - }) - t.Run("SemanticTokens", func(t *testing.T) { t.Helper() for _, spn := range data.SemanticTokens { @@ -617,19 +563,6 @@ func Run(t *testing.T, tests Tests, data *Data) { } }) - t.Run("Definition", func(t *testing.T) { - t.Helper() - for spn, d := range data.Definitions { - t.Run(SpanName(spn), func(t *testing.T) { - t.Helper() - if strings.Contains(t.Name(), "cgo") { - testenv.NeedsTool(t, "cgo") - } - tests.Definition(t, spn, d) - }) - } - }) - t.Run("InlayHints", func(t *testing.T) { t.Helper() for _, src := range data.InlayHints { @@ -727,23 +660,10 @@ func Run(t *testing.T, tests Tests, data *Data) { func checkData(t *testing.T, data *Data) { buf := &bytes.Buffer{} - diagnosticsCount := 0 - for _, want := range data.Diagnostics { - diagnosticsCount += len(want) - } linksCount := 0 for _, want := range data.Links { linksCount += len(want) } - definitionCount := 0 - typeDefinitionCount := 0 - for _, d := range data.Definitions { - if d.IsType { - typeDefinitionCount++ - } else { - definitionCount++ - } - } snippetCount := 0 for _, want := range data.CompletionSnippets { @@ -760,17 +680,13 @@ func checkData(t *testing.T, data *Data) { fmt.Fprintf(buf, "CallHierarchyCount = %v\n", len(data.CallHierarchy)) fmt.Fprintf(buf, "CompletionsCount = %v\n", countCompletions(data.Completions)) fmt.Fprintf(buf, "CompletionSnippetCount = %v\n", snippetCount) - fmt.Fprintf(buf, "UnimportedCompletionsCount = %v\n", countCompletions(data.UnimportedCompletions)) fmt.Fprintf(buf, "DeepCompletionsCount = %v\n", countCompletions(data.DeepCompletions)) fmt.Fprintf(buf, "FuzzyCompletionsCount = %v\n", countCompletions(data.FuzzyCompletions)) fmt.Fprintf(buf, "RankedCompletionsCount = %v\n", countCompletions(data.RankCompletions)) fmt.Fprintf(buf, "CaseSensitiveCompletionsCount = %v\n", countCompletions(data.CaseSensitiveCompletions)) - fmt.Fprintf(buf, "DiagnosticsCount = %v\n", diagnosticsCount) fmt.Fprintf(buf, "SemanticTokenCount = %v\n", len(data.SemanticTokens)) fmt.Fprintf(buf, "SuggestedFixCount = %v\n", len(data.SuggestedFixes)) fmt.Fprintf(buf, "MethodExtractionCount = %v\n", len(data.MethodExtractions)) - fmt.Fprintf(buf, "DefinitionsCount = %v\n", definitionCount) - fmt.Fprintf(buf, "TypeDefinitionsCount = %v\n", typeDefinitionCount) fmt.Fprintf(buf, "InlayHintsCount = %v\n", len(data.InlayHints)) fmt.Fprintf(buf, "RenamesCount = %v\n", len(data.Renames)) fmt.Fprintf(buf, "PrepareRenamesCount = %v\n", len(data.PrepareRenames)) @@ -857,27 +773,6 @@ func (data *Data) Golden(t *testing.T, tag, target string, update func() ([]byte return file.Data[:len(file.Data)-1] // drop the trailing \n } -func (data *Data) collectDiagnostics(spn span.Span, msgSource, msgPattern, msgSeverity string) { - severity := protocol.SeverityError - switch msgSeverity { - case "error": - severity = protocol.SeverityError - case "warning": - severity = protocol.SeverityWarning - case "hint": - severity = protocol.SeverityHint - case "information": - severity = protocol.SeverityInformation - } - - data.Diagnostics[spn.URI()] = append(data.Diagnostics[spn.URI()], &source.Diagnostic{ - Range: data.mustRange(spn), - Severity: severity, - Source: source.DiagnosticSource(msgSource), - Message: msgPattern, - }) -} - func (data *Data) collectCompletions(typ CompletionTestType) func(span.Span, []token.Pos) { result := func(m map[span.Span][]Completion, src span.Span, expected []token.Pos) { m[src] = append(m[src], Completion{ @@ -889,10 +784,6 @@ func (data *Data) collectCompletions(typ CompletionTestType) func(span.Span, []t return func(src span.Span, expected []token.Pos) { result(data.DeepCompletions, src, expected) } - case CompletionUnimported: - return func(src span.Span, expected []token.Pos) { - result(data.UnimportedCompletions, src, expected) - } case CompletionFuzzy: return func(src span.Span, expected []token.Pos) { result(data.FuzzyCompletions, src, expected) @@ -943,13 +834,6 @@ func (data *Data) collectMethodExtractions(start span.Span, end span.Span) { } } -func (data *Data) collectDefinitions(src, target span.Span) { - data.Definitions[src] = Definition{ - Src: src, - Def: target, - } -} - func (data *Data) collectSelectionRanges(spn span.Span) { data.SelectionRanges = append(data.SelectionRanges, spn) } @@ -988,28 +872,6 @@ func (data *Data) collectOutgoingCalls(src span.Span, calls []span.Span) { } } -func (data *Data) collectHoverDefinitions(src, target span.Span) { - data.Definitions[src] = Definition{ - Src: src, - Def: target, - OnlyHover: true, - } -} - -func (data *Data) collectTypeDefinitions(src, target span.Span) { - data.Definitions[src] = Definition{ - Src: src, - Def: target, - IsType: true, - } -} - -func (data *Data) collectDefinitionNames(src span.Span, name string) { - d := data.Definitions[src] - d.Name = name - data.Definitions[src] = d -} - func (data *Data) collectInlayHints(src span.Span) { data.InlayHints = append(data.InlayHints, src) } diff --git a/gopls/internal/lsp/tests/util.go b/gopls/internal/lsp/tests/util.go index 67b939087c6..b9a21fe9627 100644 --- a/gopls/internal/lsp/tests/util.go +++ b/gopls/internal/lsp/tests/util.go @@ -8,17 +8,13 @@ import ( "bytes" "fmt" "go/token" - "path" - "regexp" "sort" "strconv" "strings" - "testing" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "golang.org/x/tools/gopls/internal/lsp/protocol" - "golang.org/x/tools/gopls/internal/lsp/source" "golang.org/x/tools/gopls/internal/lsp/source/completion" "golang.org/x/tools/gopls/internal/lsp/tests/compare" "golang.org/x/tools/gopls/internal/span" @@ -92,73 +88,6 @@ func DiffLinks(mapper *protocol.Mapper, wantLinks []Link, gotLinks []protocol.Do return msg.String() } -// CompareDiagnostics reports testing errors to t when the diagnostic set got -// does not match want. -func CompareDiagnostics(t *testing.T, uri span.URI, want, got []*source.Diagnostic) { - t.Helper() - fileName := path.Base(string(uri)) - - // Build a helper function to match an actual diagnostic to an overlapping - // expected diagnostic (if any). - unmatched := make([]*source.Diagnostic, len(want)) - copy(unmatched, want) - source.SortDiagnostics(unmatched) - match := func(g *source.Diagnostic) *source.Diagnostic { - // Find the last expected diagnostic d for which start(d) < end(g), and - // check to see if it overlaps. - i := sort.Search(len(unmatched), func(i int) bool { - d := unmatched[i] - // See rangeOverlaps: if a range is a single point, we consider End to be - // included in the range... - if g.Range.Start == g.Range.End { - return protocol.ComparePosition(d.Range.Start, g.Range.End) > 0 - } - // ...otherwise the end position of a range is not included. - return protocol.ComparePosition(d.Range.Start, g.Range.End) >= 0 - }) - if i == 0 { - return nil - } - w := unmatched[i-1] - if rangeOverlaps(w.Range, g.Range) { - unmatched = append(unmatched[:i-1], unmatched[i:]...) - return w - } - return nil - } - - for _, g := range got { - w := match(g) - if w == nil { - t.Errorf("%s:%s: unexpected diagnostic %q", fileName, g.Range, g.Message) - continue - } - if match, err := regexp.MatchString(w.Message, g.Message); err != nil { - t.Errorf("%s:%s: invalid regular expression %q: %v", fileName, w.Range.Start, w.Message, err) - } else if !match { - t.Errorf("%s:%s: got Message %q, want match for pattern %q", fileName, g.Range.Start, g.Message, w.Message) - } - if w.Severity != g.Severity { - t.Errorf("%s:%s: got Severity %v, want %v", fileName, g.Range.Start, g.Severity, w.Severity) - } - if w.Source != g.Source { - t.Errorf("%s:%s: got Source %v, want %v", fileName, g.Range.Start, g.Source, w.Source) - } - } - - for _, w := range unmatched { - t.Errorf("%s:%s: unmatched diagnostic pattern %q", fileName, w.Range, w.Message) - } -} - -// rangeOverlaps reports whether r1 and r2 overlap. -func rangeOverlaps(r1, r2 protocol.Range) bool { - if inRange(r2.Start, r1) || inRange(r1.Start, r2) { - return true - } - return false -} - // inRange reports whether p is contained within [r.Start, r.End), or if p == // r.Start == r.End (special handling for the case where the range is a single // point). diff --git a/gopls/internal/regtest/completion/completion_test.go b/gopls/internal/regtest/completion/completion_test.go index 4294b92c2db..3949578a597 100644 --- a/gopls/internal/regtest/completion/completion_test.go +++ b/gopls/internal/regtest/completion/completion_test.go @@ -911,7 +911,44 @@ func _() { sort.Strings(got) if diff := cmp.Diff(builtins, got); diff != "" { - t.Errorf("Completion: unexpected mismatch:\n%s", diff) + t.Errorf("Completion: unexpected mismatch (-want +got):\n%s", diff) + } + }) +} + +func TestOverlayCompletion(t *testing.T) { + const files = ` +-- go.mod -- +module foo.test + +go 1.18 + +-- foo/foo.go -- +package foo + +type Foo struct{} +` + + Run(t, files, func(t *testing.T, env *Env) { + env.CreateBuffer("nodisk/nodisk.go", ` +package nodisk + +import ( + "foo.test/foo" +) + +func _() { + foo.Foo() +} +`) + list := env.Completion(env.RegexpSearch("nodisk/nodisk.go", "foo.(Foo)")) + want := []string{"Foo"} + var got []string + for _, item := range list.Items { + got = append(got, item.Label) + } + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("Completion: unexpected mismatch (-want +got):\n%s", diff) } }) } diff --git a/gopls/internal/regtest/marker/testdata/completion/foobarbaz.txt b/gopls/internal/regtest/marker/testdata/completion/foobarbaz.txt new file mode 100644 index 00000000000..24ac7171055 --- /dev/null +++ b/gopls/internal/regtest/marker/testdata/completion/foobarbaz.txt @@ -0,0 +1,541 @@ +This test ports some arbitrary tests from the old marker framework, that were +*mostly* about completion. + +-- flags -- +-ignore_extra_diags +-min_go=go1.20 + +-- settings.json -- +{ + "completeUnimported": false, + "deepCompletion": false, + "experimentalPostfixCompletions": false +} + +-- go.mod -- +module foobar.test + +go 1.18 + +-- foo/foo.go -- +package foo //@loc(PackageFoo, "foo"),item(PackageFooItem, "foo", "\"foobar.test/foo\"", "package") + +type StructFoo struct { //@loc(StructFooLoc, "StructFoo"), item(StructFoo, "StructFoo", "struct{...}", "struct") + Value int //@item(Value, "Value", "int", "field") +} + +// Pre-set this marker, as we don't have a "source" for it in this package. +/* Error() */ //@item(Error, "Error", "func() string", "method") + +func Foo() { //@item(Foo, "Foo", "func()", "func") + var err error + err.Error() //@complete("E", Error) +} + +func _() { + var sFoo StructFoo //@complete("t", StructFoo) + if x := sFoo; x.Value == 1 { //@complete("V", Value), typedef("sFoo", StructFooLoc) + return + } +} + +func _() { + shadowed := 123 + { + shadowed := "hi" //@item(shadowed, "shadowed", "string", "var") + sha //@complete("a", shadowed), diag("sha", re"(undefined|undeclared)") + _ = shadowed + } +} + +type IntFoo int //@loc(IntFooLoc, "IntFoo"), item(IntFoo, "IntFoo", "int", "type") + +-- bar/bar.go -- +package bar + +import ( + "foobar.test/foo" //@item(foo, "foo", "\"foobar.test/foo\"", "package") +) + +func helper(i foo.IntFoo) {} //@item(helper, "helper", "func(i foo.IntFoo)", "func") + +func _() { + help //@complete("l", helper) + _ = foo.StructFoo{} //@complete("S", IntFoo, StructFoo) +} + +// Bar is a function. +func Bar() { //@item(Bar, "Bar", "func()", "func", "Bar is a function.") + foo.Foo() //@complete("F", Foo, IntFoo, StructFoo) + var _ foo.IntFoo //@complete("I", IntFoo, StructFoo) + foo.() //@complete("(", Foo, IntFoo, StructFoo), diag(")", re"expected type") +} + +// These items weren't present in the old marker tests (due to settings), but +// we may as well include them. +//@item(intConversion, "int()"), item(fooFoo, "foo.Foo") +//@item(fooIntFoo, "foo.IntFoo"), item(fooStructFoo, "foo.StructFoo") + +func _() { + var Valentine int //@item(Valentine, "Valentine", "int", "var") + + _ = foo.StructFoo{ //@diag("foo", re"unkeyed fields") + Valu //@complete(" //", Value) + } + _ = foo.StructFoo{ //@diag("foo", re"unkeyed fields") + Va //@complete("a", Value, Valentine) + + } + _ = foo.StructFoo{ + Value: 5, //@complete("a", Value) + } + _ = foo.StructFoo{ + //@complete("//", Value, Valentine, intConversion, foo, helper, Bar) + } + _ = foo.StructFoo{ + Value: Valen //@complete("le", Valentine) + } + _ = foo.StructFoo{ + Value: //@complete(" //", Valentine, intConversion, foo, helper, Bar) + } + _ = foo.StructFoo{ + Value: //@complete(" ", Valentine, intConversion, foo, helper, Bar) + } +} + +-- baz/baz.go -- +package baz + +import ( + "foobar.test/bar" + + f "foobar.test/foo" +) + +var FooStruct f.StructFoo + +func Baz() { + defer bar.Bar() //@complete("B", Bar) + // TODO: Test completion here. + defer bar.B //@diag(re"bar.B()", re"must be function call") + var x f.IntFoo //@complete("n", IntFoo), typedef("x", IntFooLoc) + bar.Bar() //@complete("B", Bar) +} + +func _() { + bob := f.StructFoo{Value: 5} + if x := bob. //@complete(" //", Value) + switch true == false { + case true: + if x := bob. //@complete(" //", Value) + case false: + } + if x := bob.Va //@complete("a", Value) + switch true == true { + default: + } +} + +-- arraytype/arraytype.go -- +package arraytype + +import ( + "foobar.test/foo" +) + +func _() { + var ( + val string //@item(atVal, "val", "string", "var") + ) + + [] //@complete(" //", atVal, PackageFooItem) + + []val //@complete(" //") + + []foo.StructFoo //@complete(" //", StructFoo) + + []foo.StructFoo(nil) //@complete("(", StructFoo) + + []*foo.StructFoo //@complete(" //", StructFoo) + + [...]foo.StructFoo //@complete(" //", StructFoo) + + [2][][4]foo.StructFoo //@complete(" //", StructFoo) + + []struct { f []foo.StructFoo } //@complete(" }", StructFoo) +} + +func _() { + type myInt int //@item(atMyInt, "myInt", "int", "type") + + var mark []myInt //@item(atMark, "mark", "[]myInt", "var") + + var s []myInt //@item(atS, "s", "[]myInt", "var") + s = []m //@complete(" //", atMyInt) + + var a [1]myInt + a = [1]m //@complete(" //", atMyInt) + + var ds [][]myInt + ds = [][]m //@complete(" //", atMyInt) +} + +func _() { + var b [0]byte //@item(atByte, "b", "[0]byte", "var") + var _ []byte = b //@snippet(" //", atByte, "b[:]") +} + +-- badstmt/badstmt.go -- +package badstmt + +import ( + "foobar.test/foo" +) + +// (The syntax error causes suppression of diagnostics for type errors. +// See issue #59888.) + +func _(x int) { + defer foo.F //@complete(" //", Foo, IntFoo, StructFoo) + defer foo.F //@complete(" //", Foo, IntFoo, StructFoo) +} + +func _() { + switch true { + case true: + go foo.F //@complete(" //", Foo, IntFoo, StructFoo) + } +} + +func _() { + defer func() { + foo.F //@complete(" //", Foo, IntFoo, StructFoo), snippet(" //", Foo, "Foo()") + + foo. //@rank(" //", Foo) + } +} + +-- badstmt/badstmt_2.go -- +package badstmt + +import ( + "foobar.test/foo" +) + +func _() { + defer func() { foo. } //@rank(" }", Foo) +} + +-- badstmt/badstmt_3.go -- +package badstmt + +import ( + "foobar.test/foo" +) + +func _() { + go foo. //@rank(" //", Foo, IntFoo), snippet(" //", Foo, "Foo()") +} + +-- badstmt/badstmt_4.go -- +package badstmt + +import ( + "foobar.test/foo" +) + +func _() { + go func() { + defer foo. //@rank(" //", Foo, IntFoo) + } +} + +-- selector/selector.go -- +package selector + +import ( + "foobar.test/bar" +) + +type S struct { + B, A, C int //@item(Bf, "B", "int", "field"),item(Af, "A", "int", "field"),item(Cf, "C", "int", "field") +} + +func _() { + _ = S{}.; //@complete(";", Af, Bf, Cf) +} + +type bob struct { a int } //@item(a, "a", "int", "field") +type george struct { b int } +type jack struct { c int } //@item(c, "c", "int", "field") +type jill struct { d int } + +func (b *bob) george() *george {} //@item(george, "george", "func() *george", "method") +func (g *george) jack() *jack {} +func (j *jack) jill() *jill {} //@item(jill, "jill", "func() *jill", "method") + +func _() { + b := &bob{} + y := b.george(). + jack(); + y.; //@complete(";", c, jill) +} + +func _() { + bar. //@complete(" /", Bar) + x := 5 + + var b *bob + b. //@complete(" /", a, george) + y, z := 5, 6 + + b. //@complete(" /", a, george) + y, z, a, b, c := 5, 6 +} + +func _() { + bar. //@complete(" /", Bar) + bar.Bar() + + bar. //@complete(" /", Bar) + go f() +} + +func _() { + var b *bob + if y != b. //@complete(" /", a, george) + z := 5 + + if z + y + 1 + b. //@complete(" /", a, george) + r, s, t := 4, 5 + + if y != b. //@complete(" /", a, george) + z = 5 + + if z + y + 1 + b. //@complete(" /", a, george) + r = 4 +} + +-- literal_snippets/literal_snippets.go -- +package literal_snippets + +import ( + "bytes" + "context" + "go/ast" + "net/http" + "sort" + + "golang.org/lsptests/foo" +) + +func _() { + []int{} //@item(litIntSlice, "[]int{}", "", "var") + &[]int{} //@item(litIntSliceAddr, "&[]int{}", "", "var") + make([]int, 0) //@item(makeIntSlice, "make([]int, 0)", "", "func") + + var _ *[]int = in //@snippet(" //", litIntSliceAddr, "&[]int{$0\\}") + var _ **[]int = in //@complete(" //") + + var slice []int + slice = i //@snippet(" //", litIntSlice, "[]int{$0\\}") + slice = m //@snippet(" //", makeIntSlice, "make([]int, ${1:})") +} + +func _() { + type namedInt []int + + namedInt{} //@item(litNamedSlice, "namedInt{}", "", "var") + make(namedInt, 0) //@item(makeNamedSlice, "make(namedInt, 0)", "", "func") + + var namedSlice namedInt + namedSlice = n //@snippet(" //", litNamedSlice, "namedInt{$0\\}") + namedSlice = m //@snippet(" //", makeNamedSlice, "make(namedInt, ${1:})") +} + +func _() { + make(chan int) //@item(makeChan, "make(chan int)", "", "func") + + var ch chan int + ch = m //@snippet(" //", makeChan, "make(chan int)") +} + +func _() { + map[string]struct{}{} //@item(litMap, "map[string]struct{}{}", "", "var") + make(map[string]struct{}) //@item(makeMap, "make(map[string]struct{})", "", "func") + + var m map[string]struct{} + m = m //@snippet(" //", litMap, "map[string]struct{\\}{$0\\}") + m = m //@snippet(" //", makeMap, "make(map[string]struct{\\})") + + struct{}{} //@item(litEmptyStruct, "struct{}{}", "", "var") + + m["hi"] = s //@snippet(" //", litEmptyStruct, "struct{\\}{\\}") +} + +func _() { + type myStruct struct{ i int } //@item(myStructType, "myStruct", "struct{...}", "struct") + + myStruct{} //@item(litStruct, "myStruct{}", "", "var") + &myStruct{} //@item(litStructPtr, "&myStruct{}", "", "var") + + var ms myStruct + ms = m //@snippet(" //", litStruct, "myStruct{$0\\}") + + var msPtr *myStruct + msPtr = m //@snippet(" //", litStructPtr, "&myStruct{$0\\}") + + msPtr = &m //@snippet(" //", litStruct, "myStruct{$0\\}") + + type myStructCopy struct { i int } //@item(myStructCopyType, "myStructCopy", "struct{...}", "struct") + + // Don't offer literal completion for convertible structs. + ms = myStruct //@complete(" //", litStruct, myStructType, myStructCopyType) +} + +type myImpl struct{} + +func (myImpl) foo() {} + +func (*myImpl) bar() {} + +type myBasicImpl string + +func (myBasicImpl) foo() {} + +func _() { + type myIntf interface { + foo() + } + + myImpl{} //@item(litImpl, "myImpl{}", "", "var") + + var mi myIntf + mi = m //@snippet(" //", litImpl, "myImpl{\\}") + + myBasicImpl() //@item(litBasicImpl, "myBasicImpl()", "string", "var") + + mi = m //@snippet(" //", litBasicImpl, "myBasicImpl($0)") + + // only satisfied by pointer to myImpl + type myPtrIntf interface { + bar() + } + + &myImpl{} //@item(litImplPtr, "&myImpl{}", "", "var") + + var mpi myPtrIntf + mpi = m //@snippet(" //", litImplPtr, "&myImpl{\\}") +} + +func _() { + var s struct{ i []int } //@item(litSliceField, "i", "[]int", "field") + var foo []int + // no literal completions after selector + foo = s.i //@complete(" //", litSliceField) +} + +func _() { + type myStruct struct{ i int } //@item(litMyStructType, "myStruct", "struct{...}", "struct") + myStruct{} //@item(litMyStruct, "myStruct{}", "", "var") + + foo := func(s string, args ...myStruct) {} + // Don't give literal slice candidate for variadic arg. + // Do give literal candidates for variadic element. + foo("", myStruct) //@complete(")", litMyStruct, litMyStructType) +} + +func _() { + Buffer{} //@item(litBuffer, "Buffer{}", "", "var") + + var b *bytes.Buffer + b = bytes.Bu //@snippet(" //", litBuffer, "Buffer{\\}") +} + +func _() { + _ = "func(...) {}" //@item(litFunc, "func(...) {}", "", "var") + + // no literal "func" completions + http.Handle("", fun) //@complete(")") + + var namedReturn func(s string) (b bool) + namedReturn = f //@snippet(" //", litFunc, "func(s string) (b bool) {$0\\}") + + var multiReturn func() (bool, int) + multiReturn = f //@snippet(" //", litFunc, "func() (bool, int) {$0\\}") + + var multiNamedReturn func() (b bool, i int) + multiNamedReturn = f //@snippet(" //", litFunc, "func() (b bool, i int) {$0\\}") + + var duplicateParams func(myImpl, int, myImpl) + duplicateParams = f //@snippet(" //", litFunc, "func(mi1 myImpl, i int, mi2 myImpl) {$0\\}") + + type aliasImpl = myImpl + var aliasParams func(aliasImpl) aliasImpl + aliasParams = f //@snippet(" //", litFunc, "func(ai aliasImpl) aliasImpl {$0\\}") + + const two = 2 + var builtinTypes func([]int, [two]bool, map[string]string, struct{ i int }, interface{ foo() }, <-chan int) + builtinTypes = f //@snippet(" //", litFunc, "func(i1 []int, b [two]bool, m map[string]string, s struct{ i int \\}, i2 interface{ foo() \\}, c <-chan int) {$0\\}") + + var _ func(ast.Node) = f //@snippet(" //", litFunc, "func(n ast.Node) {$0\\}") + var _ func(error) = f //@snippet(" //", litFunc, "func(err error) {$0\\}") + var _ func(context.Context) = f //@snippet(" //", litFunc, "func(ctx context.Context) {$0\\}") + + type context struct {} + var _ func(context) = f //@snippet(" //", litFunc, "func(ctx context) {$0\\}") +} + +func _() { + float64() //@item(litFloat64, "float64()", "float64", "var") + + // don't complete to "&float64()" + var _ *float64 = float64 //@complete(" //") + + var f float64 + f = fl //@complete(" //", litFloat64),snippet(" //", litFloat64, "float64($0)") + + type myInt int + myInt() //@item(litMyInt, "myInt()", "", "var") + + var mi myInt + mi = my //@snippet(" //", litMyInt, "myInt($0)") +} + +func _() { + type ptrStruct struct { + p *ptrStruct + } + + ptrStruct{} //@item(litPtrStruct, "ptrStruct{}", "", "var") + + ptrStruct{ + p: &ptrSt, //@rank(",", litPtrStruct) + } + + &ptrStruct{} //@item(litPtrStructPtr, "&ptrStruct{}", "", "var") + + &ptrStruct{ + p: ptrSt, //@rank(",", litPtrStructPtr) + } +} + +func _() { + f := func(...[]int) {} + f() //@snippet(")", litIntSlice, "[]int{$0\\}") +} + + +func _() { + // don't complete to "untyped int()" + []int{}[untyped] //@complete("] //") +} + +type Tree[T any] struct{} + +func (tree Tree[T]) Do(f func(s T)) {} + +func _() { + var t Tree[string] + t.Do(fun) //@complete(")", litFunc), snippet(")", litFunc, "func(s string) {$0\\}") +} diff --git a/gopls/internal/regtest/marker/testdata/completion/lit.txt b/gopls/internal/regtest/marker/testdata/completion/lit.txt new file mode 100644 index 00000000000..7224f42ab77 --- /dev/null +++ b/gopls/internal/regtest/marker/testdata/completion/lit.txt @@ -0,0 +1,49 @@ + +-- flags -- +-ignore_extra_diags + +-- go.mod -- +module mod.test + +go 1.18 + +-- foo/foo.go -- +package foo + +type StructFoo struct{ F int } + +-- a.go -- +package a + +import "mod.test/foo" + +func _() { + StructFoo{} //@item(litStructFoo, "StructFoo{}", "struct{...}", "struct") + + var sfp *foo.StructFoo + // Don't insert the "&" before "StructFoo{}". + sfp = foo.Str //@snippet(" //", litStructFoo, "StructFoo{$0\\}") + + var sf foo.StructFoo + sf = foo.Str //@snippet(" //", litStructFoo, "StructFoo{$0\\}") + sf = foo. //@snippet(" //", litStructFoo, "StructFoo{$0\\}") +} + +-- http.go -- +package a + +import ( + "net/http" + "sort" +) + +func _() { + sort.Slice(nil, fun) //@snippet(")", litFunc, "func(i, j int) bool {$0\\}") + + http.HandleFunc("", f) //@snippet(")", litFunc, "func(w http.ResponseWriter, r *http.Request) {$0\\}") + + //@item(litFunc, "func(...) {}", "", "var") + http.HandlerFunc() //@item(handlerFunc, "http.HandlerFunc()", "", "var") + http.Handle("", http.HandlerFunc()) //@snippet("))", litFunc, "func(w http.ResponseWriter, r *http.Request) {$0\\}") + http.Handle("", h) //@snippet(")", handlerFunc, "http.HandlerFunc($0)") +} diff --git a/gopls/internal/regtest/marker/testdata/completion/testy.txt b/gopls/internal/regtest/marker/testdata/completion/testy.txt new file mode 100644 index 00000000000..f26b3ae1b1f --- /dev/null +++ b/gopls/internal/regtest/marker/testdata/completion/testy.txt @@ -0,0 +1,57 @@ + +-- flags -- +-ignore_extra_diags + +-- go.mod -- +module testy.test + +go 1.18 + +-- types/types.go -- +package types + + +-- signature/signature.go -- +package signature + +type Alias = int + +-- snippets/snippets.go -- +package snippets + +import ( + "testy.test/signature" + t "testy.test/types" +) + +func X(_ map[signature.Alias]t.CoolAlias) (map[signature.Alias]t.CoolAlias) { + return nil +} + +-- testy/testy.go -- +package testy + +func a() { //@item(funcA, "a", "func()", "func") + //@complete("", funcA) +} + + +-- testy/testy_test.go -- +package testy + +import ( + "testing" + + sig "testy.test/signature" + "testy.test/snippets" +) + +func TestSomething(t *testing.T) { //@item(TestSomething, "TestSomething(t *testing.T)", "", "func") + var x int //@loc(testyX, "x"), diag("x", re"x declared (and|but) not used") + a() //@loc(testyA, "a") +} + +func _() { + _ = snippets.X(nil) //@signature("nil", "X(_ map[sig.Alias]types.CoolAlias) map[sig.Alias]types.CoolAlias") + var _ sig.Alias +} diff --git a/gopls/internal/regtest/marker/testdata/completion/unimported.txt b/gopls/internal/regtest/marker/testdata/completion/unimported.txt new file mode 100644 index 00000000000..7d12269c8ba --- /dev/null +++ b/gopls/internal/regtest/marker/testdata/completion/unimported.txt @@ -0,0 +1,88 @@ + +-- flags -- +-ignore_extra_diags + +-- go.mod -- +module unimported.test + +go 1.18 + +-- unimported/export_test.go -- +package unimported + +var TestExport int //@item(testexport, "TestExport", "var (from \"unimported.test/unimported\")", "var") + +-- signature/signature.go -- +package signature + +func Foo() {} + +-- foo/foo.go -- +package foo + +type StructFoo struct{ F int } + +-- baz/baz.go -- +package baz + +import ( + f "unimported.test/foo" +) + +var FooStruct f.StructFoo + +-- unimported/unimported.go -- +package unimported + +func _() { + http //@complete("p", http, httptest, httptrace, httputil) + // container/ring is extremely unlikely to be imported by anything, so shouldn't have type information. + ring.Ring //@complete(re"R()ing", ringring) + signature.Foo //@complete("Foo", signaturefoo) + + context.Bac //@complete(" //", contextBackground) +} + +// Create markers for unimported std lib packages. Only for use by this test. +/* http */ //@item(http, "http", "\"net/http\"", "package") +/* httptest */ //@item(httptest, "httptest", "\"net/http/httptest\"", "package") +/* httptrace */ //@item(httptrace, "httptrace", "\"net/http/httptrace\"", "package") +/* httputil */ //@item(httputil, "httputil", "\"net/http/httputil\"", "package") + +/* ring.Ring */ //@item(ringring, "Ring", "(from \"container/ring\")", "var") + +/* signature.Foo */ //@item(signaturefoo, "Foo", "func (from \"unimported.test/signature\")", "func") + +/* context.Background */ //@item(contextBackground, "Background", "func (from \"context\")", "func") + +// Now that we no longer type-check imported completions, +// we don't expect the context.Background().Err method (see golang/go#58663). +/* context.Background().Err */ //@item(contextBackgroundErr, "Background().Err", "func (from \"context\")", "method") + +-- unimported/unimported_cand_type.go -- +package unimported + +import ( + _ "context" + + "unimported.test/baz" +) + +func _() { + foo.StructFoo{} //@item(litFooStructFoo, "foo.StructFoo{}", "struct{...}", "struct") + + // We get the literal completion for "foo.StructFoo{}" even though we haven't + // imported "foo" yet. + baz.FooStruct = f //@snippet(" //", litFooStructFoo, "foo.StructFoo{$0\\}") +} + +-- unimported/x_test.go -- +package unimported_test + +import ( + "testing" +) + +func TestSomething(t *testing.T) { + _ = unimported.TestExport //@complete("TestExport", testexport) +} diff --git a/gopls/internal/regtest/marker/testdata/definition/cgo.txt b/gopls/internal/regtest/marker/testdata/definition/cgo.txt new file mode 100644 index 00000000000..6d108a46656 --- /dev/null +++ b/gopls/internal/regtest/marker/testdata/definition/cgo.txt @@ -0,0 +1,62 @@ +This test is ported from the old marker tests. +It tests hover and definition for cgo declarations. + +-- flags -- +-cgo + +-- go.mod -- +module cgo.test + +go 1.18 + +-- cgo/cgo.go -- +package cgo + +/* +#include +#include + +void myprint(char* s) { + printf("%s\n", s); +} +*/ +import "C" + +import ( + "fmt" + "unsafe" +) + +func Example() { //@loc(cgoexample, "Example"), item(cgoexampleItem, "Example", "func()", "func") + fmt.Println() + cs := C.CString("Hello from stdio\n") + C.myprint(cs) + C.free(unsafe.Pointer(cs)) +} + +func _() { + Example() //@hover("ample", "Example", hoverExample), def("ample", cgoexample), complete("ample", cgoexampleItem) +} + +-- @hoverExample/hover.md -- +```go +func Example() +``` + +[`cgo.Example` on pkg.go.dev](https://pkg.go.dev/cgo.test/cgo#Example) +-- usecgo/usecgo.go -- +package cgoimport + +import ( + "cgo.test/cgo" +) + +func _() { + cgo.Example() //@hover("ample", "Example", hoverImportedExample), def("ample", cgoexample), complete("ample", cgoexampleItem) +} +-- @hoverImportedExample/hover.md -- +```go +func cgo.Example() +``` + +[`cgo.Example` on pkg.go.dev](https://pkg.go.dev/cgo.test/cgo#Example) diff --git a/gopls/internal/regtest/marker/testdata/hover/godef.txt b/gopls/internal/regtest/marker/testdata/hover/godef.txt new file mode 100644 index 00000000000..e6a67616302 --- /dev/null +++ b/gopls/internal/regtest/marker/testdata/hover/godef.txt @@ -0,0 +1,406 @@ +This test was ported from 'godef' in the old marker tests. +It tests various hover and definition requests. + +-- flags -- +-min_go=go1.20 + +-- go.mod -- +module godef.test + +go 1.18 + +-- a/a_x_test.go -- +package a_test + +import ( + "testing" +) + +func TestA2(t *testing.T) { //@hover("TestA2", "TestA2", TestA2) + Nonexistant() //@diag("Nonexistant", re"(undeclared name|undefined): Nonexistant") +} + +-- @TestA2/hover.md -- +```go +func TestA2(t *testing.T) +``` +-- @ember/hover.md -- +```go +field Member string +``` + +@loc(Member, "Member") + + +[`(a.Thing).Member` on pkg.go.dev](https://pkg.go.dev/godef.test/a#Thing.Member) +-- a/d.go -- +package a //@hover("a", _, a) + +import "fmt" + +type Thing struct { //@loc(Thing, "Thing") + Member string //@loc(Member, "Member") +} + +var Other Thing //@loc(Other, "Other") + +func Things(val []string) []Thing { //@loc(Things, "Things") + return nil +} + +func (t Thing) Method(i int) string { //@loc(Method, "Method") + return t.Member +} + +func (t Thing) Method3() { +} + +func (t *Thing) Method2(i int, j int) (error, string) { + return nil, t.Member +} + +func (t *Thing) private() { +} + +func useThings() { + t := Thing{ //@hover("ing", "Thing", ing) + Member: "string", //@hover("ember", "Member", ember), def("ember", Member) + } + fmt.Print(t.Member) //@hover("ember", "Member", ember), def("ember", Member) + fmt.Print(Other) //@hover("ther", "Other", ther), def("ther", Other) + Things(nil) //@hover("ings", "Things", ings), def("ings", Things) + t.Method(0) //@hover("eth", "Method", eth), def("eth", Method) +} + +type NextThing struct { //@loc(NextThing, "NextThing") + Thing + Value int +} + +func (n NextThing) another() string { + return n.Member +} + +// Shadows Thing.Method3 +func (n *NextThing) Method3() int { + return n.Value +} + +var nextThing NextThing //@hover("NextThing", "NextThing", NextThing), def("NextThing", NextThing) + +-- @ings/hover.md -- +```go +func Things(val []string) []Thing +``` + +[`a.Things` on pkg.go.dev](https://pkg.go.dev/godef.test/a#Things) +-- @ther/hover.md -- +```go +var Other Thing +``` + +@loc(Other, "Other") + + +[`a.Other` on pkg.go.dev](https://pkg.go.dev/godef.test/a#Other) +-- @a/hover.md -- +-- @ing/hover.md -- +```go +type Thing struct { + Member string //@loc(Member, "Member") +} + +func (Thing).Method(i int) string +func (*Thing).Method2(i int, j int) (error, string) +func (Thing).Method3() +func (*Thing).private() +``` + +[`a.Thing` on pkg.go.dev](https://pkg.go.dev/godef.test/a#Thing) +-- @NextThing/hover.md -- +```go +type NextThing struct { + Thing + Value int +} + +func (*NextThing).Method3() int +func (NextThing).another() string +``` + +[`a.NextThing` on pkg.go.dev](https://pkg.go.dev/godef.test/a#NextThing) +-- @eth/hover.md -- +```go +func (Thing).Method(i int) string +``` + +[`(a.Thing).Method` on pkg.go.dev](https://pkg.go.dev/godef.test/a#Thing.Method) +-- a/f.go -- +// Package a is a package for testing go to definition. +package a + +import "fmt" + +func TypeStuff() { + var x string + + switch y := interface{}(x).(type) { //@loc(y, "y"), hover("y", "y", y) , def("y", y) + case int: //@loc(intY, "int") + fmt.Printf("%v", y) //@hover("y", "y", inty), def("y", y) + case string: //@loc(stringY, "string") + fmt.Printf("%v", y) //@hover("y", "y", stringy), def("y", y) + } + +} +-- @inty/hover.md -- +```go +var y int +``` +-- @stringy/hover.md -- +```go +var y string +``` +-- @y/hover.md -- +```go +var y interface{} +``` +-- a/h.go -- +package a + +func _() { + type s struct { + nested struct { + // nested number + number int64 //@loc(nestedNumber, "number") + } + nested2 []struct { + // nested string + str string //@loc(nestedString, "str") + } + x struct { + x struct { + x struct { + x struct { + x struct { + // nested map + m map[string]float64 //@loc(nestedMap, "m") + } + } + } + } + } + } + + var t s + _ = t.nested.number //@hover("number", "number", nestedNumber), def("number", nestedNumber) + _ = t.nested2[0].str //@hover("str", "str", nestedString), def("str", nestedString) + _ = t.x.x.x.x.x.m //@hover("m", "m", nestedMap), def("m", nestedMap) +} + +func _() { + var s struct { + // a field + a int //@loc(structA, "a") + // b nested struct + b struct { //@loc(structB, "b") + // c field of nested struct + c int //@loc(structC, "c") + } + } + _ = s.a //@def("a", structA) + _ = s.b //@def("b", structB) + _ = s.b.c //@def("c", structC) + + var arr []struct { + // d field + d int //@loc(arrD, "d") + // e nested struct + e struct { //@loc(arrE, "e") + // f field of nested struct + f int //@loc(arrF, "f") + } + } + _ = arr[0].d //@def("d", arrD) + _ = arr[0].e //@def("e", arrE) + _ = arr[0].e.f //@def("f", arrF) + + var complex []struct { + c <-chan map[string][]struct { + // h field + h int //@loc(complexH, "h") + // i nested struct + i struct { //@loc(complexI, "i") + // j field of nested struct + j int //@loc(complexJ, "j") + } + } + } + _ = (<-complex[0].c)["0"][0].h //@def("h", complexH) + _ = (<-complex[0].c)["0"][0].i //@def("i", complexI) + _ = (<-complex[0].c)["0"][0].i.j //@def("j", complexJ) + + var mapWithStructKey map[struct { //@diag("struct", re"invalid map key") + // X key field + x []string //@loc(mapStructKeyX, "x") + }]int + for k := range mapWithStructKey { + _ = k.x //@def("x", mapStructKeyX) + } + + var mapWithStructKeyAndValue map[struct { + // Y key field + y string //@loc(mapStructKeyY, "y") + }]struct { + // X value field + x string //@loc(mapStructValueX, "x") + } + for k, v := range mapWithStructKeyAndValue { + // TODO: we don't show docs for y field because both map key and value + // are structs. And in this case, we parse only map value + _ = k.y //@hover("y", "y", hoverStructKeyY), def("y", mapStructKeyY) + _ = v.x //@hover("x", "x", hoverStructKeyX), def("x", mapStructValueX) + } + + var i []map[string]interface { + // open method comment + open() error //@loc(openMethod, "open") + } + i[0]["1"].open() //@hover("pen","open", openMethod), def("open", openMethod) +} + +func _() { + test := struct { + // test description + desc string //@loc(testDescription, "desc") + }{} + _ = test.desc //@def("desc", testDescription) + + for _, tt := range []struct { + // test input + in map[string][]struct { //@loc(testInput, "in") + // test key + key string //@loc(testInputKey, "key") + // test value + value interface{} //@loc(testInputValue, "value") + } + result struct { + v <-chan struct { + // expected test value + value int //@loc(testResultValue, "value") + } + } + }{} { + _ = tt.in //@def("in", testInput) + _ = tt.in["0"][0].key //@def("key", testInputKey) + _ = tt.in["0"][0].value //@def("value", testInputValue) + + _ = (<-tt.result.v).value //@def("value", testResultValue) + } +} + +func _() { + getPoints := func() []struct { + // X coord + x int //@loc(returnX, "x") + // Y coord + y int //@loc(returnY, "y") + } { + return nil + } + + r := getPoints() + _ = r[0].x //@def("x", returnX) + _ = r[0].y //@def("y", returnY) +} +-- @hoverStructKeyX/hover.md -- +```go +field x string +``` + +X value field +-- @hoverStructKeyY/hover.md -- +```go +field y string +``` + +Y key field +-- @nestedNumber/hover.md -- +```go +field number int64 +``` + +nested number +-- @nestedString/hover.md -- +```go +field str string +``` + +nested string +-- @openMethod/hover.md -- +```go +func (interface).open() error +``` + +open method comment +-- @nestedMap/hover.md -- +```go +field m map[string]float64 +``` + +nested map +-- b/e.go -- +package b + +import ( + "fmt" + + "godef.test/a" +) + +func useThings() { + t := a.Thing{} //@loc(bStructType, "ing") + fmt.Print(t.Member) //@loc(bMember, "ember") + fmt.Print(a.Other) //@loc(bVar, "ther") + a.Things(nil) //@loc(bFunc, "ings") +} + +/*@ +def(bStructType, Thing) +def(bMember, Member) +def(bVar, Other) +def(bFunc, Things) +*/ + +func _() { + var x interface{} + switch x := x.(type) { //@hover("x", "x", xInterface) + case string: //@loc(eString, "string") + fmt.Println(x) //@hover("x", "x", xString) + case int: //@loc(eInt, "int") + fmt.Println(x) //@hover("x", "x", xInt) + } +} +-- @xInt/hover.md -- +```go +var x int +``` +-- @xInterface/hover.md -- +```go +var x interface{} +``` +-- @xString/hover.md -- +```go +var x string +``` +-- broken/unclosedIf.go -- +package broken + +import "fmt" + +func unclosedIf() { + if false { + var myUnclosedIf string //@loc(myUnclosedIf, "myUnclosedIf") + fmt.Printf("s = %v\n", myUnclosedIf) //@def("my", myUnclosedIf) +} + +func _() {} //@diag("_", re"expected") diff --git a/gopls/internal/regtest/marker/testdata/typedef/typedef.txt b/gopls/internal/regtest/marker/testdata/typedef/typedef.txt new file mode 100644 index 00000000000..3bc9dabdb8b --- /dev/null +++ b/gopls/internal/regtest/marker/testdata/typedef/typedef.txt @@ -0,0 +1,68 @@ +This test exercises the textDocument/typeDefinition action. + +-- typedef.go -- +package typedef + +type Struct struct { //@loc(Struct, "Struct"), + Field string +} + +type Int int //@loc(Int, "Int") + +func _() { + var ( + value Struct + point *Struct + ) + _ = value //@typedef("value", Struct) + _ = point //@typedef("point", Struct) + + var ( + array [3]Struct + slice []Struct + ch chan Struct + complex [3]chan *[5][]Int + ) + _ = array //@typedef("array", Struct) + _ = slice //@typedef("slice", Struct) + _ = ch //@typedef("ch", Struct) + _ = complex //@typedef("complex", Int) + + var s struct { + x struct { + xx struct { + field1 []Struct + field2 []Int + } + } + } + _ = s.x.xx.field1 //@typedef("field1", Struct) + _ = s.x.xx.field2 //@typedef("field2", Int) +} + +func F1() Int { return 0 } +func F2() (Int, float64) { return 0, 0 } +func F3() (Struct, int, bool, error) { return Struct{}, 0, false, nil } +func F4() (**int, Int, bool, *error) { return nil, 0, false, nil } +func F5() (int, float64, error, Struct) { return 0, 0, nil, Struct{} } +func F6() (int, float64, ***Struct, error) { return 0, 0, nil, nil } + +func _() { + F1() //@typedef("F1", Int) + F2() //@typedef("F2", Int) + F3() //@typedef("F3", Struct) + F4() //@typedef("F4", Int) + F5() //@typedef("F5", Struct) + F6() //@typedef("F6", Struct) + + f := func() Int { return 0 } + f() //@typedef("f", Int) +} + +// https://github.com/golang/go/issues/38589#issuecomment-620350922 +func _() { + type myFunc func(int) Int //@loc(myFunc, "myFunc") + + var foo myFunc + _ = foo() //@typedef("foo", myFunc), diag(")", re"not enough arguments") +} diff --git a/gopls/internal/regtest/misc/definition_test.go b/gopls/internal/regtest/misc/definition_test.go index 0a36336b567..f11b2073292 100644 --- a/gopls/internal/regtest/misc/definition_test.go +++ b/gopls/internal/regtest/misc/definition_test.go @@ -351,7 +351,7 @@ func main() {} Run(t, mod, func(t *testing.T, env *Env) { env.OpenFile("main.go") - loc, err := env.Editor.GoToTypeDefinition(env.Ctx, env.RegexpSearch("main.go", tt.re)) + loc, err := env.Editor.TypeDefinition(env.Ctx, env.RegexpSearch("main.go", tt.re)) if tt.wantError { if err == nil { t.Fatal("expected error, got nil") @@ -386,10 +386,7 @@ func F[T comparable]() {} Run(t, mod, func(t *testing.T, env *Env) { env.OpenFile("main.go") - _, err := env.Editor.GoToTypeDefinition(env.Ctx, env.RegexpSearch("main.go", "comparable")) // must not panic - if err != nil { - t.Fatal(err) - } + _ = env.TypeDefinition(env.RegexpSearch("main.go", "comparable")) // must not panic }) } diff --git a/gopls/internal/regtest/misc/references_test.go b/gopls/internal/regtest/misc/references_test.go index a85bcc27d61..262284abc3d 100644 --- a/gopls/internal/regtest/misc/references_test.go +++ b/gopls/internal/regtest/misc/references_test.go @@ -133,10 +133,8 @@ var _ = unsafe.Slice(nil, 0) loc := env.RegexpSearch("a.go", `\b`+name+`\b`) // definition -> {builtin,unsafe}.go - def, err := env.Editor.GoToDefinition(env.Ctx, loc) - if err != nil { - t.Errorf("definition(%q) failed: %v", name, err) - } else if (!strings.HasSuffix(string(def.URI), "builtin.go") && + def := env.GoToDefinition(loc) + if (!strings.HasSuffix(string(def.URI), "builtin.go") && !strings.HasSuffix(string(def.URI), "unsafe.go")) || def.Range.Start.Line == 0 { t.Errorf("definition(%q) = %v, want {builtin,unsafe}.go", @@ -144,7 +142,7 @@ var _ = unsafe.Slice(nil, 0) } // "references to (builtin "Foo"|unsafe.Foo) are not supported" - _, err = env.Editor.References(env.Ctx, loc) + _, err := env.Editor.References(env.Ctx, loc) gotErr := fmt.Sprint(err) if !strings.Contains(gotErr, "references to") || !strings.Contains(gotErr, "not supported") || diff --git a/gopls/internal/regtest/modfile/modfile_test.go b/gopls/internal/regtest/modfile/modfile_test.go index 855141a7b30..076366958e6 100644 --- a/gopls/internal/regtest/modfile/modfile_test.go +++ b/gopls/internal/regtest/modfile/modfile_test.go @@ -878,7 +878,7 @@ func hello() {} } // Confirm that we no longer have metadata when the file is saved. env.SaveBufferWithoutActions("go.mod") - _, err := env.Editor.GoToDefinition(env.Ctx, env.RegexpSearch("main.go", "hello")) + _, err := env.Editor.Definition(env.Ctx, env.RegexpSearch("main.go", "hello")) if err == nil { t.Fatalf("expected error, got none") } From d2b79cdec8ff8fdd6466aec7776d7a834f3d885e Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Fri, 15 Sep 2023 13:31:24 -0400 Subject: [PATCH 111/178] gopls/internal/regtest/marker: rename to clarify marker types Implement review feedback from CL 528335, renaming dataFunc to valueMarkerFunc and markerFunc to actionMarkerFunc. Change-Id: Ia8b35d74f4eecc109b84641b28ad394134ab40c6 Reviewed-on: https://go-review.googlesource.com/c/tools/+/528297 Reviewed-by: Alan Donovan LUCI-TryBot-Result: Go LUCI --- gopls/internal/lsp/regtest/marker.go | 143 ++++++++++++++------------- 1 file changed, 73 insertions(+), 70 deletions(-) diff --git a/gopls/internal/lsp/regtest/marker.go b/gopls/internal/lsp/regtest/marker.go index 17d210c5df8..f6d462aabbb 100644 --- a/gopls/internal/lsp/regtest/marker.go +++ b/gopls/internal/lsp/regtest/marker.go @@ -136,6 +136,12 @@ var update = flag.Bool("update", false, "if set, update test data during marker // // # Marker types // +// Markers are of two kinds. A few are "value markers" (e.g. @item), which are +// processed in a first pass and each computes a value that may be referred to +// by name later. Most are "action markers", which are processed in a second +// pass and take some action such as testing an LSP operation; they may refer +// to values computed by value markers. +// // The following markers are supported within marker tests: // // - acceptcompletion(location, label, golden): specifies that accepting the @@ -430,7 +436,7 @@ func RunMarkerTests(t *testing.T, dir string) { test: test, env: newEnv(t, cache, test.files, test.proxyFiles, test.writeGoSum, config), settings: config.Settings, - data: make(map[expect.Identifier]any), + values: make(map[expect.Identifier]any), diags: make(map[protocol.Location][]protocol.Diagnostic), extraNotes: make(map[protocol.DocumentURI]map[string][]*expect.Note), } @@ -467,9 +473,9 @@ func RunMarkerTests(t *testing.T, dir string) { var markers []marker for _, note := range test.notes { mark := marker{run: run, note: note} - if fn, ok := dataFuncs[note.Name]; ok { + if fn, ok := valueMarkerFuncs[note.Name]; ok { fn(mark) - } else if _, ok := actionFuncs[note.Name]; ok { + } else if _, ok := actionMarkerFuncs[note.Name]; ok { markers = append(markers, mark) // save for later } else { uri := mark.uri() @@ -482,7 +488,7 @@ func RunMarkerTests(t *testing.T, dir string) { // Invoke each remaining marker in the test. for _, mark := range markers { - actionFuncs[mark.note.Name](mark) + actionMarkerFuncs[mark.note.Name](mark) } // Any remaining (un-eliminated) diagnostics are an error. @@ -558,43 +564,43 @@ func (mark marker) errorf(format string, args ...any) { mark.run.env.T.Errorf("%s: %s", mark.run.fmtPos(mark.note.Pos), msg) } -// A dataFunc is a func that binds an identifier to a value. The first argument -// to the provided func must be of type `marker`, and the func must return -// exactly one result. dataFuncs run before markerFuncs. +// valueMarkerFunc returns a wrapper around a function that allows it to be +// called during the processing of value markers (e.g. @value(v, 123)) with marker +// arguments converted to function parameters. The provided function's first +// parameter must be of type 'marker', and it must return a value. // -// When associated with a note, the first argument of the note must be an -// identifier, and remaining arguments are converted and passed to the provided -// function along with the mark context. +// Unlike action markers, which are executed for actions such as test +// assertions, value markers are all evaluated first, and each computes +// a value that is recorded by its identifier, which is the marker's first +// argument. These values may be referred to from an action marker by +// this identifier, e.g. @action(... , v, ...). // -// For example, given a data func with signature +// For example, given a fn with signature // // func(mark marker, label, details, kind string) CompletionItem // -// The resulting dataFunc can associated with @item notes, and invoked as follows: +// The result of valueMarkerFunc can associated with @item notes, and invoked +// as follows: // // //@item(FooCompletion, "Foo", "func() int", "func") // -// In this example, the name 'FooCompletion' is bound to the completion item -// produced with the given arguments, and may be referenced by other marker -// funcs in the test, by passing the FooCompletion identifier. -// -// dataFuncs should not mutate the test environment. -func dataFunc(fn any) func(marker) { +// The provided fn should not mutate the test environment. +func valueMarkerFunc(fn any) func(marker) { ftype := reflect.TypeOf(fn) if ftype.NumIn() == 0 || ftype.In(0) != markerType { - panic(fmt.Sprintf("data function %#v must accept marker as its first argument", ftype)) + panic(fmt.Sprintf("value marker function %#v must accept marker as its first argument", ftype)) } if ftype.NumOut() != 1 { - panic(fmt.Sprintf("data function %#v must have exactly 1 result", ftype)) + panic(fmt.Sprintf("value marker function %#v must have exactly 1 result", ftype)) } return func(mark marker) { if len(mark.note.Args) == 0 || !is[expect.Identifier](mark.note.Args[0]) { - mark.errorf("first argument to a data func must be an identifier") + mark.errorf("first argument to a value marker function must be an identifier") return } id := mark.note.Args[0].(expect.Identifier) - if alt, ok := mark.run.data[id]; ok { + if alt, ok := mark.run.values[id]; ok { mark.errorf("%s already declared as %T", id, alt) return } @@ -605,27 +611,24 @@ func dataFunc(fn any) func(marker) { return } results := reflect.ValueOf(fn).Call(argValues) - mark.run.data[id] = results[0].Interface() + mark.run.values[id] = results[0].Interface() } } -// A markerFunc is a func that executes after all dataFuncs, performs some -// operation, and verifies an assertion. -// -// The first argument of the provided function must be of type `marker`, and -// the function must not return any results. -// -// When associated with a note, the notes arguments are converted and passed to -// the provided function along with the mark context. +// actionMarkerFunc returns a wrapper around a function that allows it to be +// called during the processing of action markers (e.g. @action("abc", 123)) +// with marker arguments converted to function parameters. The provided +// function's first parameter must be of type 'marker', and it must not return +// any values. // -// markerFuncs should not mutate the test environment. -func markerFunc(fn any) func(marker) { +// The provided fn should not mutate the test environment. +func actionMarkerFunc(fn any) func(marker) { ftype := reflect.TypeOf(fn) if ftype.NumIn() == 0 || ftype.In(0) != markerType { - panic(fmt.Sprintf("marker function %#v must accept marker as its first argument", ftype)) + panic(fmt.Sprintf("action marker function %#v must accept marker as its first argument", ftype)) } if ftype.NumOut() != 0 { - panic(fmt.Sprintf("action function %#v cannot have results", ftype)) + panic(fmt.Sprintf("action marker function %#v cannot have results", ftype)) } return func(mark marker) { @@ -651,7 +654,7 @@ func convertArgs(mark marker, ftype reflect.Type, args []any) ([]reflect.Value, pnext++ } else if p == nil || !ftype.IsVariadic() { // The actual number of arguments expected by the mark varies, depending - // on whether this is a data func or an action func. + // on whether this is a value marker or an action marker. // // Since this error indicates a bug, probably OK to have an imprecise // error message here. @@ -688,36 +691,36 @@ func is[T any](arg any) bool { return ok } -// Supported data functions. See [dataFunc] for more details. -var dataFuncs = map[string]func(marker){ - "loc": dataFunc(locMarker), - "item": dataFunc(completionItemMarker), -} - -// Supported marker functions. See [markerFunc] for more details. -var actionFuncs = map[string]func(marker){ - "acceptcompletion": markerFunc(acceptCompletionMarker), - "codeaction": markerFunc(codeActionMarker), - "codeactionerr": markerFunc(codeActionErrMarker), - "codelenses": markerFunc(codeLensesMarker), - "complete": markerFunc(completeMarker), - "def": markerFunc(defMarker), - "diag": markerFunc(diagMarker), - "foldingrange": markerFunc(foldingRangeMarker), - "format": markerFunc(formatMarker), - "highlight": markerFunc(highlightMarker), - "hover": markerFunc(hoverMarker), - "implementation": markerFunc(implementationMarker), - "rank": markerFunc(rankMarker), - "refs": markerFunc(refsMarker), - "rename": markerFunc(renameMarker), - "renameerr": markerFunc(renameErrMarker), - "signature": markerFunc(signatureMarker), - "snippet": markerFunc(snippetMarker), - "suggestedfix": markerFunc(suggestedfixMarker), - "symbol": markerFunc(symbolMarker), - "typedef": markerFunc(typedefMarker), - "workspacesymbol": markerFunc(workspaceSymbolMarker), +// Supported value marker functions. See [valueMarkerFunc] for more details. +var valueMarkerFuncs = map[string]func(marker){ + "loc": valueMarkerFunc(locMarker), + "item": valueMarkerFunc(completionItemMarker), +} + +// Supported action marker functions. See [actionMarkerFunc] for more details. +var actionMarkerFuncs = map[string]func(marker){ + "acceptcompletion": actionMarkerFunc(acceptCompletionMarker), + "codeaction": actionMarkerFunc(codeActionMarker), + "codeactionerr": actionMarkerFunc(codeActionErrMarker), + "codelenses": actionMarkerFunc(codeLensesMarker), + "complete": actionMarkerFunc(completeMarker), + "def": actionMarkerFunc(defMarker), + "diag": actionMarkerFunc(diagMarker), + "foldingrange": actionMarkerFunc(foldingRangeMarker), + "format": actionMarkerFunc(formatMarker), + "highlight": actionMarkerFunc(highlightMarker), + "hover": actionMarkerFunc(hoverMarker), + "implementation": actionMarkerFunc(implementationMarker), + "rank": actionMarkerFunc(rankMarker), + "refs": actionMarkerFunc(refsMarker), + "rename": actionMarkerFunc(renameMarker), + "renameerr": actionMarkerFunc(renameErrMarker), + "signature": actionMarkerFunc(signatureMarker), + "snippet": actionMarkerFunc(snippetMarker), + "suggestedfix": actionMarkerFunc(suggestedfixMarker), + "symbol": actionMarkerFunc(symbolMarker), + "typedef": actionMarkerFunc(typedefMarker), + "workspacesymbol": actionMarkerFunc(workspaceSymbolMarker), } // markerTest holds all the test data extracted from a test txtar archive. @@ -1060,8 +1063,8 @@ type markerTestRun struct { // Collected information. // Each @diag/@suggestedfix marker eliminates an entry from diags. - data map[expect.Identifier]any - diags map[protocol.Location][]protocol.Diagnostic // diagnostics by position; location end == start + values map[expect.Identifier]any + diags map[protocol.Location][]protocol.Diagnostic // diagnostics by position; location end == start // Notes that weren't associated with a top-level marker func. They may be // consumed by another marker (e.g. @codelenses collects @codelens markers). @@ -1210,7 +1213,7 @@ func convert(mark marker, arg any, paramType reflect.Type) (any, error) { return mark.run.test.getGolden(id), nil } if id, ok := arg.(expect.Identifier); ok { - if arg, ok := mark.run.data[id]; ok { + if arg, ok := mark.run.values[id]; ok { if !reflect.TypeOf(arg).AssignableTo(paramType) { return nil, fmt.Errorf("cannot convert %v to %s", arg, paramType) } @@ -1885,7 +1888,7 @@ func codeLensesMarker(mark marker) { } var want []codeLens - mark.consumeExtraNotes("codelens", markerFunc(func(mark marker, loc protocol.Location, title string) { + mark.consumeExtraNotes("codelens", actionMarkerFunc(func(mark marker, loc protocol.Location, title string) { want = append(want, codeLens{loc.Range, title}) })) From 612889592da646100e3f42c5bccd5fea8eefbd8e Mon Sep 17 00:00:00 2001 From: "Hana (Hyang-Ah) Kim" Date: Fri, 15 Sep 2023 16:47:40 -0400 Subject: [PATCH 112/178] gopls/internal/vulncheck/scan: buffer govulncheck STDERR output Govulncheck scan API rarely outputs anything through STDERR. Messages shown in govulncheck scan API's stderr indicate real problems and it's better to show at the end when the scan fails. Buffer STDERR messages and sent them only when the command has failed and the goroutine that post-processes STDOUT stream has terminated. That will eliminate race between progress reports produced from STDERR and from STDOUT. Updates golang/go#57032 Change-Id: Ibd73668b11c07b83ea3aee55a43d6b8072c80870 Reviewed-on: https://go-review.googlesource.com/c/tools/+/528361 Reviewed-by: Alan Donovan TryBot-Result: Gopher Robot Run-TryBot: Hyang-Ah Hana Kim --- gopls/internal/vulncheck/scan/command.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/gopls/internal/vulncheck/scan/command.go b/gopls/internal/vulncheck/scan/command.go index 4a7a5262d8d..89d24e08b71 100644 --- a/gopls/internal/vulncheck/scan/command.go +++ b/gopls/internal/vulncheck/scan/command.go @@ -8,6 +8,7 @@ package scan import ( + "bytes" "context" "fmt" "io" @@ -67,6 +68,7 @@ func RunGovulncheck(ctx context.Context, pattern string, snapshot source.Snapsho ir, iw := io.Pipe() handler := &govulncheckHandler{logger: log, osvs: map[string]*osv.Entry{}} + stderr := new(bytes.Buffer) var g errgroup.Group // We run the govulncheck's analysis in a separate process as it can // consume a lot of CPUs and memory, and terminates: a separate process @@ -81,8 +83,8 @@ func RunGovulncheck(ctx context.Context, pattern string, snapshot source.Snapsho // in https://go.googlesource.com/vuln/+/v1.0.1/internal/scan/run.go#76 cmd.Env = append(cmd.Env, "GOVERSION="+goversion) } - cmd.Stderr = log // stream vulncheck's STDERR as progress reports - cmd.Stdout = iw // let the other goroutine parses the result. + cmd.Stderr = stderr // stream vulncheck's STDERR as progress reports + cmd.Stdout = iw // let the other goroutine parses the result. if err := cmd.Start(); err != nil { return fmt.Errorf("failed to start govulncheck: %v", err) @@ -96,6 +98,9 @@ func RunGovulncheck(ctx context.Context, pattern string, snapshot source.Snapsho return govulncheck.HandleJSON(ir, handler) }) if err := g.Wait(); err != nil { + if stderr.Len() > 0 { + log.Write(stderr.Bytes()) + } return nil, fmt.Errorf("failed to read govulncheck output: %v", err) } From b37e7e3c0760bffd2215e4bbb2f449058258c7b4 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Thu, 14 Sep 2023 11:20:19 -0400 Subject: [PATCH 113/178] internal/refactor/inline: test everything This change adds a test that applies the inliner to every function in one or more packages, and tests that it does something sensible: either report an error, or produce a valid code transformation. It discovered a number of bugs even when applied to just a single package, and I plan to run it over all of x/tools, then all of k8s, then over the corpus of the module proxy using our pipeline for parallel processing. Change-Id: I70660d36406bf1a03b4c97114a29ca7629e5d279 Reviewed-on: https://go-review.googlesource.com/c/tools/+/528495 LUCI-TryBot-Result: Go LUCI Auto-Submit: Alan Donovan Reviewed-by: Robert Findley --- internal/refactor/inline/everything_test.go | 227 ++++++++++++++++++++ internal/refactor/inline/inline.go | 10 +- internal/refactor/inline/inline_test.go | 60 +++--- 3 files changed, 261 insertions(+), 36 deletions(-) create mode 100644 internal/refactor/inline/everything_test.go diff --git a/internal/refactor/inline/everything_test.go b/internal/refactor/inline/everything_test.go new file mode 100644 index 00000000000..5742832ab27 --- /dev/null +++ b/internal/refactor/inline/everything_test.go @@ -0,0 +1,227 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package inline_test + +import ( + "flag" + "fmt" + "go/ast" + "go/parser" + "go/types" + "os" + "path/filepath" + "strings" + "testing" + + "golang.org/x/tools/go/packages" + "golang.org/x/tools/go/types/typeutil" + "golang.org/x/tools/internal/diff" + "golang.org/x/tools/internal/refactor/inline" + "golang.org/x/tools/internal/testenv" +) + +// Run with this command: +// +// $ go test ./internal/refactor/inline/ -run=Everything -v -packages=./internal/... + +// TODO(adonovan): +// - report counters (number of attempts, failed AnalyzeCallee, failed +// Inline, etc.) +// - Make a pretty log of the entire output so that we can peruse it +// for opportunities for systematic improvement. + +var packagesFlag = flag.String("packages", "", "set of packages for TestEverything") + +// TestEverything invokes the inliner on every single call site in a +// given package. and checks that it produces either a reasonable +// error, or output that parses and type-checks. +func TestEverything(t *testing.T) { + testenv.NeedsGoPackages(t) + if testing.Short() { + t.Skipf("skipping slow test in -short mode") + } + if *packagesFlag == "" { + return + } + + // Load this package plus dependencies from typed syntax. + cfg := &packages.Config{ + Dir: "../../..", // root of x/tools + Mode: packages.LoadAllSyntax, + Env: append(os.Environ(), + "GO111MODULES=on", + "GOPATH=", + "GOWORK=off", + "GOPROXY=off"), + } + pkgs, err := packages.Load(cfg, *packagesFlag) + if err != nil { + t.Errorf("Load: %v", err) + } + // Report parse/type errors. + // Also, build transitive dependency mapping. + deps := make(map[string]*packages.Package) // key is PkgPath + packages.Visit(pkgs, nil, func(pkg *packages.Package) { + deps[pkg.Types.Path()] = pkg + for _, err := range pkg.Errors { + t.Fatal(err) + } + }) + + // Memoize repeated calls for same file. + fileContent := make(map[string][]byte) + readFile := func(filename string) ([]byte, error) { + content, ok := fileContent[filename] + if !ok { + var err error + content, err = os.ReadFile(filename) + if err != nil { + return nil, err + } + fileContent[filename] = content + } + return content, nil + } + + for _, callerPkg := range pkgs { + // Find all static function calls in the package. + for _, callerFile := range callerPkg.Syntax { + noMutCheck := checkNoMutation(callerFile) + ast.Inspect(callerFile, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + fn := typeutil.StaticCallee(callerPkg.TypesInfo, call) + if fn == nil { + return true + } + + // Prepare caller info. + callPosn := callerPkg.Fset.Position(call.Lparen) + callerContent, err := readFile(callPosn.Filename) + if err != nil { + t.Fatal(err) + } + caller := &inline.Caller{ + Fset: callerPkg.Fset, + Types: callerPkg.Types, + Info: callerPkg.TypesInfo, + File: callerFile, + Call: call, + Content: callerContent, + } + + // Analyze callee. + calleePkg, ok := deps[fn.Pkg().Path()] + if !ok { + t.Fatalf("missing package for callee %v", fn) + } + calleePosn := callerPkg.Fset.Position(fn.Pos()) + calleeDecl, err := findFuncByPosition(calleePkg, calleePosn) + if err != nil { + t.Fatal(err) + } + calleeContent, err := readFile(calleePosn.Filename) + if err != nil { + t.Fatal(err) + } + + // Create a subtest for each inlining operation. + name := fmt.Sprintf("%s@%v", fn.Name(), filepath.Base(callPosn.String())) + t.Run(name, func(t *testing.T) { + // TODO(adonovan): add a panic handler. + + t.Logf("callee declared at %v", + filepath.Base(calleePosn.String())) + + t.Logf("run this command to reproduce locally:\n$ gopls fix -a -d %s:#%d refactor.inline", + callPosn.Filename, callPosn.Offset) + + callee, err := inline.AnalyzeCallee( + calleePkg.Fset, + calleePkg.Types, + calleePkg.TypesInfo, + calleeDecl, + calleeContent) + if err != nil { + // Ignore the expected kinds of errors. + for _, ignore := range []string{ + "has no body", + "type parameters are not yet", + "line directives", + } { + if strings.Contains(err.Error(), ignore) { + return + } + } + t.Fatalf("AnalyzeCallee: %v", err) + } + if err := checkTranscode(callee); err != nil { + t.Fatal(err) + } + + got, err := inline.Inline(t.Logf, caller, callee) + if err != nil { + // Write error to a log, but this ok. + t.Log(err) + return + } + + // Print the diff. + t.Logf("Got diff:\n%s", + diff.Unified("old", "new", string(callerContent), string(got))) + + // Parse and type-check the transformed source. + f, err := parser.ParseFile(caller.Fset, callPosn.Filename, got, parser.SkipObjectResolution) + if err != nil { + t.Fatalf("transformed source does not parse: %v", err) + } + // Splice into original file list. + syntax := append([]*ast.File(nil), callerPkg.Syntax...) + for i := range callerPkg.Syntax { + if syntax[i] == callerFile { + syntax[i] = f + break + } + } + + var typeErrors []string + conf := &types.Config{ + Error: func(err error) { + typeErrors = append(typeErrors, err.Error()) + }, + Importer: importerFunc(func(importPath string) (*types.Package, error) { + // Note: deps is properly keyed by package path, + // not import path, but we can't assume + // Package.Imports[importPath] exists in the + // case of newly added imports of indirect + // dependencies. Seems not to matter to this test. + dep, ok := deps[importPath] + if ok { + return dep.Types, nil + } + return nil, fmt.Errorf("missing package: %q", importPath) + }), + } + if _, err := conf.Check("p", caller.Fset, syntax, nil); err != nil { + t.Fatalf("transformed package has type errors:\n\n%s\n\nTransformed file:\n\n%s", + strings.Join(typeErrors, "\n"), + got) + } + }) + return true + }) + noMutCheck() + } + } + t.Errorf("Analyzed %d packages", len(pkgs)) +} + +type importerFunc func(path string) (*types.Package, error) + +func (f importerFunc) Import(path string) (*types.Package, error) { + return f(path) +} diff --git a/internal/refactor/inline/inline.go b/internal/refactor/inline/inline.go index e2f1a2af2a4..259156c13b8 100644 --- a/internal/refactor/inline/inline.go +++ b/internal/refactor/inline/inline.go @@ -227,10 +227,6 @@ // thoughtful factoring of the large design space, and thorough // test coverage. // -// - Write a fuzz-like test that selects function calls at -// random in the corpus, inlines them, and checks that the -// result is either a sensible error or a valid transformation. -// // - Compute precisely (not conservatively) when parameter // elimination would remove the last reference to a caller local // variable, and blank out the local instead of retreating from @@ -687,6 +683,9 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // If the receiver argument and parameter have // different pointerness, make the "&" or "*" explicit. // + // Also, if x.f() is shorthand for promoted method x.y.f(), + // make the .y explicit in T.f(x.y, ...). + // // Beware that: // // - a method can only be called through a selection, but only @@ -1983,7 +1982,8 @@ func findIdent(root ast.Node, pos token.Pos) *ast.Ident { return true }) if found == nil { - panic(fmt.Sprintf("findIdent %d not found", pos)) + panic(fmt.Sprintf("findIdent %d not found in %s", + pos, debugFormatNode(token.NewFileSet(), root))) } return found } diff --git a/internal/refactor/inline/inline_test.go b/internal/refactor/inline/inline_test.go index cb52f3182ac..46c473bf682 100644 --- a/internal/refactor/inline/inline_test.go +++ b/internal/refactor/inline/inline_test.go @@ -203,18 +203,11 @@ func doInlineNote(logf func(string, ...any), pkg *packages.Package, file *ast.Fi } // Find callee function. - var ( - calleePkg *packages.Package - calleeDecl *ast.FuncDecl - ) + var calleePkg *packages.Package { - var same func(*ast.FuncDecl) bool // Is the call within the package? if fn.Pkg() == caller.Types { calleePkg = pkg // same as caller - same = func(decl *ast.FuncDecl) bool { - return decl.Name.Pos() == fn.Pos() - } } else { // Different package. Load it now. // (The primary load loaded all dependencies, @@ -235,31 +228,12 @@ func doInlineNote(logf func(string, ...any), pkg *packages.Package, file *ast.Fi return fmt.Errorf("callee package had errors") // (see log) } calleePkg = roots[0] - posn := caller.Fset.Position(fn.Pos()) // callee posn wrt caller package - same = func(decl *ast.FuncDecl) bool { - // We can't rely on columns in export data: - // some variants replace it with 1. - // We can't expect file names to have the same prefix. - // export data for go1.20 std packages have $GOROOT written in - // them, so how are we supposed to find the source? Yuck! - // Ugh. need to samefile? Nope $GOROOT just won't work - // This is highly client specific anyway. - posn2 := calleePkg.Fset.Position(decl.Name.Pos()) - return posn.Filename == posn2.Filename && - posn.Line == posn2.Line - } } + } - for _, file := range calleePkg.Syntax { - for _, decl := range file.Decls { - if decl, ok := decl.(*ast.FuncDecl); ok && same(decl) { - calleeDecl = decl - goto found - } - } - } - return fmt.Errorf("can't find FuncDecl for callee") // can't happen? - found: + calleeDecl, err := findFuncByPosition(calleePkg, caller.Fset.Position(fn.Pos())) + if err != nil { + return err } // Do the inlining. For the purposes of the test, @@ -312,6 +286,30 @@ func doInlineNote(logf func(string, ...any), pkg *packages.Package, file *ast.Fi } +// findFuncByPosition returns the FuncDecl at the specified (package-agnostic) position. +func findFuncByPosition(pkg *packages.Package, posn token.Position) (*ast.FuncDecl, error) { + same := func(decl *ast.FuncDecl) bool { + // We can't rely on columns in export data: + // some variants replace it with 1. + // We can't expect file names to have the same prefix. + // export data for go1.20 std packages have $GOROOT written in + // them, so how are we supposed to find the source? Yuck! + // Ugh. need to samefile? Nope $GOROOT just won't work + // This is highly client specific anyway. + posn2 := pkg.Fset.Position(decl.Name.Pos()) + return posn.Filename == posn2.Filename && + posn.Line == posn2.Line + } + for _, file := range pkg.Syntax { + for _, decl := range file.Decls { + if decl, ok := decl.(*ast.FuncDecl); ok && same(decl) { + return decl, nil + } + } + } + return nil, fmt.Errorf("can't find FuncDecl at %v in package %q", posn, pkg.PkgPath) +} + // TestTable is a table driven test, enabling more compact expression // of single-package test cases than is possible with the txtar notation. func TestTable(t *testing.T) { From 9df3852806b4e9366db6f1ac15d222866058409d Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Tue, 19 Sep 2023 16:51:50 -0400 Subject: [PATCH 114/178] internal/refactor/inline: two minor cleanups 1. move the package doc into doc.go 2. s/elimination/substitution/g, since parameters can be eliminated by a binding decl too. Change-Id: Ie9c893b3f45a237a024470f46914941f6cfb28a5 Reviewed-on: https://go-review.googlesource.com/c/tools/+/529616 Auto-Submit: Alan Donovan LUCI-TryBot-Result: Go LUCI Reviewed-by: Robert Findley --- internal/refactor/inline/doc.go | 276 ++++++++++++++++++++++++ internal/refactor/inline/inline.go | 324 +++-------------------------- 2 files changed, 301 insertions(+), 299 deletions(-) create mode 100644 internal/refactor/inline/doc.go diff --git a/internal/refactor/inline/doc.go b/internal/refactor/inline/doc.go new file mode 100644 index 00000000000..f83759ac0c3 --- /dev/null +++ b/internal/refactor/inline/doc.go @@ -0,0 +1,276 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Package inline implements inlining of Go function calls. + +The client provides information about the caller and callee, +including the source text, syntax tree, and type information, and +the inliner returns the modified source file for the caller, or an +error if the inlining operation is invalid (for example because the +function body refers to names that are inaccessible to the caller). + +Although this interface demands more information from the client +than might seem necessary, it enables smoother integration with +existing batch and interactive tools that have their own ways of +managing the processes of reading, parsing, and type-checking +packages. In particular, this package does not assume that the +caller and callee belong to the same token.FileSet or +types.Importer realms. + +There are many aspects to a function call. It is the only construct +that can simultaneously bind multiple variables of different +explicit types, with implicit assignment conversions. (Neither var +nor := declarations can do that.) It defines the scope of control +labels, of return statements, and of defer statements. Arguments +and results of function calls may be tuples even though tuples are +not first-class values in Go, and a tuple-valued call expression +may be "spread" across the argument list of a call or the operands +of a return statement. All these unique features mean that in the +general case, not everything that can be expressed by a function +call can be expressed without one. + +So, in general, inlining consists of modifying a function or method +call expression f(a1, ..., an) so that the name of the function f +is replaced ("literalized") by a literal copy of the function +declaration, with free identifiers suitably modified to use the +locally appropriate identifiers or perhaps constant argument +values. + +Inlining must not change the semantics of the call. Semantics +preservation is crucial for clients such as codebase maintenance +tools that automatically inline all calls to designated functions +on a large scale. Such tools must not introduce subtle behavior +changes. (Fully inlining a call is dynamically observable using +reflection over the call stack, but this exception to the rule is +explicitly allowed.) + +In many cases it is possible to entirely replace ("reduce") the +call by a copy of the function's body in which parameters have been +replaced by arguments. The inliner supports a number of reduction +strategies, and we expect this set to grow. Nonetheless, sound +reduction is surprisingly tricky. + +The inliner is in some ways like an optimizing compiler. A compiler +is considered correct if it doesn't change the meaning of the +program in translation from source language to target language. An +optimizing compiler exploits the particulars of the input to +generate better code, where "better" usually means more efficient. +When a case is found in which it emits suboptimal code, the +compiler is improved to recognize more cases, or more rules, and +more exceptions to rules; this process has no end. Inlining is +similar except that "better" code means tidier code. The baseline +translation (literalization) is correct, but there are endless +rules--and exceptions to rules--by which the output can be +improved. + +The following section lists some of the challenges, and ways in +which they can be addressed. + + - All effects of the call argument expressions must be preserved, + both in their number (they must not be eliminated or repeated), + and in their order (both with respect to other arguments, and any + effects in the callee function). + + This must be the case even if the corresponding parameters are + never referenced, are referenced multiple times, referenced in + a different order from the arguments, or referenced within a + nested function that may be executed an arbitrary number of + times. + + Currently, parameter replacement is not applied to arguments + with effects, but with further analysis of the sequence of + strict effects within the callee we could relax this constraint. + + - When not all parameters can be substituted by their arguments + (e.g. due to possible effects), if the call appears in a + statement context, the inliner may introduce a var declaration + that declares the parameter variables (with the correct types) + and assigns them to their corresponding argument values. + The rest of the function body may then follow. + For example, the call + + f(1, 2) + + to the function + + func f(x, y int32) { stmts } + + may be reduced to + + { var x, y int32 = 1, 2; stmts }. + + There are many reasons why this is not always possible. For + example, true parameters are statically resolved in the same + scope, and are dynamically assigned their arguments in + parallel; but each spec in a var declaration is statically + resolved in sequence and dynamically executed in sequence, so + earlier parameters may shadow references in later ones. + + - Even an argument expression as simple as ptr.x may not be + referentially transparent, because another argument may have the + effect of changing the value of ptr. + + This constraint could be relaxed by some kind of alias or + escape analysis that proves that ptr cannot be mutated during + the call. + + - Although constants are referentially transparent, as a matter of + style we do not wish to duplicate literals that are referenced + multiple times in the body because this undoes proper factoring. + Also, string literals may be arbitrarily large. + + - If the function body consists of statements other than just + "return expr", in some contexts it may be syntactically + impossible to reduce the call. Consider: + + if x := f(); cond { ... } + + Go has no equivalent to Lisp's progn or Rust's blocks, + nor ML's let expressions (let param = arg in body); + its closest equivalent is func(param){body}(arg). + Reduction strategies must therefore consider the syntactic + context of the call. + + In such situations we could work harder to extract a statement + context for the call, by transforming it to: + + { x := f(); if cond { ... } } + + - Similarly, without the equivalent of Rust-style blocks and + first-class tuples, there is no general way to reduce a call + to a function such as + + func(params)(args)(results) { stmts; return expr } + + to an expression such as + + { var params = args; stmts; expr } + + or even a statement such as + + results = { var params = args; stmts; expr } + + Consequently the declaration and scope of the result variables, + and the assignment and control-flow implications of the return + statement, must be dealt with by cases. + + - A standalone call statement that calls a function whose body is + "return expr" cannot be simply replaced by the body expression + if it is not itself a call or channel receive expression; it is + necessary to explicitly discard the result using "_ = expr". + + Similarly, if the body is a call expression, only calls to some + built-in functions with no result (such as copy or panic) are + permitted as statements, whereas others (such as append) return + a result that must be used, even if just by discarding. + + - If a parameter or result variable is updated by an assignment + within the function body, it cannot always be safely replaced + by a variable in the caller. For example, given + + func f(a int) int { a++; return a } + + The call y = f(x) cannot be replaced by { x++; y = x } because + this would change the value of the caller's variable x. + Only if the caller is finished with x is this safe. + + A similar argument applies to parameter or result variables + that escape: by eliminating a variable, inlining would change + the identity of the variable that escapes. + + - If the function body uses 'defer' and the inlined call is not a + tail-call, inlining may delay the deferred effects. + + - Because the scope of a control label is the entire function, a + call cannot be reduced if the caller and callee have intersecting + sets of control labels. (It is possible to α-rename any + conflicting ones, but our colleagues building C++ refactoring + tools report that, when tools must choose new identifiers, they + generally do a poor job.) + + - Given + + func f() uint8 { return 0 } + + var x any = f() + + reducing the call to var x any = 0 is unsound because it + discards the implicit conversion to uint8. We may need to make + each argument-to-parameter conversion explicit if the types + differ. Assignments to variadic parameters may need to + explicitly construct a slice. + + An analogous problem applies to the implicit assignments in + return statements: + + func g() any { return f() } + + Replacing the call f() with 0 would silently lose a + conversion to uint8 and change the behavior of the program. + + - When inlining a call f(1, x, g()) where those parameters are + unreferenced, we should be able to avoid evaluating 1 and x + since they are pure and thus have no effect. But x may be the + last reference to a local variable in the caller, so removing + it would cause a compilation error. Parameter substitution must + avoid making the caller's local variables unreferenced (or must + be prepared to eliminate the declaration too---this is where an + iterative framework for simplification would really help). + +More complex callee functions are inlinable with more elaborate and +invasive changes to the statements surrounding the call expression. + +TODO(adonovan): future work: + + - Handle more of the above special cases by careful analysis, + thoughtful factoring of the large design space, and thorough + test coverage. + + - Compute precisely (not conservatively) when parameter + substitution would remove the last reference to a caller local + variable, and blank out the local instead of retreating from + the substitution. + + - Afford the client more control such as a limit on the total + increase in line count, or a refusal to inline using the + general approach (replacing name by function literal). This + could be achieved by returning metadata alongside the result + and having the client conditionally discard the change. + + - Is it acceptable to skip effects that are limited to runtime + panics? Can we avoid evaluating an argument x.f + or a[i] when the corresponding parameter is unused? + + - Support inlining of generic functions, replacing type parameters + by their instantiations. + + - Support inlining of calls to function literals ("closures"). + But note that the existing algorithm makes widespread assumptions + that the callee is a package-level function or method. + + - Eliminate parens and braces inserted conservatively when they + are redundant. + + - Allow non-'go' build systems such as Bazel/Blaze a chance to + decide whether an import is accessible using logic other than + "/internal/" path segments. This could be achieved by returning + the list of added import paths instead of a text diff. + + - Inlining a function from another module may change the + effective version of the Go language spec that governs it. We + should probably make the client responsible for rejecting + attempts to inline from newer callees to older callers, since + there's no way for this package to access module versions. + + - Use an alternative implementation of the import-organizing + operation that doesn't require operating on a complete file + (and reformatting). Then return the results in a higher-level + form as a set of import additions and deletions plus a single + diff that encloses the call expression. This interface could + perhaps be implemented atop imports.Process by post-processing + its result to obtain the abstract import changes and discarding + its formatted output. +*/ +package inline diff --git a/internal/refactor/inline/inline.go b/internal/refactor/inline/inline.go index 259156c13b8..b42a3f2891d 100644 --- a/internal/refactor/inline/inline.go +++ b/internal/refactor/inline/inline.go @@ -2,275 +2,6 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Package inline implements inlining of Go function calls. -// -// The client provides information about the caller and callee, -// including the source text, syntax tree, and type information, and -// the inliner returns the modified source file for the caller, or an -// error if the inlining operation is invalid (for example because the -// function body refers to names that are inaccessible to the caller). -// -// Although this interface demands more information from the client -// than might seem necessary, it enables smoother integration with -// existing batch and interactive tools that have their own ways of -// managing the processes of reading, parsing, and type-checking -// packages. In particular, this package does not assume that the -// caller and callee belong to the same token.FileSet or -// types.Importer realms. -// -// There are many aspects to a function call. It is the only construct -// that can simultaneously bind multiple variables of different -// explicit types, with implicit assignment conversions. (Neither var -// nor := declarations can do that.) It defines the scope of control -// labels, of return statements, and of defer statements. Arguments -// and results of function calls may be tuples even though tuples are -// not first-class values in Go, and a tuple-valued call expression -// may be "spread" across the argument list of a call or the operands -// of a return statement. All these unique features mean that in the -// general case, not everything that can be expressed by a function -// call can be expressed without one. -// -// So, in general, inlining consists of modifying a function or method -// call expression f(a1, ..., an) so that the name of the function f -// is replaced ("literalized") by a literal copy of the function -// declaration, with free identifiers suitably modified to use the -// locally appropriate identifiers or perhaps constant argument -// values. -// -// Inlining must not change the semantics of the call. Semantics -// preservation is crucial for clients such as codebase maintenance -// tools that automatically inline all calls to designated functions -// on a large scale. Such tools must not introduce subtle behavior -// changes. (Fully inlining a call is dynamically observable using -// reflection over the call stack, but this exception to the rule is -// explicitly allowed.) -// -// In many cases it is possible to entirely replace ("reduce") the -// call by a copy of the function's body in which parameters have been -// replaced by arguments. The inliner supports a number of reduction -// strategies, and we expect this set to grow. Nonetheless, sound -// reduction is surprisingly tricky. -// -// The inliner is in some ways like an optimizing compiler. A compiler -// is considered correct if it doesn't change the meaning of the -// program in translation from source language to target language. An -// optimizing compiler exploits the particulars of the input to -// generate better code, where "better" usually means more efficient. -// When a case is found in which it emits suboptimal code, the -// compiler is improved to recognize more cases, or more rules, and -// more exceptions to rules; this process has no end. Inlining is -// similar except that "better" code means tidier code. The baseline -// translation (literalization) is correct, but there are endless -// rules--and exceptions to rules--by which the output can be -// improved. -// -// The following section lists some of the challenges, and ways in -// which they can be addressed. -// -// - All effects of the call argument expressions must be preserved, -// both in their number (they must not be eliminated or repeated), -// and in their order (both with respect to other arguments, and any -// effects in the callee function). -// -// This must be the case even if the corresponding parameters are -// never referenced, are referenced multiple times, referenced in -// a different order from the arguments, or referenced within a -// nested function that may be executed an arbitrary number of -// times. -// -// Currently, parameter replacement is not applied to arguments -// with effects, but with further analysis of the sequence of -// strict effects within the callee we could relax this constraint. -// -// - When not all parameters can be substituted by their arguments -// (e.g. due to possible effects), if the call appears in a -// statement context, the inliner may introduce a var declaration -// that declares the parameter variables (with the correct types) -// and assigns them to their corresponding argument values. -// The rest of the function body may then follow. -// For example, the call -// -// f(1, 2) -// -// to the function -// -// func f(x, y int32) { stmts } -// -// may be reduced to -// -// { var x, y int32 = 1, 2; stmts }. -// -// There are many reasons why this is not always possible. For -// example, true parameters are statically resolved in the same -// scope, and are dynamically assigned their arguments in -// parallel; but each spec in a var declaration is statically -// resolved in sequence and dynamically executed in sequence, so -// earlier parameters may shadow references in later ones. -// -// - Even an argument expression as simple as ptr.x may not be -// referentially transparent, because another argument may have the -// effect of changing the value of ptr. -// -// This constraint could be relaxed by some kind of alias or -// escape analysis that proves that ptr cannot be mutated during -// the call. -// -// - Although constants are referentially transparent, as a matter of -// style we do not wish to duplicate literals that are referenced -// multiple times in the body because this undoes proper factoring. -// Also, string literals may be arbitrarily large. -// -// - If the function body consists of statements other than just -// "return expr", in some contexts it may be syntactically -// impossible to reduce the call. Consider: -// -// if x := f(); cond { ... } -// -// Go has no equivalent to Lisp's progn or Rust's blocks, -// nor ML's let expressions (let param = arg in body); -// its closest equivalent is func(param){body}(arg). -// Reduction strategies must therefore consider the syntactic -// context of the call. -// -// In such situations we could work harder to extract a statement -// context for the call, by transforming it to: -// -// { x := f(); if cond { ... } } -// -// - Similarly, without the equivalent of Rust-style blocks and -// first-class tuples, there is no general way to reduce a call -// to a function such as -// -// func(params)(args)(results) { stmts; return expr } -// -// to an expression such as -// -// { var params = args; stmts; expr } -// -// or even a statement such as -// -// results = { var params = args; stmts; expr } -// -// Consequently the declaration and scope of the result variables, -// and the assignment and control-flow implications of the return -// statement, must be dealt with by cases. -// -// - A standalone call statement that calls a function whose body is -// "return expr" cannot be simply replaced by the body expression -// if it is not itself a call or channel receive expression; it is -// necessary to explicitly discard the result using "_ = expr". -// -// Similarly, if the body is a call expression, only calls to some -// built-in functions with no result (such as copy or panic) are -// permitted as statements, whereas others (such as append) return -// a result that must be used, even if just by discarding. -// -// - If a parameter or result variable is updated by an assignment -// within the function body, it cannot always be safely replaced -// by a variable in the caller. For example, given -// -// func f(a int) int { a++; return a } -// -// The call y = f(x) cannot be replaced by { x++; y = x } because -// this would change the value of the caller's variable x. -// Only if the caller is finished with x is this safe. -// -// A similar argument applies to parameter or result variables -// that escape: by eliminating a variable, inlining would change -// the identity of the variable that escapes. -// -// - If the function body uses 'defer' and the inlined call is not a -// tail-call, inlining may delay the deferred effects. -// -// - Because the scope of a control label is the entire function, a -// call cannot be reduced if the caller and callee have intersecting -// sets of control labels. (It is possible to α-rename any -// conflicting ones, but our colleagues building C++ refactoring -// tools report that, when tools must choose new identifiers, they -// generally do a poor job.) -// -// - Given -// -// func f() uint8 { return 0 } -// -// var x any = f() -// -// reducing the call to var x any = 0 is unsound because it -// discards the implicit conversion to uint8. We may need to make -// each argument-to-parameter conversion explicit if the types -// differ. Assignments to variadic parameters may need to -// explicitly construct a slice. -// -// An analogous problem applies to the implicit assignments in -// return statements: -// -// func g() any { return f() } -// -// Replacing the call f() with 0 would silently lose a -// conversion to uint8 and change the behavior of the program. -// -// - When inlining a call f(1, x, g()) where those parameters are -// unreferenced, we should be able to avoid evaluating 1 and x -// since they are pure and thus have no effect. But x may be the -// last reference to a local variable in the caller, so removing -// it would cause a compilation error. Argument elimination must -// avoid making the caller's local variables unreferenced (or must -// be prepared to eliminate the declaration too---this is where an -// iterative framework for simplification would really help). -// -// More complex callee functions are inlinable with more elaborate and -// invasive changes to the statements surrounding the call expression. -// -// TODO(adonovan): future work: -// -// - Handle more of the above special cases by careful analysis, -// thoughtful factoring of the large design space, and thorough -// test coverage. -// -// - Compute precisely (not conservatively) when parameter -// elimination would remove the last reference to a caller local -// variable, and blank out the local instead of retreating from -// the elimination. -// -// - Afford the client more control such as a limit on the total -// increase in line count, or a refusal to inline using the -// general approach (replacing name by function literal). This -// could be achieved by returning metadata alongside the result -// and having the client conditionally discard the change. -// -// - Is it acceptable to skip effects that are limited to runtime -// panics? Can we avoid evaluating an argument x.f -// or a[i] when the corresponding parameter is unused? -// -// - Support inlining of generic functions, replacing type parameters -// by their instantiations. -// -// - Support inlining of calls to function literals ("closures"). -// But note that the existing algorithm makes widespread assumptions -// that the callee is a package-level function or method. -// -// - Eliminate parens and braces inserted conservatively when they -// are redundant. -// -// - Allow non-'go' build systems such as Bazel/Blaze a chance to -// decide whether an import is accessible using logic other than -// "/internal/" path segments. This could be achieved by returning -// the list of added import paths instead of a text diff. -// -// - Inlining a function from another module may change the -// effective version of the Go language spec that governs it. We -// should probably make the client responsible for rejecting -// attempts to inline from newer callees to older callers, since -// there's no way for this package to access module versions. -// -// - Use an alternative implementation of the import-organizing -// operation that doesn't require operating on a complete file -// (and reformatting). Then return the results in a higher-level -// form as a set of import additions and deletions plus a single -// diff that encloses the call expression. This interface could -// perhaps be implemented atop imports.Process by post-processing -// its result to obtain the abstract import changes and discarding -// its formatted output. package inline import ( @@ -716,7 +447,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu duplicable bool // expr may be duplicated freevars map[string]bool // free names of expr } - var args []*argument // effective arguments; nil => eliminated + var args []*argument // effective arguments; nil => substituted { // TODO(adonovan): extract to a function (in a separate CL). callArgs := caller.Call.Args @@ -817,7 +548,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu info *paramInfo // information from AnalyzeCallee variadic bool // (final) parameter is unsimplified ...T } - var params []*parameter // including receiver; nil => parameter eliminated + var params []*parameter // including receiver; nil => parameter substituted { sig := calleeSymbol.Type().(*types.Signature) if sig.Recv() != nil { @@ -889,12 +620,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // Note: computation below should be expressed in terms of // the args and params slices, not the raw material. - // Parameter elimination - // - // TODO(adonovan): use the term "parameter substitution" - // instead because a "binding decl" (see below) is an - // alternative way that parameters can be eliminated even when - // they can't be substituted. + // Parameter substitution // // Consider each parameter and its corresponding argument in turn // and evaluate these conditions: @@ -907,7 +633,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // - the argument (or types.Default(argument) if it's untyped) has // the same type as the parameter. // - // If all conditions are met then the parameter can be eliminated + // If all conditions are met then the parameter can be substituted // and each reference to it replaced by the argument. { // Inv: @@ -1006,8 +732,8 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu for _, ref := range param.info.Refs { replaceCalleeID(ref, cloneNode(arg.expr).(ast.Expr)) } - params[i] = nil // eliminated - args[i] = nil // eliminated + params[i] = nil // substituted + args[i] = nil // substituted } } @@ -1018,7 +744,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu } } - // Modify callee's FuncDecl.Type.Params to remove eliminated + // Modify callee's FuncDecl.Type.Params to remove substituted // parameters and move the receiver (if any) to the head of // the ordinary parameters. // @@ -1046,8 +772,8 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu paramIdx++ } else { // Named parameter field e.g. func f(x, y int) - // Remove eliminated parameters in place. - // If all were eliminated, delete field. + // Remove substituted parameters in place. + // If all were substituted, delete field. for _, id := range field.Names { if pinfo := params[paramIdx]; pinfo != nil { // Rename unreferenced parameters with "_". @@ -1083,7 +809,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // // If we succeed, the declaration may be used by reduction // strategies to relax the requirement that all parameters - // have been eliminated. + // have been substituted. // // For example, a call: // f(a0, a1, a2) @@ -1239,7 +965,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // Attempt to reduce parameterless calls // whose result variables do not escape. - allParamsEliminated := forall(params, func(i int, p *parameter) bool { + allParamsSubstituted := forall(params, func(i int, p *parameter) bool { return p == nil }) noResultEscapes := !exists(callee.Results, func(i int, r *paramInfo) bool { @@ -1255,8 +981,8 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // // If: // - the body is just "return expr" with trivial implicit conversions, - // - all parameters have been eliminated or - // can be replaced by a binding decl, and + // - all parameters can be eliminated + // (by substitution, or a binding decl), // - no result var escapes, // then the call expression can be replaced by the // callee's body expression, suitably substituted. @@ -1267,14 +993,14 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // statement context if stmt, ok := context.(*ast.ExprStmt); ok && - (allParamsEliminated && noResultEscapes || bindingDeclStmt != nil) { + (allParamsSubstituted && noResultEscapes || bindingDeclStmt != nil) { logf("strategy: reduce stmt-context call to { return exprs }") clearPositions(calleeDecl.Body) if callee.ValidForCallStmt { logf("callee body is valid as statement") // Inv: len(results) == 1 - if allParamsEliminated && noResultEscapes { + if allParamsSubstituted && noResultEscapes { // Reduces to: expr res.old = caller.Call res.new = results[0] @@ -1300,7 +1026,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu Rhs: results, } res.old = stmt - if allParamsEliminated && noResultEscapes { + if allParamsSubstituted && noResultEscapes { // Reduces to: _, _ = exprs res.new = discard } else { @@ -1317,7 +1043,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu } // expression context - if allParamsEliminated && noResultEscapes { + if allParamsSubstituted && noResultEscapes { clearPositions(calleeDecl.Body) if callee.NumResults == 1 { @@ -1382,8 +1108,8 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // { var (bindings); body } // { body } // so long as: - // - all parameters have been eliminated or - // can be replaced by a binding declaration; + // - all parameters can be eliminated + // (by substitution, or a binding decl), // - call is a tail-call; // - all returns in body have trivial result conversions; // - there is no label conflict; @@ -1401,7 +1127,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu if ret, ok := callContext(callerPath).(*ast.ReturnStmt); ok && len(ret.Results) == 1 && callee.TrivialReturns == callee.TotalReturns && - (allParamsEliminated && noResultEscapes || bindingDeclStmt != nil) && + (allParamsSubstituted && noResultEscapes || bindingDeclStmt != nil) && !hasLabelConflict(callerPath, callee.Labels) && forall(callee.Results, func(i int, p *paramInfo) bool { // all result vars are unreferenced @@ -1410,7 +1136,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu logf("strategy: reduce tail-call") body := calleeDecl.Body clearPositions(body) - if !(allParamsEliminated && noResultEscapes) { + if !(allParamsSubstituted && noResultEscapes) { body.List = prepend(bindingDeclStmt, body.List...) } res.old = ret @@ -1431,12 +1157,12 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // - callee is a void function (no returns) // - callee does not use defer // - there is no label conflict between caller and callee - // - all parameters have been eliminated or - // can be replaced by a binding decl. + // - all parameters can be eliminated + // (by substitution, or a binding decl), // // If there is only a single statement, the braces are omitted. if stmt := callStmt(callerPath); stmt != nil && - (allParamsEliminated && noResultEscapes || bindingDeclStmt != nil) && + (allParamsSubstituted && noResultEscapes || bindingDeclStmt != nil) && !callee.HasDefer && !hasLabelConflict(callerPath, callee.Labels) && callee.TotalReturns == 0 { @@ -1444,7 +1170,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu body := calleeDecl.Body var repl ast.Stmt = body clearPositions(repl) - if !(allParamsEliminated && noResultEscapes) { + if !(allParamsSubstituted && noResultEscapes) { body.List = prepend(bindingDeclStmt, body.List...) } if len(body.List) == 1 { From c00d71d8bbb72614748372f0ab27fe29ab7d53c5 Mon Sep 17 00:00:00 2001 From: cui fliter Date: Fri, 26 May 2023 11:55:33 +0800 Subject: [PATCH 115/178] go/analysis: add a new analyzer for check missing values after append Currently, if there is no second parameter added during append, there will be no prompt when executing go vet. Add an analyzer to detect this situation Update golang/go#60448 Change-Id: If213fc25fb6521a0f9777ae047fbfbac97e5365f Reviewed-on: https://go-review.googlesource.com/c/tools/+/498475 TryBot-Result: Gopher Robot Reviewed-by: Tim King gopls-CI: kokoro LUCI-TryBot-Result: Go LUCI Reviewed-by: Damien Neil Run-TryBot: shuang cui --- go/analysis/passes/append/append.go | 49 +++++++++++++++++++ go/analysis/passes/append/append_test.go | 18 +++++++ go/analysis/passes/append/doc.go | 20 ++++++++ go/analysis/passes/append/testdata/src/a/a.go | 32 ++++++++++++ go/analysis/passes/append/testdata/src/b/b.go | 18 +++++++ 5 files changed, 137 insertions(+) create mode 100644 go/analysis/passes/append/append.go create mode 100644 go/analysis/passes/append/append_test.go create mode 100644 go/analysis/passes/append/doc.go create mode 100644 go/analysis/passes/append/testdata/src/a/a.go create mode 100644 go/analysis/passes/append/testdata/src/b/b.go diff --git a/go/analysis/passes/append/append.go b/go/analysis/passes/append/append.go new file mode 100644 index 00000000000..aa40fcd8973 --- /dev/null +++ b/go/analysis/passes/append/append.go @@ -0,0 +1,49 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package append defines an Analyzer that detects +// if there is only one variable in append. +package append + +import ( + _ "embed" + "go/ast" + "go/types" + + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/analysis/passes/inspect" + "golang.org/x/tools/go/analysis/passes/internal/analysisutil" + "golang.org/x/tools/go/ast/inspector" +) + +//go:embed doc.go +var doc string + +var Analyzer = &analysis.Analyzer{ + Name: "append", + Doc: analysisutil.MustExtractDoc(doc, "append"), + URL: "https://pkg.go.dev/golang.org/x/tools/go/analysis/passes/append", + Requires: []*analysis.Analyzer{inspect.Analyzer}, + Run: run, +} + +func run(pass *analysis.Pass) (interface{}, error) { + inspect := pass.ResultOf[inspect.Analyzer].(*inspector.Inspector) + + nodeFilter := []ast.Node{ + (*ast.CallExpr)(nil), + } + inspect.Preorder(nodeFilter, func(n ast.Node) { + call := n.(*ast.CallExpr) + if ident, ok := call.Fun.(*ast.Ident); ok && ident.Name == "append" { + if _, ok := pass.TypesInfo.Uses[ident].(*types.Builtin); ok { + if len(call.Args) == 1 { + pass.ReportRangef(call, "append with no values") + } + } + } + }) + + return nil, nil +} diff --git a/go/analysis/passes/append/append_test.go b/go/analysis/passes/append/append_test.go new file mode 100644 index 00000000000..f05aa866c93 --- /dev/null +++ b/go/analysis/passes/append/append_test.go @@ -0,0 +1,18 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package append_test + +import ( + "testing" + + "golang.org/x/tools/go/analysis/analysistest" + "golang.org/x/tools/go/analysis/passes/append" +) + +func Test(t *testing.T) { + testdata := analysistest.TestData() + tests := []string{"a", "b"} + analysistest.Run(t, testdata, append.Analyzer, tests...) +} diff --git a/go/analysis/passes/append/doc.go b/go/analysis/passes/append/doc.go new file mode 100644 index 00000000000..d88c5d88fdd --- /dev/null +++ b/go/analysis/passes/append/doc.go @@ -0,0 +1,20 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package append defines an Analyzer that detects +// if there is only one variable in append. +// +// # Analyzer append +// +// append: check for missing values after append +// +// This checker reports append with no values. +// +// For example: +// +// sli := []string{"a", "b", "c"} +// sli = append(sli) +// +// it would report "append with no values" +package append diff --git a/go/analysis/passes/append/testdata/src/a/a.go b/go/analysis/passes/append/testdata/src/a/a.go new file mode 100644 index 00000000000..9239c48571f --- /dev/null +++ b/go/analysis/passes/append/testdata/src/a/a.go @@ -0,0 +1,32 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains tests for the append checker. + +package a + +func badAppendSlice1() { + sli := []string{"a", "b", "c"} + sli = append(sli) // want "append with no values" +} + +func badAppendSlice2() { + _ = append([]string{"a"}) // want "append with no values" +} + +func goodAppendSlice1() { + sli := []string{"a", "b", "c"} + sli = append(sli, "d") +} + +func goodAppendSlice2() { + sli1 := []string{"a", "b", "c"} + sli2 := []string{"d", "e", "f"} + sli1 = append(sli1, sli2...) +} + +func goodAppendSlice3() { + sli := []string{"a", "b", "c"} + sli = append(sli, "d", "e", "f") +} diff --git a/go/analysis/passes/append/testdata/src/b/b.go b/go/analysis/passes/append/testdata/src/b/b.go new file mode 100644 index 00000000000..9a297916341 --- /dev/null +++ b/go/analysis/passes/append/testdata/src/b/b.go @@ -0,0 +1,18 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains tests for the append checker. + +package b + +func append(args ...interface{}) []int { + println(args) + return []int{0} +} + +func userdefine() { + sli := []int{1, 2, 3} + sli = append(sli, 4, 5, 6) + sli = append(sli) +} From a3c6fd8de24ddd1fc9e1dcf2d31df86db192bab9 Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Tue, 12 Sep 2023 13:08:54 -0400 Subject: [PATCH 116/178] gopls/internal/lsp: add an idle prompt asking users to enable telemetry Wire in (but do not enable) a prompt to ask users if they want to enable Go telemetry. There are several aspects of this change. - A new "telemetryPrompt" setting controlling whether prompting is allowed. For now, this defaults to false. - A new mechanism for persisting data to the user config dir (e.g. .config/gopls on linux). - Logic to determine whether to prompt. - ...the prompt itself. - Various unit and integration tests. Wherever possible, the new logic attempts to be defensive against accidentally spamming the user. Fixes golang/go#62576 Change-Id: Ia738bdeb6bbaf0d972cd6d7234948ef67f029355 Reviewed-on: https://go-review.googlesource.com/c/tools/+/529355 LUCI-TryBot-Result: Go LUCI Reviewed-by: Alan Donovan --- gopls/internal/lsp/fake/client.go | 6 +- gopls/internal/lsp/fake/editor.go | 4 + gopls/internal/lsp/general.go | 5 + gopls/internal/lsp/prompt.go | 281 ++++++++++++++++++ gopls/internal/lsp/prompt_test.go | 82 +++++ gopls/internal/lsp/regtest/expectation.go | 75 +++-- gopls/internal/lsp/regtest/options.go | 13 +- gopls/internal/lsp/regtest/wrappers.go | 2 +- gopls/internal/lsp/source/options.go | 10 + .../regtest/diagnostics/diagnostics_test.go | 8 +- gopls/internal/regtest/misc/prompt_test.go | 179 +++++++++++ .../internal/regtest/workspace/broken_test.go | 6 +- gopls/internal/telemetry/telemetry.go | 11 + gopls/internal/telemetry/telemetry_go118.go | 8 + 14 files changed, 652 insertions(+), 38 deletions(-) create mode 100644 gopls/internal/lsp/prompt.go create mode 100644 gopls/internal/lsp/prompt_test.go create mode 100644 gopls/internal/regtest/misc/prompt_test.go diff --git a/gopls/internal/lsp/fake/client.go b/gopls/internal/lsp/fake/client.go index 1c073727109..cedd5884386 100644 --- a/gopls/internal/lsp/fake/client.go +++ b/gopls/internal/lsp/fake/client.go @@ -63,10 +63,10 @@ func (c *Client) ShowMessageRequest(ctx context.Context, params *protocol.ShowMe return nil, err } } - if len(params.Actions) == 0 || len(params.Actions) > 1 { - return nil, fmt.Errorf("fake editor cannot handle multiple action items") + if c.editor.config.MessageResponder != nil { + return c.editor.config.MessageResponder(params) } - return ¶ms.Actions[0], nil + return nil, nil // don't choose, which is effectively dismissing the message } func (c *Client) LogMessage(ctx context.Context, params *protocol.LogMessageParams) error { diff --git a/gopls/internal/lsp/fake/editor.go b/gopls/internal/lsp/fake/editor.go index 2cf77d19ff3..12d07a753e7 100644 --- a/gopls/internal/lsp/fake/editor.go +++ b/gopls/internal/lsp/fake/editor.go @@ -117,6 +117,10 @@ type EditorConfig struct { // Specifically, this JSON string will be unmarshalled into the editor's // client capabilities struct, before sending to the server. CapabilitiesJSON []byte + + // If non-nil, MessageResponder is used to respond to ShowMessageRequest + // messages. + MessageResponder func(params *protocol.ShowMessageRequestParams) (*protocol.MessageActionItem, error) } // NewEditor creates a new Editor. diff --git a/gopls/internal/lsp/general.go b/gopls/internal/lsp/general.go index 61c16c0531a..b1d6850b3c6 100644 --- a/gopls/internal/lsp/general.go +++ b/gopls/internal/lsp/general.go @@ -238,6 +238,11 @@ func (s *Server) initialized(ctx context.Context, params *protocol.InitializedPa return err } } + + // Ask (maybe) about enabling telemetry. Do this asynchronously, as it's OK + // for users to ignore or dismiss the question. + go s.maybePromptForTelemetry(ctx) + return nil } diff --git a/gopls/internal/lsp/prompt.go b/gopls/internal/lsp/prompt.go new file mode 100644 index 00000000000..9b7555cbe6a --- /dev/null +++ b/gopls/internal/lsp/prompt.go @@ -0,0 +1,281 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lsp + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + "golang.org/x/tools/gopls/internal/lsp/protocol" + "golang.org/x/tools/gopls/internal/telemetry" + "golang.org/x/tools/internal/event" +) + +// promptTimeout is the amount of time we wait for an ongoing prompt before +// prompting again. This gives the user time to reply. However, at some point +// we must assume that the client is not displaying the prompt, the user is +// ignoring it, or the prompt has been disrupted in some way (e.g. by a gopls +// crash). +const promptTimeout = 24 * time.Hour + +// The following constants are used for testing telemetry integration. +const ( + TelemetryPromptWorkTitle = "Checking telemetry prompt" // progress notification title, for awaiting in tests + GoplsConfigDirEnvvar = "GOPLS_CONFIG_DIR" // overridden for testing + FakeTelemetryModefileEnvvar = "GOPLS_FAKE_TELEMETRY_MODEFILE" // overridden for testing +) + +// getenv returns the effective environment variable value for the provided +// key, looking up the key in the session environment before falling back on +// the process environment. +func (s *Server) getenv(key string) string { + if v, ok := s.Options().Env[key]; ok { + return v + } + return os.Getenv(key) +} + +// configDir returns the root of the gopls configuration dir. By default this +// is os.UserConfigDir/gopls, but it may be overridden for tests. +func (s *Server) configDir() (string, error) { + if d := s.getenv(GoplsConfigDirEnvvar); d != "" { + return d, nil + } + userDir, err := os.UserConfigDir() + if err != nil { + return "", err + } + return filepath.Join(userDir, "gopls"), nil +} + +// telemetryMode returns the current effective telemetry mode. +// By default this is x/telemetry.Mode(), but it may be overridden for tests. +func (s *Server) telemetryMode() string { + if fake := s.getenv(FakeTelemetryModefileEnvvar); fake != "" { + if data, err := os.ReadFile(fake); err == nil { + return string(data) + } + return "local" + } + return telemetry.Mode() +} + +// setTelemetryMode sets the current telemetry mode. +// By default this calls x/telemetry.SetMode, but it may be overridden for +// tests. +func (s *Server) setTelemetryMode(mode string) error { + if fake := s.getenv(FakeTelemetryModefileEnvvar); fake != "" { + return os.WriteFile(fake, []byte(mode), 0666) + } + return telemetry.SetMode(mode) +} + +// maybePromptForTelemetry checks for the right conditions, and then prompts +// the user to ask if they want to enable Go telemetry uploading. If the user +// responds 'Yes', the telemetry mode is set to "on". +// +// The actual conditions for prompting are defensive, erring on the side of not +// prompting. +func (s *Server) maybePromptForTelemetry(ctx context.Context) { + if s.Options().VerboseWorkDoneProgress { + work := s.progress.Start(ctx, TelemetryPromptWorkTitle, "Checking if gopls should prompt about telemetry...", nil, nil) + defer work.End(ctx, "Done.") + } + + if !s.Options().TelemetryPrompt { + return // prompt is disabled + } + + // Only prompt if telemetry is in the default "local" mode. If it is already + // "on" there's nothing to ask about, and if it is explicitly "off" let's + // assume the user doesn't want it. + if s.telemetryMode() != "local" { + return + } + + errorf := func(format string, args ...any) { + err := fmt.Errorf(format, args...) + event.Error(ctx, "telemetry prompt failed", err) + } + + // Only prompt if we can read/write the prompt config file. + configDir, err := s.configDir() + if err != nil { + errorf("unable to determine config dir: %v", err) + return + } + + var ( + promptDir = filepath.Join(configDir, "prompt") // prompt configuration directory + promptFile = filepath.Join(promptDir, "telemetry") // telemetry prompt file + ) + + // prompt states, to be written to the prompt file + const ( + pYes = "yes" // user said yes + pNo = "no" // user said no + pPending = "pending" // current prompt is still pending + pFailed = "failed" // prompt was asked but failed + ) + validStates := map[string]bool{ + pYes: true, + pNo: true, + pPending: true, + pFailed: true, + } + + // parse the current prompt file + var ( + state string + attempts = 0 // number of times we've asked already + ) + if result, err := os.ReadFile(promptFile); err == nil { + if _, err := fmt.Sscanf(string(result), "%s %d", &state, &attempts); err == nil && validStates[state] { + if state == pYes || state == pNo { + // Prompt has been answered. Nothing to do. + return + } + } else { + state, attempts = "", 0 + errorf("malformed prompt result %q", string(result)) + } + } else if !os.IsNotExist(err) { + errorf("reading prompt file: %v", err) + // Something went wrong. Since we don't know how many times we've asked the + // prompt, err on the side of not spamming. + return + } + + if attempts >= 5 { + // We've tried asking enough; give up. + return + } + if attempts == 0 { + // First time asking the prompt; we may need to make the prompt dir. + if err := os.MkdirAll(promptDir, 0777); err != nil { + errorf("creating prompt dir: %v", err) + return + } + } + + // Acquire the lock and write "pending" to the prompt file before actually + // prompting. + // + // This ensures that the prompt file is writeable, and that we increment the + // attempt counter before we prompt, so that we don't end up in a failure + // mode where we keep prompting and then failing to record the response. + + release, ok, err := acquireLockFile(promptFile) + if err != nil { + errorf("acquiring prompt: %v", err) + return + } + if !ok { + // Another prompt is currently pending. + return + } + defer release() + + attempts++ + + pendingContent := []byte(fmt.Sprintf("%s %d", pPending, attempts)) + if err := os.WriteFile(promptFile, pendingContent, 0666); err != nil { + errorf("writing pending state: %v", err) + return + } + + const prompt = `Go telemetry helps us improve Go by periodically sending anonymous metrics and crash reports to the Go team. Learn more at https://telemetry.go.dev/privacy. + +Would you like to enable Go telemetry? +` + + // TODO(rfindley): investigate a "tell me more" action in combination with ShowDocument. + params := &protocol.ShowMessageRequestParams{ + Type: protocol.Info, + Message: prompt, + Actions: []protocol.MessageActionItem{{Title: "Yes"}, {Title: "No"}}, + } + item, err := s.client.ShowMessageRequest(ctx, params) + if err != nil { + errorf("ShowMessageRequest failed: %v", err) + } + + result := pFailed + if item == nil { + // e.g. dialog was dismissed + errorf("no response") + } else { + // Response matches MessageActionItem.Title. + switch item.Title { + case "Yes": + result = pYes + s.setTelemetryMode("on") + case "No": + result = pNo + default: + errorf("unrecognized response %q", item.Title) + if err := s.client.ShowMessage(ctx, &protocol.ShowMessageParams{ + Type: protocol.Error, + Message: fmt.Sprintf("Unrecognized response %q", item.Title), + }); err != nil { + errorf("ShowMessage failed: %v", err) + } + } + } + resultContent := []byte(fmt.Sprintf("%s %d", result, attempts)) + if err := os.WriteFile(promptFile, resultContent, 0666); err != nil { + errorf("error writing result state to prompt file: %v", err) + } +} + +// acquireLockFile attempts to "acquire a lock" for writing to path. +// +// This is achieved by creating an exclusive lock file at .lock. Lock +// files expire after a period, at which point acquireLockFile will remove and +// recreate the lock file. +// +// acquireLockFile fails if path is in a directory that doesn't exist. +func acquireLockFile(path string) (func(), bool, error) { + lockpath := path + ".lock" + fi, err := os.Stat(lockpath) + if err == nil { + if time.Since(fi.ModTime()) > promptTimeout { + _ = os.Remove(lockpath) // ignore error + } else { + return nil, false, nil + } + } else if !os.IsNotExist(err) { + return nil, false, fmt.Errorf("statting lockfile: %v", err) + } + + f, err := os.OpenFile(lockpath, os.O_CREATE|os.O_EXCL, 0666) + if err != nil { + if os.IsExist(err) { + return nil, false, nil + } + return nil, false, fmt.Errorf("creating lockfile: %v", err) + } + fi, err = f.Stat() + if err != nil { + return nil, false, err + } + release := func() { + _ = f.Close() // ignore error + fi2, err := os.Stat(lockpath) + if err == nil && os.SameFile(fi, fi2) { + // Only clean up the lockfile if it's the same file we created. + // Otherwise, our lock has expired and something else has the lock. + // + // There's a race here, in that the file could have changed since the + // stat above; but given that we've already waited 24h this is extremely + // unlikely, and acceptable. + _ = os.Remove(lockpath) + } + } + return release, true, nil +} diff --git a/gopls/internal/lsp/prompt_test.go b/gopls/internal/lsp/prompt_test.go new file mode 100644 index 00000000000..d268d1f3a0c --- /dev/null +++ b/gopls/internal/lsp/prompt_test.go @@ -0,0 +1,82 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package lsp + +import ( + "path/filepath" + "sync" + "sync/atomic" + "testing" +) + +func TestAcquireFileLock(t *testing.T) { + name := filepath.Join(t.TempDir(), "config.json") + + const concurrency = 100 + var acquired int32 + var releasers [concurrency]func() + defer func() { + for _, r := range releasers { + if r != nil { + r() + } + } + }() + + var wg sync.WaitGroup + for i := range releasers { + i := i + wg.Add(1) + go func() { + defer wg.Done() + + release, ok, err := acquireLockFile(name) + if err != nil { + t.Errorf("Acquire failed: %v", err) + return + } + if ok { + atomic.AddInt32(&acquired, 1) + releasers[i] = release + } + }() + } + + wg.Wait() + + if acquired != 1 { + t.Errorf("Acquire succeeded %d times, expected exactly 1", acquired) + } +} + +func TestReleaseAndAcquireFileLock(t *testing.T) { + name := filepath.Join(t.TempDir(), "config.json") + + acquire := func() (func(), bool) { + t.Helper() + release, ok, err := acquireLockFile(name) + if err != nil { + t.Fatal(err) + } + return release, ok + } + + release, ok := acquire() + if !ok { + t.Fatal("failed to Acquire") + } + if release2, ok := acquire(); ok { + release() + release2() + t.Fatalf("Acquire succeeded unexpectedly") + } + + release() + release3, ok := acquire() + release3() + if !ok { + t.Fatalf("failed to Acquire") + } +} diff --git a/gopls/internal/lsp/regtest/expectation.go b/gopls/internal/lsp/regtest/expectation.go index 922f9a0b8a1..7238eb0e832 100644 --- a/gopls/internal/lsp/regtest/expectation.go +++ b/gopls/internal/lsp/regtest/expectation.go @@ -212,21 +212,6 @@ func ReadAllDiagnostics(into *map[string]*protocol.PublishDiagnosticsParams) Exp } } -// NoOutstandingWork asserts that there is no work initiated using the LSP -// $/progress API that has not completed. -func NoOutstandingWork() Expectation { - check := func(s State) Verdict { - if len(s.outstandingWork()) == 0 { - return Met - } - return Unmet - } - return Expectation{ - Check: check, - Description: "no outstanding work", - } -} - // ShownDocument asserts that the client has received a // ShowDocumentRequest for the given URI. func ShownDocument(uri protocol.URI) Expectation { @@ -277,26 +262,24 @@ func ShownMessage(containing string) Expectation { } } -// ShowMessageRequest asserts that the editor has received a ShowMessageRequest -// with an action item that has the given title. -func ShowMessageRequest(title string) Expectation { +// ShownMessageRequest asserts that the editor has received a +// ShowMessageRequest with message matching the given regular expression. +func ShownMessageRequest(messageRegexp string) Expectation { + msgRE := regexp.MustCompile(messageRegexp) check := func(s State) Verdict { if len(s.showMessageRequest) == 0 { return Unmet } - // Only check the most recent one. - m := s.showMessageRequest[len(s.showMessageRequest)-1] - if len(m.Actions) == 0 || len(m.Actions) > 1 { - return Unmet - } - if m.Actions[0].Title == title { - return Met + for _, m := range s.showMessageRequest { + if msgRE.MatchString(m.Message) { + return Met + } } return Unmet } return Expectation{ Check: check, - Description: "received ShowMessageRequest", + Description: fmt.Sprintf("ShowMessageRequest matching %q", messageRegexp), } } @@ -496,6 +479,46 @@ func OutstandingWork(title, msg string) Expectation { } } +// NoOutstandingWork asserts that there is no work initiated using the LSP +// $/progress API that has not completed. +// +// If non-nil, the ignore func is used to ignore certain work items for the +// purpose of this check. +// +// TODO(rfindley): consider refactoring to treat outstanding work the same way +// we treat diagnostics: with an algebra of filters. +func NoOutstandingWork(ignore func(title, msg string) bool) Expectation { + check := func(s State) Verdict { + for _, w := range s.work { + if w.complete { + continue + } + if w.title == "" { + // A token that has been created but not yet used. + // + // TODO(rfindley): this should be separated in the data model: until + // the "begin" notification, work should not be in progress. + continue + } + if ignore(w.title, w.msg) { + continue + } + return Unmet + } + return Met + } + return Expectation{ + Check: check, + Description: "no outstanding work", + } +} + +// IgnoreTelemetryPromptWork may be used in conjunction with NoOutStandingWork +// to ignore the telemetry prompt. +func IgnoreTelemetryPromptWork(title, msg string) bool { + return title == lsp.TelemetryPromptWorkTitle +} + // NoErrorLogs asserts that the client has not received any log messages of // error severity. func NoErrorLogs() Expectation { diff --git a/gopls/internal/lsp/regtest/options.go b/gopls/internal/lsp/regtest/options.go index f55fd5b1150..7084d621f81 100644 --- a/gopls/internal/lsp/regtest/options.go +++ b/gopls/internal/lsp/regtest/options.go @@ -4,7 +4,10 @@ package regtest -import "golang.org/x/tools/gopls/internal/lsp/fake" +import ( + "golang.org/x/tools/gopls/internal/lsp/fake" + "golang.org/x/tools/gopls/internal/lsp/protocol" +) type runConfig struct { editor fake.EditorConfig @@ -121,3 +124,11 @@ func InGOPATH() RunOption { opts.sandbox.InGoPath = true }) } + +// MessageResponder configures the editor to respond to +// window/showMessageRequest messages using the provided function. +func MessageResponder(f func(*protocol.ShowMessageRequestParams) (*protocol.MessageActionItem, error)) RunOption { + return optionSetter(func(opts *runConfig) { + opts.editor.MessageResponder = f + }) +} diff --git a/gopls/internal/lsp/regtest/wrappers.go b/gopls/internal/lsp/regtest/wrappers.go index d56d0c00335..0220d30d390 100644 --- a/gopls/internal/lsp/regtest/wrappers.go +++ b/gopls/internal/lsp/regtest/wrappers.go @@ -256,7 +256,7 @@ func (e *Env) RunGenerate(dir string) { if err := e.Editor.RunGenerate(e.Ctx, dir); err != nil { e.T.Fatal(err) } - e.Await(NoOutstandingWork()) + e.Await(NoOutstandingWork(IgnoreTelemetryPromptWork)) // Ideally the fake.Workspace would handle all synthetic file watching, but // we help it out here as we need to wait for the generate command to // complete before checking the filesystem. diff --git a/gopls/internal/lsp/source/options.go b/gopls/internal/lsp/source/options.go index a0802361bf2..02a6057d29f 100644 --- a/gopls/internal/lsp/source/options.go +++ b/gopls/internal/lsp/source/options.go @@ -176,6 +176,7 @@ func DefaultOptions(overrides ...func(*Options)) *Options { NewDiff: "new", SubdirWatchPatterns: SubdirWatchPatternsAuto, ReportAnalysisProgressAfter: 5 * time.Second, + TelemetryPrompt: false, }, Hooks: Hooks{ // TODO(adonovan): switch to new diff.Strings implementation. @@ -675,6 +676,12 @@ type InternalOptions struct { // // It is intended to be used for testing only. ReportAnalysisProgressAfter time.Duration + + // TelemetryPrompt controls whether gopls prompts about enabling Go telemetry. + // + // Once the prompt is answered, gopls doesn't ask again, but TelemetryPrompt + // can prevent the question from ever being asked in the first place. + TelemetryPrompt bool } type SubdirWatchPatterns string @@ -1259,6 +1266,9 @@ func (o *Options) set(name string, value interface{}, seen map[string]struct{}) case "reportAnalysisProgressAfter": result.setDuration(&o.ReportAnalysisProgressAfter) + case "telemetryPrompt": + result.setBool(&o.TelemetryPrompt) + // Replaced settings. case "experimentalDisabledAnalyses": result.deprecated("analyses") diff --git a/gopls/internal/regtest/diagnostics/diagnostics_test.go b/gopls/internal/regtest/diagnostics/diagnostics_test.go index 8066b7502c2..f5aa240e49d 100644 --- a/gopls/internal/regtest/diagnostics/diagnostics_test.go +++ b/gopls/internal/regtest/diagnostics/diagnostics_test.go @@ -559,7 +559,7 @@ func f() { // Deleting the import dismisses the warning. env.RegexpReplace("a.go", `import "mod.com/hello"`, "") env.AfterChange( - NoOutstandingWork(), + NoOutstandingWork(IgnoreTelemetryPromptWork), ) }) } @@ -576,7 +576,7 @@ hi mom ).Run(t, files, func(t *testing.T, env *Env) { env.OnceMet( InitialWorkspaceLoad, - NoOutstandingWork(), + NoOutstandingWork(IgnoreTelemetryPromptWork), ) }) }) @@ -1469,7 +1469,7 @@ package foo_ env.RegexpReplace("foo/foo_test.go", "_t", "_test") env.AfterChange( NoDiagnostics(ForFile("foo/foo_test.go")), - NoOutstandingWork(), + NoOutstandingWork(IgnoreTelemetryPromptWork), ) }) } @@ -1503,7 +1503,7 @@ go 1.hello env.RegexpReplace("go.mod", "go 1.hello", "go 1.12") env.SaveBufferWithoutActions("go.mod") env.AfterChange( - NoOutstandingWork(), + NoOutstandingWork(IgnoreTelemetryPromptWork), ) }) } diff --git a/gopls/internal/regtest/misc/prompt_test.go b/gopls/internal/regtest/misc/prompt_test.go new file mode 100644 index 00000000000..efe4bdf999b --- /dev/null +++ b/gopls/internal/regtest/misc/prompt_test.go @@ -0,0 +1,179 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package misc + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "testing" + + "golang.org/x/tools/gopls/internal/lsp" + "golang.org/x/tools/gopls/internal/lsp/protocol" + . "golang.org/x/tools/gopls/internal/lsp/regtest" +) + +// Test that gopls prompts for telemetry only when it is supposed to. +func TestTelemetryPrompt_Conditions(t *testing.T) { + const src = ` +-- go.mod -- +module mod.com + +go 1.12 +-- main.go -- +package main + +func main() { +} +` + + for _, enabled := range []bool{true, false} { + t.Run(fmt.Sprintf("telemetryPrompt=%v", enabled), func(t *testing.T) { + for _, initialMode := range []string{"", "off", "local", "on"} { + t.Run(fmt.Sprintf("initial_mode=%s", initialMode), func(t *testing.T) { + modeFile := filepath.Join(t.TempDir(), "mode") + if initialMode != "" { + if err := os.WriteFile(modeFile, []byte(initialMode), 0666); err != nil { + t.Fatal(err) + } + } + WithOptions( + Modes(Default), // no need to run this in all modes + EnvVars{ + lsp.GoplsConfigDirEnvvar: t.TempDir(), + lsp.FakeTelemetryModefileEnvvar: modeFile, + }, + Settings{ + "telemetryPrompt": enabled, + }, + ).Run(t, src, func(t *testing.T, env *Env) { + wantPrompt := enabled && (initialMode == "" || initialMode == "local") + expectation := ShownMessageRequest(".*Would you like to enable Go telemetry?") + if !wantPrompt { + expectation = Not(expectation) + } + env.OnceMet( + CompletedWork(lsp.TelemetryPromptWorkTitle, 1, true), + expectation, + ) + }) + }) + } + }) + } +} + +// Test that responding to the telemetry prompt results in the expected state. +func TestTelemetryPrompt_Response(t *testing.T) { + const src = ` +-- go.mod -- +module mod.com + +go 1.12 +-- main.go -- +package main + +func main() { +} +` + + tests := []struct { + response string + wantMode string + }{ + {"Yes", "on"}, + {"No", ""}, + {"", ""}, + } + for _, test := range tests { + t.Run(fmt.Sprintf("response=%s", test.response), func(t *testing.T) { + modeFile := filepath.Join(t.TempDir(), "mode") + msgRE := regexp.MustCompile(".*Would you like to enable Go telemetry?") + respond := func(m *protocol.ShowMessageRequestParams) (*protocol.MessageActionItem, error) { + if msgRE.MatchString(m.Message) { + for _, item := range m.Actions { + if item.Title == test.response { + return &item, nil + } + } + if test.response != "" { + t.Errorf("action item %q not found", test.response) + } + } + return nil, nil + } + WithOptions( + Modes(Default), // no need to run this in all modes + EnvVars{ + lsp.GoplsConfigDirEnvvar: t.TempDir(), + lsp.FakeTelemetryModefileEnvvar: modeFile, + }, + Settings{ + "telemetryPrompt": true, + }, + MessageResponder(respond), + ).Run(t, src, func(t *testing.T, env *Env) { + env.Await( + CompletedWork(lsp.TelemetryPromptWorkTitle, 1, true), + ) + gotMode := "" + if contents, err := os.ReadFile(modeFile); err == nil { + gotMode = string(contents) + } else if !os.IsNotExist(err) { + t.Fatal(err) + } + if gotMode != test.wantMode { + t.Errorf("after prompt, mode=%s, want %s", gotMode, test.wantMode) + } + }) + }) + } +} + +// Test that we stop asking about telemetry after the user ignores the question +// 5 times. +func TestTelemetryPrompt_GivingUp(t *testing.T) { + const src = ` +-- go.mod -- +module mod.com + +go 1.12 +-- main.go -- +package main + +func main() { +} +` + + // For this test, we want to share state across gopls sessions. + modeFile := filepath.Join(t.TempDir(), "mode") + configDir := t.TempDir() + + const maxPrompts = 5 // internal prompt limit defined by gopls + + for i := 0; i < maxPrompts+1; i++ { + WithOptions( + Modes(Default), // no need to run this in all modes + EnvVars{ + lsp.GoplsConfigDirEnvvar: configDir, + lsp.FakeTelemetryModefileEnvvar: modeFile, + }, + Settings{ + "telemetryPrompt": true, + }, + ).Run(t, src, func(t *testing.T, env *Env) { + wantPrompt := i < maxPrompts + expectation := ShownMessageRequest(".*Would you like to enable Go telemetry?") + if !wantPrompt { + expectation = Not(expectation) + } + env.OnceMet( + CompletedWork(lsp.TelemetryPromptWorkTitle, 1, true), + expectation, + ) + }) + } +} diff --git a/gopls/internal/regtest/workspace/broken_test.go b/gopls/internal/regtest/workspace/broken_test.go index d7d54c4d139..baa6ec1384a 100644 --- a/gopls/internal/regtest/workspace/broken_test.go +++ b/gopls/internal/regtest/workspace/broken_test.go @@ -109,7 +109,7 @@ const CompleteMe = 222 ./package2/vendor/example.com/foo ) `) - env.AfterChange(NoOutstandingWork()) + env.AfterChange(NoOutstandingWork(IgnoreTelemetryPromptWork)) // Check that definitions in package1 go to the copy vendored in package2. location := string(env.GoToDefinition(env.RegexpSearch("package1/main.go", "CompleteMe")).URI) @@ -220,7 +220,7 @@ package b env.Await( NoDiagnostics(ForFile("a/a.go")), NoDiagnostics(ForFile("b/go.mod")), - NoOutstandingWork(), + NoOutstandingWork(IgnoreTelemetryPromptWork), ) env.ChangeWorkspaceFolders(".") @@ -256,7 +256,7 @@ package b env.OpenFile("a/a.go") env.AfterChange( NoDiagnostics(ForFile("a/a.go")), - NoOutstandingWork(), + NoOutstandingWork(IgnoreTelemetryPromptWork), ) }) }) diff --git a/gopls/internal/telemetry/telemetry.go b/gopls/internal/telemetry/telemetry.go index 840896d516d..dc6f7c23372 100644 --- a/gopls/internal/telemetry/telemetry.go +++ b/gopls/internal/telemetry/telemetry.go @@ -10,11 +10,22 @@ package telemetry import ( "fmt" + "golang.org/x/telemetry" "golang.org/x/telemetry/counter" "golang.org/x/telemetry/upload" "golang.org/x/tools/gopls/internal/lsp/protocol" ) +// Mode calls x/telemetry.Mode. +func Mode() string { + return telemetry.Mode() +} + +// SetMode calls x/telemetry.SetMode. +func SetMode(mode string) error { + return telemetry.SetMode(mode) +} + // Start starts telemetry instrumentation. func Start() { counter.Open() diff --git a/gopls/internal/telemetry/telemetry_go118.go b/gopls/internal/telemetry/telemetry_go118.go index 2d1d6040490..53394002f76 100644 --- a/gopls/internal/telemetry/telemetry_go118.go +++ b/gopls/internal/telemetry/telemetry_go118.go @@ -9,6 +9,14 @@ package telemetry import "golang.org/x/tools/gopls/internal/lsp/protocol" +func Mode() string { + return "local" +} + +func SetMode(mode string) error { + return nil +} + func Start() { } From 9125a0fc193264aa70c0985e95ee0e61b4947e83 Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Wed, 20 Sep 2023 09:38:26 -0400 Subject: [PATCH 117/178] gopls/internal/lsp: minor cleanup to prompt logic This fixes a couple nits found during follow-up review of CL 529355. Change-Id: I0d8331b037d5a4ba91ad06b47375fe2f127164a3 Reviewed-on: https://go-review.googlesource.com/c/tools/+/529559 LUCI-TryBot-Result: Go LUCI Reviewed-by: Alan Donovan --- gopls/internal/lsp/prompt.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/gopls/internal/lsp/prompt.go b/gopls/internal/lsp/prompt.go index 9b7555cbe6a..c87bfb5a0ba 100644 --- a/gopls/internal/lsp/prompt.go +++ b/gopls/internal/lsp/prompt.go @@ -134,15 +134,15 @@ func (s *Server) maybePromptForTelemetry(ctx context.Context) { state string attempts = 0 // number of times we've asked already ) - if result, err := os.ReadFile(promptFile); err == nil { - if _, err := fmt.Sscanf(string(result), "%s %d", &state, &attempts); err == nil && validStates[state] { + if content, err := os.ReadFile(promptFile); err == nil { + if _, err := fmt.Sscanf(string(content), "%s %d", &state, &attempts); err == nil && validStates[state] { if state == pYes || state == pNo { // Prompt has been answered. Nothing to do. return } } else { state, attempts = "", 0 - errorf("malformed prompt result %q", string(result)) + errorf("malformed prompt result %q", string(content)) } } else if !os.IsNotExist(err) { errorf("reading prompt file: %v", err) @@ -203,6 +203,8 @@ Would you like to enable Go telemetry? item, err := s.client.ShowMessageRequest(ctx, params) if err != nil { errorf("ShowMessageRequest failed: %v", err) + // Defensive: ensure item == nil for the logic below. + item = nil } result := pFailed From 0669fa31266dedc85938170fb26bfca839d5f1d0 Mon Sep 17 00:00:00 2001 From: Than McIntosh Date: Wed, 20 Sep 2023 11:22:47 -0400 Subject: [PATCH 118/178] cmd/compilebench: fix assembler invocation When building assembly for the "reflect" package, we should no longer be passing the "-compiling-runtime" flag (it is no longer supported, since the toolchain now infers the runtime property based on the import path). Change-Id: I364959b86038816f34811ab450edc1bc5c0382d0 Reviewed-on: https://go-review.googlesource.com/c/tools/+/529300 Auto-Submit: Than McIntosh Reviewed-by: Cherry Mui LUCI-TryBot-Result: Go LUCI --- cmd/compilebench/main.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/cmd/compilebench/main.go b/cmd/compilebench/main.go index 2509253791b..40344ba442d 100644 --- a/cmd/compilebench/main.go +++ b/cmd/compilebench/main.go @@ -578,9 +578,6 @@ func genSymAbisFile(pkg *Pkg, symAbisFile, incdir string) error { "-I", incdir, "-D", "GOOS_" + runtime.GOOS, "-D", "GOARCH_" + runtime.GOARCH} - if pkg.ImportPath == "reflect" { - args = append(args, "-compiling-runtime") - } args = append(args, pkg.SFiles...) if *flagTrace { fmt.Fprintf(os.Stderr, "running: %s %+v\n", From 6120b4501ae373a825751e0026061a38db9087bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Wed, 13 Sep 2023 10:30:58 +0100 Subject: [PATCH 119/178] go/packages: fix a typo of TypesInfo The field is Package.TypesInfo, not Package.TypeInfo. Change-Id: I97331b0713bc9323834c9e0b01ca2740b48c9856 Reviewed-on: https://go-review.googlesource.com/c/tools/+/527915 LUCI-TryBot-Result: Go LUCI Reviewed-by: Robert Findley Reviewed-by: Heschi Kreinick --- go/packages/doc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/packages/doc.go b/go/packages/doc.go index da4ab89fe63..a7a8f73e3d1 100644 --- a/go/packages/doc.go +++ b/go/packages/doc.go @@ -35,7 +35,7 @@ The Package struct provides basic information about the package, including - Imports, a map from source import strings to the Packages they name; - Types, the type information for the package's exported symbols; - Syntax, the parsed syntax trees for the package's source code; and - - TypeInfo, the result of a complete type-check of the package syntax trees. + - TypesInfo, the result of a complete type-check of the package syntax trees. (See the documentation for type Package for the complete list of fields and more detailed descriptions.) From f8acb7f1e8f8bc81e03e20fd107317c8d4e6c174 Mon Sep 17 00:00:00 2001 From: "Hana (Hyang-Ah) Kim" Date: Tue, 19 Sep 2023 16:37:59 -0400 Subject: [PATCH 120/178] gopls: update golang.org/x/telemetry dependency Fixes golang/go#62706 Change-Id: Ia34fcab36b7dd36d8e329868c9e200bbbb1ec5a3 Reviewed-on: https://go-review.googlesource.com/c/tools/+/529615 Run-TryBot: Hyang-Ah Hana Kim TryBot-Result: Gopher Robot Reviewed-by: Robert Findley --- gopls/go.mod | 2 +- gopls/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/gopls/go.mod b/gopls/go.mod index 542619f4a2a..ad1d0f4a5cf 100644 --- a/gopls/go.mod +++ b/gopls/go.mod @@ -10,7 +10,7 @@ require ( golang.org/x/mod v0.12.0 golang.org/x/sync v0.3.0 golang.org/x/sys v0.12.0 - golang.org/x/telemetry v0.0.0-20230914213300-46be8a514e94 + golang.org/x/telemetry v0.0.0-20230919143304-dd3d43c296e4 golang.org/x/text v0.13.0 golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 golang.org/x/vuln v1.0.1 diff --git a/gopls/go.sum b/gopls/go.sum index 347d4e06dac..39b838916cc 100644 --- a/gopls/go.sum +++ b/gopls/go.sum @@ -44,8 +44,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/telemetry v0.0.0-20230914213300-46be8a514e94 h1:wwhvP22pWmWAdBy6nMmHj0rgDU99Br9H9YVV+rRj7E4= -golang.org/x/telemetry v0.0.0-20230914213300-46be8a514e94/go.mod h1:ppZ76JTkRgJC2GQEgtVY3fiuJR+N8FU2MAlp+gfN1E4= +golang.org/x/telemetry v0.0.0-20230919143304-dd3d43c296e4 h1:8UKgM6RV6tVUWPOKlSEunJpIzYW3oy9aV//8cJz0UE4= +golang.org/x/telemetry v0.0.0-20230919143304-dd3d43c296e4/go.mod h1:ppZ76JTkRgJC2GQEgtVY3fiuJR+N8FU2MAlp+gfN1E4= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From d7442965d343e498fb2b6b70b206485899144538 Mon Sep 17 00:00:00 2001 From: Than McIntosh Date: Wed, 20 Sep 2023 12:25:22 -0400 Subject: [PATCH 121/178] cmd/compilebench: check asm support for -compiling-runtime Rather than just omitting the -compiling-runtime flag (which is needed for tip of master as of CL 521697), do a test run of the assembler to see whether the flag is accepted. If it is, then assume we need it when running the assembler for package 'reflect', so as to ensure compilebench keeps working properly for old Go versions. Change-Id: I17fb54c37a94aef08e0b855579d1d1c333880092 Reviewed-on: https://go-review.googlesource.com/c/tools/+/529560 LUCI-TryBot-Result: Go LUCI Reviewed-by: Cherry Mui --- cmd/compilebench/main.go | 58 +++++++++++++++++++++++++++++++++++----- 1 file changed, 52 insertions(+), 6 deletions(-) diff --git a/cmd/compilebench/main.go b/cmd/compilebench/main.go index 40344ba442d..681ffd346b2 100644 --- a/cmd/compilebench/main.go +++ b/cmd/compilebench/main.go @@ -94,12 +94,13 @@ import ( ) var ( - goroot string - compiler string - assembler string - linker string - runRE *regexp.Regexp - is6g bool + goroot string + compiler string + assembler string + linker string + runRE *regexp.Regexp + is6g bool + needCompilingRuntimeFlag bool ) var ( @@ -185,6 +186,9 @@ func main() { if assembler == "" { _, assembler = toolPath("asm") } + if err := checkCompilingRuntimeFlag(assembler); err != nil { + log.Fatalf("checkCompilingRuntimeFlag: %v", err) + } linker = *flagLinker if linker == "" && !is6g { // TODO: Support 6l @@ -566,6 +570,45 @@ func runBuildCmd(name string, count int, dir, tool string, args []string) error return nil } +func checkCompilingRuntimeFlag(assembler string) error { + td, err := os.MkdirTemp("", "asmsrcd") + if err != nil { + return fmt.Errorf("MkdirTemp failed: %v", err) + } + defer os.RemoveAll(td) + src := filepath.Join(td, "asm.s") + obj := filepath.Join(td, "asm.o") + const code = ` +TEXT ·foo(SB),$0-0 +RET +` + if err := os.WriteFile(src, []byte(code), 0644); err != nil { + return fmt.Errorf("writing %s failed: %v", src, err) + } + + // Try compiling the assembly source file passing + // -compiling-runtime; if it succeeds, then we'll need it + // when doing assembly of the reflect package later on. + // If it does not succeed, the assumption is that it's not + // needed. + args := []string{"-o", obj, "-p", "reflect", "-compiling-runtime", src} + cmd := exec.Command(assembler, args...) + cmd.Dir = td + out, aerr := cmd.CombinedOutput() + if aerr != nil { + if strings.Contains(string(out), "flag provided but not defined: -compiling-runtime") { + // flag not defined: assume we're using a recent assembler, so + // don't use -compiling-runtime. + return nil + } + // error is not flag-related; report it. + return fmt.Errorf("problems invoking assembler with args %+v: error %v\n%s\n", args, aerr, out) + } + // asm invocation succeeded -- assume we need the flag. + needCompilingRuntimeFlag = true + return nil +} + // genSymAbisFile runs the assembler on the target package asm files // with "-gensymabis" to produce a symabis file that will feed into // the Go source compilation. This is fairly hacky in that if the @@ -578,6 +621,9 @@ func genSymAbisFile(pkg *Pkg, symAbisFile, incdir string) error { "-I", incdir, "-D", "GOOS_" + runtime.GOOS, "-D", "GOARCH_" + runtime.GOARCH} + if pkg.ImportPath == "reflect" && needCompilingRuntimeFlag { + args = append(args, "-compiling-runtime") + } args = append(args, pkg.SFiles...) if *flagTrace { fmt.Fprintf(os.Stderr, "running: %s %+v\n", From f9b8da7b22beace0978523ec7c472430639cbd71 Mon Sep 17 00:00:00 2001 From: Tim King Date: Wed, 20 Sep 2023 13:37:48 -0700 Subject: [PATCH 122/178] go/analysis/passes/appends: rename package to appends Renaming the package to appends to avoid overriding a builtin symbol. Minor additional changes to doc.go. Change-Id: Ia5d0c92a044fc61391ec1428a595885534d1f11b Reviewed-on: https://go-review.googlesource.com/c/tools/+/529915 LUCI-TryBot-Result: Go LUCI Reviewed-by: Alan Donovan Run-TryBot: Tim King TryBot-Result: Gopher Robot --- go/analysis/passes/append/doc.go | 20 ------------------- .../{append/append.go => appends/appends.go} | 10 +++++----- .../appends_test.go} | 6 +++--- go/analysis/passes/appends/doc.go | 20 +++++++++++++++++++ .../{append => appends}/testdata/src/a/a.go | 2 +- .../{append => appends}/testdata/src/b/b.go | 2 +- 6 files changed, 30 insertions(+), 30 deletions(-) delete mode 100644 go/analysis/passes/append/doc.go rename go/analysis/passes/{append/append.go => appends/appends.go} (87%) rename go/analysis/passes/{append/append_test.go => appends/appends_test.go} (71%) create mode 100644 go/analysis/passes/appends/doc.go rename go/analysis/passes/{append => appends}/testdata/src/a/a.go (92%) rename go/analysis/passes/{append => appends}/testdata/src/b/b.go (86%) diff --git a/go/analysis/passes/append/doc.go b/go/analysis/passes/append/doc.go deleted file mode 100644 index d88c5d88fdd..00000000000 --- a/go/analysis/passes/append/doc.go +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2023 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package append defines an Analyzer that detects -// if there is only one variable in append. -// -// # Analyzer append -// -// append: check for missing values after append -// -// This checker reports append with no values. -// -// For example: -// -// sli := []string{"a", "b", "c"} -// sli = append(sli) -// -// it would report "append with no values" -package append diff --git a/go/analysis/passes/append/append.go b/go/analysis/passes/appends/appends.go similarity index 87% rename from go/analysis/passes/append/append.go rename to go/analysis/passes/appends/appends.go index aa40fcd8973..f0b90a4920e 100644 --- a/go/analysis/passes/append/append.go +++ b/go/analysis/passes/appends/appends.go @@ -2,9 +2,9 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Package append defines an Analyzer that detects +// Package appends defines an Analyzer that detects // if there is only one variable in append. -package append +package appends import ( _ "embed" @@ -21,9 +21,9 @@ import ( var doc string var Analyzer = &analysis.Analyzer{ - Name: "append", - Doc: analysisutil.MustExtractDoc(doc, "append"), - URL: "https://pkg.go.dev/golang.org/x/tools/go/analysis/passes/append", + Name: "appends", + Doc: analysisutil.MustExtractDoc(doc, "appends"), + URL: "https://pkg.go.dev/golang.org/x/tools/go/analysis/passes/appends", Requires: []*analysis.Analyzer{inspect.Analyzer}, Run: run, } diff --git a/go/analysis/passes/append/append_test.go b/go/analysis/passes/appends/appends_test.go similarity index 71% rename from go/analysis/passes/append/append_test.go rename to go/analysis/passes/appends/appends_test.go index f05aa866c93..bb95aca605c 100644 --- a/go/analysis/passes/append/append_test.go +++ b/go/analysis/passes/appends/appends_test.go @@ -2,17 +2,17 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package append_test +package appends_test import ( "testing" "golang.org/x/tools/go/analysis/analysistest" - "golang.org/x/tools/go/analysis/passes/append" + "golang.org/x/tools/go/analysis/passes/appends" ) func Test(t *testing.T) { testdata := analysistest.TestData() tests := []string{"a", "b"} - analysistest.Run(t, testdata, append.Analyzer, tests...) + analysistest.Run(t, testdata, appends.Analyzer, tests...) } diff --git a/go/analysis/passes/appends/doc.go b/go/analysis/passes/appends/doc.go new file mode 100644 index 00000000000..2e6a2e010ba --- /dev/null +++ b/go/analysis/passes/appends/doc.go @@ -0,0 +1,20 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package appends defines an Analyzer that detects +// if there is only one variable in append. +// +// # Analyzer appends +// +// appends: check for missing values after append +// +// This checker reports calls to append that pass +// no values to be appended to the slice. +// +// s := []string{"a", "b", "c"} +// _ = append(s) +// +// Such calls are always no-ops and often indicate an +// underlying mistake. +package appends diff --git a/go/analysis/passes/append/testdata/src/a/a.go b/go/analysis/passes/appends/testdata/src/a/a.go similarity index 92% rename from go/analysis/passes/append/testdata/src/a/a.go rename to go/analysis/passes/appends/testdata/src/a/a.go index 9239c48571f..5d61620d4e0 100644 --- a/go/analysis/passes/append/testdata/src/a/a.go +++ b/go/analysis/passes/appends/testdata/src/a/a.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// This file contains tests for the append checker. +// This file contains tests for the appends checker. package a diff --git a/go/analysis/passes/append/testdata/src/b/b.go b/go/analysis/passes/appends/testdata/src/b/b.go similarity index 86% rename from go/analysis/passes/append/testdata/src/b/b.go rename to go/analysis/passes/appends/testdata/src/b/b.go index 9a297916341..87a04c4a7bd 100644 --- a/go/analysis/passes/append/testdata/src/b/b.go +++ b/go/analysis/passes/appends/testdata/src/b/b.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// This file contains tests for the append checker. +// This file contains tests for the appends checker. package b From f096129eac638b0ff8cd01f22e51b9119eb514eb Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Wed, 20 Sep 2023 14:01:07 -0400 Subject: [PATCH 123/178] internal/refactor/inline: use escape analysis in purity This change factors the escape analysis out of AnalyzeCallee so it can be used on the caller side too, where it provides information to the pure() predicate on which local variables enclosing the call have the "single assignment" property, meaning they are never updated once they are initialized. Expressions that read only single-assignment variables may be safely considered pure. Also, the definition of pure has been refined to mean an expression that does not depend on the value of any mutable variable. This disentangles it from the related concept of having no effect, which is about updating (not reading) variables. We introduce a separate predicate for that. A follow-up change will make richer use of them for analysis of effects in the calleee. Also: - plumb a logger into AnalyzeCallee too. The logger is now mandatory. - binary operators are now pure. - a few tests have regressed in tidiness due to the stricter definition of pure, but this will be fixed by a follow-up analysis of callee effects. Change-Id: I5256bcff86829dd02067ce2ef81b8cb58faa1f50 Reviewed-on: https://go-review.googlesource.com/c/tools/+/530055 Auto-Submit: Alan Donovan LUCI-TryBot-Result: Go LUCI Reviewed-by: Robert Findley --- gopls/internal/lsp/source/inline.go | 13 +- internal/refactor/inline/analyzer/analyzer.go | 6 +- internal/refactor/inline/callee.go | 146 +++------ internal/refactor/inline/escape.go | 94 ++++++ internal/refactor/inline/everything_test.go | 1 + internal/refactor/inline/inline.go | 304 +++++++++++++----- internal/refactor/inline/inline_test.go | 23 +- .../refactor/inline/testdata/basic-err.txtar | 8 +- .../inline/testdata/param-subst.txtar | 2 +- 9 files changed, 398 insertions(+), 199 deletions(-) create mode 100644 internal/refactor/inline/escape.go diff --git a/gopls/internal/lsp/source/inline.go b/gopls/internal/lsp/source/inline.go index 23389e1163d..4e6e16f9159 100644 --- a/gopls/internal/lsp/source/inline.go +++ b/gopls/internal/lsp/source/inline.go @@ -104,7 +104,13 @@ func inlineCall(ctx context.Context, snapshot Snapshot, fh FileHandle, rng proto }() } - callee, err := inline.AnalyzeCallee(calleePkg.FileSet(), calleePkg.GetTypes(), calleePkg.GetTypesInfo(), calleeDecl, calleePGF.Src) + // Users can consult the gopls event log to see + // why a particular inlining strategy was chosen. + logf := func(format string, args ...any) { + event.Log(ctx, "inliner: "+fmt.Sprintf(format, args...)) + } + + callee, err := inline.AnalyzeCallee(logf, calleePkg.FileSet(), calleePkg.GetTypes(), calleePkg.GetTypesInfo(), calleeDecl, calleePGF.Src) if err != nil { return nil, nil, err } @@ -119,11 +125,6 @@ func inlineCall(ctx context.Context, snapshot Snapshot, fh FileHandle, rng proto Content: callerPGF.Src, } - // Users can consult the gopls event log to see - // why a particular inlining strategy was chosen. - logf := func(format string, args ...any) { - event.Log(ctx, "inliner: "+fmt.Sprintf(format, args...)) - } got, err := inline.Inline(logf, caller, callee) if err != nil { return nil, nil, err diff --git a/internal/refactor/inline/analyzer/analyzer.go b/internal/refactor/inline/analyzer/analyzer.go index 4758a4275fb..bb7d9d0d512 100644 --- a/internal/refactor/inline/analyzer/analyzer.go +++ b/internal/refactor/inline/analyzer/analyzer.go @@ -72,7 +72,7 @@ func run(pass *analysis.Pass) (interface{}, error) { pass.Reportf(decl.Doc.Pos(), "invalid inlining candidate: cannot read source file: %v", err) continue } - callee, err := inline.AnalyzeCallee(pass.Fset, pass.Pkg, pass.TypesInfo, decl, content) + callee, err := inline.AnalyzeCallee(discard, pass.Fset, pass.Pkg, pass.TypesInfo, decl, content) if err != nil { pass.Reportf(decl.Doc.Pos(), "invalid inlining candidate: %v", err) continue @@ -126,7 +126,7 @@ func run(pass *analysis.Pass) (interface{}, error) { Call: call, Content: content, } - got, err := inline.Inline(nil, caller, callee) + got, err := inline.Inline(discard, caller, callee) if err != nil { pass.Reportf(call.Lparen, "%v", err) return @@ -161,3 +161,5 @@ type inlineMeFact struct{ callee *inline.Callee } func (f *inlineMeFact) String() string { return "inlineme " + f.callee.String() } func (*inlineMeFact) AFact() {} + +func discard(string, ...any) {} diff --git a/internal/refactor/inline/callee.go b/internal/refactor/inline/callee.go index 9fff8493bc3..64a6903479f 100644 --- a/internal/refactor/inline/callee.go +++ b/internal/refactor/inline/callee.go @@ -74,7 +74,7 @@ type object struct { // The content should be the actual input to the compiler, not the // apparent source file according to any //line directives that // may be present within it. -func AnalyzeCallee(fset *token.FileSet, pkg *types.Package, info *types.Info, decl *ast.FuncDecl, content []byte) (*Callee, error) { +func AnalyzeCallee(logf func(string, ...any), fset *token.FileSet, pkg *types.Package, info *types.Info, decl *ast.FuncDecl, content []byte) (*Callee, error) { checkInfoFields(info) // The client is expected to have determined that the callee @@ -82,6 +82,8 @@ func AnalyzeCallee(fset *token.FileSet, pkg *types.Package, info *types.Info, de fn := info.Defs[decl.Name].(*types.Func) sig := fn.Type().(*types.Signature) + logf("analyzeCallee %v @ %v", fn, fset.PositionFor(decl.Pos(), false)) + // Create user-friendly name ("pkg.Func" or "(pkg.T).Method") var name string if sig.Recv() == nil { @@ -336,7 +338,7 @@ func AnalyzeCallee(fset *token.FileSet, pkg *types.Package, info *types.Info, de return nil, err } - params, results := analyzeParams(fset, info, decl) + params, results := analyzeParams(logf, fset, info, decl) return &Callee{gobCallee{ Content: content, PkgPath: pkg.Path(), @@ -384,7 +386,7 @@ type paramInfo struct { // the other of the result variables of function fn. // // The input must be well-typed. -func analyzeParams(fset *token.FileSet, info *types.Info, decl *ast.FuncDecl) (params, results []*paramInfo) { +func analyzeParams(logf func(string, ...any), fset *token.FileSet, info *types.Info, decl *ast.FuncDecl) (params, results []*paramInfo) { fnobj, ok := info.Defs[decl.Name] if !ok { panic(fmt.Sprintf("%s: no func object for %q", @@ -410,117 +412,57 @@ func analyzeParams(fset *token.FileSet, info *types.Info, decl *ast.FuncDecl) (p } } - // lvalue is called for each address-taken expression or LHS of assignment. - // Supported forms are: x, (x), x[i], x.f, *x, T{}. - var lvalue func(e ast.Expr, escapes bool) - lvalue = func(e ast.Expr, escapes bool) { - switch e := e.(type) { - case *ast.Ident: - if v, ok := info.Uses[e].(*types.Var); ok { - if info := paramInfos[v]; info != nil { - // e is a use of parameter v. - if escapes { - info.Escapes = true - } else { - info.Assigned = true - } - } - } - case *ast.ParenExpr: - lvalue(e.X, escapes) - case *ast.IndexExpr: - // TODO(adonovan): support generics without assuming e.X has a core type. - // Consider: - // - // func Index[T interface{ [3]int | []int }](t T, i int) *int { - // return &t[i] - // } - // - // We must traverse the normal terms and check - // whether any of them is an array. - if _, ok := info.TypeOf(e.X).Underlying().(*types.Array); ok { - lvalue(e.X, escapes) // &a[i] on array - } - case *ast.SelectorExpr: - if _, ok := info.TypeOf(e.X).Underlying().(*types.Struct); ok { - lvalue(e.X, escapes) // &s.f on struct + // Search function body for operations &x, x.f(), and x = y + // where x is a parameter, and record it. + escape(info, decl, func(v *types.Var, escapes bool) { + if info := paramInfos[v]; info != nil { + if escapes { + info.Escapes = true + } else { + info.Assigned = true } - case *ast.StarExpr: - // *ptr indirects an existing pointer - case *ast.CompositeLit: - // &T{...} creates a new variable - default: - panic(fmt.Sprintf("&x on %T", e)) // unreachable in well-typed code } - } + }) - // Search function body for operations &x, x.f(), and x = y - // where x is a parameter. Each of these treats x as an address. - // - // Also record locations of all references to parameters. + // Record locations of all references to parameters. // And record the set of intervening definitions for each parameter. - if decl.Body != nil { - var stack []ast.Node - stack = append(stack, decl.Type) // for scope of function itself - ast.Inspect(decl.Body, func(n ast.Node) bool { - if n != nil { - stack = append(stack, n) // push - } else { - stack = stack[:len(stack)-1] // pop - } + var stack []ast.Node + stack = append(stack, decl.Type) // for scope of function itself + ast.Inspect(decl.Body, func(n ast.Node) bool { + if n != nil { + stack = append(stack, n) // push + } else { + stack = stack[:len(stack)-1] // pop + } - switch n := n.(type) { - case *ast.Ident: - if v, ok := info.Uses[n].(*types.Var); ok { - if pinfo, ok := paramInfos[v]; ok { - // Record location of ref to parameter. - offset := int(n.Pos() - decl.Pos()) - pinfo.Refs = append(pinfo.Refs, offset) - - // Find set of names shadowed within body - // (excluding the parameter itself). - // If these names are free in the arg expression, - // we can't substitute the parameter. - for _, n := range stack { - if scope, ok := info.Scopes[n]; ok { - for _, name := range scope.Names() { - if name != pinfo.Name { - if pinfo.Shadow == nil { - pinfo.Shadow = make(map[string]bool) - } - pinfo.Shadow[name] = true + if id, ok := n.(*ast.Ident); ok { + if v, ok := info.Uses[id].(*types.Var); ok { + if pinfo, ok := paramInfos[v]; ok { + // Record location of ref to parameter. + offset := int(n.Pos() - decl.Pos()) + pinfo.Refs = append(pinfo.Refs, offset) + + // Find set of names shadowed within body + // (excluding the parameter itself). + // If these names are free in the arg expression, + // we can't substitute the parameter. + for _, n := range stack { + if scope, ok := info.Scopes[n]; ok { + for _, name := range scope.Names() { + if name != pinfo.Name { + if pinfo.Shadow == nil { + pinfo.Shadow = make(map[string]bool) } + pinfo.Shadow[name] = true } } } } } - - case *ast.UnaryExpr: - if n.Op == token.AND { - lvalue(n.X, true) // &x - } - - case *ast.CallExpr: - // implicit &x in method call x.f(), - // where x has type T and method is (*T).f - if sel, ok := n.Fun.(*ast.SelectorExpr); ok { - if seln, ok := info.Selections[sel]; ok && - seln.Kind() == types.MethodVal && - !seln.Indirect() && - is[*types.Pointer](seln.Obj().Type().(*types.Signature).Recv().Type()) { - lvalue(sel.X, true) // &x.f - } - } - - case *ast.AssignStmt: - for _, lhs := range n.Lhs { - lvalue(lhs, false) - } } - return true - }) - } + } + return true + }) return params, results } diff --git a/internal/refactor/inline/escape.go b/internal/refactor/inline/escape.go new file mode 100644 index 00000000000..f88be27a86f --- /dev/null +++ b/internal/refactor/inline/escape.go @@ -0,0 +1,94 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package inline + +import ( + "fmt" + "go/ast" + "go/token" + "go/types" +) + +// escape implements a simple "address-taken" escape analysis. It +// calls f for each local variable that appears on the left side of an +// assignment (escapes=false) or has its address taken (escapes=true). +// The initialization of a variable by its declaration does not count +// as an assignment. +func escape(info *types.Info, root ast.Node, f func(v *types.Var, escapes bool)) { + + // lvalue is called for each address-taken expression or LHS of assignment. + // Supported forms are: x, (x), x[i], x.f, *x, T{}. + var lvalue func(e ast.Expr, escapes bool) + lvalue = func(e ast.Expr, escapes bool) { + switch e := e.(type) { + case *ast.Ident: + if v, ok := info.Uses[e].(*types.Var); ok { + if !isPkgLevel(v) { + f(v, escapes) + } + } + case *ast.ParenExpr: + lvalue(e.X, escapes) + case *ast.IndexExpr: + // TODO(adonovan): support generics without assuming e.X has a core type. + // Consider: + // + // func Index[T interface{ [3]int | []int }](t T, i int) *int { + // return &t[i] + // } + // + // We must traverse the normal terms and check + // whether any of them is an array. + if _, ok := info.TypeOf(e.X).Underlying().(*types.Array); ok { + lvalue(e.X, escapes) // &a[i] on array + } + case *ast.SelectorExpr: + if _, ok := info.TypeOf(e.X).Underlying().(*types.Struct); ok { + lvalue(e.X, escapes) // &s.f on struct + } + case *ast.StarExpr: + // *ptr indirects an existing pointer + case *ast.CompositeLit: + // &T{...} creates a new variable + default: + panic(fmt.Sprintf("&x on %T", e)) // unreachable in well-typed code + } + } + + // Search function body for operations &x, x.f(), and x = y + // where x is a parameter. Each of these treats x as an address. + ast.Inspect(root, func(n ast.Node) bool { + switch n := n.(type) { + case *ast.UnaryExpr: + if n.Op == token.AND { + lvalue(n.X, true) // &x + } + + case *ast.CallExpr: + // implicit &x in method call x.f(), + // where x has type T and method is (*T).f + if sel, ok := n.Fun.(*ast.SelectorExpr); ok { + if seln, ok := info.Selections[sel]; ok && + seln.Kind() == types.MethodVal && + !seln.Indirect() && + is[*types.Pointer](seln.Obj().Type().(*types.Signature).Recv().Type()) { + lvalue(sel.X, true) // &x.f + } + } + + case *ast.AssignStmt: + for _, lhs := range n.Lhs { + if id, ok := lhs.(*ast.Ident); ok && + info.Defs[id] != nil && + n.Tok == token.DEFINE { + // declaration: doesn't count + } else { + lvalue(lhs, false) + } + } + } + return true + }) +} diff --git a/internal/refactor/inline/everything_test.go b/internal/refactor/inline/everything_test.go index 5742832ab27..7166c1baad6 100644 --- a/internal/refactor/inline/everything_test.go +++ b/internal/refactor/inline/everything_test.go @@ -141,6 +141,7 @@ func TestEverything(t *testing.T) { callPosn.Filename, callPosn.Offset) callee, err := inline.AnalyzeCallee( + t.Logf, calleePkg.Fset, calleePkg.Types, calleePkg.TypesInfo, diff --git a/internal/refactor/inline/inline.go b/internal/refactor/inline/inline.go index b42a3f2891d..70fa1d1981e 100644 --- a/internal/refactor/inline/inline.go +++ b/internal/refactor/inline/inline.go @@ -40,16 +40,13 @@ type Caller struct { // // Inline does not mutate any part of Caller or Callee. // -// The caller may supply a log function to observe the decision-making process. +// The log records the decision-making process. // // TODO(adonovan): provide an API for clients that want structured // output: a list of import additions and deletions plus one or more // localized diffs (or even AST transformations, though ownership and // mutation are tricky) near the call site. func Inline(logf func(string, ...any), caller *Caller, callee *Callee) ([]byte, error) { - if logf == nil { - logf = func(string, ...any) {} // discard - } logf("inline %s @ %v", debugFormatNode(caller.Fset, caller.Call), caller.Fset.PositionFor(caller.Call.Lparen, false)) @@ -250,7 +247,25 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu return nil } - // Import map, initially populated with caller imports. + // Find the outermost function enclosing the call site (if any). + // Analyze all its local vars for the "single assignment" property + // (Taking the address &v counts as a potential assignment.) + var assign1 func(v *types.Var) bool // reports whether v a single-assignment local var + { + updatedLocals := make(map[*types.Var]bool) + for _, n := range callerPath { + if decl, ok := n.(*ast.FuncDecl); ok { + escape(caller.Info, decl.Body, func(v *types.Var, _ bool) { + updatedLocals[v] = true + }) + break + } + } + logf("multiple-assignment vars: %v", updatedLocals) + assign1 = func(v *types.Var) bool { return !updatedLocals[v] } + } + + // import map, initially populated with caller imports. // // For simplicity we ignore existing dot imports, so that a // qualified identifier (QI) in the callee is always @@ -443,7 +458,8 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu expr ast.Expr typ types.Type // may be tuple for sole non-receiver arg in spread call spread bool // final arg is call() assigned to multiple params - pure bool // expr has no effects + pure bool // expr is pure (doesn't read variables) + effects bool // expr has effects (updates variables) duplicable bool // expr may be duplicated freevars map[string]bool // free names of expr } @@ -469,7 +485,8 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu arg := &argument{ expr: recvArg, typ: caller.Info.TypeOf(recvArg), - pure: pure(caller.Info, recvArg), + pure: pure(caller.Info, assign1, recvArg), + effects: effects(caller.Info, recvArg), duplicable: duplicable(caller.Info, recvArg), freevars: freeVars(caller.Info, recvArg), } @@ -524,7 +541,6 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // is also pure; this is very common. // // arg.pure = false - // arg.duplicable = false } } } @@ -534,7 +550,8 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu expr: expr, typ: typ, spread: is[*types.Tuple](typ), // => last - pure: pure(caller.Info, expr), + pure: pure(caller.Info, assign1, expr), + effects: effects(caller.Info, expr), duplicable: duplicable(caller.Info, expr), freevars: freeVars(caller.Info, expr), }) @@ -597,10 +614,11 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu n := len(params) - 1 ordinary, extra := args[:n], args[n:] var elts []ast.Expr - pure := true + pure, effects := true, false for _, arg := range extra { elts = append(elts, arg.expr) pure = pure && arg.pure + effects = effects || arg.effects } args = append(ordinary, &argument{ expr: &ast.CompositeLit{ @@ -609,6 +627,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu }, typ: lastParam.obj.Type(), pure: pure, + effects: effects, duplicable: false, freevars: nil, // not needed }) @@ -635,6 +654,8 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // // If all conditions are met then the parameter can be substituted // and each reference to it replaced by the argument. + // + // TODO(adonovan): extract this into a separate function. { // Inv: // in calls to variadic, len(args) >= len(params)-1 @@ -646,6 +667,13 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu next: for i, param := range params { arg := args[i] + // Check argument against parameter. + // + // Beware: don't use types.Info on arg since + // the syntax may be synthetic (not created by parser) + // and thus lacking positions and types; + // do it earlier (see pure/duplicable/freevars). + if arg.spread { logf("keeping param %q and following ones: argument %s is spread", param.info.Name, debugFormatNode(caller.Fset, arg.expr)) @@ -660,14 +688,9 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu logf("keeping param %q: assigned by callee", param.info.Name) continue // callee needs the parameter variable } - - // Check argument against parameter. - // - // Beware: don't use types.Info on arg since - // the syntax may be synthetic (not created by parser) - // and thus lacking positions and types; - // do it earlier (see pure/duplicable/freevars). - if !arg.pure { + if !arg.pure || arg.effects { + // TODO(adonovan): conduct a deeper analysis of callee effects + // to relax this constraint. logf("keeping param %q: argument %s is impure", param.info.Name, debugFormatNode(caller.Fset, arg.expr)) continue // unsafe to change order or cardinality of effects @@ -684,7 +707,8 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // TODO(adonovan): be more precise and check // that v is defined within the body of the caller // function (if any) and is indeed referenced - // only by the call. + // only by the call. (See assign1 for analysis + // of enclosing func.) logf("keeping param %q: arg contains perhaps the last reference to possible caller local %v @ %v", param.info.Name, v, caller.Fset.PositionFor(v.Pos(), false)) continue next @@ -1278,83 +1302,196 @@ func freeishNames(free map[string]bool, t ast.Expr) { ast.Inspect(t, visit) } -// pure reports whether the expression is pure, that is, -// has no side effects nor potential to panic. +// effects reports whether an expression might change the state of the +// program (through function calls and channel receives) and affect +// the evaluation of subsequent expressions. +func effects(info *types.Info, expr ast.Expr) bool { + effects := false + ast.Inspect(expr, func(n ast.Node) bool { + switch n := n.(type) { + case *ast.FuncLit: + return false // prune descent + + case *ast.CallExpr: + if !info.Types[n.Fun].IsType() { + // A conversion T(x) has only the effect of its operand. + } else if !callsPureBuiltin(info, n) { + // A handful of built-ins have no effect + // beyond those of their arguments. + // All other calls (including append, copy, recover) + // have unknown effects. + effects = true + } + + case *ast.UnaryExpr: + if n.Op == token.ARROW { // <-ch + effects = true + } + } + return true + }) + return effects +} + +// pure reports whether an expression has the same result no matter +// when it is executed relative to other expressions, so it can be +// commuted with any other expression or statement without changing +// its meaning. // -// Beware that pure does not imply referentially transparent: for -// example, new(T) is a pure expression but it returns a different -// value each time it is evaluated. (One could say that is has effects -// on the memory allocator.) +// An expression is considered impure if it reads the contents of any +// variable, with the exception of "single assignment" local variables +// (as classified by the provided callback), which are never updated +// after their initialization. // -// TODO(adonovan): -// - add a unit test of this function. -// - "potential to panic": I'm not sure this is an important -// criterion. We should be allowed to assume that good programs -// don't rely on runtime panics for correct behavior. -// - Should a binary + operator be considered pure? For strings, it -// allocates memory, but so does a composite literal and that's pure -// (but not duplicable). We need clearer definitions here. -func pure(info *types.Info, e ast.Expr) bool { - switch e := e.(type) { - case *ast.ParenExpr: - return pure(info, e.X) - case *ast.Ident: - return true - case *ast.FuncLit: - return true - case *ast.BasicLit: - return true - case *ast.UnaryExpr: // + - ! ^ & but not <- - return e.Op != token.ARROW && pure(info, e.X) - case *ast.CallExpr: - // A conversion is considered pure - if info.Types[e.Fun].IsType() { - // TODO(adonovan): fix: reject the newly allowed - // conversions between T[] and *[k]T, as they may panic. - return pure(info, e.Args[0]) - } +// Pure does not imply duplicable: for example, new(T) and T{} are +// pure expressions but both return a different value each time they +// are evaluated, so they are not safe to duplicate. +// +// Purity does not imply freedom from run-time panics. We assume that +// target programs do not encounter run-time panics nor depend on them +// for correct operation. +// +// TODO(adonovan): add unit tests of this function. +func pure(info *types.Info, assign1 func(*types.Var) bool, e ast.Expr) bool { + var pure func(e ast.Expr) bool + pure = func(e ast.Expr) bool { + switch e := e.(type) { + case *ast.ParenExpr: + return pure(e.X) - // Call to these built-ins are pure if their arguments are pure. - if id, ok := astutil.Unparen(e.Fun).(*ast.Ident); ok { - if b, ok := info.ObjectOf(id).(*types.Builtin); ok { - switch b.Name() { - case "len", "cap", "complex", "imag", "real", "make", "new", "max", "min": - for _, arg := range e.Args { - if !pure(info, arg) { - return false + case *ast.Ident: + if v, ok := info.Uses[e].(*types.Var); ok { + // In general variables are impure + // as they may be updated, but + // single-assignment local variables + // never change value. + // + // We assume all package-level variables + // may be updated, but for non-exported + // ones we could do better by analyzing + // the complete package. + return !isPkgLevel(v) && assign1(v) + } + + // All other kinds of reference are pure. + return true + + case *ast.FuncLit: + // A function literal may allocate a closure that + // references mutable variables, but mutation + // cannot be observed without calling the function, + // and calls are considered impure. + return true + + case *ast.BasicLit: + return true + + case *ast.UnaryExpr: // + - ! ^ & but not <- + return e.Op != token.ARROW && pure(e.X) + + case *ast.BinaryExpr: // arithmetic, shifts, comparisons, &&/|| + return pure(e.X) && pure(e.Y) + + case *ast.CallExpr: + // A conversion is as pure as its operand. + if info.Types[e.Fun].IsType() { + return pure(e.Args[0]) + } + + // Calls to some built-ins are as pure as their arguments. + if callsPureBuiltin(info, e) { + for _, arg := range e.Args { + if !pure(arg) { + return false + } + } + return true + } + + // All other calls are impure, so we can + // reject them without even looking at e.Fun. + // + // More sophisticated analysis could infer purity in + // commonly used functions such as strings.Contains; + // perhaps we could offer the client a hook so that + // go/analysis-based implementation could exploit the + // results of a purity analysis. But that would make + // the inliner's choices harder to explain. + return false + + case *ast.CompositeLit: + // T{...} is as pure as its elements. + for _, elt := range e.Elts { + if kv, ok := elt.(*ast.KeyValueExpr); ok { + if !pure(kv.Value) { + return false + } + if id, ok := kv.Key.(*ast.Ident); ok { + if v, ok := info.Uses[id].(*types.Var); ok && v.IsField() { + continue // struct {field: value} } } + // map/slice/array {key: value} + if !pure(kv.Key) { + return false + } + + } else if !pure(elt) { + return false + } + } + return true + + case *ast.SelectorExpr: + if sel, ok := info.Selections[e]; ok { + switch sel.Kind() { + case types.MethodExpr: + // A method expression T.f acts like a + // reference to a func decl, so it is pure. return true + + case types.MethodVal: + // A method value x.f acts like a + // closure around a T.f(x, ...) call, + // so it is as pure as x. + return pure(e.X) + + case types.FieldVal: + // A field selection x.f is pure if + // x is pure and the selection does + // not indirect a pointer. + return !sel.Indirect() && pure(e.X) + + default: + panic(sel) } + } else { + // A qualified identifier is + // treated like an unqualified one. + return pure(e.Sel) } + + case *ast.StarExpr: + return false // *ptr depends on the state of the heap + + default: + return false } + } + return pure(e) +} - return false - case *ast.KeyValueExpr: - // map {key: value} or struct {field: value} - return pure(info, e.Key) && pure(info, e.Value) - case *ast.CompositeLit: - // T{x: 0} is pure (though it may imply - // an allocation, so it is not duplicable). - for _, elt := range e.Elts { - if !pure(info, elt) { - return false +func callsPureBuiltin(info *types.Info, call *ast.CallExpr) bool { + if id, ok := astutil.Unparen(call.Fun).(*ast.Ident); ok { + if b, ok := info.ObjectOf(id).(*types.Builtin); ok { + switch b.Name() { + case "len", "cap", "complex", "imag", "real", "make", "new", "max", "min": + return true } + // Not: append clear close copy delete panic print println recover } - return true - case *ast.SelectorExpr: - if sel, ok := info.Selections[e]; ok { - // A field or method selection x.f is pure - // if it does not indirect a pointer. - return !sel.Indirect() - } - // A qualified identifier pkg.Name is pure. - return true - case *ast.StarExpr: - return false // *ptr may panic - default: - return false } + return false } // duplicable reports whether it is appropriate for the expression to @@ -1437,6 +1574,7 @@ func importedPkgName(info *types.Info, imp *ast.ImportSpec) (*types.PkgName, boo } func isPkgLevel(obj types.Object) bool { + // TODO(adonovan): why not simply: obj.Parent() == obj.Pkg().Scope()? return obj.Pkg().Scope().Lookup(obj.Name()) == obj } diff --git a/internal/refactor/inline/inline_test.go b/internal/refactor/inline/inline_test.go index 46c473bf682..3cad64920ca 100644 --- a/internal/refactor/inline/inline_test.go +++ b/internal/refactor/inline/inline_test.go @@ -245,6 +245,7 @@ func doInlineNote(logf func(string, ...any), pkg *packages.Package, file *ast.Fi return nil, err } callee, err := inline.AnalyzeCallee( + logf, calleePkg.Fset, calleePkg.Types, calleePkg.TypesInfo, @@ -474,11 +475,20 @@ func TestTable(t *testing.T) { `func _(v V) { v.f() }`, `func _(v V) { print(*v.U.T) }`, }, + // TODO(adonovan): due to former unsoundness in pure(), + // the previous outputs of two tests below used to be neater. + // A followup analysis (strict effects) will restore tidiness. { "Embedded fields in x.f method selection (implicit &).", `type ( T int; U struct{T}; V struct {U} ); func (t *T) f() { print(t) }`, `func _(v V) { v.f() }`, - `func _(v V) { print(&v.U.T) }`, + // was `func _(v V) { print(&v.U.T) }`, + `func _(v V) { + { + var t *T = &v.U.T + print(t) + } +}`, }, // Now the same tests again with T.f(recv). { @@ -497,9 +507,14 @@ func TestTable(t *testing.T) { "Embedded fields in (*T).f method selection.", `type ( T int; U struct{T}; V struct {U} ); func (t *T) f() { print(t) }`, `func _(v V) { (*V).f(&v) }`, - `func _(v V) { print(&(&v).U.T) }`, + // was `func _(v V) { print(&(&v).U.T) }`, + `func _(v V) { + { + var t *T = &(&v).U.T + print(t) + } +}`, }, - // TODO(adonovan): improve coverage of the cross // product of each strategy with the checklist of // concerns enumerated in the package doc comment. @@ -570,7 +585,7 @@ func TestTable(t *testing.T) { // Analyze callee and inline call. doIt := func() ([]byte, error) { - callee, err := inline.AnalyzeCallee(fset, pkg, info, decl, []byte(calleeContent)) + callee, err := inline.AnalyzeCallee(t.Logf, fset, pkg, info, decl, []byte(calleeContent)) if err != nil { return nil, err } diff --git a/internal/refactor/inline/testdata/basic-err.txtar b/internal/refactor/inline/testdata/basic-err.txtar index c289e9bb544..c811fe47101 100644 --- a/internal/refactor/inline/testdata/basic-err.txtar +++ b/internal/refactor/inline/testdata/basic-err.txtar @@ -1,6 +1,12 @@ Test of inlining a function that references err.Error, which is often a special case because it has no position. +Previously the output was + var _ = (io.EOF.Error()) +but this relied on a bug in pure(). +A follow-up analysis of callee effect ordering +will re-enable this "style optimization". + -- go.mod -- module testdata go 1.12 @@ -19,6 +25,6 @@ package a import "io" -var _ = (io.EOF.Error()) //@ inline(re"getError", getError) +var _ = func(err error) string { return err.Error() }(io.EOF) //@ inline(re"getError", getError) func getError(err error) string { return err.Error() } diff --git a/internal/refactor/inline/testdata/param-subst.txtar b/internal/refactor/inline/testdata/param-subst.txtar index 28e5effe712..135313b865a 100644 --- a/internal/refactor/inline/testdata/param-subst.txtar +++ b/internal/refactor/inline/testdata/param-subst.txtar @@ -14,6 +14,6 @@ func add(x, y int) int { return x + 2*y } -- add -- package a -var _ = func(y int) int { return 2 + 2*y }(1 + 1) //@ inline(re"add", add) +var _ = (2 + 2*(1+1)) //@ inline(re"add", add) func add(x, y int) int { return x + 2*y } \ No newline at end of file From a490c54ecfddbd272be3368ec2b18fd3b2869714 Mon Sep 17 00:00:00 2001 From: cui fliter Date: Fri, 4 Aug 2023 14:08:22 +0800 Subject: [PATCH 124/178] all: register the appends analyzer in a few places and update gopls/go.mod Change-Id: I472e00e88972f91082f1cf16cd9a6e68c82b5b81 Reviewed-on: https://go-review.googlesource.com/c/tools/+/529695 TryBot-Result: Gopher Robot Reviewed-by: Robert Findley Reviewed-by: Tim King Run-TryBot: shuang cui LUCI-TryBot-Result: Go LUCI --- go/analysis/unitchecker/main.go | 2 ++ go/analysis/unitchecker/vet_std_test.go | 2 ++ gopls/doc/analyzers.md | 15 +++++++++++++++ gopls/go.mod | 2 +- gopls/internal/lsp/source/api_json.go | 11 +++++++++++ gopls/internal/lsp/source/options.go | 2 ++ 6 files changed, 33 insertions(+), 1 deletion(-) diff --git a/go/analysis/unitchecker/main.go b/go/analysis/unitchecker/main.go index 6e08ce94a3d..4374e7bf945 100644 --- a/go/analysis/unitchecker/main.go +++ b/go/analysis/unitchecker/main.go @@ -19,6 +19,7 @@ package main import ( "golang.org/x/tools/go/analysis/unitchecker" + "golang.org/x/tools/go/analysis/passes/appends" "golang.org/x/tools/go/analysis/passes/asmdecl" "golang.org/x/tools/go/analysis/passes/assign" "golang.org/x/tools/go/analysis/passes/atomic" @@ -52,6 +53,7 @@ import ( func main() { unitchecker.Main( + appends.Analyzer, asmdecl.Analyzer, assign.Analyzer, atomic.Analyzer, diff --git a/go/analysis/unitchecker/vet_std_test.go b/go/analysis/unitchecker/vet_std_test.go index e0fb41c77ed..64d4378fe57 100644 --- a/go/analysis/unitchecker/vet_std_test.go +++ b/go/analysis/unitchecker/vet_std_test.go @@ -11,6 +11,7 @@ import ( "strings" "testing" + "golang.org/x/tools/go/analysis/passes/appends" "golang.org/x/tools/go/analysis/passes/asmdecl" "golang.org/x/tools/go/analysis/passes/assign" "golang.org/x/tools/go/analysis/passes/atomic" @@ -47,6 +48,7 @@ import ( // Keep consistent with the actual vet in GOROOT/src/cmd/vet/main.go. func vet() { unitchecker.Main( + appends.Analyzer, asmdecl.Analyzer, assign.Analyzer, atomic.Analyzer, diff --git a/gopls/doc/analyzers.md b/gopls/doc/analyzers.md index da8814f4b16..55c199ce3eb 100644 --- a/gopls/doc/analyzers.md +++ b/gopls/doc/analyzers.md @@ -6,6 +6,21 @@ Details about how to enable/disable these analyses can be found [here](settings.md#analyses). +## **appends** + +check for missing values after append + +This checker reports calls to append that pass +no values to be appended to the slice. + + s := []string{"a", "b", "c"} + _ = append(s) + +Such calls are always no-ops and often indicate an +underlying mistake. + +**Enabled by default.** + ## **asmdecl** report mismatches between assembly files and Go declarations diff --git a/gopls/go.mod b/gopls/go.mod index ad1d0f4a5cf..99c5c137529 100644 --- a/gopls/go.mod +++ b/gopls/go.mod @@ -12,7 +12,7 @@ require ( golang.org/x/sys v0.12.0 golang.org/x/telemetry v0.0.0-20230919143304-dd3d43c296e4 golang.org/x/text v0.13.0 - golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 + golang.org/x/tools v0.13.1-0.20230920233436-f9b8da7b22be golang.org/x/vuln v1.0.1 gopkg.in/yaml.v3 v3.0.1 honnef.co/go/tools v0.4.5 diff --git a/gopls/internal/lsp/source/api_json.go b/gopls/internal/lsp/source/api_json.go index d0fdb2008ce..0bca2e6d6a8 100644 --- a/gopls/internal/lsp/source/api_json.go +++ b/gopls/internal/lsp/source/api_json.go @@ -218,6 +218,11 @@ var GeneratedAPIJSON = &APIJSON{ EnumKeys: EnumKeys{ ValueType: "bool", Keys: []EnumKey{ + { + Name: "\"appends\"", + Doc: "check for missing values after append\n\nThis checker reports calls to append that pass\nno values to be appended to the slice.\n\n\ts := []string{\"a\", \"b\", \"c\"}\n\t_ = append(s)\n\nSuch calls are always no-ops and often indicate an\nunderlying mistake.", + Default: "true", + }, { Name: "\"asmdecl\"", Doc: "report mismatches between assembly files and Go declarations", @@ -922,6 +927,12 @@ var GeneratedAPIJSON = &APIJSON{ }, }, Analyzers: []*AnalyzerJSON{ + { + Name: "appends", + Doc: "check for missing values after append\n\nThis checker reports calls to append that pass\nno values to be appended to the slice.\n\n\ts := []string{\"a\", \"b\", \"c\"}\n\t_ = append(s)\n\nSuch calls are always no-ops and often indicate an\nunderlying mistake.", + URL: "https://pkg.go.dev/golang.org/x/tools/go/analysis/passes/appends", + Default: true, + }, { Name: "asmdecl", Doc: "report mismatches between assembly files and Go declarations", diff --git a/gopls/internal/lsp/source/options.go b/gopls/internal/lsp/source/options.go index 02a6057d29f..5cb24830017 100644 --- a/gopls/internal/lsp/source/options.go +++ b/gopls/internal/lsp/source/options.go @@ -16,6 +16,7 @@ import ( "time" "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/analysis/passes/appends" "golang.org/x/tools/go/analysis/passes/asmdecl" "golang.org/x/tools/go/analysis/passes/assign" "golang.org/x/tools/go/analysis/passes/atomic" @@ -1547,6 +1548,7 @@ func convenienceAnalyzers() map[string]*Analyzer { func defaultAnalyzers() map[string]*Analyzer { return map[string]*Analyzer{ // The traditional vet suite: + appends.Analyzer.Name: {Analyzer: appends.Analyzer, Enabled: true}, asmdecl.Analyzer.Name: {Analyzer: asmdecl.Analyzer, Enabled: true}, assign.Analyzer.Name: {Analyzer: assign.Analyzer, Enabled: true}, atomic.Analyzer.Name: {Analyzer: atomic.Analyzer, Enabled: true}, From 771061d3f8ed35a1800186940045fe861df91498 Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Thu, 21 Sep 2023 14:45:20 -0400 Subject: [PATCH 125/178] gopls: instrument telemetry for latency of important operations Add instrumentation for the telemetry described in golang/go#63129, along with a test. Also isolate the existing telemetry regtest. Updates golang/go#63129 Updates golang/go#62665 Change-Id: If641b44d430eaaf96c469c804f42fd72cd821bed Reviewed-on: https://go-review.googlesource.com/c/tools/+/530215 LUCI-TryBot-Result: Go LUCI Reviewed-by: Hyang-Ah Hana Kim --- gopls/internal/lsp/completion.go | 8 +- gopls/internal/lsp/definition.go | 8 +- gopls/internal/lsp/hover.go | 8 +- gopls/internal/lsp/implementation.go | 8 +- gopls/internal/lsp/references.go | 8 +- gopls/internal/lsp/workspace_symbol.go | 8 +- gopls/internal/telemetry/latency.go | 102 +++++++++++++++++ gopls/internal/telemetry/telemetry_test.go | 124 +++++++++++++++++++-- 8 files changed, 260 insertions(+), 14 deletions(-) create mode 100644 gopls/internal/telemetry/latency.go diff --git a/gopls/internal/lsp/completion.go b/gopls/internal/lsp/completion.go index 209f26be3cb..66b3a3945bf 100644 --- a/gopls/internal/lsp/completion.go +++ b/gopls/internal/lsp/completion.go @@ -14,11 +14,17 @@ import ( "golang.org/x/tools/gopls/internal/lsp/source/completion" "golang.org/x/tools/gopls/internal/lsp/template" "golang.org/x/tools/gopls/internal/lsp/work" + "golang.org/x/tools/gopls/internal/telemetry" "golang.org/x/tools/internal/event" "golang.org/x/tools/internal/event/tag" ) -func (s *Server) completion(ctx context.Context, params *protocol.CompletionParams) (*protocol.CompletionList, error) { +func (s *Server) completion(ctx context.Context, params *protocol.CompletionParams) (_ *protocol.CompletionList, rerr error) { + recordLatency := telemetry.StartLatencyTimer("completion") + defer func() { + recordLatency(ctx, rerr) + }() + ctx, done := event.Start(ctx, "lsp.Server.completion", tag.URI.Of(params.TextDocument.URI)) defer done() diff --git a/gopls/internal/lsp/definition.go b/gopls/internal/lsp/definition.go index fb691ef9d16..a438bebc8d3 100644 --- a/gopls/internal/lsp/definition.go +++ b/gopls/internal/lsp/definition.go @@ -12,11 +12,17 @@ import ( "golang.org/x/tools/gopls/internal/lsp/protocol" "golang.org/x/tools/gopls/internal/lsp/source" "golang.org/x/tools/gopls/internal/lsp/template" + "golang.org/x/tools/gopls/internal/telemetry" "golang.org/x/tools/internal/event" "golang.org/x/tools/internal/event/tag" ) -func (s *Server) definition(ctx context.Context, params *protocol.DefinitionParams) ([]protocol.Location, error) { +func (s *Server) definition(ctx context.Context, params *protocol.DefinitionParams) (_ []protocol.Location, rerr error) { + recordLatency := telemetry.StartLatencyTimer("definition") + defer func() { + recordLatency(ctx, rerr) + }() + ctx, done := event.Start(ctx, "lsp.Server.definition", tag.URI.Of(params.TextDocument.URI)) defer done() diff --git a/gopls/internal/lsp/hover.go b/gopls/internal/lsp/hover.go index eef59920ae4..263a1c8ac72 100644 --- a/gopls/internal/lsp/hover.go +++ b/gopls/internal/lsp/hover.go @@ -12,11 +12,17 @@ import ( "golang.org/x/tools/gopls/internal/lsp/source" "golang.org/x/tools/gopls/internal/lsp/template" "golang.org/x/tools/gopls/internal/lsp/work" + "golang.org/x/tools/gopls/internal/telemetry" "golang.org/x/tools/internal/event" "golang.org/x/tools/internal/event/tag" ) -func (s *Server) hover(ctx context.Context, params *protocol.HoverParams) (*protocol.Hover, error) { +func (s *Server) hover(ctx context.Context, params *protocol.HoverParams) (_ *protocol.Hover, rerr error) { + recordLatency := telemetry.StartLatencyTimer("hover") + defer func() { + recordLatency(ctx, rerr) + }() + ctx, done := event.Start(ctx, "lsp.Server.hover", tag.URI.Of(params.TextDocument.URI)) defer done() diff --git a/gopls/internal/lsp/implementation.go b/gopls/internal/lsp/implementation.go index e231d96bccb..bc527b3b58a 100644 --- a/gopls/internal/lsp/implementation.go +++ b/gopls/internal/lsp/implementation.go @@ -9,11 +9,17 @@ import ( "golang.org/x/tools/gopls/internal/lsp/protocol" "golang.org/x/tools/gopls/internal/lsp/source" + "golang.org/x/tools/gopls/internal/telemetry" "golang.org/x/tools/internal/event" "golang.org/x/tools/internal/event/tag" ) -func (s *Server) implementation(ctx context.Context, params *protocol.ImplementationParams) ([]protocol.Location, error) { +func (s *Server) implementation(ctx context.Context, params *protocol.ImplementationParams) (_ []protocol.Location, rerr error) { + recordLatency := telemetry.StartLatencyTimer("implementation") + defer func() { + recordLatency(ctx, rerr) + }() + ctx, done := event.Start(ctx, "lsp.Server.implementation", tag.URI.Of(params.TextDocument.URI)) defer done() diff --git a/gopls/internal/lsp/references.go b/gopls/internal/lsp/references.go index cc89b381088..d3d36235697 100644 --- a/gopls/internal/lsp/references.go +++ b/gopls/internal/lsp/references.go @@ -10,11 +10,17 @@ import ( "golang.org/x/tools/gopls/internal/lsp/protocol" "golang.org/x/tools/gopls/internal/lsp/source" "golang.org/x/tools/gopls/internal/lsp/template" + "golang.org/x/tools/gopls/internal/telemetry" "golang.org/x/tools/internal/event" "golang.org/x/tools/internal/event/tag" ) -func (s *Server) references(ctx context.Context, params *protocol.ReferenceParams) ([]protocol.Location, error) { +func (s *Server) references(ctx context.Context, params *protocol.ReferenceParams) (_ []protocol.Location, rerr error) { + recordLatency := telemetry.StartLatencyTimer("references") + defer func() { + recordLatency(ctx, rerr) + }() + ctx, done := event.Start(ctx, "lsp.Server.references", tag.URI.Of(params.TextDocument.URI)) defer done() diff --git a/gopls/internal/lsp/workspace_symbol.go b/gopls/internal/lsp/workspace_symbol.go index 451289f1cad..eb690b047e5 100644 --- a/gopls/internal/lsp/workspace_symbol.go +++ b/gopls/internal/lsp/workspace_symbol.go @@ -9,10 +9,16 @@ import ( "golang.org/x/tools/gopls/internal/lsp/protocol" "golang.org/x/tools/gopls/internal/lsp/source" + "golang.org/x/tools/gopls/internal/telemetry" "golang.org/x/tools/internal/event" ) -func (s *Server) symbol(ctx context.Context, params *protocol.WorkspaceSymbolParams) ([]protocol.SymbolInformation, error) { +func (s *Server) symbol(ctx context.Context, params *protocol.WorkspaceSymbolParams) (_ []protocol.SymbolInformation, rerr error) { + recordLatency := telemetry.StartLatencyTimer("symbol") + defer func() { + recordLatency(ctx, rerr) + }() + ctx, done := event.Start(ctx, "lsp.Server.symbol") defer done() diff --git a/gopls/internal/telemetry/latency.go b/gopls/internal/telemetry/latency.go new file mode 100644 index 00000000000..b0e2da73165 --- /dev/null +++ b/gopls/internal/telemetry/latency.go @@ -0,0 +1,102 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package telemetry + +import ( + "context" + "errors" + "fmt" + "sort" + "sync" + "time" + + "golang.org/x/telemetry/counter" +) + +// latencyKey is used for looking up latency counters. +type latencyKey struct { + operation, bucket string + isError bool +} + +var ( + latencyBuckets = []struct { + end time.Duration + name string + }{ + {10 * time.Millisecond, "<10ms"}, + {50 * time.Millisecond, "<50ms"}, + {100 * time.Millisecond, "<100ms"}, + {200 * time.Millisecond, "<200ms"}, + {500 * time.Millisecond, "<500ms"}, + {1 * time.Second, "<1s"}, + {5 * time.Second, "<5s"}, + {24 * time.Hour, "<24h"}, + } + + latencyCounterMu sync.Mutex + latencyCounters = make(map[latencyKey]*counter.Counter) // lazily populated +) + +// ForEachLatencyCounter runs the provided function for each current latency +// counter measuring the given operation. +// +// Exported for testing. +func ForEachLatencyCounter(operation string, isError bool, f func(*counter.Counter)) { + latencyCounterMu.Lock() + defer latencyCounterMu.Unlock() + + for k, v := range latencyCounters { + if k.operation == operation && k.isError == isError { + f(v) + } + } +} + +// getLatencyCounter returns the counter used to record latency of the given +// operation in the given bucket. +func getLatencyCounter(operation, bucket string, isError bool) *counter.Counter { + latencyCounterMu.Lock() + defer latencyCounterMu.Unlock() + + key := latencyKey{operation, bucket, isError} + c, ok := latencyCounters[key] + if !ok { + var name string + if isError { + name = fmt.Sprintf("gopls/%s/error-latency:%s", operation, bucket) + } else { + name = fmt.Sprintf("gopls/%s/latency:%s", operation, bucket) + } + c = counter.New(name) + latencyCounters[key] = c + } + return c +} + +// StartLatencyTimer starts a timer for the gopls operation with the given +// name, and returns a func to stop the timer and record the latency sample. +// +// If the context provided to the the resulting func is done, no observation is +// recorded. +func StartLatencyTimer(operation string) func(context.Context, error) { + start := time.Now() + return func(ctx context.Context, err error) { + if errors.Is(ctx.Err(), context.Canceled) { + // Ignore timing where the operation is cancelled, it may be influenced + // by client behavior. + return + } + latency := time.Since(start) + bucketIdx := sort.Search(len(latencyBuckets), func(i int) bool { + bucket := latencyBuckets[i] + return latency < bucket.end + }) + if bucketIdx < len(latencyBuckets) { // ignore latency longer than a day :) + bucketName := latencyBuckets[bucketIdx].name + getLatencyCounter(operation, bucketName, err != nil).Inc() + } + } +} diff --git a/gopls/internal/telemetry/telemetry_test.go b/gopls/internal/telemetry/telemetry_test.go index 57fff1c4798..25e94f6284f 100644 --- a/gopls/internal/telemetry/telemetry_test.go +++ b/gopls/internal/telemetry/telemetry_test.go @@ -8,6 +8,8 @@ package telemetry_test import ( + "context" + "errors" "os" "strconv" "strings" @@ -21,6 +23,7 @@ import ( "golang.org/x/tools/gopls/internal/lsp/command" "golang.org/x/tools/gopls/internal/lsp/protocol" . "golang.org/x/tools/gopls/internal/lsp/regtest" + "golang.org/x/tools/gopls/internal/telemetry" ) func TestMain(m *testing.M) { @@ -39,6 +42,28 @@ func TestTelemetry(t *testing.T) { editor = "vscode" // We set ClientName("Visual Studio Code") below. ) + // Run gopls once to determine the Go version. + WithOptions( + Modes(Default), + ).Run(t, "", func(_ *testing.T, env *Env) { + goversion = strconv.Itoa(env.GoVersion()) + }) + + // counters that should be incremented once per session + sessionCounters := []*counter.Counter{ + counter.New("gopls/client:" + editor), + counter.New("gopls/goversion:1." + goversion), + counter.New("fwd/vscode/linter:a"), + } + initialCounts := make([]uint64, len(sessionCounters)) + for i, c := range sessionCounters { + count, err := countertest.ReadCounter(c) + if err != nil { + t.Fatalf("ReadCounter(%s): %v", c.Name(), err) + } + initialCounts[i] = count + } + // Verify that a properly configured session gets notified of a bug on the // server. WithOptions( @@ -56,14 +81,11 @@ func TestTelemetry(t *testing.T) { // gopls/editor:client // gopls/goversion:1.x // fwd/vscode/linter:a - for _, c := range []*counter.Counter{ - counter.New("gopls/client:" + editor), - counter.New("gopls/goversion:1." + goversion), - counter.New("fwd/vscode/linter:a"), - } { - count, err := countertest.ReadCounter(c) - if err != nil || count != 1 { - t.Errorf("ReadCounter(%q) = (%v, %v), want (1, nil)", c.Name(), count, err) + for i, c := range sessionCounters { + want := initialCounts[i] + 1 + got, err := countertest.ReadCounter(c) + if err != nil || got != want { + t.Errorf("ReadCounter(%q) = (%v, %v), want (%v, nil)", c.Name(), got, err, want) t.Logf("Current timestamp = %v", time.Now().UTC()) } } @@ -105,3 +127,89 @@ func hasEntry(counts map[string]uint64, pattern string, want uint64) bool { } return false } + +func TestLatencyCounter(t *testing.T) { + const operation = "TestLatencyCounter" // a unique operation name + + stop := telemetry.StartLatencyTimer(operation) + stop(context.Background(), nil) + + for isError, want := range map[bool]uint64{false: 1, true: 0} { + if got := totalLatencySamples(t, operation, isError); got != want { + t.Errorf("totalLatencySamples(operation=%v, isError=%v) = %d, want %d", operation, isError, got, want) + } + } +} + +func TestLatencyCounter_Error(t *testing.T) { + const operation = "TestLatencyCounter_Error" // a unique operation name + + stop := telemetry.StartLatencyTimer(operation) + stop(context.Background(), errors.New("bad")) + + for isError, want := range map[bool]uint64{false: 0, true: 1} { + if got := totalLatencySamples(t, operation, isError); got != want { + t.Errorf("totalLatencySamples(operation=%v, isError=%v) = %d, want %d", operation, isError, got, want) + } + } +} + +func TestLatencyCounter_Cancellation(t *testing.T) { + const operation = "TestLatencyCounter_Cancellation" + + stop := telemetry.StartLatencyTimer(operation) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + stop(ctx, nil) + + for isError, want := range map[bool]uint64{false: 0, true: 0} { + if got := totalLatencySamples(t, operation, isError); got != want { + t.Errorf("totalLatencySamples(operation=%v, isError=%v) = %d, want %d", operation, isError, got, want) + } + } +} + +func totalLatencySamples(t *testing.T, operation string, isError bool) uint64 { + var total uint64 + telemetry.ForEachLatencyCounter(operation, isError, func(c *counter.Counter) { + count, err := countertest.ReadCounter(c) + if err != nil { + t.Errorf("ReadCounter(%s) failed: %v", c.Name(), err) + } else { + total += count + } + }) + return total +} + +func TestLatencyInstrumentation(t *testing.T) { + const files = ` +-- go.mod -- +module mod.test/a +go 1.18 +-- a.go -- +package a + +func _() { + x := 0 + _ = x +} +` + + // Verify that a properly configured session gets notified of a bug on the + // server. + WithOptions( + Modes(Default), // must be in-process to receive the bug report below + ).Run(t, files, func(_ *testing.T, env *Env) { + env.OpenFile("a.go") + before := totalLatencySamples(t, "completion", false) + loc := env.RegexpSearch("a.go", "x") + for i := 0; i < 10; i++ { + env.Completion(loc) + } + after := totalLatencySamples(t, "completion", false) + if after-before < 10 { + t.Errorf("after 10 completions, completion counter went from %d to %d", before, after) + } + }) +} From e3bbe43a8b110939c56607876410c6e2f4e56491 Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Thu, 21 Sep 2023 14:53:00 -0400 Subject: [PATCH 126/178] gopls/internal/lsp: update prompting logic for local/off mode We're eliminating the "local" telemetry mode, in favor of just on/off. Update gopls logic accordingly. For golang/go#62576 Change-Id: I41089062bf6963391ace059efe5ab4775a438c9e Reviewed-on: https://go-review.googlesource.com/c/tools/+/530057 Reviewed-by: Hyang-Ah Hana Kim LUCI-TryBot-Result: Go LUCI --- gopls/internal/lsp/prompt.go | 8 +++----- gopls/internal/regtest/misc/prompt_test.go | 4 ++-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/gopls/internal/lsp/prompt.go b/gopls/internal/lsp/prompt.go index c87bfb5a0ba..ad9d404d5a6 100644 --- a/gopls/internal/lsp/prompt.go +++ b/gopls/internal/lsp/prompt.go @@ -60,7 +60,7 @@ func (s *Server) telemetryMode() string { if data, err := os.ReadFile(fake); err == nil { return string(data) } - return "local" + return "off" } return telemetry.Mode() } @@ -91,10 +91,8 @@ func (s *Server) maybePromptForTelemetry(ctx context.Context) { return // prompt is disabled } - // Only prompt if telemetry is in the default "local" mode. If it is already - // "on" there's nothing to ask about, and if it is explicitly "off" let's - // assume the user doesn't want it. - if s.telemetryMode() != "local" { + if s.telemetryMode() == "on" { + // Telemetry is already on -- nothing to ask about. return } diff --git a/gopls/internal/regtest/misc/prompt_test.go b/gopls/internal/regtest/misc/prompt_test.go index efe4bdf999b..e173fa3c6d5 100644 --- a/gopls/internal/regtest/misc/prompt_test.go +++ b/gopls/internal/regtest/misc/prompt_test.go @@ -32,7 +32,7 @@ func main() { for _, enabled := range []bool{true, false} { t.Run(fmt.Sprintf("telemetryPrompt=%v", enabled), func(t *testing.T) { - for _, initialMode := range []string{"", "off", "local", "on"} { + for _, initialMode := range []string{"", "off", "on"} { t.Run(fmt.Sprintf("initial_mode=%s", initialMode), func(t *testing.T) { modeFile := filepath.Join(t.TempDir(), "mode") if initialMode != "" { @@ -50,7 +50,7 @@ func main() { "telemetryPrompt": enabled, }, ).Run(t, src, func(t *testing.T, env *Env) { - wantPrompt := enabled && (initialMode == "" || initialMode == "local") + wantPrompt := enabled && (initialMode == "" || initialMode == "off") expectation := ShownMessageRequest(".*Would you like to enable Go telemetry?") if !wantPrompt { expectation = Not(expectation) From d5538db06d35bd6ed228f4f3b16c6d00a5405391 Mon Sep 17 00:00:00 2001 From: "Hana (Hyang-Ah) Kim" Date: Thu, 21 Sep 2023 21:43:31 -0400 Subject: [PATCH 127/178] gopls/internal/lsp/cmd: undocument gopls vulncheck This is meant to be internal. Users must use `golang.org/x/vuln/cmd/govulncheck`. Remove this command from the default usage output. But `gopls -v -h` will show the list of internal commands. Change-Id: I19bb0295f4c3fc05d62e6ad4b52f0c7f0d6b3d4f Reviewed-on: https://go-review.googlesource.com/c/tools/+/530455 Reviewed-by: Robert Findley Run-TryBot: Hyang-Ah Hana Kim TryBot-Result: Gopher Robot --- gopls/internal/lsp/cmd/cmd.go | 15 +++- gopls/internal/lsp/cmd/help_test.go | 26 +++++++ gopls/internal/lsp/cmd/usage/usage-v.hlp | 82 ++++++++++++++++++++++ gopls/internal/lsp/cmd/usage/usage.hlp | 1 - gopls/internal/lsp/cmd/usage/vulncheck.hlp | 4 +- gopls/internal/lsp/cmd/vulncheck.go | 4 +- 6 files changed, 126 insertions(+), 6 deletions(-) create mode 100644 gopls/internal/lsp/cmd/usage/usage-v.hlp diff --git a/gopls/internal/lsp/cmd/cmd.go b/gopls/internal/lsp/cmd/cmd.go index 12976f04579..d3d4069a412 100644 --- a/gopls/internal/lsp/cmd/cmd.go +++ b/gopls/internal/lsp/cmd/cmd.go @@ -158,6 +158,12 @@ Command: for _, c := range app.featureCommands() { fmt.Fprintf(w, " %s\t%s\n", c.Name(), c.ShortHelp()) } + if app.verbose() { + fmt.Fprint(w, "\t\nInternal Use Only\t\n") + for _, c := range app.internalCommands() { + fmt.Fprintf(w, " %s\t%s\n", c.Name(), c.ShortHelp()) + } + } fmt.Fprint(w, "\nflags:\n") printFlagDefaults(f) } @@ -263,6 +269,7 @@ func (app *Application) Commands() []tool.Application { var commands []tool.Application commands = append(commands, app.mainCommands()...) commands = append(commands, app.featureCommands()...) + commands = append(commands, app.internalCommands()...) return commands } @@ -277,6 +284,12 @@ func (app *Application) mainCommands() []tool.Application { } } +func (app *Application) internalCommands() []tool.Application { + return []tool.Application{ + &vulncheck{app: app}, + } +} + func (app *Application) featureCommands() []tool.Application { return []tool.Application{ &callHierarchy{app: app}, @@ -298,8 +311,8 @@ func (app *Application) featureCommands() []tool.Application { &stats{app: app}, &suggestedFix{app: app}, &symbols{app: app}, + &workspaceSymbol{app: app}, - &vulncheck{app: app}, } } diff --git a/gopls/internal/lsp/cmd/help_test.go b/gopls/internal/lsp/cmd/help_test.go index a8feb6e9001..6d8f10af46f 100644 --- a/gopls/internal/lsp/cmd/help_test.go +++ b/gopls/internal/lsp/cmd/help_test.go @@ -56,3 +56,29 @@ func TestHelpFiles(t *testing.T) { }) } } + +func TestVerboseHelp(t *testing.T) { + testenv.NeedsGoBuild(t) // This is a lie. We actually need the source code. + app := cmd.New(appName, "", nil, nil) + ctx := context.Background() + var buf bytes.Buffer + s := flag.NewFlagSet(appName, flag.ContinueOnError) + s.SetOutput(&buf) + tool.Run(ctx, s, app, []string{"-v", "-h"}) + got := buf.Bytes() + + helpFile := filepath.Join("usage", "usage-v.hlp") + if *updateHelpFiles { + if err := os.WriteFile(helpFile, got, 0666); err != nil { + t.Errorf("Failed writing %v: %v", helpFile, err) + } + return + } + want, err := os.ReadFile(helpFile) + if err != nil { + t.Fatalf("Missing help file %q", helpFile) + } + if diff := cmp.Diff(string(want), string(got)); diff != "" { + t.Errorf("Help file %q did not match, run with -update-help-files to fix (-want +got)\n%s", helpFile, diff) + } +} diff --git a/gopls/internal/lsp/cmd/usage/usage-v.hlp b/gopls/internal/lsp/cmd/usage/usage-v.hlp new file mode 100644 index 00000000000..0edb37e9300 --- /dev/null +++ b/gopls/internal/lsp/cmd/usage/usage-v.hlp @@ -0,0 +1,82 @@ + +gopls is a Go language server. + +It is typically used with an editor to provide language features. When no +command is specified, gopls will default to the 'serve' command. The language +features can also be accessed via the gopls command-line interface. + +Usage: + gopls help [] + +Command: + +Main + serve run a server for Go code using the Language Server Protocol + version print the gopls version information + bug report a bug in gopls + help print usage information for subcommands + api-json print json describing gopls API + licenses print licenses of included software + +Features + call_hierarchy display selected identifier's call hierarchy + check show diagnostic results for the specified file + definition show declaration of selected identifier + folding_ranges display selected file's folding ranges + format format the code according to the go standard + highlight display selected identifier's highlights + implementation display selected identifier's implementation + imports updates import statements + remote interact with the gopls daemon + inspect interact with the gopls daemon (deprecated: use 'remote') + links list links in a file + prepare_rename test validity of a rename operation at location + references display selected identifier's references + rename rename selected identifier + semtok show semantic tokens for the specified file + signature display selected identifier's signature + stats print workspace statistics + fix apply suggested fixes + symbols display selected file's symbols + workspace_symbol search symbols in workspace + +Internal Use Only + vulncheck run vulncheck analysis (internal-use only) + +flags: + -debug=string + serve debug information on the supplied address + -listen=string + address on which to listen for remote connections. If prefixed by 'unix;', the subsequent address is assumed to be a unix domain socket. Otherwise, TCP is used. + -listen.timeout=duration + when used with -listen, shut down the server when there are no connected clients for this duration + -logfile=string + filename to log to. if value is "auto", then logging to a default output file is enabled + -mode=string + no effect + -ocagent=string + the address of the ocagent (e.g. http://localhost:55678), or off (default "off") + -port=int + port on which to run gopls for debugging purposes + -profile.alloc=string + write alloc profile to this file + -profile.cpu=string + write CPU profile to this file + -profile.mem=string + write memory profile to this file + -profile.trace=string + write trace log to this file + -remote=string + forward all commands to a remote lsp specified by this flag. With no special prefix, this is assumed to be a TCP address. If prefixed by 'unix;', the subsequent address is assumed to be a unix domain socket. If 'auto', or prefixed by 'auto;', the remote address is automatically resolved based on the executing environment. + -remote.debug=string + when used with -remote=auto, the -debug value used to start the daemon + -remote.listen.timeout=duration + when used with -remote=auto, the -listen.timeout value used to start the daemon (default 1m0s) + -remote.logfile=string + when used with -remote=auto, the -logfile value used to start the daemon + -rpc.trace + print the full rpc trace in lsp inspector format + -v,-verbose + verbose output + -vv,-veryverbose + very verbose output diff --git a/gopls/internal/lsp/cmd/usage/usage.hlp b/gopls/internal/lsp/cmd/usage/usage.hlp index 7f1bfb46a87..c9cc12a943f 100644 --- a/gopls/internal/lsp/cmd/usage/usage.hlp +++ b/gopls/internal/lsp/cmd/usage/usage.hlp @@ -39,7 +39,6 @@ Features fix apply suggested fixes symbols display selected file's symbols workspace_symbol search symbols in workspace - vulncheck run experimental vulncheck analysis (experimental: under development) flags: -debug=string diff --git a/gopls/internal/lsp/cmd/usage/vulncheck.hlp b/gopls/internal/lsp/cmd/usage/vulncheck.hlp index 1f5800f0ae1..d16cb130871 100644 --- a/gopls/internal/lsp/cmd/usage/vulncheck.hlp +++ b/gopls/internal/lsp/cmd/usage/vulncheck.hlp @@ -1,9 +1,9 @@ -run experimental vulncheck analysis (experimental: under development) +run vulncheck analysis (internal-use only) Usage: gopls [flags] vulncheck - WARNING: this command is experimental. + WARNING: this command is for internal-use only. By default, the command outputs a JSON-encoded golang.org/x/tools/gopls/internal/lsp/command.VulncheckResult diff --git a/gopls/internal/lsp/cmd/vulncheck.go b/gopls/internal/lsp/cmd/vulncheck.go index bb53e0c82b3..855b9eef830 100644 --- a/gopls/internal/lsp/cmd/vulncheck.go +++ b/gopls/internal/lsp/cmd/vulncheck.go @@ -23,11 +23,11 @@ func (v *vulncheck) Name() string { return "vulncheck" } func (v *vulncheck) Parent() string { return v.app.Name() } func (v *vulncheck) Usage() string { return "" } func (v *vulncheck) ShortHelp() string { - return "run experimental vulncheck analysis (experimental: under development)" + return "run vulncheck analysis (internal-use only)" } func (v *vulncheck) DetailedHelp(f *flag.FlagSet) { fmt.Fprint(f.Output(), ` - WARNING: this command is experimental. + WARNING: this command is for internal-use only. By default, the command outputs a JSON-encoded golang.org/x/tools/gopls/internal/lsp/command.VulncheckResult From 1c59c38f41500844672e5a9da3d690f8b50a34b8 Mon Sep 17 00:00:00 2001 From: "Hana (Hyang-Ah) Kim" Date: Fri, 22 Sep 2023 08:32:23 -0400 Subject: [PATCH 128/178] gopls/internal/lsp/source: add linkifyShowMessage (internal option) VS Code supports `[text](url)` syntax in its showMessage UI. Allow the editor to inform gopls and use it when prompting for telemetry. Change-Id: I4fbf80958affd54fa9b3802b72495e96efaac8fc Reviewed-on: https://go-review.googlesource.com/c/tools/+/530456 Run-TryBot: Hyang-Ah Hana Kim Reviewed-by: Robert Findley TryBot-Result: Gopher Robot --- gopls/internal/lsp/prompt.go | 7 ++++++- gopls/internal/lsp/source/options.go | 7 +++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/gopls/internal/lsp/prompt.go b/gopls/internal/lsp/prompt.go index ad9d404d5a6..df90cfb5e36 100644 --- a/gopls/internal/lsp/prompt.go +++ b/gopls/internal/lsp/prompt.go @@ -187,11 +187,16 @@ func (s *Server) maybePromptForTelemetry(ctx context.Context) { return } - const prompt = `Go telemetry helps us improve Go by periodically sending anonymous metrics and crash reports to the Go team. Learn more at https://telemetry.go.dev/privacy. + var prompt = `Go telemetry helps us improve Go by periodically sending anonymous metrics and crash reports to the Go team. Learn more at https://telemetry.go.dev/privacy. Would you like to enable Go telemetry? ` + if s.Options().LinkifyShowMessage { + prompt = `Go telemetry helps us improve Go by periodically sending anonymous metrics and crash reports to the Go team. Learn more at [telemetry.go.dev/privacy](https://telemetry.go.dev/privacy). + Would you like to enable Go telemetry? + ` + } // TODO(rfindley): investigate a "tell me more" action in combination with ShowDocument. params := &protocol.ShowMessageRequestParams{ Type: protocol.Info, diff --git a/gopls/internal/lsp/source/options.go b/gopls/internal/lsp/source/options.go index 5cb24830017..0d610a42e30 100644 --- a/gopls/internal/lsp/source/options.go +++ b/gopls/internal/lsp/source/options.go @@ -178,6 +178,7 @@ func DefaultOptions(overrides ...func(*Options)) *Options { SubdirWatchPatterns: SubdirWatchPatternsAuto, ReportAnalysisProgressAfter: 5 * time.Second, TelemetryPrompt: false, + LinkifyShowMessage: false, }, Hooks: Hooks{ // TODO(adonovan): switch to new diff.Strings implementation. @@ -683,6 +684,10 @@ type InternalOptions struct { // Once the prompt is answered, gopls doesn't ask again, but TelemetryPrompt // can prevent the question from ever being asked in the first place. TelemetryPrompt bool + + // LinkifyShowMessage controls whether the client wants gopls + // to linkify links in showMessage. e.g. [go.dev](https://go.dev). + LinkifyShowMessage bool } type SubdirWatchPatterns string @@ -1269,6 +1274,8 @@ func (o *Options) set(name string, value interface{}, seen map[string]struct{}) case "telemetryPrompt": result.setBool(&o.TelemetryPrompt) + case "linkifyShowMessage": + result.setBool(&o.LinkifyShowMessage) // Replaced settings. case "experimentalDisabledAnalyses": From 64beb9515cd59e0acf92bb3fd0ae91fa7828734a Mon Sep 17 00:00:00 2001 From: "Hana (Hyang-Ah) Kim" Date: Fri, 22 Sep 2023 09:19:20 -0400 Subject: [PATCH 129/178] gopls/internal/lsp/command: add maybePromptForTelemetry VS Code collapses notifications when there are multiple notification messages or progress messages. In order to increase the visibility, VS Code can pick the quiet time (when there was no message exchanges for a while and show prompt). This internal command makes the invocation easy. I thought about setting the telemetryPrompt option when VS Code wants gopls to prompt, but that is a configuration change and may cause other progress notifications to get in the way. Change-Id: Id9c18118a27931e41ef96e1648c0e8b6fe27f7ae Reviewed-on: https://go-review.googlesource.com/c/tools/+/530457 Reviewed-by: Robert Findley TryBot-Result: Gopher Robot Run-TryBot: Hyang-Ah Hana Kim --- gopls/doc/commands.md | 6 ++ gopls/internal/lsp/command.go | 6 +- gopls/internal/lsp/command/command_gen.go | 74 +++++++++++++--------- gopls/internal/lsp/command/interface.go | 5 ++ gopls/internal/lsp/general.go | 2 +- gopls/internal/lsp/prompt.go | 6 +- gopls/internal/lsp/source/api_json.go | 5 ++ gopls/internal/regtest/misc/prompt_test.go | 46 ++++++++++++++ 8 files changed, 117 insertions(+), 33 deletions(-) diff --git a/gopls/doc/commands.md b/gopls/doc/commands.md index a2ffda56a6e..833dad9a1d0 100644 --- a/gopls/doc/commands.md +++ b/gopls/doc/commands.md @@ -238,6 +238,12 @@ Result: } ``` +### **checks for the right conditions, and then prompts** +Identifier: `gopls.maybe_prompt_for_telemetry` + +the user to ask if they want to enable Go telemetry uploading. If the user +responds 'Yes', the telemetry mode is set to "on". + ### **fetch memory statistics** Identifier: `gopls.mem_stats` diff --git a/gopls/internal/lsp/command.go b/gopls/internal/lsp/command.go index a64007b988a..f4d4a9e4ba2 100644 --- a/gopls/internal/lsp/command.go +++ b/gopls/internal/lsp/command.go @@ -64,7 +64,11 @@ type commandHandler struct { params *protocol.ExecuteCommandParams } -// AddTelemetryCounters implements command.Interface. +func (h *commandHandler) MaybePromptForTelemetry(ctx context.Context) error { + go h.s.maybePromptForTelemetry(ctx, true) + return nil +} + func (*commandHandler) AddTelemetryCounters(_ context.Context, args command.AddTelemetryCountersArgs) error { if len(args.Names) != len(args.Values) { return fmt.Errorf("Names and Values must have the same length") diff --git a/gopls/internal/lsp/command/command_gen.go b/gopls/internal/lsp/command/command_gen.go index 40eda278190..5dd2a9dd452 100644 --- a/gopls/internal/lsp/command/command_gen.go +++ b/gopls/internal/lsp/command/command_gen.go @@ -19,35 +19,36 @@ import ( ) const ( - AddDependency Command = "add_dependency" - AddImport Command = "add_import" - AddTelemetryCounters Command = "add_telemetry_counters" - ApplyFix Command = "apply_fix" - CheckUpgrades Command = "check_upgrades" - EditGoDirective Command = "edit_go_directive" - FetchVulncheckResult Command = "fetch_vulncheck_result" - GCDetails Command = "gc_details" - Generate Command = "generate" - GoGetPackage Command = "go_get_package" - ListImports Command = "list_imports" - ListKnownPackages Command = "list_known_packages" - MemStats Command = "mem_stats" - RegenerateCgo Command = "regenerate_cgo" - RemoveDependency Command = "remove_dependency" - ResetGoModDiagnostics Command = "reset_go_mod_diagnostics" - RunGoWorkCommand Command = "run_go_work_command" - RunGovulncheck Command = "run_govulncheck" - RunTests Command = "run_tests" - StartDebugging Command = "start_debugging" - StartProfile Command = "start_profile" - StopProfile Command = "stop_profile" - Test Command = "test" - Tidy Command = "tidy" - ToggleGCDetails Command = "toggle_gc_details" - UpdateGoSum Command = "update_go_sum" - UpgradeDependency Command = "upgrade_dependency" - Vendor Command = "vendor" - WorkspaceStats Command = "workspace_stats" + AddDependency Command = "add_dependency" + AddImport Command = "add_import" + AddTelemetryCounters Command = "add_telemetry_counters" + ApplyFix Command = "apply_fix" + CheckUpgrades Command = "check_upgrades" + EditGoDirective Command = "edit_go_directive" + FetchVulncheckResult Command = "fetch_vulncheck_result" + GCDetails Command = "gc_details" + Generate Command = "generate" + GoGetPackage Command = "go_get_package" + ListImports Command = "list_imports" + ListKnownPackages Command = "list_known_packages" + MaybePromptForTelemetry Command = "maybe_prompt_for_telemetry" + MemStats Command = "mem_stats" + RegenerateCgo Command = "regenerate_cgo" + RemoveDependency Command = "remove_dependency" + ResetGoModDiagnostics Command = "reset_go_mod_diagnostics" + RunGoWorkCommand Command = "run_go_work_command" + RunGovulncheck Command = "run_govulncheck" + RunTests Command = "run_tests" + StartDebugging Command = "start_debugging" + StartProfile Command = "start_profile" + StopProfile Command = "stop_profile" + Test Command = "test" + Tidy Command = "tidy" + ToggleGCDetails Command = "toggle_gc_details" + UpdateGoSum Command = "update_go_sum" + UpgradeDependency Command = "upgrade_dependency" + Vendor Command = "vendor" + WorkspaceStats Command = "workspace_stats" ) var Commands = []Command{ @@ -63,6 +64,7 @@ var Commands = []Command{ GoGetPackage, ListImports, ListKnownPackages, + MaybePromptForTelemetry, MemStats, RegenerateCgo, RemoveDependency, @@ -156,6 +158,8 @@ func Dispatch(ctx context.Context, params *protocol.ExecuteCommandParams, s Inte return nil, err } return s.ListKnownPackages(ctx, a0) + case "gopls.maybe_prompt_for_telemetry": + return nil, s.MaybePromptForTelemetry(ctx) case "gopls.mem_stats": return s.MemStats(ctx) case "gopls.regenerate_cgo": @@ -400,6 +404,18 @@ func NewListKnownPackagesCommand(title string, a0 URIArg) (protocol.Command, err }, nil } +func NewMaybePromptForTelemetryCommand(title string) (protocol.Command, error) { + args, err := MarshalArgs() + if err != nil { + return protocol.Command{}, err + } + return protocol.Command{ + Title: title, + Command: "gopls.maybe_prompt_for_telemetry", + Arguments: args, + }, nil +} + func NewMemStatsCommand(title string) (protocol.Command, error) { args, err := MarshalArgs() if err != nil { diff --git a/gopls/internal/lsp/command/interface.go b/gopls/internal/lsp/command/interface.go index 2ae50fb0e87..603e6121c8a 100644 --- a/gopls/internal/lsp/command/interface.go +++ b/gopls/internal/lsp/command/interface.go @@ -195,6 +195,11 @@ type Interface interface { // Gopls will prepend "fwd/" to all the counters updated using this command // to avoid conflicts with other counters gopls collects. AddTelemetryCounters(context.Context, AddTelemetryCountersArgs) error + + // MaybePromptForTelemetry: checks for the right conditions, and then prompts + // the user to ask if they want to enable Go telemetry uploading. If the user + // responds 'Yes', the telemetry mode is set to "on". + MaybePromptForTelemetry(context.Context) error } type RunTestsArgs struct { diff --git a/gopls/internal/lsp/general.go b/gopls/internal/lsp/general.go index b1d6850b3c6..7d9ed35325e 100644 --- a/gopls/internal/lsp/general.go +++ b/gopls/internal/lsp/general.go @@ -241,7 +241,7 @@ func (s *Server) initialized(ctx context.Context, params *protocol.InitializedPa // Ask (maybe) about enabling telemetry. Do this asynchronously, as it's OK // for users to ignore or dismiss the question. - go s.maybePromptForTelemetry(ctx) + go s.maybePromptForTelemetry(ctx, options.TelemetryPrompt) return nil } diff --git a/gopls/internal/lsp/prompt.go b/gopls/internal/lsp/prompt.go index df90cfb5e36..c0e3a5b5184 100644 --- a/gopls/internal/lsp/prompt.go +++ b/gopls/internal/lsp/prompt.go @@ -81,13 +81,15 @@ func (s *Server) setTelemetryMode(mode string) error { // // The actual conditions for prompting are defensive, erring on the side of not // prompting. -func (s *Server) maybePromptForTelemetry(ctx context.Context) { +// If enabled is false, this will not prompt the user in any condition, +// but will send work progress reports to help testing. +func (s *Server) maybePromptForTelemetry(ctx context.Context, enabled bool) { if s.Options().VerboseWorkDoneProgress { work := s.progress.Start(ctx, TelemetryPromptWorkTitle, "Checking if gopls should prompt about telemetry...", nil, nil) defer work.End(ctx, "Done.") } - if !s.Options().TelemetryPrompt { + if !enabled { // check this after the work progress message for testing. return // prompt is disabled } diff --git a/gopls/internal/lsp/source/api_json.go b/gopls/internal/lsp/source/api_json.go index 0bca2e6d6a8..45ebabb56dd 100644 --- a/gopls/internal/lsp/source/api_json.go +++ b/gopls/internal/lsp/source/api_json.go @@ -777,6 +777,11 @@ var GeneratedAPIJSON = &APIJSON{ ArgDoc: "{\n\t// The file URI.\n\t\"URI\": string,\n}", ResultDoc: "{\n\t// Packages is a list of packages relative\n\t// to the URIArg passed by the command request.\n\t// In other words, it omits paths that are already\n\t// imported or cannot be imported due to compiler\n\t// restrictions.\n\t\"Packages\": []string,\n}", }, + { + Command: "gopls.maybe_prompt_for_telemetry", + Title: "checks for the right conditions, and then prompts", + Doc: "the user to ask if they want to enable Go telemetry uploading. If the user\nresponds 'Yes', the telemetry mode is set to \"on\".", + }, { Command: "gopls.mem_stats", Title: "fetch memory statistics", diff --git a/gopls/internal/regtest/misc/prompt_test.go b/gopls/internal/regtest/misc/prompt_test.go index e173fa3c6d5..e053cc9d7a6 100644 --- a/gopls/internal/regtest/misc/prompt_test.go +++ b/gopls/internal/regtest/misc/prompt_test.go @@ -12,6 +12,7 @@ import ( "testing" "golang.org/x/tools/gopls/internal/lsp" + "golang.org/x/tools/gopls/internal/lsp/command" "golang.org/x/tools/gopls/internal/lsp/protocol" . "golang.org/x/tools/gopls/internal/lsp/regtest" ) @@ -177,3 +178,48 @@ func main() { }) } } + +// Test that gopls prompts for telemetry only when it is supposed to. +func TestTelemetryPrompt_Conditions2(t *testing.T) { + const src = ` +-- go.mod -- +module mod.com + +go 1.12 +-- main.go -- +package main + +func main() { +} +` + modeFile := filepath.Join(t.TempDir(), "mode") + WithOptions( + Modes(Default), // no need to run this in all modes + EnvVars{ + lsp.GoplsConfigDirEnvvar: t.TempDir(), + lsp.FakeTelemetryModefileEnvvar: modeFile, + }, + Settings{ + // off because we are testing + // if we can trigger the prompt with command. + "telemetryPrompt": false, + }, + ).Run(t, src, func(t *testing.T, env *Env) { + cmd, err := command.NewMaybePromptForTelemetryCommand("prompt") + if err != nil { + t.Fatal(err) + } + var result error + env.ExecuteCommand(&protocol.ExecuteCommandParams{ + Command: cmd.Command, + }, &result) + if result != nil { + t.Fatal(err) + } + expectation := ShownMessageRequest(".*Would you like to enable Go telemetry?") + env.OnceMet( + CompletedWork(lsp.TelemetryPromptWorkTitle, 2, true), + expectation, + ) + }) +} From f9759353eed3270bf007753dd44fb217fbc6aca3 Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Fri, 22 Sep 2023 10:27:32 -0400 Subject: [PATCH 130/178] gopls/internal/lsp: update telemetry prompt and add a follow-up message Update the telemetry prompt per discussion, and add a follow up message with more information after telemetry is enabled. For golang/go#62576 Change-Id: If03cade72b27a765da43db293c74d53d19b95a9c Reviewed-on: https://go-review.googlesource.com/c/tools/+/530459 LUCI-TryBot-Result: Go LUCI Reviewed-by: Hyang-Ah Hana Kim --- gopls/internal/lsp/prompt.go | 48 +++++++++++++++++----- gopls/internal/regtest/misc/prompt_test.go | 18 +++++--- 2 files changed, 50 insertions(+), 16 deletions(-) diff --git a/gopls/internal/lsp/prompt.go b/gopls/internal/lsp/prompt.go index c0e3a5b5184..c3cc8f91645 100644 --- a/gopls/internal/lsp/prompt.go +++ b/gopls/internal/lsp/prompt.go @@ -28,6 +28,8 @@ const ( TelemetryPromptWorkTitle = "Checking telemetry prompt" // progress notification title, for awaiting in tests GoplsConfigDirEnvvar = "GOPLS_CONFIG_DIR" // overridden for testing FakeTelemetryModefileEnvvar = "GOPLS_FAKE_TELEMETRY_MODEFILE" // overridden for testing + TelemetryYes = "Yes, I'd like to help." + TelemetryNo = "No, thanks." ) // getenv returns the effective environment variable value for the provided @@ -203,8 +205,12 @@ Would you like to enable Go telemetry? params := &protocol.ShowMessageRequestParams{ Type: protocol.Info, Message: prompt, - Actions: []protocol.MessageActionItem{{Title: "Yes"}, {Title: "No"}}, + Actions: []protocol.MessageActionItem{ + {Title: TelemetryYes}, + {Title: TelemetryNo}, + }, } + item, err := s.client.ShowMessageRequest(ctx, params) if err != nil { errorf("ShowMessageRequest failed: %v", err) @@ -212,6 +218,15 @@ Would you like to enable Go telemetry? item = nil } + message := func(typ protocol.MessageType, msg string) { + if err := s.client.ShowMessage(ctx, &protocol.ShowMessageParams{ + Type: typ, + Message: msg, + }); err != nil { + errorf("ShowMessage(unrecognize) failed: %v", err) + } + } + result := pFailed if item == nil { // e.g. dialog was dismissed @@ -219,19 +234,21 @@ Would you like to enable Go telemetry? } else { // Response matches MessageActionItem.Title. switch item.Title { - case "Yes": + case TelemetryYes: result = pYes - s.setTelemetryMode("on") - case "No": + if err := s.setTelemetryMode("on"); err == nil { + message(protocol.Info, telemetryOnMessage()) + } else { + errorf("enabling telemetry failed: %v", err) + msg := fmt.Sprintf("Failed to enable Go telemetry: %v\nTo enable telemetry manually, please run `go run golang.org/x/telemetry/cmd/gotelemetry@latest on`", err) + message(protocol.Error, msg) + } + + case TelemetryNo: result = pNo default: errorf("unrecognized response %q", item.Title) - if err := s.client.ShowMessage(ctx, &protocol.ShowMessageParams{ - Type: protocol.Error, - Message: fmt.Sprintf("Unrecognized response %q", item.Title), - }); err != nil { - errorf("ShowMessage failed: %v", err) - } + message(protocol.Error, fmt.Sprintf("Unrecognized response %q", item.Title)) } } resultContent := []byte(fmt.Sprintf("%s %d", result, attempts)) @@ -240,6 +257,17 @@ Would you like to enable Go telemetry? } } +func telemetryOnMessage() string { + reportDate := time.Now().AddDate(0, 0, 7).Format("2006-01-02") + return fmt.Sprintf(`Telemetry uploading is now enabled and may be sent to https://telemetry.go.dev/ starting %s. Uploaded data is used to help improve the Go toolchain and related tools, and it will be published as part of a public dataset. + +For more details, see https://telemetry.go.dev/privacy. +This data is collected in accordance with the Google Privacy Policy (https://policies.google.com/privacy). + +To disable telemetry uploading, run %s. +`, reportDate, "`go run golang.org/x/telemetry/cmd/gotelemetry@latest off`") +} + // acquireLockFile attempts to "acquire a lock" for writing to path. // // This is achieved by creating an exclusive lock file at .lock. Lock diff --git a/gopls/internal/regtest/misc/prompt_test.go b/gopls/internal/regtest/misc/prompt_test.go index e053cc9d7a6..7a262ad934e 100644 --- a/gopls/internal/regtest/misc/prompt_test.go +++ b/gopls/internal/regtest/misc/prompt_test.go @@ -82,12 +82,13 @@ func main() { ` tests := []struct { - response string - wantMode string + response string // response to choose for the telemetry dialog + wantMode string // resulting telemetry mode + wantMsg string // substring contained in the follow-up popup (if empty, no popup is expected) }{ - {"Yes", "on"}, - {"No", ""}, - {"", ""}, + {lsp.TelemetryYes, "on", "uploading is now enabled"}, + {lsp.TelemetryNo, "", ""}, + {"", "", ""}, } for _, test := range tests { t.Run(fmt.Sprintf("response=%s", test.response), func(t *testing.T) { @@ -117,8 +118,13 @@ func main() { }, MessageResponder(respond), ).Run(t, src, func(t *testing.T, env *Env) { - env.Await( + var postConditions []Expectation + if test.wantMsg != "" { + postConditions = append(postConditions, ShownMessage(test.wantMsg)) + } + env.OnceMet( CompletedWork(lsp.TelemetryPromptWorkTitle, 1, true), + postConditions..., ) gotMode := "" if contents, err := os.ReadFile(modeFile); err == nil { From 0ceab5c177facc03da1c3c32a0c2f865bee01d81 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Thu, 21 Sep 2023 10:30:33 -0400 Subject: [PATCH 131/178] internal/refactor/inline: split up the main function This CL is a refactoring of the inline function, extracting steps such as gathering of arguments and of parameters, substitution, creation of the binding decl, and the callerLookup, into top-level functions. Other than code motion and doc tweaks, the only significant change is the movement of callerPath to a field Caller.path, allowing Caller.lookup to be expressed as a method. This means the Caller object _is_ mutated by the operation, but only its internal state. Change-Id: I5bd2d463d01765e553186a796300b12c7fed0dd9 Reviewed-on: https://go-review.googlesource.com/c/tools/+/530056 Auto-Submit: Alan Donovan LUCI-TryBot-Result: Go LUCI Reviewed-by: Robert Findley --- internal/refactor/inline/inline.go | 931 +++++++++++++++-------------- 1 file changed, 487 insertions(+), 444 deletions(-) diff --git a/internal/refactor/inline/inline.go b/internal/refactor/inline/inline.go index 70fa1d1981e..cb98d58da6d 100644 --- a/internal/refactor/inline/inline.go +++ b/internal/refactor/inline/inline.go @@ -33,12 +33,14 @@ type Caller struct { File *ast.File Call *ast.CallExpr Content []byte // source of file containing + + path []ast.Node } // Inline inlines the called function (callee) into the function call (caller) // and returns the updated, formatted content of the caller source file. // -// Inline does not mutate any part of Caller or Callee. +// Inline does not mutate any public fields of Caller or Callee. // // The log records the decision-making process. // @@ -229,23 +231,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // -- analyze callee's free references in caller context -- // syntax path enclosing Call, innermost first (Path[0]=Call) - callerPath, _ := astutil.PathEnclosingInterval(caller.File, caller.Call.Pos(), caller.Call.End()) - callerLookup := func(name string) types.Object { - pos := caller.Call.Pos() - for _, n := range callerPath { - // The function body scope (containing not just params) - // is associated with FuncDecl.Type, not FuncDecl.Body. - if decl, ok := n.(*ast.FuncDecl); ok { - n = decl.Type - } - if scope := caller.Info.Scopes[n]; scope != nil { - if _, obj := scope.LookupParent(name, pos); obj != nil { - return obj - } - } - } - return nil - } + caller.path, _ = astutil.PathEnclosingInterval(caller.File, caller.Call.Pos(), caller.Call.End()) // Find the outermost function enclosing the call site (if any). // Analyze all its local vars for the "single assignment" property @@ -253,7 +239,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu var assign1 func(v *types.Var) bool // reports whether v a single-assignment local var { updatedLocals := make(map[*types.Var]bool) - for _, n := range callerPath { + for _, n := range caller.path { if decl, ok := n.(*ast.FuncDecl); ok { escape(caller.Info, decl.Body, func(v *types.Var, _ bool) { updatedLocals[v] = true @@ -288,7 +274,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu for _, name := range importMap[path] { // Check that either the import preexisted, // or that it was newly added (no PkgName) but is not shadowed. - found := callerLookup(name) + found := caller.lookup(name) if is[*types.PkgName](found) || found == nil { return name } @@ -305,7 +291,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // use the package's declared name. base := pathpkg.Base(path) name := base - for n := 0; callerLookup(name) != nil; n++ { + for n := 0; caller.lookup(name) != nil; n++ { name = fmt.Sprintf("%s%d", base, n) } @@ -353,7 +339,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu } else if !obj.ValidPos { // Built-in function, type, or value (e.g. nil, zero): // check not shadowed at caller. - found := callerLookup(obj.Name) // always finds something + found := caller.lookup(obj.Name) // always finds something if found.Pos().IsValid() { return nil, fmt.Errorf("cannot inline because built-in %q is shadowed in caller by a %s (line %d)", obj.Name, objectKind(found), @@ -369,7 +355,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu if samePkg { // Caller and callee are in same package. // Check caller has not shadowed the decl. - found := callerLookup(obj.Name) // can't fail + found := caller.lookup(obj.Name) // can't fail if !isPkgLevel(found) { return nil, fmt.Errorf("cannot inline because %q is shadowed in caller by a %s (line %d)", obj.Name, objectKind(found), @@ -424,147 +410,15 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu } } - // Gather effective argument tuple, including receiver. - // - // If the receiver argument and parameter have - // different pointerness, make the "&" or "*" explicit. - // - // Also, if x.f() is shorthand for promoted method x.y.f(), - // make the .y explicit in T.f(x.y, ...). - // - // Beware that: - // - // - a method can only be called through a selection, but only - // the first of these two forms needs special treatment: - // - // expr.f(args) -> ([&*]expr, args) MethodVal - // T.f(recv, args) -> ( expr, args) MethodExpr - // - // - the presence of a value in receiver-position in the call - // is a property of the caller, not the callee. A method - // (calleeDecl.Recv != nil) may be called like an ordinary - // function. - // - // - the types.Signatures seen by the caller (from - // StaticCallee) and by the callee (from decl type) - // differ in this case. - // - // In a spread call f(g()), the sole ordinary argument g(), - // always last in args, has a tuple type. - // - // We compute type-based predicates like pure, duplicable, - // freevars, etc, now, before we start modifying things. - type argument struct { - expr ast.Expr - typ types.Type // may be tuple for sole non-receiver arg in spread call - spread bool // final arg is call() assigned to multiple params - pure bool // expr is pure (doesn't read variables) - effects bool // expr has effects (updates variables) - duplicable bool // expr may be duplicated - freevars map[string]bool // free names of expr - } - var args []*argument // effective arguments; nil => substituted - { - // TODO(adonovan): extract to a function (in a separate CL). - callArgs := caller.Call.Args - if calleeDecl.Recv != nil { - sel := astutil.Unparen(caller.Call.Fun).(*ast.SelectorExpr) - seln := caller.Info.Selections[sel] - var recvArg ast.Expr - switch seln.Kind() { - case types.MethodVal: // recv.f(callArgs) - recvArg = sel.X - case types.MethodExpr: // T.f(recv, callArgs) - recvArg = callArgs[0] - callArgs = callArgs[1:] - } - if recvArg != nil { - // Compute all the type-based predicates now, - // before we start meddling with the syntax; - // the meddling will update them. - arg := &argument{ - expr: recvArg, - typ: caller.Info.TypeOf(recvArg), - pure: pure(caller.Info, assign1, recvArg), - effects: effects(caller.Info, recvArg), - duplicable: duplicable(caller.Info, recvArg), - freevars: freeVars(caller.Info, recvArg), - } - recvArg = nil // prevent accidental use - - // Move receiver argument recv.f(args) to argument list f(&recv, args). - args = append(args, arg) - - // Make field selections explicit (recv.f -> recv.y.f), - // updating arg.{expr,typ}. - indices := seln.Index() - for _, index := range indices[:len(indices)-1] { - t := deref(arg.typ) - fld := typeparams.CoreType(t).(*types.Struct).Field(index) - if fld.Pkg() != caller.Types && !fld.Exported() { - return nil, fmt.Errorf("in %s, implicit reference to unexported field .%s cannot be made explicit", - debugFormatNode(caller.Fset, caller.Call.Fun), - fld.Name()) - } - arg.expr = &ast.SelectorExpr{ - X: arg.expr, - Sel: makeIdent(fld.Name()), - } - arg.typ = fld.Type() - } - - // Make * or & explicit. - argIsPtr := arg.typ != deref(arg.typ) - paramIsPtr := is[*types.Pointer](seln.Obj().Type().(*types.Signature).Recv().Type()) - if !argIsPtr && paramIsPtr { - // &recv - arg.expr = &ast.UnaryExpr{Op: token.AND, X: arg.expr} - arg.typ = types.NewPointer(arg.typ) - } else if argIsPtr && !paramIsPtr { - // *recv - arg.expr = &ast.StarExpr{X: arg.expr} - arg.typ = deref(arg.typ) - - // Technically *recv is non-pure and - // non-duplicable, as side effects - // could change the pointer between - // multiple reads. But unfortunately - // this really degrades many of our tests. - // (The indices loop above should similarly - // update these flags when traversing pointers.) - // - // TODO(adonovan): improve the precision of - // purity and duplicability. - // For example, *new(T) is actually pure. - // And *ptr, where ptr doesn't escape and - // has no assignments other than its decl, - // is also pure; this is very common. - // - // arg.pure = false - } - } - } - for _, expr := range callArgs { - typ := caller.Info.TypeOf(expr) - args = append(args, &argument{ - expr: expr, - typ: typ, - spread: is[*types.Tuple](typ), // => last - pure: pure(caller.Info, assign1, expr), - effects: effects(caller.Info, expr), - duplicable: duplicable(caller.Info, expr), - freevars: freeVars(caller.Info, expr), - }) - } + // Gather the effective call arguments, including the receiver. + // Later, elements will be eliminated (=> nil) by parameter substitution. + args, err := arguments(caller, calleeDecl, assign1) + if err != nil { + return nil, err // e.g. implicit field selection cannot be made explicit } // Gather effective parameter tuple, including the receiver if any. // Simplify variadic parameters to slices (in all cases but one). - type parameter struct { - obj *types.Var // parameter var from caller's signature - info *paramInfo // information from AnalyzeCallee - variadic bool // (final) parameter is unsimplified ...T - } var params []*parameter // including receiver; nil => parameter substituted { sig := calleeSymbol.Type().(*types.Signature) @@ -592,6 +446,8 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // the third is tricky and cannot be reduced, and (if // a receiver is present) cannot even be literalized. // Fortunately it is vanishingly rare. + // + // TODO(adonovan): extract this to a function. if sig.Variadic() { lastParam := last(params) if len(args) > 0 && last(args).spread { @@ -639,127 +495,15 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // Note: computation below should be expressed in terms of // the args and params slices, not the raw material. - // Parameter substitution - // - // Consider each parameter and its corresponding argument in turn - // and evaluate these conditions: - // - // - the parameter is neither address-taken nor assigned; - // - the argument is pure; - // - if the parameter refcount is zero, the argument must - // not contain the last use of a local var; - // - if the parameter refcount is > 1, the argument must be duplicable; - // - the argument (or types.Default(argument) if it's untyped) has - // the same type as the parameter. - // - // If all conditions are met then the parameter can be substituted - // and each reference to it replaced by the argument. - // - // TODO(adonovan): extract this into a separate function. - { - // Inv: - // in calls to variadic, len(args) >= len(params)-1 - // in spread calls to non-variadic, len(args) < len(params) - // in spread calls to variadic, len(args) <= len(params) - // (In spread calls len(args) = 1, or 2 if call has receiver.) - // Non-spread variadics have been simplified away already, - // so the args[i] lookup is safe if we stop after the spread arg. - next: - for i, param := range params { - arg := args[i] - // Check argument against parameter. - // - // Beware: don't use types.Info on arg since - // the syntax may be synthetic (not created by parser) - // and thus lacking positions and types; - // do it earlier (see pure/duplicable/freevars). - - if arg.spread { - logf("keeping param %q and following ones: argument %s is spread", - param.info.Name, debugFormatNode(caller.Fset, arg.expr)) - break // spread => last argument, but not always last parameter - } - assert(!param.variadic, "unsimplified variadic parameter") - if param.info.Escapes { - logf("keeping param %q: escapes from callee", param.info.Name) - continue - } - if param.info.Assigned { - logf("keeping param %q: assigned by callee", param.info.Name) - continue // callee needs the parameter variable - } - if !arg.pure || arg.effects { - // TODO(adonovan): conduct a deeper analysis of callee effects - // to relax this constraint. - logf("keeping param %q: argument %s is impure", - param.info.Name, debugFormatNode(caller.Fset, arg.expr)) - continue // unsafe to change order or cardinality of effects - } - if len(param.info.Refs) > 1 && !arg.duplicable { - logf("keeping param %q: argument is not duplicable", param.info.Name) - continue // incorrect or poor style to duplicate an expression - } - if len(param.info.Refs) == 0 { - // Eliminating an unreferenced parameter might - // remove the last reference to a caller local var. - for free := range arg.freevars { - if v, ok := callerLookup(free).(*types.Var); ok { - // TODO(adonovan): be more precise and check - // that v is defined within the body of the caller - // function (if any) and is indeed referenced - // only by the call. (See assign1 for analysis - // of enclosing func.) - logf("keeping param %q: arg contains perhaps the last reference to possible caller local %v @ %v", - param.info.Name, v, caller.Fset.PositionFor(v.Pos(), false)) - continue next - } - } - } + // Perform parameter substitution. + // May eliminate some elements of params/args. + substitute(logf, caller, params, args, replaceCalleeID) - // Check that eliminating the parameter wouldn't materially - // change the type. - // - // (We don't simply wrap the argument in an explicit conversion - // to the parameter type because that could increase allocation - // in the number of (e.g.) string -> any conversions. - // Even when Uses = 1, the sole ref might be in a loop or lambda that - // is multiply executed.) - if len(param.info.Refs) > 0 && !trivialConversion(args[i].typ, params[i].obj) { - logf("keeping param %q: argument passing converts %s to type %s", - param.info.Name, args[i].typ, params[i].obj.Type()) - continue // implicit conversion is significant - } + // Update the callee's signature syntax. + updateCalleeParams(calleeDecl, params) - // Check for shadowing. - // - // Consider inlining a call f(z, 1) to - // func f(x, y int) int { z := y; return x + y + z }: - // we can't replace x in the body by z (or any - // expression that has z as a free identifier) - // because there's an intervening declaration of z - // that would shadow the caller's one. - for free := range arg.freevars { - if param.info.Shadow[free] { - logf("keeping param %q: cannot replace with argument as it has free ref to %s that is shadowed", param.info.Name, free) - continue next // shadowing conflict - } - } - - // It is safe to eliminate param and replace it with arg. - // No additional parens are required around arg for - // the supported "pure" expressions. - // - // Because arg.expr belongs to the caller, - // we clone it before splicing it into the callee tree. - logf("replacing parameter %q by argument %q", - param.info.Name, debugFormatNode(caller.Fset, arg.expr)) - for _, ref := range param.info.Refs { - replaceCalleeID(ref, cloneNode(arg.expr).(ast.Expr)) - } - params[i] = nil // substituted - args[i] = nil // substituted - } - } + // Create a var (param = arg; ...) decl for use by some strategies. + bindingDeclStmt := createBindingDecl(logf, caller, args, calleeDecl) var remainingArgs []ast.Expr for _, arg := range args { @@ -768,165 +512,6 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu } } - // Modify callee's FuncDecl.Type.Params to remove substituted - // parameters and move the receiver (if any) to the head of - // the ordinary parameters. - // - // The logic is fiddly because of the three forms of ast.Field: - // func(int), func(x int), func(x, y int) - // - // Also, ensure that all remaining parameters are named - // to avoid a mix of named/unnamed when joining (recv, params...). - // func (T) f(int, bool) -> (_ T, _ int, _ bool) - // (Strictly, we need do this only for methods and only when - // the namednesses of Recv and Params differ; that might be tidier.) - { - paramIdx := 0 // index in original parameter list (incl. receiver) - var newParams []*ast.Field - filterParams := func(field *ast.Field) { - var names []*ast.Ident - if field.Names == nil { - // Unnamed parameter field (e.g. func f(int) - if params[paramIdx] != nil { - // Give it an explicit name "_" since we will - // make the receiver (if any) a regular parameter - // and one cannot mix named and unnamed parameters. - names = append(names, makeIdent("_")) - } - paramIdx++ - } else { - // Named parameter field e.g. func f(x, y int) - // Remove substituted parameters in place. - // If all were substituted, delete field. - for _, id := range field.Names { - if pinfo := params[paramIdx]; pinfo != nil { - // Rename unreferenced parameters with "_". - // This is crucial for binding decls, since - // unlike parameters, they are subject to - // "unreferenced var" checks. - if len(pinfo.info.Refs) == 0 { - id = makeIdent("_") - } - names = append(names, id) - } - paramIdx++ - } - } - if names != nil { - newParams = append(newParams, &ast.Field{ - Names: names, - Type: field.Type, - }) - } - } - if calleeDecl.Recv != nil { - filterParams(calleeDecl.Recv.List[0]) - calleeDecl.Recv = nil - } - for _, field := range calleeDecl.Type.Params.List { - filterParams(field) - } - calleeDecl.Type.Params.List = newParams - } - - // Construct a "binding decl" that implements parameter assignment. - // - // If we succeed, the declaration may be used by reduction - // strategies to relax the requirement that all parameters - // have been substituted. - // - // For example, a call: - // f(a0, a1, a2) - // where: - // func f(p0, p1 T0, p2 T1) { body } - // reduces to: - // { - // var ( - // p0, p1 T0 = a0, a1 - // p2 T1 = a2 - // ) - // body - // } - // - // so long as p0, p1 ∉ freevars(T1) or freevars(a2), and so on, - // because each spec is statically resolved in sequence and - // dynamically assigned in sequence. By contrast, all - // parameters are resolved simultaneously and assigned - // simultaneously. - // - // The pX names should already be blank ("_") if the parameter - // is unreferenced; this avoids "unreferenced local var" checks. - // - // Strategies may impose additional checks on return - // conversions, labels, defer, etc. - bindingDeclStmt := func() ast.Stmt { - // Spread calls are tricky as they may not align with the - // parameters' field groupings nor types. - // For example, given - // func g() (int, string) - // the call - // f(g()) - // is legal with these decls of f: - // func f(int, string) - // func f(x, y any) - // func f(x, y ...any) - // TODO(adonovan): support binding decls for spread calls by - // splitting parameter groupings as needed. - if lastArg := last(args); lastArg != nil && lastArg.spread { - logf("binding decls not yet supported for spread calls") - return nil - } - - var ( - values = remainingArgs - specs []ast.Spec - shadowed = make(map[string]bool) // names defined by previous specs - ) - for _, field := range calleeDecl.Type.Params.List { - // Each field (param group) becomes a ValueSpec. - spec := &ast.ValueSpec{ - Names: field.Names, - Type: field.Type, - Values: values[:len(field.Names)], - } - values = values[len(field.Names):] - - // Compute union of free names of type and values - // and detect shadowing. Values is the arguments - // (caller syntax), so we can use type info. - // But Type is the untyped callee syntax, - // so we have to use a syntax-only algorithm. - free := make(map[string]bool) - for _, value := range spec.Values { - for name := range freeVars(caller.Info, value) { - free[name] = true - } - } - freeishNames(free, field.Type) - for name := range free { - if shadowed[name] { - logf("binding decl would shadow free name %q", name) - return nil - } - } - for _, id := range spec.Names { - if id.Name != "_" { - shadowed[id.Name] = true - } - } - - specs = append(specs, spec) - } - decl := &ast.DeclStmt{ - Decl: &ast.GenDecl{ - Tok: token.VAR, - Specs: specs, - }, - } - logf("binding decl: %s", debugFormatNode(caller.Fset, decl)) - return decl - }() - // -- let the inlining strategies begin -- // // When we commit to a strategy, we log a message of the form: @@ -955,7 +540,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu logf("strategy: reduce call to empty body") // Evaluate the arguments for effects and delete the call entirely. - stmt := callStmt(callerPath) // cannot fail + stmt := callStmt(caller.path) // cannot fail res.old = stmt if nargs := len(remainingArgs); nargs > 0 { // Emit "_, _ = args" to discard results. @@ -1013,7 +598,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu if callee.BodyIsReturnExpr { results := calleeDecl.Body.List[0].(*ast.ReturnStmt).Results - context := callContext(callerPath) + context := callContext(caller.path) // statement context if stmt, ok := context.(*ast.ExprStmt); ok && @@ -1148,11 +733,11 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // TODO(adonovan): add a strategy for a 'void tail // call', i.e. a call statement prior to an (explicit // or implicit) return. - if ret, ok := callContext(callerPath).(*ast.ReturnStmt); ok && + if ret, ok := callContext(caller.path).(*ast.ReturnStmt); ok && len(ret.Results) == 1 && callee.TrivialReturns == callee.TotalReturns && (allParamsSubstituted && noResultEscapes || bindingDeclStmt != nil) && - !hasLabelConflict(callerPath, callee.Labels) && + !hasLabelConflict(caller.path, callee.Labels) && forall(callee.Results, func(i int, p *paramInfo) bool { // all result vars are unreferenced return len(p.Refs) == 0 @@ -1185,10 +770,10 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // (by substitution, or a binding decl), // // If there is only a single statement, the braces are omitted. - if stmt := callStmt(callerPath); stmt != nil && + if stmt := callStmt(caller.path); stmt != nil && (allParamsSubstituted && noResultEscapes || bindingDeclStmt != nil) && !callee.HasDefer && - !hasLabelConflict(callerPath, callee.Labels) && + !hasLabelConflict(caller.path, callee.Labels) && callee.TotalReturns == 0 { logf("strategy: reduce stmt-context call to { stmts }") body := calleeDecl.Body @@ -1253,6 +838,464 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu return res, nil } +type argument struct { + expr ast.Expr + typ types.Type // may be tuple for sole non-receiver arg in spread call + spread bool // final arg is call() assigned to multiple params + pure bool // expr is pure (doesn't read variables) + effects bool // expr has effects (updates variables) + duplicable bool // expr may be duplicated + freevars map[string]bool // free names of expr +} + +// arguments returns the effective arguments of the call. +// +// If the receiver argument and parameter have +// different pointerness, make the "&" or "*" explicit. +// +// Also, if x.f() is shorthand for promoted method x.y.f(), +// make the .y explicit in T.f(x.y, ...). +// +// Beware that: +// +// - a method can only be called through a selection, but only +// the first of these two forms needs special treatment: +// +// expr.f(args) -> ([&*]expr, args) MethodVal +// T.f(recv, args) -> ( expr, args) MethodExpr +// +// - the presence of a value in receiver-position in the call +// is a property of the caller, not the callee. A method +// (calleeDecl.Recv != nil) may be called like an ordinary +// function. +// +// - the types.Signatures seen by the caller (from +// StaticCallee) and by the callee (from decl type) +// differ in this case. +// +// In a spread call f(g()), the sole ordinary argument g(), +// always last in args, has a tuple type. +// +// We compute type-based predicates like pure, duplicable, +// freevars, etc, now, before we start modifying syntax. +func arguments(caller *Caller, calleeDecl *ast.FuncDecl, assign1 func(*types.Var) bool) ([]*argument, error) { + var args []*argument + + callArgs := caller.Call.Args + if calleeDecl.Recv != nil { + sel := astutil.Unparen(caller.Call.Fun).(*ast.SelectorExpr) + seln := caller.Info.Selections[sel] + var recvArg ast.Expr + switch seln.Kind() { + case types.MethodVal: // recv.f(callArgs) + recvArg = sel.X + case types.MethodExpr: // T.f(recv, callArgs) + recvArg = callArgs[0] + callArgs = callArgs[1:] + } + if recvArg != nil { + // Compute all the type-based predicates now, + // before we start meddling with the syntax; + // the meddling will update them. + arg := &argument{ + expr: recvArg, + typ: caller.Info.TypeOf(recvArg), + pure: pure(caller.Info, assign1, recvArg), + effects: effects(caller.Info, recvArg), + duplicable: duplicable(caller.Info, recvArg), + freevars: freeVars(caller.Info, recvArg), + } + recvArg = nil // prevent accidental use + + // Move receiver argument recv.f(args) to argument list f(&recv, args). + args = append(args, arg) + + // Make field selections explicit (recv.f -> recv.y.f), + // updating arg.{expr,typ}. + indices := seln.Index() + for _, index := range indices[:len(indices)-1] { + t := deref(arg.typ) + fld := typeparams.CoreType(t).(*types.Struct).Field(index) + if fld.Pkg() != caller.Types && !fld.Exported() { + return nil, fmt.Errorf("in %s, implicit reference to unexported field .%s cannot be made explicit", + debugFormatNode(caller.Fset, caller.Call.Fun), + fld.Name()) + } + arg.expr = &ast.SelectorExpr{ + X: arg.expr, + Sel: makeIdent(fld.Name()), + } + arg.typ = fld.Type() + } + + // Make * or & explicit. + argIsPtr := arg.typ != deref(arg.typ) + paramIsPtr := is[*types.Pointer](seln.Obj().Type().(*types.Signature).Recv().Type()) + if !argIsPtr && paramIsPtr { + // &recv + arg.expr = &ast.UnaryExpr{Op: token.AND, X: arg.expr} + arg.typ = types.NewPointer(arg.typ) + } else if argIsPtr && !paramIsPtr { + // *recv + arg.expr = &ast.StarExpr{X: arg.expr} + arg.typ = deref(arg.typ) + + // Technically *recv is non-pure and + // non-duplicable, as side effects + // could change the pointer between + // multiple reads. But unfortunately + // this really degrades many of our tests. + // (The indices loop above should similarly + // update these flags when traversing pointers.) + // + // TODO(adonovan): improve the precision of + // purity and duplicability. + // For example, *new(T) is actually pure. + // And *ptr, where ptr doesn't escape and + // has no assignments other than its decl, + // is also pure; this is very common. + // + // arg.pure = false + } + } + } + for _, expr := range callArgs { + typ := caller.Info.TypeOf(expr) + args = append(args, &argument{ + expr: expr, + typ: typ, + spread: is[*types.Tuple](typ), // => last + pure: pure(caller.Info, assign1, expr), + effects: effects(caller.Info, expr), + duplicable: duplicable(caller.Info, expr), + freevars: freeVars(caller.Info, expr), + }) + } + return args, nil +} + +type parameter struct { + obj *types.Var // parameter var from caller's signature + info *paramInfo // information from AnalyzeCallee + variadic bool // (final) parameter is unsimplified ...T +} + +// substitute implements parameter elimination by substitution. +// +// It considers each parameter and its corresponding argument in turn +// and evaluate these conditions: +// +// - the parameter is neither address-taken nor assigned; +// - the argument is pure; +// - if the parameter refcount is zero, the argument must +// not contain the last use of a local var; +// - if the parameter refcount is > 1, the argument must be duplicable; +// - the argument (or types.Default(argument) if it's untyped) has +// the same type as the parameter. +// +// If all conditions are met then the parameter can be substituted and +// each reference to it replaced by the argument. In that case, the +// replaceCalleeID function is called for each reference to the +// parameter, and is provided with its relative offset and replacement +// expression (argument), and the corresponding elements of params and +// args are replaced by nil. +func substitute(logf func(string, ...any), caller *Caller, params []*parameter, args []*argument, replaceCalleeID func(offset int, repl ast.Expr)) { + // Inv: + // in calls to variadic, len(args) >= len(params)-1 + // in spread calls to non-variadic, len(args) < len(params) + // in spread calls to variadic, len(args) <= len(params) + // (In spread calls len(args) = 1, or 2 if call has receiver.) + // Non-spread variadics have been simplified away already, + // so the args[i] lookup is safe if we stop after the spread arg. +next: + for i, param := range params { + arg := args[i] + // Check argument against parameter. + // + // Beware: don't use types.Info on arg since + // the syntax may be synthetic (not created by parser) + // and thus lacking positions and types; + // do it earlier (see pure/duplicable/freevars). + + if arg.spread { + logf("keeping param %q and following ones: argument %s is spread", + param.info.Name, debugFormatNode(caller.Fset, arg.expr)) + break // spread => last argument, but not always last parameter + } + assert(!param.variadic, "unsimplified variadic parameter") + if param.info.Escapes { + logf("keeping param %q: escapes from callee", param.info.Name) + continue + } + if param.info.Assigned { + logf("keeping param %q: assigned by callee", param.info.Name) + continue // callee needs the parameter variable + } + if !arg.pure || arg.effects { + // TODO(adonovan): conduct a deeper analysis of callee effects + // to relax this constraint. + logf("keeping param %q: argument %s is impure", + param.info.Name, debugFormatNode(caller.Fset, arg.expr)) + continue // unsafe to change order or cardinality of effects + } + if len(param.info.Refs) > 1 && !arg.duplicable { + logf("keeping param %q: argument is not duplicable", param.info.Name) + continue // incorrect or poor style to duplicate an expression + } + if len(param.info.Refs) == 0 { + // Eliminating an unreferenced parameter might + // remove the last reference to a caller local var. + for free := range arg.freevars { + if v, ok := caller.lookup(free).(*types.Var); ok { + // TODO(adonovan): be more precise and check + // that v is defined within the body of the caller + // function (if any) and is indeed referenced + // only by the call. (See assign1 for analysis + // of enclosing func.) + logf("keeping param %q: arg contains perhaps the last reference to possible caller local %v @ %v", + param.info.Name, v, caller.Fset.PositionFor(v.Pos(), false)) + continue next + } + } + } + + // Check that eliminating the parameter wouldn't materially + // change the type. + // + // (We don't simply wrap the argument in an explicit conversion + // to the parameter type because that could increase allocation + // in the number of (e.g.) string -> any conversions. + // Even when Uses = 1, the sole ref might be in a loop or lambda that + // is multiply executed.) + if len(param.info.Refs) > 0 && !trivialConversion(args[i].typ, params[i].obj) { + logf("keeping param %q: argument passing converts %s to type %s", + param.info.Name, args[i].typ, params[i].obj.Type()) + continue // implicit conversion is significant + } + + // Check for shadowing. + // + // Consider inlining a call f(z, 1) to + // func f(x, y int) int { z := y; return x + y + z }: + // we can't replace x in the body by z (or any + // expression that has z as a free identifier) + // because there's an intervening declaration of z + // that would shadow the caller's one. + for free := range arg.freevars { + if param.info.Shadow[free] { + logf("keeping param %q: cannot replace with argument as it has free ref to %s that is shadowed", param.info.Name, free) + continue next // shadowing conflict + } + } + + // It is safe to eliminate param and replace it with arg. + // No additional parens are required around arg for + // the supported "pure" expressions. + // + // Because arg.expr belongs to the caller, + // we clone it before splicing it into the callee tree. + logf("replacing parameter %q by argument %q", + param.info.Name, debugFormatNode(caller.Fset, arg.expr)) + for _, ref := range param.info.Refs { + replaceCalleeID(ref, cloneNode(arg.expr).(ast.Expr)) + } + params[i] = nil // substituted + args[i] = nil // substituted + } +} + +// updateCalleeParams updates the calleeDecl syntax to remove +// substituted parameters and move the receiver (if any) to the head +// of the ordinary parameters. +func updateCalleeParams(calleeDecl *ast.FuncDecl, params []*parameter) { + // The logic is fiddly because of the three forms of ast.Field: + // + // func(int), func(x int), func(x, y int) + // + // Also, ensure that all remaining parameters are named + // to avoid a mix of named/unnamed when joining (recv, params...). + // func (T) f(int, bool) -> (_ T, _ int, _ bool) + // (Strictly, we need do this only for methods and only when + // the namednesses of Recv and Params differ; that might be tidier.) + + paramIdx := 0 // index in original parameter list (incl. receiver) + var newParams []*ast.Field + filterParams := func(field *ast.Field) { + var names []*ast.Ident + if field.Names == nil { + // Unnamed parameter field (e.g. func f(int) + if params[paramIdx] != nil { + // Give it an explicit name "_" since we will + // make the receiver (if any) a regular parameter + // and one cannot mix named and unnamed parameters. + names = append(names, makeIdent("_")) + } + paramIdx++ + } else { + // Named parameter field e.g. func f(x, y int) + // Remove substituted parameters in place. + // If all were substituted, delete field. + for _, id := range field.Names { + if pinfo := params[paramIdx]; pinfo != nil { + // Rename unreferenced parameters with "_". + // This is crucial for binding decls, since + // unlike parameters, they are subject to + // "unreferenced var" checks. + if len(pinfo.info.Refs) == 0 { + id = makeIdent("_") + } + names = append(names, id) + } + paramIdx++ + } + } + if names != nil { + newParams = append(newParams, &ast.Field{ + Names: names, + Type: field.Type, + }) + } + } + if calleeDecl.Recv != nil { + filterParams(calleeDecl.Recv.List[0]) + calleeDecl.Recv = nil + } + for _, field := range calleeDecl.Type.Params.List { + filterParams(field) + } + calleeDecl.Type.Params.List = newParams +} + +// createBindingDecl constructs a "binding decl" that implements +// parameter assignment. +// +// If we succeed, the declaration may be used by reduction +// strategies to relax the requirement that all parameters +// have been substituted. +// +// For example, a call: +// +// f(a0, a1, a2) +// +// where: +// +// func f(p0, p1 T0, p2 T1) { body } +// +// reduces to: +// +// { +// var ( +// p0, p1 T0 = a0, a1 +// p2 T1 = a2 +// ) +// body +// } +// +// so long as p0, p1 ∉ freevars(T1) or freevars(a2), and so on, +// because each spec is statically resolved in sequence and +// dynamically assigned in sequence. By contrast, all +// parameters are resolved simultaneously and assigned +// simultaneously. +// +// The pX names should already be blank ("_") if the parameter +// is unreferenced; this avoids "unreferenced local var" checks. +// +// Strategies may impose additional checks on return +// conversions, labels, defer, etc. +func createBindingDecl(logf func(string, ...any), caller *Caller, args []*argument, calleeDecl *ast.FuncDecl) ast.Stmt { + // Spread calls are tricky as they may not align with the + // parameters' field groupings nor types. + // For example, given + // func g() (int, string) + // the call + // f(g()) + // is legal with these decls of f: + // func f(int, string) + // func f(x, y any) + // func f(x, y ...any) + // TODO(adonovan): support binding decls for spread calls by + // splitting parameter groupings as needed. + if lastArg := last(args); lastArg != nil && lastArg.spread { + logf("binding decls not yet supported for spread calls") + return nil + } + + // Compute remaining argument expressions. + var values []ast.Expr + for _, arg := range args { + if arg != nil { + values = append(values, arg.expr) + } + } + + var ( + specs []ast.Spec + shadowed = make(map[string]bool) // names defined by previous specs + ) + for _, field := range calleeDecl.Type.Params.List { + // Each field (param group) becomes a ValueSpec. + spec := &ast.ValueSpec{ + Names: field.Names, + Type: field.Type, + Values: values[:len(field.Names)], + } + values = values[len(field.Names):] + + // Compute union of free names of type and values + // and detect shadowing. Values is the arguments + // (caller syntax), so we can use type info. + // But Type is the untyped callee syntax, + // so we have to use a syntax-only algorithm. + free := make(map[string]bool) + for _, value := range spec.Values { + for name := range freeVars(caller.Info, value) { + free[name] = true + } + } + freeishNames(free, field.Type) + for name := range free { + if shadowed[name] { + logf("binding decl would shadow free name %q", name) + return nil + } + } + for _, id := range spec.Names { + if id.Name != "_" { + shadowed[id.Name] = true + } + } + + specs = append(specs, spec) + } + assert(len(values) == 0, "args/params mismatch") + decl := &ast.DeclStmt{ + Decl: &ast.GenDecl{ + Tok: token.VAR, + Specs: specs, + }, + } + logf("binding decl: %s", debugFormatNode(caller.Fset, decl)) + return decl +} + +// lookup does a symbol lookup in the lexical environment of the caller. +func (caller *Caller) lookup(name string) types.Object { + pos := caller.Call.Pos() + for _, n := range caller.path { + // The function body scope (containing not just params) + // is associated with FuncDecl.Type, not FuncDecl.Body. + if decl, ok := n.(*ast.FuncDecl); ok { + n = decl.Type + } + if scope := caller.Info.Scopes[n]; scope != nil { + if _, obj := scope.LookupParent(name, pos); obj != nil { + return obj + } + } + } + return nil +} + // -- predicates over expressions -- // freeVars returns the names of all free identifiers of e: From ebe11dfa27514d8075233a8d068676010ef2cb88 Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Mon, 7 Aug 2023 17:17:52 -0400 Subject: [PATCH 132/178] gopls/internal/regtest/bench: add additional completion benchmarks Our completion benchmarks were too narrowly focused on type checking speed, and did not provide adequate coverage of unimported completion or completion budget. Run the CompletionFollowingEdit benchmarks with unimported completion enabled, and with a budget of 100ms. The former still seems to have little impact on performance, likely due to my environment, whereas the latter demonstrates a stark regression: with gopls@v0.11, completion timing is bang-on 100ms, whereas with gopls@tip timing is largely unaffected by the budget. Also add a completion test in identifier context, as this exercises different code paths, and further highlights the regression in budget implementation. For golang/go#62665 Change-Id: I8b771225c7e2979ae1eb9e67e79b9b6165e2712b Reviewed-on: https://go-review.googlesource.com/c/tools/+/530015 Reviewed-by: Alan Donovan LUCI-TryBot-Result: Go LUCI --- .../internal/regtest/bench/completion_test.go | 187 +++++++++++------- 1 file changed, 114 insertions(+), 73 deletions(-) diff --git a/gopls/internal/regtest/bench/completion_test.go b/gopls/internal/regtest/bench/completion_test.go index 0400e70b0bd..02e640423b9 100644 --- a/gopls/internal/regtest/bench/completion_test.go +++ b/gopls/internal/regtest/bench/completion_test.go @@ -5,6 +5,7 @@ package bench import ( + "flag" "fmt" "sync/atomic" "testing" @@ -145,105 +146,145 @@ func (c *completer) _() { }, b) } -// Benchmark completion following an arbitrary edit. -// -// Edits force type-checked packages to be invalidated, so we want to measure -// how long it takes before completion results are available. -func BenchmarkCompletionFollowingEdit(b *testing.B) { - tests := []struct { - repo string - file string // repo-relative file to create - content string // file content - locationRegexp string // regexp for completion - }{ - { - "tools", - "internal/lsp/source/completion/completion2.go", - ` +type completionFollowingEditTest struct { + repo string + name string + file string // repo-relative file to create + content string // file content + locationRegexp string // regexp for completion +} + +var completionFollowingEditTests = []completionFollowingEditTest{ + { + "tools", + "selector", + "internal/lsp/source/completion/completion2.go", + ` package completion func (c *completer) _() { c.inference.kindMatches(c.) } `, - `func \(c \*completer\) _\(\) {\n\tc\.inference\.kindMatches\((c)`, - }, - { - "kubernetes", - "pkg/kubelet/kubelet2.go", - ` + `func \(c \*completer\) _\(\) {\n\tc\.inference\.kindMatches\((c)`, + }, + { + "kubernetes", + "selector", + "pkg/kubelet/kubelet2.go", + ` package kubelet func (kl *Kubelet) _() { kl. } `, - `kl\.()`, - }, - { - "oracle", - "dataintegration/pivot2.go", - ` + `kl\.()`, + }, + { + "kubernetes", + "identifier", + "pkg/kubelet/kubelet2.go", + ` +package kubelet + +func (kl *Kubelet) _() { + k // here +} +`, + `k() // here`, + }, + { + "oracle", + "selector", + "dataintegration/pivot2.go", + ` package dataintegration func (p *Pivot) _() { p. } `, - `p\.()`, - }, - } + `p\.()`, + }, +} - for _, test := range tests { - b.Run(test.repo, func(b *testing.B) { - repo := getRepo(b, test.repo) - sharedEnv := repo.sharedEnv(b) // ensure cache is warm - env := repo.newEnv(b, fake.EditorConfig{ - Env: map[string]string{ - "GOPATH": sharedEnv.Sandbox.GOPATH(), // use the warm cache - }, - Settings: map[string]interface{}{ - "completeUnimported": false, - }, - }, "completionFollowingEdit", false) - defer env.Close() - - env.CreateBuffer(test.file, "// __REGTEST_PLACEHOLDER_0__\n"+test.content) - editPlaceholder := func() { - edits := atomic.AddInt64(&editID, 1) - env.EditBuffer(test.file, protocol.TextEdit{ - Range: protocol.Range{ - Start: protocol.Position{Line: 0, Character: 0}, - End: protocol.Position{Line: 1, Character: 0}, - }, - // Increment the placeholder text, to ensure cache misses. - NewText: fmt.Sprintf("// __REGTEST_PLACEHOLDER_%d__\n", edits), +// Benchmark completion following an arbitrary edit. +// +// Edits force type-checked packages to be invalidated, so we want to measure +// how long it takes before completion results are available. +func BenchmarkCompletionFollowingEdit(b *testing.B) { + for _, test := range completionFollowingEditTests { + b.Run(fmt.Sprintf("%s_%s", test.repo, test.name), func(b *testing.B) { + for _, completeUnimported := range []bool{true, false} { + b.Run(fmt.Sprintf("completeUnimported=%v", completeUnimported), func(b *testing.B) { + for _, budget := range []string{"0s", "100ms"} { + b.Run(fmt.Sprintf("budget=%s", budget), func(b *testing.B) { + runCompletionFollowingEdit(b, test, completeUnimported, budget) + }) + } }) } - env.AfterChange() + }) + } +} - // Run a completion to make sure the system is warm. - loc := env.RegexpSearch(test.file, test.locationRegexp) - completions := env.Completion(loc) +var gomodcache = flag.String("gomodcache", "", "optional GOMODCACHE for unimported completion benchmarks") - if testing.Verbose() { - fmt.Println("Results:") - for i := 0; i < len(completions.Items); i++ { - fmt.Printf("\t%d. %v\n", i, completions.Items[i]) - } - } +func runCompletionFollowingEdit(b *testing.B, test completionFollowingEditTest, completeUnimported bool, budget string) { + repo := getRepo(b, test.repo) + sharedEnv := repo.sharedEnv(b) // ensure cache is warm + envvars := map[string]string{ + "GOPATH": sharedEnv.Sandbox.GOPATH(), // use the warm cache + } - b.ResetTimer() + if *gomodcache != "" { + envvars["GOMODCACHE"] = *gomodcache + } - if stopAndRecord := startProfileIfSupported(b, env, qualifiedName(test.repo, "completionFollowingEdit")); stopAndRecord != nil { - defer stopAndRecord() - } + env := repo.newEnv(b, fake.EditorConfig{ + Env: envvars, + Settings: map[string]interface{}{ + "completeUnimported": completeUnimported, + "completionBudget": budget, + }, + }, "completionFollowingEdit", false) + defer env.Close() - for i := 0; i < b.N; i++ { - editPlaceholder() - loc := env.RegexpSearch(test.file, test.locationRegexp) - env.Completion(loc) - } + env.CreateBuffer(test.file, "// __REGTEST_PLACEHOLDER_0__\n"+test.content) + editPlaceholder := func() { + edits := atomic.AddInt64(&editID, 1) + env.EditBuffer(test.file, protocol.TextEdit{ + Range: protocol.Range{ + Start: protocol.Position{Line: 0, Character: 0}, + End: protocol.Position{Line: 1, Character: 0}, + }, + // Increment the placeholder text, to ensure cache misses. + NewText: fmt.Sprintf("// __REGTEST_PLACEHOLDER_%d__\n", edits), }) } + env.AfterChange() + + // Run a completion to make sure the system is warm. + loc := env.RegexpSearch(test.file, test.locationRegexp) + completions := env.Completion(loc) + + if testing.Verbose() { + fmt.Println("Results:") + for i, item := range completions.Items { + fmt.Printf("\t%d. %v\n", i, item) + } + } + + b.ResetTimer() + + if stopAndRecord := startProfileIfSupported(b, env, qualifiedName(test.repo, "completionFollowingEdit")); stopAndRecord != nil { + defer stopAndRecord() + } + + for i := 0; i < b.N; i++ { + editPlaceholder() + loc := env.RegexpSearch(test.file, test.locationRegexp) + env.Completion(loc) + } } From 455b761848a577d756910ec24524303a331c746d Mon Sep 17 00:00:00 2001 From: "Hana (Hyang-Ah) Kim" Date: Fri, 22 Sep 2023 15:08:42 -0400 Subject: [PATCH 133/178] gopls/internal/lsp: use linkifyShowMessage in telemetryOnMessage Linkify the links in the followup prompts. Change-Id: Idc57f3e1b1ab61166638037c8b984b43b33063da Reviewed-on: https://go-review.googlesource.com/c/tools/+/530460 Run-TryBot: Hyang-Ah Hana Kim Reviewed-by: Peter Weinberger Commit-Queue: Hyang-Ah Hana Kim Reviewed-by: Robert Findley Auto-Submit: Hyang-Ah Hana Kim TryBot-Result: Gopher Robot --- gopls/internal/lsp/prompt.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/gopls/internal/lsp/prompt.go b/gopls/internal/lsp/prompt.go index c3cc8f91645..1e63337d391 100644 --- a/gopls/internal/lsp/prompt.go +++ b/gopls/internal/lsp/prompt.go @@ -237,7 +237,7 @@ Would you like to enable Go telemetry? case TelemetryYes: result = pYes if err := s.setTelemetryMode("on"); err == nil { - message(protocol.Info, telemetryOnMessage()) + message(protocol.Info, telemetryOnMessage(s.Options().LinkifyShowMessage)) } else { errorf("enabling telemetry failed: %v", err) msg := fmt.Sprintf("Failed to enable Go telemetry: %v\nTo enable telemetry manually, please run `go run golang.org/x/telemetry/cmd/gotelemetry@latest on`", err) @@ -257,15 +257,25 @@ Would you like to enable Go telemetry? } } -func telemetryOnMessage() string { +func telemetryOnMessage(linkify bool) string { reportDate := time.Now().AddDate(0, 0, 7).Format("2006-01-02") - return fmt.Sprintf(`Telemetry uploading is now enabled and may be sent to https://telemetry.go.dev/ starting %s. Uploaded data is used to help improve the Go toolchain and related tools, and it will be published as part of a public dataset. + format := `Telemetry uploading is now enabled and may be sent to https://telemetry.go.dev starting %s. Uploaded data is used to help improve the Go toolchain and related tools, and it will be published as part of a public dataset. For more details, see https://telemetry.go.dev/privacy. This data is collected in accordance with the Google Privacy Policy (https://policies.google.com/privacy). To disable telemetry uploading, run %s. -`, reportDate, "`go run golang.org/x/telemetry/cmd/gotelemetry@latest off`") +` + if linkify { + format = `Telemetry uploading is now enabled and may be sent to [telemetry.go.dev](https://telemetry.go.dev) starting %s. Uploaded data is used to help improve the Go toolchain and related tools, and it will be published as part of a public dataset. + +For more details, see [telemetry.go.dev/privacy](https://telemetry.go.dev/privacy). +This data is collected in accordance with the [Google Privacy Policy](https://policies.google.com/privacy). + +To disable telemetry uploading, run [%s](https://golang.org/x/telemetry/cmd/gotelemetry). +` + } + return fmt.Sprintf(format, reportDate, "`go run golang.org/x/telemetry/cmd/gotelemetry@latest off`") } // acquireLockFile attempts to "acquire a lock" for writing to path. From 903e689025c136c48fa5d0b92d38dde5a40a35c2 Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Wed, 20 Sep 2023 14:20:53 -0400 Subject: [PATCH 134/178] gopls/internal/lsp/source/completion: start timing before type checking Partially revert CL 503016, to go back to starting the completion timer before type checking. As discussed in slack and golang/go#62665, even if this leads to inconsistent behavior across various LSP clients (due to order of requests) we should still do our best to interpret the completion budget from the user perspective. For golang/go#62665 Change-Id: I2035e2ecb7776cead7a19bd37b9df512fdbf3d17 Reviewed-on: https://go-review.googlesource.com/c/tools/+/530016 LUCI-TryBot-Result: Go LUCI Reviewed-by: Alan Donovan --- .../lsp/source/completion/completion.go | 23 ++++++++++++++----- .../lsp/source/completion/deep_completion.go | 4 ++-- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/gopls/internal/lsp/source/completion/completion.go b/gopls/internal/lsp/source/completion/completion.go index e0a221f6017..1ea7718c9f7 100644 --- a/gopls/internal/lsp/source/completion/completion.go +++ b/gopls/internal/lsp/source/completion/completion.go @@ -232,6 +232,15 @@ type completer struct { // mapper converts the positions in the file from which the completion originated. mapper *protocol.Mapper + // startTime is when we started processing this completion request. It does + // not include any time the request spent in the queue. + // + // Note: in CL 503016, startTime move to *after* type checking, but it was + // subsequently determined that it was better to keep setting it *before* + // type checking, so that the completion budget best approximates the user + // experience. See golang/go#62665 for more details. + startTime time.Time + // scopes contains all scopes defined by nodes in our path, // including nil values for nodes that don't defined a scope. It // also includes our package scope and the universal scope at the @@ -437,6 +446,8 @@ func Completion(ctx context.Context, snapshot source.Snapshot, fh source.FileHan ctx, done := event.Start(ctx, "completion.Completion") defer done() + startTime := time.Now() + pkg, pgf, err := source.NarrowestPackageForFile(ctx, snapshot, fh.URI()) if err != nil || pgf.File.Package == token.NoPos { // If we can't parse this file or find position for the package @@ -451,6 +462,7 @@ func Completion(ctx context.Context, snapshot source.Snapshot, fh source.FileHan } return items, surrounding, nil } + pos, err := pgf.PositionPos(protoPos) if err != nil { return nil, nil, err @@ -545,10 +557,12 @@ func Completion(ctx context.Context, snapshot source.Snapshot, fh source.FileHan matcher: prefixMatcher(""), methodSetCache: make(map[methodSetKey]*types.MethodSet), mapper: pgf.Mapper, + startTime: startTime, scopes: scopes, } ctx, cancel := context.WithCancel(ctx) + defer cancel() // Compute the deadline for this operation. Deadline is relative to the // search operation, not the entire completion RPC, as the work up until this @@ -562,15 +576,12 @@ func Completion(ctx context.Context, snapshot source.Snapshot, fh source.FileHan // Don't overload the context with this deadline, as we don't want to // conflate user cancellation (=fail the operation) with our time limit // (=stop searching and succeed with partial results). - start := time.Now() var deadline *time.Time if c.opts.budget > 0 { - d := start.Add(c.opts.budget) + d := startTime.Add(c.opts.budget) deadline = &d } - defer cancel() - if surrounding := c.containingIdent(pgf.Src); surrounding != nil { c.setSurrounding(surrounding) } @@ -583,7 +594,7 @@ func Completion(ctx context.Context, snapshot source.Snapshot, fh source.FileHan } // Deep search collected candidates and their members for more candidates. - c.deepSearch(ctx, start, deadline) + c.deepSearch(ctx, deadline) for _, callback := range c.completionCallbacks { if err := c.snapshot.RunProcessEnvFunc(ctx, callback); err != nil { @@ -593,7 +604,7 @@ func Completion(ctx context.Context, snapshot source.Snapshot, fh source.FileHan // Search candidates populated by expensive operations like // unimportedMembers etc. for more completion items. - c.deepSearch(ctx, start, deadline) + c.deepSearch(ctx, deadline) // Statement candidates offer an entire statement in certain contexts, as // opposed to a single object. Add statement candidates last because they diff --git a/gopls/internal/lsp/source/completion/deep_completion.go b/gopls/internal/lsp/source/completion/deep_completion.go index 66309530e73..e9e31c9209b 100644 --- a/gopls/internal/lsp/source/completion/deep_completion.go +++ b/gopls/internal/lsp/source/completion/deep_completion.go @@ -113,7 +113,7 @@ func (s *deepCompletionState) newPath(cand candidate, obj types.Object) []types. // deepSearch searches a candidate and its subordinate objects for completion // items if deep completion is enabled and adds the valid candidates to // completion items. -func (c *completer) deepSearch(ctx context.Context, start time.Time, deadline *time.Time) { +func (c *completer) deepSearch(ctx context.Context, deadline *time.Time) { defer func() { // We can return early before completing the search, so be sure to // clear out our queues to not impact any further invocations. @@ -172,7 +172,7 @@ func (c *completer) deepSearch(ctx context.Context, start time.Time, deadline *t c.deepState.candidateCount++ if c.opts.budget > 0 && c.deepState.candidateCount%100 == 0 { - spent := float64(time.Since(start)) / float64(c.opts.budget) + spent := float64(time.Since(c.startTime)) / float64(c.opts.budget) select { case <-ctx.Done(): return From 6ccb09c054185e349ed607260dade4508463256f Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Wed, 20 Sep 2023 14:23:39 -0400 Subject: [PATCH 135/178] gopls/internal/lsp/source/completion: fixes for completion budget Address a few considerations that were overlooked in CL 503016: - don't run goimports callbacks if the budget has expired - don't use a min depth for the second call to deepSearch For golang/go#62665 Change-Id: I23e2a3154049eabaff14c9c5071e7eea26e7c619 Reviewed-on: https://go-review.googlesource.com/c/tools/+/530017 LUCI-TryBot-Result: Go LUCI Reviewed-by: Alan Donovan --- .../lsp/source/completion/completion.go | 10 +++-- .../lsp/source/completion/deep_completion.go | 42 ++++++++++++------- 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/gopls/internal/lsp/source/completion/completion.go b/gopls/internal/lsp/source/completion/completion.go index 1ea7718c9f7..594f234f4e5 100644 --- a/gopls/internal/lsp/source/completion/completion.go +++ b/gopls/internal/lsp/source/completion/completion.go @@ -594,17 +594,19 @@ func Completion(ctx context.Context, snapshot source.Snapshot, fh source.FileHan } // Deep search collected candidates and their members for more candidates. - c.deepSearch(ctx, deadline) + c.deepSearch(ctx, 1, deadline) for _, callback := range c.completionCallbacks { - if err := c.snapshot.RunProcessEnvFunc(ctx, callback); err != nil { - return nil, nil, err + if deadline == nil || time.Now().Before(*deadline) { + if err := c.snapshot.RunProcessEnvFunc(ctx, callback); err != nil { + return nil, nil, err + } } } // Search candidates populated by expensive operations like // unimportedMembers etc. for more completion items. - c.deepSearch(ctx, deadline) + c.deepSearch(ctx, 0, deadline) // Statement candidates offer an entire statement in certain contexts, as // opposed to a single object. Add statement candidates last because they diff --git a/gopls/internal/lsp/source/completion/deep_completion.go b/gopls/internal/lsp/source/completion/deep_completion.go index e9e31c9209b..fac11bf4117 100644 --- a/gopls/internal/lsp/source/completion/deep_completion.go +++ b/gopls/internal/lsp/source/completion/deep_completion.go @@ -113,7 +113,7 @@ func (s *deepCompletionState) newPath(cand candidate, obj types.Object) []types. // deepSearch searches a candidate and its subordinate objects for completion // items if deep completion is enabled and adds the valid candidates to // completion items. -func (c *completer) deepSearch(ctx context.Context, deadline *time.Time) { +func (c *completer) deepSearch(ctx context.Context, minDepth int, deadline *time.Time) { defer func() { // We can return early before completing the search, so be sure to // clear out our queues to not impact any further invocations. @@ -121,9 +121,25 @@ func (c *completer) deepSearch(ctx context.Context, deadline *time.Time) { c.deepState.nextQueue = c.deepState.nextQueue[:0] }() - first := true // always fully process the first set of candidates - for len(c.deepState.nextQueue) > 0 && (first || deadline == nil || time.Now().Before(*deadline)) { - first = false + depth := 0 // current depth being processed + // Stop reports whether we should stop the search immediately. + stop := func() bool { + // Context cancellation indicates that the actual completion operation was + // cancelled, so ignore minDepth and deadline. + select { + case <-ctx.Done(): + return true + default: + } + // Otherwise, only stop if we've searched at least minDepth and reached the deadline. + return depth > minDepth && deadline != nil && time.Now().After(*deadline) + } + + for len(c.deepState.nextQueue) > 0 { + depth++ + if stop() { + return + } c.deepState.thisQueue, c.deepState.nextQueue = c.deepState.nextQueue, c.deepState.thisQueue[:0] outer: @@ -172,17 +188,15 @@ func (c *completer) deepSearch(ctx context.Context, deadline *time.Time) { c.deepState.candidateCount++ if c.opts.budget > 0 && c.deepState.candidateCount%100 == 0 { - spent := float64(time.Since(c.startTime)) / float64(c.opts.budget) - select { - case <-ctx.Done(): + if stop() { return - default: - // If we are almost out of budgeted time, no further elements - // should be added to the queue. This ensures remaining time is - // used for processing current queue. - if !c.deepState.queueClosed && spent >= 0.85 { - c.deepState.queueClosed = true - } + } + spent := float64(time.Since(c.startTime)) / float64(c.opts.budget) + // If we are almost out of budgeted time, no further elements + // should be added to the queue. This ensures remaining time is + // used for processing current queue. + if !c.deepState.queueClosed && spent >= 0.85 { + c.deepState.queueClosed = true } } From fb7463ac1de5f0c80ee4dfe30e992c37a98f2ef7 Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Mon, 25 Sep 2023 15:07:06 -0400 Subject: [PATCH 136/178] gopls: upgrade x/telemetry to latest Change-Id: I293c7f9621230bc4a607af1389421530ca7dd67a Reviewed-on: https://go-review.googlesource.com/c/tools/+/530976 LUCI-TryBot-Result: Go LUCI Reviewed-by: Peter Weinberger --- gopls/go.mod | 2 +- gopls/go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/gopls/go.mod b/gopls/go.mod index 99c5c137529..64cd7cf2387 100644 --- a/gopls/go.mod +++ b/gopls/go.mod @@ -10,7 +10,7 @@ require ( golang.org/x/mod v0.12.0 golang.org/x/sync v0.3.0 golang.org/x/sys v0.12.0 - golang.org/x/telemetry v0.0.0-20230919143304-dd3d43c296e4 + golang.org/x/telemetry v0.0.0-20230923135512-f45a5404d02c golang.org/x/text v0.13.0 golang.org/x/tools v0.13.1-0.20230920233436-f9b8da7b22be golang.org/x/vuln v1.0.1 diff --git a/gopls/go.sum b/gopls/go.sum index 39b838916cc..e9ea5d282a7 100644 --- a/gopls/go.sum +++ b/gopls/go.sum @@ -46,6 +46,8 @@ golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/telemetry v0.0.0-20230919143304-dd3d43c296e4 h1:8UKgM6RV6tVUWPOKlSEunJpIzYW3oy9aV//8cJz0UE4= golang.org/x/telemetry v0.0.0-20230919143304-dd3d43c296e4/go.mod h1:ppZ76JTkRgJC2GQEgtVY3fiuJR+N8FU2MAlp+gfN1E4= +golang.org/x/telemetry v0.0.0-20230923135512-f45a5404d02c h1:az7Rs3XV7P68bKMPT50p2X4su02nhHqtDOi9T8f3IEw= +golang.org/x/telemetry v0.0.0-20230923135512-f45a5404d02c/go.mod h1:ppZ76JTkRgJC2GQEgtVY3fiuJR+N8FU2MAlp+gfN1E4= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From d6f1bb71099c0dd23ec36bafbb5f3d2c4f52b528 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Mon, 25 Sep 2023 17:40:59 -0400 Subject: [PATCH 137/178] internal/refactor/inline: ignore line directives in testing The problems found by everything_test were in some cases spurious, caused by its failure to ignore line directives. Change-Id: I1501d9858e0c877f0488580c28aa7d636a004bd1 Reviewed-on: https://go-review.googlesource.com/c/tools/+/530439 LUCI-TryBot-Result: Go LUCI Reviewed-by: Robert Findley --- internal/refactor/inline/everything_test.go | 4 ++-- internal/refactor/inline/inline_test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/refactor/inline/everything_test.go b/internal/refactor/inline/everything_test.go index 7166c1baad6..b3a6769063c 100644 --- a/internal/refactor/inline/everything_test.go +++ b/internal/refactor/inline/everything_test.go @@ -100,7 +100,7 @@ func TestEverything(t *testing.T) { } // Prepare caller info. - callPosn := callerPkg.Fset.Position(call.Lparen) + callPosn := callerPkg.Fset.PositionFor(call.Lparen, false) callerContent, err := readFile(callPosn.Filename) if err != nil { t.Fatal(err) @@ -119,7 +119,7 @@ func TestEverything(t *testing.T) { if !ok { t.Fatalf("missing package for callee %v", fn) } - calleePosn := callerPkg.Fset.Position(fn.Pos()) + calleePosn := callerPkg.Fset.PositionFor(fn.Pos(), false) calleeDecl, err := findFuncByPosition(calleePkg, calleePosn) if err != nil { t.Fatal(err) diff --git a/internal/refactor/inline/inline_test.go b/internal/refactor/inline/inline_test.go index 3cad64920ca..176303c8d16 100644 --- a/internal/refactor/inline/inline_test.go +++ b/internal/refactor/inline/inline_test.go @@ -231,7 +231,7 @@ func doInlineNote(logf func(string, ...any), pkg *packages.Package, file *ast.Fi } } - calleeDecl, err := findFuncByPosition(calleePkg, caller.Fset.Position(fn.Pos())) + calleeDecl, err := findFuncByPosition(calleePkg, caller.Fset.PositionFor(fn.Pos(), false)) if err != nil { return err } @@ -297,7 +297,7 @@ func findFuncByPosition(pkg *packages.Package, posn token.Position) (*ast.FuncDe // them, so how are we supposed to find the source? Yuck! // Ugh. need to samefile? Nope $GOROOT just won't work // This is highly client specific anyway. - posn2 := pkg.Fset.Position(decl.Name.Pos()) + posn2 := pkg.Fset.PositionFor(decl.Name.Pos(), false) return posn.Filename == posn2.Filename && posn.Line == posn2.Line } From 313150aa08e21d106b8dccbc2ab42426381b6efa Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Mon, 25 Sep 2023 17:46:25 -0400 Subject: [PATCH 138/178] internal/refactor/inline: x++ counts as assignment in escape Somehow I overlooked this case. Change-Id: I95471bd2254ca557f069993fdb4f2ad2908ec78a Reviewed-on: https://go-review.googlesource.com/c/tools/+/529876 LUCI-TryBot-Result: Go LUCI Reviewed-by: Robert Findley --- internal/refactor/inline/escape.go | 5 ++++- internal/refactor/inline/inline_test.go | 11 +++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/internal/refactor/inline/escape.go b/internal/refactor/inline/escape.go index f88be27a86f..d05d2b927c0 100644 --- a/internal/refactor/inline/escape.go +++ b/internal/refactor/inline/escape.go @@ -57,7 +57,7 @@ func escape(info *types.Info, root ast.Node, f func(v *types.Var, escapes bool)) } } - // Search function body for operations &x, x.f(), and x = y + // Search function body for operations &x, x.f(), x++, and x = y // where x is a parameter. Each of these treats x as an address. ast.Inspect(root, func(n ast.Node) bool { switch n := n.(type) { @@ -88,6 +88,9 @@ func escape(info *types.Info, root ast.Node, f func(v *types.Var, escapes bool)) lvalue(lhs, false) } } + + case *ast.IncDecStmt: + lvalue(n.X, false) } return true }) diff --git a/internal/refactor/inline/inline_test.go b/internal/refactor/inline/inline_test.go index 176303c8d16..488faecea1d 100644 --- a/internal/refactor/inline/inline_test.go +++ b/internal/refactor/inline/inline_test.go @@ -430,6 +430,17 @@ func TestTable(t *testing.T) { `func _() { f(g()) }`, `func _() { func(x, y int, rest ...int) { println(x, y, rest) }(g()) }`, }, + { + "IncDec counts as assignment.", + `func f(x int) { x++ }`, + `func _() { f(1) }`, + `func _() { + { + var x int = 1 + x++ + } +}`, + }, { "Binding declaration (x eliminated).", `func f(w, x, y any, z int) { println(w, y, z) }; func g(int) int`, From c42ed47c5ec27ba3eb0dddf52f475622e6a68b96 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Mon, 25 Sep 2023 18:26:36 -0400 Subject: [PATCH 139/178] internal/refactor/inline: reject attempts to inline in cgo code Plus a test. Change-Id: I2d5542ed2c6e0cabae740279d09ceaf0d22b8710 Reviewed-on: https://go-review.googlesource.com/c/tools/+/529877 LUCI-TryBot-Result: Go LUCI Reviewed-by: Robert Findley --- internal/refactor/inline/callee.go | 12 ++++++ internal/refactor/inline/inline.go | 5 +++ internal/refactor/inline/testdata/cgo.txtar | 45 +++++++++++++++++++++ 3 files changed, 62 insertions(+) create mode 100644 internal/refactor/inline/testdata/cgo.txtar diff --git a/internal/refactor/inline/callee.go b/internal/refactor/inline/callee.go index 64a6903479f..fb3e6c7264f 100644 --- a/internal/refactor/inline/callee.go +++ b/internal/refactor/inline/callee.go @@ -14,6 +14,7 @@ import ( "go/parser" "go/token" "go/types" + "strings" "golang.org/x/tools/go/ast/astutil" "golang.org/x/tools/go/types/typeutil" @@ -322,6 +323,17 @@ func AnalyzeCallee(logf func(string, ...any), fset *token.FileSet, pkg *types.Pa return true }) + // Reject attempts to inline cgo-generated functions. + for _, obj := range freeObjs { + // There are others (iconst fconst sconst fpvar macro) + // but this is probably sufficient. + if strings.HasPrefix(obj.Name, "_Cfunc_") || + strings.HasPrefix(obj.Name, "_Ctype_") || + strings.HasPrefix(obj.Name, "_Cvar_") { + return nil, fmt.Errorf("cannot inline cgo-generated functions") + } + } + // Compact content to just the FuncDecl. // // As a space optimization, we don't retain the complete diff --git a/internal/refactor/inline/inline.go b/internal/refactor/inline/inline.go index cb98d58da6d..c30e2829c3a 100644 --- a/internal/refactor/inline/inline.go +++ b/internal/refactor/inline/inline.go @@ -57,6 +57,11 @@ func Inline(logf func(string, ...any), caller *Caller, callee *Callee) ([]byte, return nil, fmt.Errorf("internal error: caller syntax positions are inconsistent with file content (did you forget to use FileSet.PositionFor when computing the file name?)") } + // TODO(adonovan): use go1.21's ast.IsGenerated. + if bytes.Contains(caller.Content, []byte("// Code generated by cmd/cgo; DO NOT EDIT.")) { + return nil, fmt.Errorf("cannot inline calls from files that import \"C\"") + } + res, err := inline(logf, caller, &callee.impl) if err != nil { return nil, err diff --git a/internal/refactor/inline/testdata/cgo.txtar b/internal/refactor/inline/testdata/cgo.txtar new file mode 100644 index 00000000000..41567ed7cbb --- /dev/null +++ b/internal/refactor/inline/testdata/cgo.txtar @@ -0,0 +1,45 @@ +Test that attempts to inline with caller or callee in a cgo-generated +file are rejected. + +-- go.mod -- +module testdata +go 1.12 + +-- a/a.go -- +package a + +/* +static void f() {} +*/ +import "C" + +func a() { + C.f() //@ inline(re"f", re"cannot inline cgo-generated functions") + g() //@ inline(re"g", re`cannot inline calls from files that import "C"`) +} + +func g() { + println() +} + +-- a/a2.go -- +package a + +func b() { + a() //@ inline(re"a", re"cannot inline cgo-generated functions") +} + +func c() { + b() //@ inline(re"b", result) +} + +-- result -- +package a + +func b() { + a() //@ inline(re"a", re"cannot inline cgo-generated functions") +} + +func c() { + a() //@ inline(re"b", result) +} From f4abeaefa78d297311af6e1d296e2c1a903fdcd9 Mon Sep 17 00:00:00 2001 From: qiulaidongfeng <2645477756@qq.com> Date: Sat, 23 Sep 2023 09:21:30 +0000 Subject: [PATCH 140/178] go/analysis/passes/directive: use strings.Cut Change-Id: Ie3273daca752b8c22443edfdbde635a9f61577a3 GitHub-Last-Rev: 1e19b61b07df339cabcd2f6cf7cf799b6dd7cf32 GitHub-Pull-Request: golang/tools#450 Reviewed-on: https://go-review.googlesource.com/c/tools/+/530696 Reviewed-by: Tim King TryBot-Result: Gopher Robot LUCI-TryBot-Result: Go LUCI Reviewed-by: Than McIntosh Run-TryBot: Tim King --- go/analysis/passes/directive/directive.go | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/go/analysis/passes/directive/directive.go b/go/analysis/passes/directive/directive.go index 1146d7be457..2691f189aae 100644 --- a/go/analysis/passes/directive/directive.go +++ b/go/analysis/passes/directive/directive.go @@ -124,7 +124,7 @@ func (check *checker) nonGoFile(pos token.Pos, fullText string) { for text != "" { offset := len(fullText) - len(text) var line string - line, text, _ = stringsCut(text, "\n") + line, text, _ = strings.Cut(text, "\n") if !inStar && strings.HasPrefix(line, "//") { check.comment(pos+token.Pos(offset), line) @@ -137,7 +137,7 @@ func (check *checker) nonGoFile(pos token.Pos, fullText string) { line = strings.TrimSpace(line) if inStar { var ok bool - _, line, ok = stringsCut(line, "*/") + _, line, ok = strings.Cut(line, "*/") if !ok { break } @@ -200,14 +200,6 @@ func (check *checker) comment(pos token.Pos, line string) { } } -// Go 1.18 strings.Cut. -func stringsCut(s, sep string) (before, after string, found bool) { - if i := strings.Index(s, sep); i >= 0 { - return s[:i], s[i+len(sep):], true - } - return s, "", false -} - // Go 1.20 strings.CutPrefix. func stringsCutPrefix(s, prefix string) (after string, found bool) { if !strings.HasPrefix(s, prefix) { From d32f97a6d27c5f8ca0449df61482f57668208b3a Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Sun, 24 Sep 2023 15:34:46 -0400 Subject: [PATCH 141/178] internal/refactor/inline: eliminate Callee.BodyIsReturnExpr This CL is a minor cleanup. There's no need for the Callee.BodyIsReturnExpr field now that the caller has syntax (but not types) for the callee and has the pair of Callee.{Total,Trivial}Returns. In general, only type-derived information needs to be recorded in the Callee. Change-Id: Ib9c7661fb113bc043154bee59bf7cae872ad6691 Reviewed-on: https://go-review.googlesource.com/c/tools/+/530815 LUCI-TryBot-Result: Go LUCI Reviewed-by: Robert Findley --- internal/refactor/inline/callee.go | 83 +++++++++--------------------- internal/refactor/inline/inline.go | 5 +- 2 files changed, 28 insertions(+), 60 deletions(-) diff --git a/internal/refactor/inline/callee.go b/internal/refactor/inline/callee.go index fb3e6c7264f..980034cef07 100644 --- a/internal/refactor/inline/callee.go +++ b/internal/refactor/inline/callee.go @@ -37,8 +37,7 @@ type gobCallee struct { Unexported []string // names of free objects that are unexported FreeRefs []freeRef // locations of references to free objects FreeObjs []object // descriptions of free objects - BodyIsReturnExpr bool // function body is "return expr(s)" with trivial conversion - ValidForCallStmt bool // => bodyIsReturnExpr and sole expr is f() or <-ch + ValidForCallStmt bool // function body is "return expr" where expr is f() or <-ch NumResults int // number of results (according to type, not ast.FieldList) Params []*paramInfo // information about parameters (incl. receiver) Results []*paramInfo // information about result variables @@ -208,67 +207,34 @@ func AnalyzeCallee(logf func(string, ...any), fset *token.FileSet, pkg *types.Pa } visit(decl) - // Analyze callee body for "return results" form, where - // results is one or more expressions or an n-ary call, - // and the implied conversions are trivial. + // Analyze callee body for "return expr" form, + // where expr is f() or <-ch. These forms are + // safe to inline as a standalone statement. validForCallStmt := false - bodyIsReturnExpr := func() bool { - if decl.Type.Results != nil && - len(decl.Type.Results.List) > 0 && - len(decl.Body.List) == 1 { - if ret, ok := decl.Body.List[0].(*ast.ReturnStmt); ok && len(ret.Results) > 0 { - // Don't reduce calls to functions whose - // return statement has non trivial conversions. - argType := func(i int) types.Type { - return info.TypeOf(ret.Results[i]) - } - if len(ret.Results) == 1 && sig.Results().Len() > 1 { - // Spread return: return f() where f.Results > 1. - tuple := info.TypeOf(ret.Results[0]).(*types.Tuple) - argType = func(i int) types.Type { - return tuple.At(i).Type() - } - } - for i := 0; i < sig.Results().Len(); i++ { - if !trivialConversion(argType(i), sig.Results().At(i)) { - return false - } - } - - return true - } - } - return false - }() - if bodyIsReturnExpr { - ret := decl.Body.List[0].(*ast.ReturnStmt) - - // Ascertain whether the results expression(s) - // would be safe to inline as a standalone statement. - // (This is true only for a single call or receive expression.) + if len(decl.Body.List) != 1 { + // not just a return statement + } else if ret, ok := decl.Body.List[0].(*ast.ReturnStmt); ok && len(ret.Results) == 1 { validForCallStmt = func() bool { - if len(ret.Results) == 1 { - switch expr := astutil.Unparen(ret.Results[0]).(type) { - case *ast.CallExpr: // f(x) - callee := typeutil.Callee(info, expr) - if callee == nil { - return false // conversion T(x) - } + switch expr := astutil.Unparen(ret.Results[0]).(type) { + case *ast.CallExpr: // f(x) + callee := typeutil.Callee(info, expr) + if callee == nil { + return false // conversion T(x) + } - // The only non-void built-in functions that may be - // called as a statement are copy and recover - // (though arguably a call to recover should never - // be inlined as that changes its behavior). - if builtin, ok := callee.(*types.Builtin); ok { - return builtin.Name() == "copy" || - builtin.Name() == "recover" - } + // The only non-void built-in functions that may be + // called as a statement are copy and recover + // (though arguably a call to recover should never + // be inlined as that changes its behavior). + if builtin, ok := callee.(*types.Builtin); ok { + return builtin.Name() == "copy" || + builtin.Name() == "recover" + } - return true // ordinary call f() + return true // ordinary call f() - case *ast.UnaryExpr: // <-x - return expr.Op == token.ARROW // channel receive <-ch - } + case *ast.UnaryExpr: // <-x + return expr.Op == token.ARROW // channel receive <-ch } // No other expressions are valid statements. @@ -358,7 +324,6 @@ func AnalyzeCallee(logf func(string, ...any), fset *token.FileSet, pkg *types.Pa Unexported: unexported, FreeObjs: freeObjs, FreeRefs: freeRefs, - BodyIsReturnExpr: bodyIsReturnExpr, ValidForCallStmt: validForCallStmt, NumResults: sig.Results().Len(), Params: params, diff --git a/internal/refactor/inline/inline.go b/internal/refactor/inline/inline.go index c30e2829c3a..e951b83672c 100644 --- a/internal/refactor/inline/inline.go +++ b/internal/refactor/inline/inline.go @@ -600,7 +600,10 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // - no result var escapes, // then the call expression can be replaced by the // callee's body expression, suitably substituted. - if callee.BodyIsReturnExpr { + if len(calleeDecl.Body.List) == 1 && + is[*ast.ReturnStmt](calleeDecl.Body.List[0]) && + len(calleeDecl.Body.List[0].(*ast.ReturnStmt).Results) > 0 && // not a bare return + callee.TrivialReturns == callee.TotalReturns { results := calleeDecl.Body.List[0].(*ast.ReturnStmt).Results context := callContext(caller.path) From 1c8e684dd5e3f11cd31f9cd421a094e75b1ba154 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Mon, 25 Sep 2023 15:56:20 -0400 Subject: [PATCH 142/178] internal/refactor/inline: sound treatment of named results Previously the condition for eliminating named result vars was that they do not escape, but that's not strong enough: it should be that they are never referenced. If they are referenced, they can be accommodated in a binding decl, as done in this change. Also, tests for the interaction of named results with each strategy. Change-Id: I2a4bdeec17c98efd488f9939aebe6ab1d72a0814 Reviewed-on: https://go-review.googlesource.com/c/tools/+/530438 Reviewed-by: Robert Findley Auto-Submit: Alan Donovan LUCI-TryBot-Result: Go LUCI --- internal/refactor/inline/callee.go | 5 + internal/refactor/inline/inline.go | 147 +++++++++++++++--------- internal/refactor/inline/inline_test.go | 76 ++++++++++++ 3 files changed, 175 insertions(+), 53 deletions(-) diff --git a/internal/refactor/inline/callee.go b/internal/refactor/inline/callee.go index 980034cef07..d3af09b5f12 100644 --- a/internal/refactor/inline/callee.go +++ b/internal/refactor/inline/callee.go @@ -42,6 +42,7 @@ type gobCallee struct { Params []*paramInfo // information about parameters (incl. receiver) Results []*paramInfo // information about result variables HasDefer bool // uses defer + HasBareReturn bool // uses bare return in non-void function TotalReturns int // number of return statements TrivialReturns int // number of return statements with trivial result conversions Labels []string // names of all control labels @@ -246,6 +247,7 @@ func AnalyzeCallee(logf func(string, ...any), fset *token.FileSet, pkg *types.Pa // (but not any nested functions). var ( hasDefer = false + hasBareReturn = false totalReturns = 0 trivialReturns = 0 labels []string @@ -281,6 +283,8 @@ func AnalyzeCallee(logf func(string, ...any), fset *token.FileSet, pkg *types.Pa break } } + } else if sig.Results().Len() > 0 { + hasBareReturn = true } if trivial { trivialReturns++ @@ -329,6 +333,7 @@ func AnalyzeCallee(logf func(string, ...any), fset *token.FileSet, pkg *types.Pa Params: params, Results: results, HasDefer: hasDefer, + HasBareReturn: hasBareReturn, TotalReturns: totalReturns, TrivialReturns: trivialReturns, Labels: labels, diff --git a/internal/refactor/inline/inline.go b/internal/refactor/inline/inline.go index e951b83672c..c6362a272f0 100644 --- a/internal/refactor/inline/inline.go +++ b/internal/refactor/inline/inline.go @@ -508,7 +508,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu updateCalleeParams(calleeDecl, params) // Create a var (param = arg; ...) decl for use by some strategies. - bindingDeclStmt := createBindingDecl(logf, caller, args, calleeDecl) + bindingDeclStmt := createBindingDecl(logf, caller, args, calleeDecl, callee.Results) var remainingArgs []ast.Expr for _, arg := range args { @@ -577,14 +577,12 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu return res, nil } - // Attempt to reduce parameterless calls - // whose result variables do not escape. - allParamsSubstituted := forall(params, func(i int, p *parameter) bool { - return p == nil - }) - noResultEscapes := !exists(callee.Results, func(i int, r *paramInfo) bool { - return r.Escapes - }) + // If all parameters have been substituted and no result + // variable is referenced, we don't need a binding decl. + // This may enable better reduction strategies. + allResultsUnreferenced := forall(callee.Results, func(i int, r *paramInfo) bool { return len(r.Refs) == 0 }) + needBindingDecl := !allResultsUnreferenced || + exists(params, func(i int, p *parameter) bool { return p != nil }) // Special case: call to { return exprs }. // @@ -595,9 +593,8 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // // If: // - the body is just "return expr" with trivial implicit conversions, - // - all parameters can be eliminated - // (by substitution, or a binding decl), - // - no result var escapes, + // - all parameters and result vars can be eliminated + // or replaced by a binding decl, // then the call expression can be replaced by the // callee's body expression, suitably substituted. if len(calleeDecl.Body.List) == 1 && @@ -610,14 +607,14 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // statement context if stmt, ok := context.(*ast.ExprStmt); ok && - (allParamsSubstituted && noResultEscapes || bindingDeclStmt != nil) { + (!needBindingDecl || bindingDeclStmt != nil) { logf("strategy: reduce stmt-context call to { return exprs }") clearPositions(calleeDecl.Body) if callee.ValidForCallStmt { logf("callee body is valid as statement") // Inv: len(results) == 1 - if allParamsSubstituted && noResultEscapes { + if !needBindingDecl { // Reduces to: expr res.old = caller.Call res.new = results[0] @@ -643,7 +640,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu Rhs: results, } res.old = stmt - if allParamsSubstituted && noResultEscapes { + if !needBindingDecl { // Reduces to: _, _ = exprs res.new = discard } else { @@ -660,7 +657,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu } // expression context - if allParamsSubstituted && noResultEscapes { + if !needBindingDecl { clearPositions(calleeDecl.Body) if callee.NumResults == 1 { @@ -725,12 +722,12 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // { var (bindings); body } // { body } // so long as: - // - all parameters can be eliminated - // (by substitution, or a binding decl), + // - all parameters can be eliminated or replaced by a binding decl, // - call is a tail-call; // - all returns in body have trivial result conversions; // - there is no label conflict; - // - no result variable is referenced by name. + // - no result variable is referenced by name, + // or implicitly by a bare return. // // The body may use defer, arbitrary control flow, and // multiple returns. @@ -744,16 +741,14 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu if ret, ok := callContext(caller.path).(*ast.ReturnStmt); ok && len(ret.Results) == 1 && callee.TrivialReturns == callee.TotalReturns && - (allParamsSubstituted && noResultEscapes || bindingDeclStmt != nil) && + !callee.HasBareReturn && + (!needBindingDecl || bindingDeclStmt != nil) && !hasLabelConflict(caller.path, callee.Labels) && - forall(callee.Results, func(i int, p *paramInfo) bool { - // all result vars are unreferenced - return len(p.Refs) == 0 - }) { + allResultsUnreferenced { logf("strategy: reduce tail-call") body := calleeDecl.Body clearPositions(body) - if !(allParamsSubstituted && noResultEscapes) { + if needBindingDecl { body.List = prepend(bindingDeclStmt, body.List...) } res.old = ret @@ -774,12 +769,12 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // - callee is a void function (no returns) // - callee does not use defer // - there is no label conflict between caller and callee - // - all parameters can be eliminated - // (by substitution, or a binding decl), + // - all parameters and result vars can be eliminated + // or replaced by a binding decl, // // If there is only a single statement, the braces are omitted. if stmt := callStmt(caller.path); stmt != nil && - (allParamsSubstituted && noResultEscapes || bindingDeclStmt != nil) && + (!needBindingDecl || bindingDeclStmt != nil) && !callee.HasDefer && !hasLabelConflict(caller.path, callee.Labels) && callee.TotalReturns == 0 { @@ -787,7 +782,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu body := calleeDecl.Body var repl ast.Stmt = body clearPositions(repl) - if !(allParamsSubstituted && noResultEscapes) { + if needBindingDecl { body.List = prepend(bindingDeclStmt, body.List...) } if len(body.List) == 1 { @@ -1175,11 +1170,13 @@ func updateCalleeParams(calleeDecl *ast.FuncDecl, params []*parameter) { } // createBindingDecl constructs a "binding decl" that implements -// parameter assignment. +// parameter assignment and declares any named result variables +// referenced by the callee. // -// If we succeed, the declaration may be used by reduction -// strategies to relax the requirement that all parameters -// have been substituted. +// It may not always be possible to create the decl (e.g. due to +// shadowing), in which case it returns nil; but if it succeeds, the +// declaration may be used by reduction strategies to relax the +// requirement that all parameters have been substituted. // // For example, a call: // @@ -1210,7 +1207,7 @@ func updateCalleeParams(calleeDecl *ast.FuncDecl, params []*parameter) { // // Strategies may impose additional checks on return // conversions, labels, defer, etc. -func createBindingDecl(logf func(string, ...any), caller *Caller, args []*argument, calleeDecl *ast.FuncDecl) ast.Stmt { +func createBindingDecl(logf func(string, ...any), caller *Caller, args []*argument, calleeDecl *ast.FuncDecl, results []*paramInfo) ast.Stmt { // Spread calls are tricky as they may not align with the // parameters' field groupings nor types. // For example, given @@ -1228,27 +1225,15 @@ func createBindingDecl(logf func(string, ...any), caller *Caller, args []*argume return nil } - // Compute remaining argument expressions. - var values []ast.Expr - for _, arg := range args { - if arg != nil { - values = append(values, arg.expr) - } - } - var ( specs []ast.Spec shadowed = make(map[string]bool) // names defined by previous specs ) - for _, field := range calleeDecl.Type.Params.List { - // Each field (param group) becomes a ValueSpec. - spec := &ast.ValueSpec{ - Names: field.Names, - Type: field.Type, - Values: values[:len(field.Names)], - } - values = values[len(field.Names):] - + // shadow reports whether any name referenced by spec is + // shadowed by a name declared by a previous spec (since, + // unlike parameters, each spec of a var decl is within the + // scope of the previous specs). + shadow := func(spec *ast.ValueSpec) bool { // Compute union of free names of type and values // and detect shadowing. Values is the arguments // (caller syntax), so we can use type info. @@ -1260,11 +1245,11 @@ func createBindingDecl(logf func(string, ...any), caller *Caller, args []*argume free[name] = true } } - freeishNames(free, field.Type) + freeishNames(free, spec.Type) for name := range free { if shadowed[name] { logf("binding decl would shadow free name %q", name) - return nil + return true } } for _, id := range spec.Names { @@ -1272,10 +1257,66 @@ func createBindingDecl(logf func(string, ...any), caller *Caller, args []*argume shadowed[id.Name] = true } } + return false + } + // parameters + // + // Bind parameters that were not eliminated through + // substitution. (Non-nil arguments correspond to the + // remaining parameters in calleeDecl.) + var values []ast.Expr + for _, arg := range args { + if arg != nil { + values = append(values, arg.expr) + } + } + for _, field := range calleeDecl.Type.Params.List { + // Each field (param group) becomes a ValueSpec. + spec := &ast.ValueSpec{ + Names: field.Names, + Type: field.Type, + Values: values[:len(field.Names)], + } + values = values[len(field.Names):] + if shadow(spec) { + return nil + } specs = append(specs, spec) } assert(len(values) == 0, "args/params mismatch") + + // results + // + // Add specs to declare any named result + // variables that are referenced by the body. + if calleeDecl.Type.Results != nil { + resultIdx := 0 + for _, field := range calleeDecl.Type.Results.List { + if field.Names == nil { + resultIdx++ + continue // unnamed field + } + var names []*ast.Ident + for _, id := range field.Names { + if len(results[resultIdx].Refs) > 0 { + names = append(names, id) + } + resultIdx++ + } + if len(names) > 0 { + spec := &ast.ValueSpec{ + Names: names, + Type: field.Type, + } + if shadow(spec) { + return nil + } + specs = append(specs, spec) + } + } + } + decl := &ast.DeclStmt{ Decl: &ast.GenDecl{ Tok: token.VAR, diff --git a/internal/refactor/inline/inline_test.go b/internal/refactor/inline/inline_test.go index 488faecea1d..b5be22b00d0 100644 --- a/internal/refactor/inline/inline_test.go +++ b/internal/refactor/inline/inline_test.go @@ -526,6 +526,82 @@ func TestTable(t *testing.T) { } }`, }, + // Treatment of named result vars across strategies: + { + "Stmt-context call to {return g()} that mentions named result.", + `func f() (x int) { return g(x) }; func g(int) int`, + `func _() { f() }`, + `func _() { + { + var x int + g(x) + } +}`, + }, + { + "Ditto, with binding decl (due to repeated y refs).", + `func f(y string) (x string) { return x+y+y }`, + `func _() { f("") }`, + `func _() { + { + var ( + y string = "" + x string + ) + _ = x + y + y + } +}`, + }, + { + "Stmt-context call to {return binary} that mentions named result.", + `func f() (x int) { return x+x }`, + `func _() { f() }`, + `func _() { + { + var x int + _ = x + x + } +}`, + }, + { + "Ditto, with binding decl again.", + `func f(y string) (x int) { return x+x+len(y+y) }`, + `func _() { f("") }`, + `func _() { + { + var ( + y string = "" + x int + ) + _ = x + x + len(y+y) + } +}`, + }, + { + "Tail call to {return expr} that mentions named result.", + `func f() (x int) { return x }`, + `func _() int { return f() }`, + `func _() int { return func() (x int) { return x }() }`, + }, + { + "Tail call to {return} that implicitly reads named result.", + `func f() (x int) { return }`, + `func _() int { return f() }`, + `func _() int { return func() (x int) { return }() }`, + }, + { + "Spread-context call to {return expr} that mentions named result.", + `func f() (x, y int) { return x, y }`, + `func _() { var _, _ = f() }`, + `func _() { var _, _ = func() (x, y int) { return x, y }() }`, + }, + { + "Shadowing in binding decl for named results => literalization.", + `func f(y string) (x y) { return x+x+len(y+y) }; type y = int`, + `func _() { f("") }`, + `func _() { func(y string) (x y) { return x + x + len(y+y) }("") }`, + }, + // TODO(adonovan): improve coverage of the cross // product of each strategy with the checklist of // concerns enumerated in the package doc comment. From 160210312b25df782c7da7b74b6723adb8b01df0 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Tue, 26 Sep 2023 14:32:33 -0400 Subject: [PATCH 143/178] internal/refactor/inline: skip cgo tests on non-cgo builders Change-Id: I23beaea36053df131b4f6b13c32cba4fe1d89bc6 Reviewed-on: https://go-review.googlesource.com/c/tools/+/530978 LUCI-TryBot-Result: Go LUCI Reviewed-by: Robert Findley --- internal/refactor/inline/inline_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/refactor/inline/inline_test.go b/internal/refactor/inline/inline_test.go index b5be22b00d0..1371afd9d05 100644 --- a/internal/refactor/inline/inline_test.go +++ b/internal/refactor/inline/inline_test.go @@ -45,6 +45,12 @@ func TestData(t *testing.T) { t.Run(filepath.Base(file), func(t *testing.T) { t.Parallel() + // The few tests that use cgo should be in + // files whose name includes "cgo". + if strings.Contains(t.Name(), "cgo") { + testenv.NeedsTool(t, "cgo") + } + // Extract archive to temporary tree. ar, err := txtar.ParseFile(file) if err != nil { From b3ada30db78a535dcc1cf4371d16043e1e39945c Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Thu, 21 Sep 2023 13:33:31 -0400 Subject: [PATCH 144/178] internal/refactor/inline: analyze callee effects This changes adds an analysis of the order of evaluation of parameter variables and other read or write effects in the callee, and uses it to optimize inlining by allowing parameter substitution of argument expressions that are impure (modelled as reads, R) or have side effects modelled as writes, W), so long as this does not cause reordering of R-W, W-W, or W-R pairs of operations. R-R pairs of operations may be reordered freely. Pure expressions, which are neither R nor W, may be ordered arbitrarily. The caller-side hazard algorithm is due to Rob Findley. Also: - tests of parameter substitution in the presence of reads and effects; - unit test of callee effects algorithm. - update TODO comments re: ParenExpr removal. Change-Id: If0342f4b1edc22c8b3445e39b54403d5b58eb33b Reviewed-on: https://go-review.googlesource.com/c/tools/+/530655 Reviewed-by: Robert Findley LUCI-TryBot-Result: Go LUCI --- internal/refactor/inline/callee.go | 32 +- internal/refactor/inline/calleefx.go | 334 ++++++++++++++++++ internal/refactor/inline/calleefx_test.go | 159 +++++++++ internal/refactor/inline/export_test.go | 9 + internal/refactor/inline/inline.go | 228 +++++++++--- internal/refactor/inline/inline_test.go | 215 +++++++++-- .../refactor/inline/testdata/basic-err.txtar | 8 +- internal/refactor/inline/util.go | 21 ++ 8 files changed, 907 insertions(+), 99 deletions(-) create mode 100644 internal/refactor/inline/calleefx.go create mode 100644 internal/refactor/inline/calleefx_test.go create mode 100644 internal/refactor/inline/export_test.go diff --git a/internal/refactor/inline/callee.go b/internal/refactor/inline/callee.go index d3af09b5f12..23ec98e8a52 100644 --- a/internal/refactor/inline/callee.go +++ b/internal/refactor/inline/callee.go @@ -41,6 +41,7 @@ type gobCallee struct { NumResults int // number of results (according to type, not ast.FieldList) Params []*paramInfo // information about parameters (incl. receiver) Results []*paramInfo // information about result variables + Effects []int // order in which parameters are evaluated (see calleefx) HasDefer bool // uses defer HasBareReturn bool // uses bare return in non-void function TotalReturns int // number of return statements @@ -320,7 +321,7 @@ func AnalyzeCallee(logf func(string, ...any), fset *token.FileSet, pkg *types.Pa return nil, err } - params, results := analyzeParams(logf, fset, info, decl) + params, results, effects := analyzeParams(logf, fset, info, decl) return &Callee{gobCallee{ Content: content, PkgPath: pkg.Path(), @@ -332,6 +333,7 @@ func AnalyzeCallee(logf func(string, ...any), fset *token.FileSet, pkg *types.Pa NumResults: sig.Results().Len(), Params: params, Results: results, + Effects: effects, HasDefer: hasDefer, HasBareReturn: hasBareReturn, TotalReturns: totalReturns, @@ -353,8 +355,11 @@ func parseCompact(content []byte) (*token.FileSet, *ast.FuncDecl, error) { } // A paramInfo records information about a callee receiver, parameter, or result variable. +// TODO(adonovan): rename to sigVarInfo or paramOrResultInfo? type paramInfo struct { Name string // parameter name (may be blank, or even "") + Index int // index within signature + IsResult bool // false for receiver or parameter, true for result variable Assigned bool // parameter appears on left side of an assignment statement Escapes bool // parameter has its address taken Refs []int // FuncDecl-relative byte offset of parameter ref within body @@ -368,7 +373,7 @@ type paramInfo struct { // the other of the result variables of function fn. // // The input must be well-typed. -func analyzeParams(logf func(string, ...any), fset *token.FileSet, info *types.Info, decl *ast.FuncDecl) (params, results []*paramInfo) { +func analyzeParams(logf func(string, ...any), fset *token.FileSet, info *types.Info, decl *ast.FuncDecl) (params, results []*paramInfo, effects []int) { fnobj, ok := info.Defs[decl.Name] if !ok { panic(fmt.Sprintf("%s: no func object for %q", @@ -378,19 +383,23 @@ func analyzeParams(logf func(string, ...any), fset *token.FileSet, info *types.I paramInfos := make(map[*types.Var]*paramInfo) { sig := fnobj.Type().(*types.Signature) - newParamInfo := func(param *types.Var) *paramInfo { - info := ¶mInfo{Name: param.Name()} + newParamInfo := func(param *types.Var, isResult bool) *paramInfo { + info := ¶mInfo{ + Name: param.Name(), + IsResult: isResult, + Index: len(paramInfos), + } paramInfos[param] = info return info } if sig.Recv() != nil { - params = append(params, newParamInfo(sig.Recv())) + params = append(params, newParamInfo(sig.Recv(), false)) } for i := 0; i < sig.Params().Len(); i++ { - params = append(params, newParamInfo(sig.Params().At(i))) + params = append(params, newParamInfo(sig.Params().At(i), false)) } for i := 0; i < sig.Results().Len(); i++ { - results = append(results, newParamInfo(sig.Results().At(i))) + results = append(results, newParamInfo(sig.Results().At(i), true)) } } @@ -420,7 +429,7 @@ func analyzeParams(logf func(string, ...any), fset *token.FileSet, info *types.I if id, ok := n.(*ast.Ident); ok { if v, ok := info.Uses[id].(*types.Var); ok { if pinfo, ok := paramInfos[v]; ok { - // Record location of ref to parameter. + // Record location of ref to parameter/result. offset := int(n.Pos() - decl.Pos()) pinfo.Refs = append(pinfo.Refs, offset) @@ -446,7 +455,12 @@ func analyzeParams(logf func(string, ...any), fset *token.FileSet, info *types.I return true }) - return params, results + // Compute subset and order of parameters that are strictly evaluated. + // (Depends on Refs computed above.) + effects = calleefx(info, decl.Body, paramInfos) + logf("effects list = %v", effects) + + return params, results, effects } // -- callee helpers -- diff --git a/internal/refactor/inline/calleefx.go b/internal/refactor/inline/calleefx.go new file mode 100644 index 00000000000..6e3dc7994be --- /dev/null +++ b/internal/refactor/inline/calleefx.go @@ -0,0 +1,334 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package inline + +// This file defines the analysis of callee effects. + +import ( + "go/ast" + "go/token" + "go/types" +) + +const ( + rinf = -1 // R∞: arbitrary read from memory + winf = -2 // W∞: arbitrary write to memory (or unknown control) +) + +// calleefx returns a list of parameter indices indicating the order +// in which parameters are first referenced during evaluation of the +// callee, relative both to each other and to other effects of the +// callee (if any), such as arbitrary reads (rinf) and arbitrary +// effects (winf), including unknown control flow. Each parameter +// that is referenced appears once in the list. +// +// For example, the effects list of this function: +// +// func f(x, y, z int) int { +// return y + x + g() + z +// } +// +// is [1 0 -2 2], indicating reads of y and x, followed by the unknown +// effects of the g() call. and finally the read of parameter z. This +// information is used during inlining to ascertain when it is safe +// for parameter references to be replaced by their corresponding +// argument expressions. Such substitutions are permitted only when +// they do not cause "write" operations (those with effects) to +// commute with "read" operations (those that have no effect but are +// not pure). Impure operations may be reordered with other impure +// operations, and pure operations may be reordered arbitrarily. +// +// The analysis ignores the effects of runtime panics, on the +// assumption that well-behaved programs shouldn't encounter them. +func calleefx(info *types.Info, body *ast.BlockStmt, paramInfos map[*types.Var]*paramInfo) []int { + // This traversal analyzes the callee's statements (in syntax + // form, though one could do better with SSA) to compute the + // sequence of events of the following kinds: + // + // 1 read of a parameter variable. + // 2. reads from other memory. + // 3. writes to memory + + var effects []int // indices of parameters, or rinf/winf (-ve) + seen := make(map[int]bool) + effect := func(i int) { + if !seen[i] { + seen[i] = true + effects = append(effects, i) + } + } + + // unknown is called for statements of unknown effects (or control). + unknown := func() { + effect(winf) + + // Ensure that all remaining parameters are "seen" + // after we go into the unknown (unless they are + // unreferenced by the function body). This lets us + // not bother implementing the complete traversal into + // control structures. + // + // TODO(adonovan): add them in a deterministic order. + // (This is not a bug but determinism is good.) + for _, pinfo := range paramInfos { + if !pinfo.IsResult && len(pinfo.Refs) > 0 { + effect(pinfo.Index) + } + } + } + + var visitExpr func(n ast.Expr) + var visitStmt func(n ast.Stmt) bool + visitExpr = func(n ast.Expr) { + switch n := n.(type) { + case *ast.Ident: + if v, ok := info.Uses[n].(*types.Var); ok && !v.IsField() { + // Use of global? + if v.Parent() == v.Pkg().Scope() { + effect(rinf) // read global var + } + + // Use of parameter? + if pinfo, ok := paramInfos[v]; ok && !pinfo.IsResult { + effect(pinfo.Index) // read parameter var + } + + // Use of local variables is ok. + } + + case *ast.BasicLit: + // no effect + + case *ast.FuncLit: + // A func literal has no read or write effect + // until called, and (most) function calls are + // considered to have arbitrary effects. + // So, no effect. + + case *ast.CompositeLit: + for _, elt := range n.Elts { + visitExpr(elt) // note: visits KeyValueExpr + } + + case *ast.ParenExpr: + visitExpr(n.X) + + case *ast.SelectorExpr: + if sel, ok := info.Selections[n]; ok { + visitExpr(n.X) + if sel.Indirect() { + effect(rinf) // indirect read x.f of heap variable + } + } else { + // qualified identifier: treat like unqualified + visitExpr(n.Sel) + } + + case *ast.IndexExpr: + if tv := info.Types[n.Index]; tv.IsType() { + // no effect (G[T] instantiation) + } else { + visitExpr(n.X) + visitExpr(n.Index) + switch tv.Type.Underlying().(type) { + case *types.Slice, *types.Pointer: // []T, *[n]T (not string, [n]T) + effect(rinf) // indirect read of slice/array element + } + } + + case *ast.IndexListExpr: + // no effect (M[K,V] instantiation) + + case *ast.SliceExpr: + visitExpr(n.X) + visitExpr(n.Low) + visitExpr(n.High) + visitExpr(n.Max) + + case *ast.TypeAssertExpr: + visitExpr(n.X) + + case *ast.CallExpr: + if info.Types[n.Fun].IsType() { + // conversion T(x) + visitExpr(n.Args[0]) + } else { + // call f(args) + visitExpr(n.Fun) + for i, arg := range n.Args { + if i == 0 && info.Types[arg].IsType() { + continue // new(T), make(T, n) + } + visitExpr(arg) + } + + // The pure built-ins have no effects beyond + // those of their operands (not even memory reads). + // All other calls have unknown effects. + if !callsPureBuiltin(info, n) { + unknown() // arbitrary effects + } + } + + case *ast.StarExpr: + visitExpr(n.X) + effect(rinf) // *ptr load or store depends on state of heap + + case *ast.UnaryExpr: // + - ! ^ & ~ <- + visitExpr(n.X) + if n.Op == token.ARROW { + unknown() // effect: channel receive + } + + case *ast.BinaryExpr: + visitExpr(n.X) + visitExpr(n.Y) + + case *ast.KeyValueExpr: + visitExpr(n.Key) // may be a struct field + visitExpr(n.Value) + + case *ast.BadExpr: + // no effect + + case nil: + // optional subtree + + default: + // type syntax: unreachable given traversal + panic(n) + } + } + + // visitStmt's result indicates the continuation: + // false for return, true for the next statement. + // + // We could treat return as an unknown, but this way + // yields definite effects for simple sequences like + // {S1; S2; return}, so unreferenced parameters are + // not spuriously added to the effects list, and thus + // not spuriously disqualified from elimination. + visitStmt = func(n ast.Stmt) bool { + switch n := n.(type) { + case *ast.DeclStmt: + decl := n.Decl.(*ast.GenDecl) + for _, spec := range decl.Specs { + switch spec := spec.(type) { + case *ast.ValueSpec: + for _, v := range spec.Values { + visitExpr(v) + } + + case *ast.TypeSpec: + // no effect + } + } + + case *ast.LabeledStmt: + return visitStmt(n.Stmt) + + case *ast.ExprStmt: + visitExpr(n.X) + + case *ast.SendStmt: + visitExpr(n.Chan) + visitExpr(n.Value) + unknown() // effect: channel send + + case *ast.IncDecStmt: + visitExpr(n.X) + unknown() // effect: variable increment + + case *ast.AssignStmt: + for _, lhs := range n.Lhs { + visitExpr(lhs) + } + for _, rhs := range n.Rhs { + visitExpr(rhs) + } + for _, lhs := range n.Lhs { + id, _ := lhs.(*ast.Ident) + if id != nil && id.Name == "_" { + continue // blank assign has no effect + } + if n.Tok == token.DEFINE && id != nil && info.Defs[id] != nil { + continue // new var declared by := has no effect + } + unknown() // assignment to existing var + break + } + + case *ast.GoStmt: + visitExpr(n.Call.Fun) + for _, arg := range n.Call.Args { + visitExpr(arg) + } + unknown() // effect: create goroutine + + case *ast.DeferStmt: + visitExpr(n.Call.Fun) + for _, arg := range n.Call.Args { + visitExpr(arg) + } + unknown() // effect: push defer + + case *ast.ReturnStmt: + for _, res := range n.Results { + visitExpr(res) + } + return false + + case *ast.BlockStmt: + for _, stmt := range n.List { + if !visitStmt(stmt) { + return false + } + } + + case *ast.BranchStmt: + unknown() // control flow + + case *ast.IfStmt: + visitStmt(n.Init) + visitExpr(n.Cond) + unknown() // control flow + + case *ast.SwitchStmt: + visitStmt(n.Init) + visitExpr(n.Tag) + unknown() // control flow + + case *ast.TypeSwitchStmt: + visitStmt(n.Init) + visitStmt(n.Assign) + unknown() // control flow + + case *ast.SelectStmt: + unknown() // control flow + + case *ast.ForStmt: + visitStmt(n.Init) + visitExpr(n.Cond) + unknown() // control flow + + case *ast.RangeStmt: + visitExpr(n.X) + unknown() // control flow + + case *ast.EmptyStmt, *ast.BadStmt: + // no effect + + case nil: + // optional subtree + + default: + panic(n) + } + return true + } + visitStmt(body) + + return effects +} diff --git a/internal/refactor/inline/calleefx_test.go b/internal/refactor/inline/calleefx_test.go new file mode 100644 index 00000000000..1fc16aebaac --- /dev/null +++ b/internal/refactor/inline/calleefx_test.go @@ -0,0 +1,159 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package inline_test + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "go/types" + "testing" + + "golang.org/x/tools/internal/refactor/inline" +) + +// TestCalleeEffects is a unit test of the calleefx analysis. +func TestCalleeEffects(t *testing.T) { + // Each callee must declare a function or method named f. + const funcName = "f" + + var tests = []struct { + descr string + callee string // Go source file (sans package decl) containing callee decl + want string // expected effects string (-1=R∞ -2=W∞) + }{ + { + "Assignments have unknown effects.", + `func f(x, y int) { x = y }`, + `[0 1 -2]`, + }, + { + "Reads from globals are impure.", + `func f() { _ = g }; var g int`, + `[-1]`, + }, + { + "Writes to globals have effects.", + `func f() { g = 0 }; var g int`, + `[-1 -2]`, // the -1 is spurious but benign + }, + { + "Blank assign has no effect.", + `func f(x int) { _ = x }`, + `[0]`, + }, + { + "Short decl of new var has has no effect.", + `func f(x int) { y := x; _ = y }`, + `[0]`, + }, + { + "Short decl of existing var (y) is an assignment.", + `func f(x int) { y := x; y, z := 1, 2; _, _ = y, z }`, + `[0 -2]`, + }, + { + "Unreferenced parameters are excluded.", + `func f(x, y, z int) { _ = z + x }`, + `[2 0]`, + }, + { + "Built-in len has no effect.", + `func f(x, y string) { _ = len(y) + len(x) }`, + `[1 0]`, + }, + { + "Built-in println has effects.", + `func f(x, y int) { println(y, x) }`, + `[1 0 -2]`, + }, + { + "Return has no effect, and no control successor.", + `func f(x, y int) int { return x + y; panic(1) }`, + `[0 1]`, + }, + { + "Loops (etc) have unknown effects.", + `func f(x, y bool) { for x { _ = y } }`, + `[0 -2 1]`, + }, + { + "Calls have unknown effects.", + `func f(x, y int) { _, _, _ = x, g(), y }; func g() int`, + `[0 -2 1]`, + }, + { + "Calls to some built-ins are pure.", + `func f(x, y int) { _, _, _ = x, len("hi"), y }`, + `[0 1]`, + }, + { + "Calls to some built-ins are pure (variant).", + `func f(x, y int) { s := "hi"; _, _, _ = x, len(s), y; s = "bye" }`, + `[0 1 -2]`, + }, + { + "Calls to some built-ins are pure (another variants).", + `func f(x, y int) { s := "hi"; _, _, _ = x, len(s), y }`, + `[0 1]`, + }, + { + "Reading a local var is impure but does not have effects.", + `func f(x, y bool) { for x { _ = y } }`, + `[0 -2 1]`, + }, + } + for _, test := range tests { + test := test + t.Run(test.descr, func(t *testing.T) { + fset := token.NewFileSet() + mustParse := func(filename string, content any) *ast.File { + f, err := parser.ParseFile(fset, filename, content, parser.ParseComments|parser.SkipObjectResolution) + if err != nil { + t.Fatalf("ParseFile: %v", err) + } + return f + } + + // Parse callee file and find first func decl named f. + calleeContent := "package p\n" + test.callee + calleeFile := mustParse("callee.go", calleeContent) + var decl *ast.FuncDecl + for _, d := range calleeFile.Decls { + if d, ok := d.(*ast.FuncDecl); ok && d.Name.Name == funcName { + decl = d + break + } + } + if decl == nil { + t.Fatalf("declaration of func %s not found: %s", funcName, test.callee) + } + + info := &types.Info{ + Defs: make(map[*ast.Ident]types.Object), + Uses: make(map[*ast.Ident]types.Object), + Types: make(map[ast.Expr]types.TypeAndValue), + Implicits: make(map[ast.Node]types.Object), + Selections: make(map[*ast.SelectorExpr]*types.Selection), + Scopes: make(map[ast.Node]*types.Scope), + } + conf := &types.Config{Error: func(err error) { t.Error(err) }} + pkg, err := conf.Check("p", fset, []*ast.File{calleeFile}, info) + if err != nil { + t.Fatal(err) + } + + callee, err := inline.AnalyzeCallee(t.Logf, fset, pkg, info, decl, []byte(calleeContent)) + if err != nil { + t.Fatal(err) + } + if got := fmt.Sprint(callee.Effects()); got != test.want { + t.Errorf("for effects of %s, got %s want %s", + test.callee, got, test.want) + } + }) + } +} diff --git a/internal/refactor/inline/export_test.go b/internal/refactor/inline/export_test.go new file mode 100644 index 00000000000..7b2cec7f19d --- /dev/null +++ b/internal/refactor/inline/export_test.go @@ -0,0 +1,9 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package inline + +// This file opens back doors for testing. + +func (callee *Callee) Effects() []int { return callee.impl.Effects } diff --git a/internal/refactor/inline/inline.go b/internal/refactor/inline/inline.go index c6362a272f0..918cb3acfeb 100644 --- a/internal/refactor/inline/inline.go +++ b/internal/refactor/inline/inline.go @@ -83,7 +83,19 @@ func Inline(logf func(string, ...any), caller *Caller, callee *Callee) ([]byte, var out bytes.Buffer out.Write(caller.Content[:start]) // TODO(adonovan): might it make more sense to use - // callee.Fset when formatting res.new?? + // callee.Fset when formatting res.new? + // The new tree is a mix of (cloned) caller nodes for + // the argument expressions and callee nodes for the + // function body. In essence the question is: which + // is more likely to have comments? + // Usually the callee body will be larger and more + // statement-heavy than the the arguments, but a + // strategy may widen the scope of the replacement + // (res.old) from CallExpr to, say, its enclosing + // block, so the caller nodes dominate. + // Precise comment handling would make this a + // non-issue. Formatting wouldn't really need a + // FileSet at all. if err := format.Node(&out, caller.Fset, res.new); err != nil { return nil, err } @@ -497,12 +509,19 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu } } + // Log effective arguments. + for i, arg := range args { + logf("arg #%d: %s pure=%t effects=%t duplicable=%t free=%v", + i, debugFormatNode(caller.Fset, arg.expr), + arg.pure, arg.effects, arg.duplicable, arg.freevars) + } + // Note: computation below should be expressed in terms of // the args and params slices, not the raw material. // Perform parameter substitution. // May eliminate some elements of params/args. - substitute(logf, caller, params, args, replaceCalleeID) + substitute(logf, caller, params, args, callee.Effects, replaceCalleeID) // Update the callee's signature syntax. updateCalleeParams(calleeDecl, params) @@ -668,6 +687,16 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // func two() int { return 1+1 } // print(-two()) => print(-1+1) // oops! // + // Usually it is not necessary to insert ParenExprs + // as the formatter is smart enough to insert them as + // needed by the context. But the res.{old,new} + // substitution is done by formatting res.new in isolation + // and then splicing its text over res.old, so the + // formatter doesn't see the parent node and cannot do + // the right thing. (One solution would be to always + // format the enclosing node of old, but that requires + // non-lossy comment handling, #20744.) + // // TODO(adonovan): do better by analyzing 'context' // to see whether ambiguity is possible. // For example, if the context is x[y:z], then @@ -842,13 +871,14 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu } type argument struct { - expr ast.Expr - typ types.Type // may be tuple for sole non-receiver arg in spread call - spread bool // final arg is call() assigned to multiple params - pure bool // expr is pure (doesn't read variables) - effects bool // expr has effects (updates variables) - duplicable bool // expr may be duplicated - freevars map[string]bool // free names of expr + expr ast.Expr + typ types.Type // may be tuple for sole non-receiver arg in spread call + spread bool // final arg is call() assigned to multiple params + pure bool // expr is pure (doesn't read variables) + effects bool // expr has effects (updates variables) + duplicable bool // expr may be duplicated + freevars map[string]bool // free names of expr + substitutable bool // is candidate for substitution } // arguments returns the effective arguments of the call. @@ -929,6 +959,10 @@ func arguments(caller *Caller, calleeDecl *ast.FuncDecl, assign1 func(*types.Var Sel: makeIdent(fld.Name()), } arg.typ = fld.Type() + arg.duplicable = false + } + if seln.Indirect() { + arg.pure = false // one or more implicit *ptr operation => impure } // Make * or & explicit. @@ -942,23 +976,7 @@ func arguments(caller *Caller, calleeDecl *ast.FuncDecl, assign1 func(*types.Var // *recv arg.expr = &ast.StarExpr{X: arg.expr} arg.typ = deref(arg.typ) - - // Technically *recv is non-pure and - // non-duplicable, as side effects - // could change the pointer between - // multiple reads. But unfortunately - // this really degrades many of our tests. - // (The indices loop above should similarly - // update these flags when traversing pointers.) - // - // TODO(adonovan): improve the precision of - // purity and duplicability. - // For example, *new(T) is actually pure. - // And *ptr, where ptr doesn't escape and - // has no assignments other than its decl, - // is also pure; this is very common. - // - // arg.pure = false + arg.duplicable = false } } } @@ -1002,7 +1020,7 @@ type parameter struct { // parameter, and is provided with its relative offset and replacement // expression (argument), and the corresponding elements of params and // args are replaced by nil. -func substitute(logf func(string, ...any), caller *Caller, params []*parameter, args []*argument, replaceCalleeID func(offset int, repl ast.Expr)) { +func substitute(logf func(string, ...any), caller *Caller, params []*parameter, args []*argument, effects []int, replaceCalleeID func(offset int, repl ast.Expr)) { // Inv: // in calls to variadic, len(args) >= len(params)-1 // in spread calls to non-variadic, len(args) < len(params) @@ -1021,9 +1039,10 @@ next: // do it earlier (see pure/duplicable/freevars). if arg.spread { + // spread => last argument, but not always last parameter logf("keeping param %q and following ones: argument %s is spread", param.info.Name, debugFormatNode(caller.Fset, arg.expr)) - break // spread => last argument, but not always last parameter + return // give up } assert(!param.variadic, "unsimplified variadic parameter") if param.info.Escapes { @@ -1034,18 +1053,16 @@ next: logf("keeping param %q: assigned by callee", param.info.Name) continue // callee needs the parameter variable } - if !arg.pure || arg.effects { - // TODO(adonovan): conduct a deeper analysis of callee effects - // to relax this constraint. - logf("keeping param %q: argument %s is impure", - param.info.Name, debugFormatNode(caller.Fset, arg.expr)) - continue // unsafe to change order or cardinality of effects - } if len(param.info.Refs) > 1 && !arg.duplicable { logf("keeping param %q: argument is not duplicable", param.info.Name) continue // incorrect or poor style to duplicate an expression } if len(param.info.Refs) == 0 { + if arg.effects { + logf("keeping param %q: though unreferenced, it has effects", param.info.Name) + continue + } + // Eliminating an unreferenced parameter might // remove the last reference to a caller local var. for free := range arg.freevars { @@ -1091,19 +1108,122 @@ next: } } - // It is safe to eliminate param and replace it with arg. - // No additional parens are required around arg for - // the supported "pure" expressions. - // - // Because arg.expr belongs to the caller, - // we clone it before splicing it into the callee tree. - logf("replacing parameter %q by argument %q", - param.info.Name, debugFormatNode(caller.Fset, arg.expr)) - for _, ref := range param.info.Refs { - replaceCalleeID(ref, cloneNode(arg.expr).(ast.Expr)) + arg.substitutable = true // may be substituted, if effects permit + } + + // As a final step, introduce bindings to resolve any + // evaluation order hazards. This must be done last, as + // additional subsequent bindings could introduce new hazards. + resolveEffects(logf, args, effects) + + // The remaining candidates are safe to substitute. + for i, param := range params { + if arg := args[i]; arg.substitutable { + // It is safe to substitute param and replace it with arg. + // The formatter introduces parens as needed for precedence. + logf("replacing parameter %q by argument %q", + param.info.Name, debugFormatNode(caller.Fset, arg.expr)) + for _, ref := range param.info.Refs { + replaceCalleeID(ref, cloneNode(arg.expr).(ast.Expr)) + } + params[i] = nil // substituted + args[i] = nil // substituted + } + } +} + +// resolveEffects marks arguments as non-substitutable to resolve +// hazards resulting from the callee evaluation order described by the +// effects list. +// +// To do this, each argument is categorized as a read (R), write (W), +// or pure. A hazard occurs when the order of evaluation of a W +// changes with respect to any R or W. Pure arguments can be +// effectively ignored, as they can be safely evaluated in any order. +// +// The callee effects list contains the index of each parameter in the +// order it is first evaluated during execution of the callee. In +// addition, the two special values R∞ and W∞ indicate the relative +// position of the callee's first non-parameter read and its first +// effects (or other unknown behavior). +// For example, the list [0 2 1 R∞ 3 W∞] for func(a, b, c, d) +// indicates that the callee referenced parameters a, c, and b, +// followed by an arbitrary read, then parameter d, and finally +// unknown behavior. +// +// When an argument is marked as not substitutable, we say that it is +// 'bound', in the sense that its evaluation occurs in a binding decl +// or literalized call. Such bindings always occur in the original +// callee parameter order. +// +// In this context, "resolving hazards" means binding arguments so +// that they are evaluated in a valid, hazard-free order. A trivial +// solution to this problem would be to bind all arguments, but of +// course that's not useful. The goal is to bind as few arguments as +// possible. +// +// The algorithm proceeds by inspecting arguments in reverse parameter +// order (right to left), preserving the invariant that every +// higher-ordered argument is either already substituted or does not +// need to be substituted. At each iteration, if there is an +// evaluation hazard in the callee effects relative to the current +// argument, the argument must be bound. Subsequently, if the argument +// is bound for any reason, each lower-ordered argument must also be +// bound if either the argument or lower-order argument is a +// W---otherwise the binding itself would introduce a hazard. +// +// Thus, after each iteration, there are no hazards relative to the +// current argument. Subsequent iterations cannot introduce hazards +// with that argument because they can result only in additional +// binding of lower-ordered arguments. +func resolveEffects(logf func(string, ...any), args []*argument, effects []int) { + effectStr := func(effects bool, idx int) string { + i := fmt.Sprint(idx) + if idx == len(args) { + i = "∞" + } + return string("RW"[btoi(effects)]) + i + } + for i := len(args) - 1; i >= 0; i-- { + argi := args[i] + if argi.substitutable && !argi.pure { + // i is not bound: check whether it must be bound due to hazards. + idx := index(effects, i) + if idx >= 0 { + for _, j := range effects[:idx] { + var ( + ji int // effective param index + jw bool // j is a write + ) + if j == winf || j == rinf { + jw = j == winf + ji = len(args) + } else { + jw = args[j].effects + ji = j + } + if ji > i && (jw || argi.effects) { // out of order evaluation + logf("binding argument %s: preceded by %s", + effectStr(argi.effects, i), effectStr(jw, ji)) + argi.substitutable = false + break + } + } + } + } + if !argi.substitutable { + for j := 0; j < i; j++ { + argj := args[j] + if argj.pure { + continue + } + if (argi.effects || argj.effects) && argj.substitutable { + logf("binding argument %s: %s is bound", + effectStr(argj.effects, j), effectStr(argi.effects, i)) + argj.substitutable = false + } + } } - params[i] = nil // substituted - args[i] = nil // substituted } } @@ -1405,13 +1525,16 @@ func effects(info *types.Info, expr ast.Expr) bool { return false // prune descent case *ast.CallExpr: - if !info.Types[n.Fun].IsType() { + if info.Types[n.Fun].IsType() { // A conversion T(x) has only the effect of its operand. } else if !callsPureBuiltin(info, n) { // A handful of built-ins have no effect // beyond those of their arguments. // All other calls (including append, copy, recover) // have unknown effects. + // + // As with 'pure', there is room for + // improvement by inspecting the callee. effects = true } @@ -1573,6 +1696,11 @@ func pure(info *types.Info, assign1 func(*types.Var) bool, e ast.Expr) bool { return pure(e) } +// callsPureBuiltin reports whether call is a call of a built-in +// function that is a pure computation over its operands (analogous to +// a + operator). Because it does not depend on program state, it may +// be evaluated at any point--though not necessarily at multiple +// points (consider new, make). func callsPureBuiltin(info *types.Info, call *ast.CallExpr) bool { if id, ok := astutil.Unparen(call.Fun).(*ast.Ident); ok { if b, ok := info.ObjectOf(id).(*types.Builtin); ok { @@ -1666,7 +1794,9 @@ func importedPkgName(info *types.Info, imp *ast.ImportSpec) (*types.PkgName, boo } func isPkgLevel(obj types.Object) bool { - // TODO(adonovan): why not simply: obj.Parent() == obj.Pkg().Scope()? + // TODO(adonovan): consider using the simpler obj.Parent() == + // obj.Pkg().Scope() instead. But be sure to test carefully + // with instantiations of generics. return obj.Pkg().Scope().Lookup(obj.Name()) == obj } diff --git a/internal/refactor/inline/inline_test.go b/internal/refactor/inline/inline_test.go index 1371afd9d05..5c31bfc7ce5 100644 --- a/internal/refactor/inline/inline_test.go +++ b/internal/refactor/inline/inline_test.go @@ -448,35 +448,39 @@ func TestTable(t *testing.T) { }`, }, { - "Binding declaration (x eliminated).", + "Binding declaration (x, z eliminated).", `func f(w, x, y any, z int) { println(w, y, z) }; func g(int) int`, `func _() { f(g(0), g(1), g(2), g(3)) }`, `func _() { { - var ( - w, _, y any = g(0), g(1), g(2) - z int = g(3) - ) - println(w, y, z) + var w, _, y any = g(0), g(1), g(2) + println(w, y, g(3)) } }`, }, { - "Binding decl in reduction of stmt-context call to { return exprs }", + "Reduction of stmt-context call to { return exprs }, with substitution", `func f(ch chan int) int { return <-ch }; func g() chan int`, `func _() { f(g()) }`, + `func _() { <-g() }`, + }, + { + // Same again, with callee effects: + "Binding decl in reduction of stmt-context call to { return exprs }", + `func f(x int) int { return <-h(g(2), x) }; func g(int) int; func h(int, int) chan int`, + `func _() { f(g(1)) }`, `func _() { { - var ch chan int = g() - <-ch + var x int = g(1) + <-h(g(2), x) } }`, }, { "No binding decl due to shadowing of int", - `func f(int, y any, z int) { defer println(int, y, z) }; func g() int`, - `func _() { f(g(), g(), g()) }`, - `func _() { func(int, y any, z int) { defer println(int, y, z) }(g(), g(), g()) } + `func f(int, y any, z int) { defer println(int, y, z) }; func g(int) int`, + `func _() { f(g(1), g(2), g(3)) }`, + `func _() { func(int, y any) { defer println(int, y, g(3)) }(g(1), g(2)) } `, }, // Embedded fields: @@ -492,20 +496,11 @@ func TestTable(t *testing.T) { `func _(v V) { v.f() }`, `func _(v V) { print(*v.U.T) }`, }, - // TODO(adonovan): due to former unsoundness in pure(), - // the previous outputs of two tests below used to be neater. - // A followup analysis (strict effects) will restore tidiness. { "Embedded fields in x.f method selection (implicit &).", `type ( T int; U struct{T}; V struct {U} ); func (t *T) f() { print(t) }`, `func _(v V) { v.f() }`, - // was `func _(v V) { print(&v.U.T) }`, - `func _(v V) { - { - var t *T = &v.U.T - print(t) - } -}`, + `func _(v V) { print(&v.U.T) }`, }, // Now the same tests again with T.f(recv). { @@ -524,14 +519,61 @@ func TestTable(t *testing.T) { "Embedded fields in (*T).f method selection.", `type ( T int; U struct{T}; V struct {U} ); func (t *T) f() { print(t) }`, `func _(v V) { (*V).f(&v) }`, - // was `func _(v V) { print(&(&v).U.T) }`, - `func _(v V) { + `func _(v V) { print(&(&v).U.T) }`, + }, + // Parameter effect ordering. + { + "Arguments have effects, but parameters are evaluated in order.", + `func f(a, b, c int) { print(a, b, c) }; func g(int) int`, + `func _() { f(g(1), g(2), g(3)) }`, + `func _() { print(g(1), g(2), g(3)) }`, + }, + { + "Arguments have effects, and parameters are evaluated out of order.", + `func f(a, b, c int) { print(a, c, b) }; func g(int) int`, + `func _() { f(g(1), g(2), g(3)) }`, + `func _() { + { + var a, b int = g(1), g(2) + print(a, g(3), b) + } +}`, + }, + { + "Pure arguments may commute with argument that have effects.", + `func f(a, b, c int) { print(a, c, b) }; func g(int) int`, + `func _() { f(g(1), 2, g(3)) }`, + `func _() { print(g(1), g(3), 2) }`, + }, + { + "Impure arguments may commute with each other.", + `func f(a, b, c, d int) { print(a, c, b, d) }; func g(int) int; var x, y int`, + `func _() { f(g(1), x, y, g(2)) }`, + `func _() { print(g(1), y, x, g(2)) }`, + }, + { + "Impure arguments do not commute with arguments that have effects (1)", + `func f(a, b, c, d int) { print(a, c, b, d) }; func g(int) int; var x, y int`, + `func _() { f(g(1), g(2), y, g(3)) }`, + `func _() { { - var t *T = &(&v).U.T - print(t) + var a, b int = g(1), g(2) + print(a, y, b, g(3)) } }`, }, + { + "Impure arguments do not commute with those that have effects (2).", + `func f(a, b, c, d int) { print(a, c, b, d) }; func g(int) int; var x, y int`, + `func _() { f(g(1), y, g(2), g(3)) }`, + `func _() { + { + var a, b int = g(1), y + print(a, g(2), b, g(3)) + } +}`, + }, + // Treatment of named result vars across strategies: { "Stmt-context call to {return g()} that mentions named result.", @@ -544,6 +586,21 @@ func TestTable(t *testing.T) { } }`, }, + { + "Ditto, with binding decl again.", + `func f(y string) (x int) { return x+x+len(y+y) }`, + `func _() { f("") }`, + `func _() { + { + var ( + y string = "" + x int + ) + _ = x + x + len(y+y) + } +}`, + }, + { "Ditto, with binding decl (due to repeated y refs).", `func f(y string) (x string) { return x+y+y }`, @@ -570,16 +627,36 @@ func TestTable(t *testing.T) { }`, }, { - "Ditto, with binding decl again.", - `func f(y string) (x int) { return x+x+len(y+y) }`, - `func _() { f("") }`, + "Callee effects commute with pure arguments.", + `func f(a, b, c int) { print(a, c, recover().(int), b) }; func g(int) int`, + `func _() { f(g(1), 2, g(3)) }`, + `func _() { print(g(1), g(3), recover().(int), 2) }`, + }, + { + "Callee reads may commute with impure arguments.", + `func f(a, b int) { print(a, x, b) }; func g(int) int; var x, y int`, + `func _() { f(g(1), y) }`, + `func _() { print(g(1), x, y) }`, + }, + { + "All impure parameters preceding a read hazard must be kept.", + `func f(a, b, c int) { print(a, b, recover().(int), c) }; var x, y, z int`, + `func _() { f(x, y, z) }`, `func _() { { - var ( - y string = "" - x int - ) - _ = x + x + len(y+y) + var c int = z + print(x, y, recover().(int), c) + } +}`, + }, + { + "All parameters preceding a write hazard must be kept.", + `func f(a, b, c int) { print(a, b, recover().(int), c) }; func g(int) int; var x, y, z int`, + `func _() { f(x, y, g(0)) }`, + `func _() { + { + var a, b, c int = x, y, g(0) + print(a, b, recover().(int), c) } }`, }, @@ -607,6 +684,76 @@ func TestTable(t *testing.T) { `func _() { f("") }`, `func _() { func(y string) (x y) { return x + x + len(y+y) }("") }`, }, + { + "[W1 R0 W2 W4 R3] -- test case for second iteration of effect loop", + `func f(a, b, c, d, e int) { print(b, a, c, e, d) }; func g(int) int; var x, y int`, + `func _() { f(x, g(1), g(2), y, g(3)) }`, + `func _() { + { + var a, b, c, d int = x, g(1), g(2), y + print(b, a, c, g(3), d) + } +}`, + }, + { + // In this example, the set() call is rejected as a substitution + // candidate due to a shadowing conflict (x). This must entail that the + // selection x.y (R) is also rejected, because it is lower numbered. + // + // Incidentally this program (which panics when executed) illustrates + // that although effects occur left-to-right, read operations such + // as x.y are not ordered wrt writes, depending on the compiler. + // Changing x.y to identity(x).y forces the ordering and avoids the panic. + "Hazards with args already rejected (e.g. due to shadowing) are detected too.", + `func f(x, y int) int { return x + y }; func set[T any](ptr *T, old, new T) int { println(old); *ptr = new; return 0; }`, + `func _() { x := new(struct{ y int }); f(x.y, set(&x, x, nil)) }`, + `func _() { + x := new(struct{ y int }) + { + var x, y int = x.y, set(&x, x, nil) + _ = x + y + } +}`, + }, + { + // Rejection of a later parameter for reasons other than callee + // effects (e.g. escape) may create hazards with lower-numbered + // parameters that require them to be rejected too. + "Hazards with already eliminated parameters (variant)", + `func f(x, y int) { _ = &y }; func g(int) int`, + `func _() { f(g(1), g(2)) }`, + `func _() { + { + var _, y int = g(1), g(2) + _ = &y + } +}`, + }, + { + // In this case g(2) is rejected for substitution because it is + // unreferenced but has effects, so parameter x must also be rejected + // so that its argument v can be evaluated earlier in the binding decl. + "Hazards with already eliminated parameters (unreferenced fx variant)", + `func f(x, y int) { _ = x }; func g(int) int; var v int`, + `func _() { f(v, g(2)) }`, + `func _() { + { + var x, _ int = v, g(2) + _ = x + } +}`, + }, + { + "Non-duplicable arguments are not substituted even if pure.", + `func f(s string, i int) { print(s, s, i, i) }`, + `func _() { f("", 0) }`, + `func _() { + { + var s string = "" + print(s, s, 0, 0) + } +}`, + }, // TODO(adonovan): improve coverage of the cross // product of each strategy with the checklist of diff --git a/internal/refactor/inline/testdata/basic-err.txtar b/internal/refactor/inline/testdata/basic-err.txtar index c811fe47101..c289e9bb544 100644 --- a/internal/refactor/inline/testdata/basic-err.txtar +++ b/internal/refactor/inline/testdata/basic-err.txtar @@ -1,12 +1,6 @@ Test of inlining a function that references err.Error, which is often a special case because it has no position. -Previously the output was - var _ = (io.EOF.Error()) -but this relied on a bug in pure(). -A follow-up analysis of callee effect ordering -will re-enable this "style optimization". - -- go.mod -- module testdata go 1.12 @@ -25,6 +19,6 @@ package a import "io" -var _ = func(err error) string { return err.Error() }(io.EOF) //@ inline(re"getError", getError) +var _ = (io.EOF.Error()) //@ inline(re"getError", getError) func getError(err error) string { return err.Error() } diff --git a/internal/refactor/inline/util.go b/internal/refactor/inline/util.go index 7ca5b084eec..82be0fb8ac6 100644 --- a/internal/refactor/inline/util.go +++ b/internal/refactor/inline/util.go @@ -19,6 +19,27 @@ func is[T any](x any) bool { return ok } +// TODO(adonovan): use go1.21's slices.Clone. +func clone[T any](slice []T) []T { return append([]T{}, slice...) } + +// TODO(adonovan): use go1.21's slices.Index. +func index[T comparable](slice []T, x T) int { + for i, elem := range slice { + if elem == x { + return i + } + } + return -1 +} + +func btoi(b bool) int { + if b { + return 1 + } else { + return 0 + } +} + func offsetOf(fset *token.FileSet, pos token.Pos) int { return fset.PositionFor(pos, false).Offset } From 169105a907f344a76994b4b42a5d6ecf56920c89 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Mon, 25 Sep 2023 14:21:50 -0400 Subject: [PATCH 145/178] internal/refactor/inline: insert conversions during substitution Previously, we rejected arguments as substitution candidates if their type (or its types.Default) did not exactly match the parameter type. Now, we allow those substitutions to proceed but we wrap the argument in an explicit conversion so that the meaning of the program doesn't change. We had initially been reluctant to do this out of concern that it could cause a single conversion during argument passing to be moved into a loop, where its dynamic cost might be multiplied; for example, a string->any conversion allocates memory. But we decided that this modest dynamic cost is acceptable so long as the meaning does not change. Also, this change includes a fix for golang/go#63193, in which the type inferred for the argument expression in func(int16){}(1) is not "untyped int" but "int16". In other words, the type checker has incorporated knowledge of the parameter type. It is unsafe to assume that the expression 1 will have the same type in a different context, so we recompute the type of each argument expression in a neutral context (using CheckExpr). Fixes golang/go#63193 Change-Id: I0fd072cc7611d113af77193f6d3362268d9af158 Reviewed-on: https://go-review.googlesource.com/c/tools/+/530975 Reviewed-by: Robert Findley LUCI-TryBot-Result: Go LUCI Auto-Submit: Alan Donovan --- internal/refactor/inline/inline.go | 100 ++++++++++++++++++------ internal/refactor/inline/inline_test.go | 75 ++++++++++++++++-- 2 files changed, 145 insertions(+), 30 deletions(-) diff --git a/internal/refactor/inline/inline.go b/internal/refactor/inline/inline.go index 918cb3acfeb..78677e30364 100644 --- a/internal/refactor/inline/inline.go +++ b/internal/refactor/inline/inline.go @@ -441,14 +441,29 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu sig := calleeSymbol.Type().(*types.Signature) if sig.Recv() != nil { params = append(params, ¶meter{ - obj: sig.Recv(), - info: callee.Params[0], + obj: sig.Recv(), + fieldType: calleeDecl.Recv.List[0].Type, + info: callee.Params[0], }) } + + // Flatten the list of syntactic types. + var types []ast.Expr + for _, field := range calleeDecl.Type.Params.List { + if field.Names == nil { + types = append(types, field.Type) + } else { + for range field.Names { + types = append(types, field.Type) + } + } + } + for i := 0; i < sig.Params().Len(); i++ { params = append(params, ¶meter{ - obj: sig.Params().At(i), - info: callee.Params[len(params)], + obj: sig.Params().At(i), + fieldType: types[i], + info: callee.Params[len(params)], }) } @@ -511,9 +526,9 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // Log effective arguments. for i, arg := range args { - logf("arg #%d: %s pure=%t effects=%t duplicable=%t free=%v", + logf("arg #%d: %s pure=%t effects=%t duplicable=%t free=%v type=%v", i, debugFormatNode(caller.Fset, arg.expr), - arg.pure, arg.effects, arg.duplicable, arg.freevars) + arg.pure, arg.effects, arg.duplicable, arg.freevars, arg.typ) } // Note: computation below should be expressed in terms of @@ -992,13 +1007,44 @@ func arguments(caller *Caller, calleeDecl *ast.FuncDecl, assign1 func(*types.Var freevars: freeVars(caller.Info, expr), }) } + + // Re-typecheck each constant argument expression in a neutral context. + // + // In a call such as func(int16){}(1), the type checker infers + // the type "int16", not "untyped int", for the argument 1, + // because it has incorporated information from the left-hand + // side of the assignment implicit in parameter passing, but + // of course in a different context, the expression 1 may have + // a different type. + // + // So, we must use CheckExpr to recompute the type of the + // argument in a neutral context to find its inherent type. + // (This is arguably a bug in go/types, but I'm pretty certain + // I requested it be this way long ago... -adonovan) + // + // This is only needed for constants. Other implicit + // assignment conversions, such as unnamed-to-named struct or + // chan to <-chan, do not result in the type-checker imposing + // the LHS type on the RHS value. + for _, arg := range args { + if caller.Info.Types[arg.expr].Value == nil { + continue // not a constant + } + info := &types.Info{Types: make(map[ast.Expr]types.TypeAndValue)} + if err := types.CheckExpr(caller.Fset, caller.Types, caller.Call.Pos(), arg.expr, info); err != nil { + return nil, err + } + arg.typ = info.TypeOf(arg.expr) + } + return args, nil } type parameter struct { - obj *types.Var // parameter var from caller's signature - info *paramInfo // information from AnalyzeCallee - variadic bool // (final) parameter is unsimplified ...T + obj *types.Var // parameter var from caller's signature + fieldType ast.Expr // syntax of type, from calleeDecl.Type.{Recv,Params} + info *paramInfo // information from AnalyzeCallee + variadic bool // (final) parameter is unsimplified ...T } // substitute implements parameter elimination by substitution. @@ -1079,20 +1125,6 @@ next: } } - // Check that eliminating the parameter wouldn't materially - // change the type. - // - // (We don't simply wrap the argument in an explicit conversion - // to the parameter type because that could increase allocation - // in the number of (e.g.) string -> any conversions. - // Even when Uses = 1, the sole ref might be in a loop or lambda that - // is multiply executed.) - if len(param.info.Refs) > 0 && !trivialConversion(args[i].typ, params[i].obj) { - logf("keeping param %q: argument passing converts %s to type %s", - param.info.Name, args[i].typ, params[i].obj.Type()) - continue // implicit conversion is significant - } - // Check for shadowing. // // Consider inlining a call f(z, 1) to @@ -1119,8 +1151,30 @@ next: // The remaining candidates are safe to substitute. for i, param := range params { if arg := args[i]; arg.substitutable { + + // Wrap the argument in an explicit conversion if + // substitution might materially change its type. + // (We already did the necessary shadowing check + // on the parameter type syntax.) + // + // This is only needed for substituted arguments. All + // other arguments are given explicit types in either + // a binding decl or when using the literalization + // strategy. + if len(param.info.Refs) > 0 && !trivialConversion(args[i].typ, params[i].obj) { + arg.expr = &ast.CallExpr{ + Fun: params[i].fieldType, // formatter adds parens as needed + Args: []ast.Expr{arg.expr}, + } + logf("param %q: adding explicit %s -> %s conversion around argument", + param.info.Name, args[i].typ, params[i].obj.Type()) + } + // It is safe to substitute param and replace it with arg. // The formatter introduces parens as needed for precedence. + // + // Because arg.expr belongs to the caller, + // we clone it before splicing it into the callee tree. logf("replacing parameter %q by argument %q", param.info.Name, debugFormatNode(caller.Fset, arg.expr)) for _, ref := range param.info.Refs { diff --git a/internal/refactor/inline/inline_test.go b/internal/refactor/inline/inline_test.go index 5c31bfc7ce5..c9cffb61a31 100644 --- a/internal/refactor/inline/inline_test.go +++ b/internal/refactor/inline/inline_test.go @@ -410,7 +410,7 @@ func TestTable(t *testing.T) { "Variadic elimination (literalization).", `func f(x any, rest ...any) { defer println(x, rest) }`, // defer => literalization `func _() { f(1, 2, 3) }`, - `func _() { func(x any) { defer println(x, []any{2, 3}) }(1) }`, + `func _() { func() { defer println(any(1), []any{2, 3}) }() }`, }, { "Variadic elimination (reduction).", @@ -448,13 +448,13 @@ func TestTable(t *testing.T) { }`, }, { - "Binding declaration (x, z eliminated).", + "Binding declaration (x, y, z eliminated).", `func f(w, x, y any, z int) { println(w, y, z) }; func g(int) int`, `func _() { f(g(0), g(1), g(2), g(3)) }`, `func _() { { - var w, _, y any = g(0), g(1), g(2) - println(w, y, g(3)) + var w, _ any = g(0), g(1) + println(w, any(g(2)), g(3)) } }`, }, @@ -477,11 +477,16 @@ func TestTable(t *testing.T) { }`, }, { - "No binding decl due to shadowing of int", + "Defer f() evaluates f() before unknown effects", `func f(int, y any, z int) { defer println(int, y, z) }; func g(int) int`, `func _() { f(g(1), g(2), g(3)) }`, - `func _() { func(int, y any) { defer println(int, y, g(3)) }(g(1), g(2)) } -`, + `func _() { func() { defer println(any(g(1)), any(g(2)), g(3)) }() }`, + }, + { + "No binding decl due to shadowing of int", + `func f(int, y any, z int) { defer g(0); println(int, y, z) }; func g(int) int`, + `func _() { f(g(1), g(2), g(3)) }`, + `func _() { func(int, y any, z int) { defer g(0); println(int, y, z) }(g(1), g(2), g(3)) }`, }, // Embedded fields: { @@ -754,6 +759,62 @@ func TestTable(t *testing.T) { } }`, }, + { + "Substitution preserves argument type (#63193).", + `func f(x int16) { y := x; _ = (*int16)(&y) }`, + `func _() { f(1) }`, + `func _() { + { + y := int16(1) + _ = (*int16)(&y) + } +}`, + }, + { + "Same, with non-constant (unnamed to named struct) conversion.", + `func f(x T) { y := x; _ = (*T)(&y) }; type T struct{}`, + `func _() { f(struct{}{}) }`, + `func _() { + { + y := T(struct{}{}) + _ = (*T)(&y) + } +}`, + }, + { + "Same, with non-constant (chan to <-chan) conversion.", + `func f(x T) { y := x; _ = (*T)(&y) }; type T = <-chan int; var ch chan int`, + `func _() { f(ch) }`, + `func _() { + { + y := T(ch) + _ = (*T)(&y) + } +}`, + }, + { + "Same, with untyped nil to typed nil conversion.", + `func f(x *int) { y := x; _ = (**int)(&y) }`, + `func _() { f(nil) }`, + `func _() { + { + y := (*int)(nil) + _ = (**int)(&y) + } +}`, + }, + { + "Conversion of untyped int to named type is made explicit.", + `type T int; func (x T) f() { x.g() }; func (T) g() {}`, + `func _() { T.f(1) }`, + `func _() { T(1).g() }`, + }, + { + "Check for shadowing error on type used in the conversion.", + `func f(x T) { _ = &x == (*T)(nil) }; type T int16`, + `func _() { type T bool; f(1) }`, + `error: T.*shadowed.*by.*type`, + }, // TODO(adonovan): improve coverage of the cross // product of each strategy with the checklist of From 9d2d0e85371fc00fe5c517db71d70d068d7b679b Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Tue, 26 Sep 2023 18:06:17 -0400 Subject: [PATCH 146/178] gopls: set a context deadline after minimal completion results Previously we avoided using context cancellation for implementing completion budget, because it can result in accidentally omitted results (because e.g. constructing of completion items was canceled). But once we have a minimum required set of results, it is OK to use context cancellation to more strictly enforce the completion budget. For golang/go#62665 Change-Id: Ibd10e1f70a396567f8e095bd6824c637cf973518 Reviewed-on: https://go-review.googlesource.com/c/tools/+/530599 LUCI-TryBot-Result: Go LUCI Reviewed-by: Peter Weinberger --- gopls/internal/lsp/source/completion/completion.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/gopls/internal/lsp/source/completion/completion.go b/gopls/internal/lsp/source/completion/completion.go index 594f234f4e5..6e12cb5d99e 100644 --- a/gopls/internal/lsp/source/completion/completion.go +++ b/gopls/internal/lsp/source/completion/completion.go @@ -596,6 +596,17 @@ func Completion(ctx context.Context, snapshot source.Snapshot, fh source.FileHan // Deep search collected candidates and their members for more candidates. c.deepSearch(ctx, 1, deadline) + // At this point we have a sufficiently complete set of results, and want to + // return as close to the completion budget as possible. Previously, we + // avoided cancelling the context because it could result in partial results + // for e.g. struct fields. At this point, we have a minimal valid set of + // candidates, and so truncating due to context cancellation is acceptable. + if c.opts.budget > 0 { + timeoutDuration := time.Until(c.startTime.Add(c.opts.budget)) + ctx, cancel = context.WithTimeout(ctx, timeoutDuration) + defer cancel() + } + for _, callback := range c.completionCallbacks { if deadline == nil || time.Now().Before(*deadline) { if err := c.snapshot.RunProcessEnvFunc(ctx, callback); err != nil { From 486787ef462965ccf87fe122fd2bfaa422defab7 Mon Sep 17 00:00:00 2001 From: Eric Loh Date: Wed, 23 Aug 2023 23:21:42 -0400 Subject: [PATCH 147/178] gopls/internal/lsp/source: Add ui.complete.completeFunctionCalls toggle Add new completion option that controls if function completions should invoke function call with open and close brackets. This is similar to Typescript's typescript.suggest.completeFunctionCalls. Option defaults to true to keep existing behavior. Fixes golang/go#58022 Change-Id: Ie3f01b4c0578c4b38afaa8c8ea02d21cd106a031 Reviewed-on: https://go-review.googlesource.com/c/tools/+/522181 Reviewed-by: Robert Findley LUCI-TryBot-Result: Go LUCI Reviewed-by: Than McIntosh gopls-CI: kokoro --- gopls/doc/settings.md | 10 +++++ gopls/internal/lsp/source/api_json.go | 7 ++++ .../lsp/source/completion/completion.go | 38 ++++++++++--------- .../internal/lsp/source/completion/snippet.go | 5 +++ gopls/internal/lsp/source/options.go | 10 +++++ 5 files changed, 52 insertions(+), 18 deletions(-) diff --git a/gopls/doc/settings.md b/gopls/doc/settings.md index 5bc692ee05b..eca3ee803d2 100644 --- a/gopls/doc/settings.md +++ b/gopls/doc/settings.md @@ -269,6 +269,16 @@ such as "someSlice.sort!". Default: `true`. +##### **completeFunctionCalls** *bool* + +completeFunctionCalls enables function call completion. + +When completing a statement, or when a function return type matches the +expected of the expression being completed, completion may suggest call +expressions (i.e. may include parentheses). + +Default: `true`. + #### Diagnostic ##### **analyses** *map[string]bool* diff --git a/gopls/internal/lsp/source/api_json.go b/gopls/internal/lsp/source/api_json.go index 45ebabb56dd..0fd6b07c54b 100644 --- a/gopls/internal/lsp/source/api_json.go +++ b/gopls/internal/lsp/source/api_json.go @@ -146,6 +146,13 @@ var GeneratedAPIJSON = &APIJSON{ Status: "experimental", Hierarchy: "ui.completion", }, + { + Name: "completeFunctionCalls", + Type: "bool", + Doc: "completeFunctionCalls enables function call completion.\n\nWhen completing a statement, or when a function return type matches the\nexpected of the expression being completed, completion may suggest call\nexpressions (i.e. may include parentheses).\n", + Default: "true", + Hierarchy: "ui.completion", + }, { Name: "importShortcut", Type: "enum", diff --git a/gopls/internal/lsp/source/completion/completion.go b/gopls/internal/lsp/source/completion/completion.go index 6e12cb5d99e..b67b6772f79 100644 --- a/gopls/internal/lsp/source/completion/completion.go +++ b/gopls/internal/lsp/source/completion/completion.go @@ -104,15 +104,16 @@ type CompletionItem struct { // completionOptions holds completion specific configuration. type completionOptions struct { - unimported bool - documentation bool - fullDocumentation bool - placeholders bool - literal bool - snippets bool - postfix bool - matcher source.Matcher - budget time.Duration + unimported bool + documentation bool + fullDocumentation bool + placeholders bool + literal bool + snippets bool + postfix bool + matcher source.Matcher + budget time.Duration + completeFunctionCalls bool } // Snippet is a convenience returns the snippet if available, otherwise @@ -543,15 +544,16 @@ func Completion(ctx context.Context, snapshot source.Snapshot, fh source.FileHan enabled: opts.DeepCompletion, }, opts: &completionOptions{ - matcher: opts.Matcher, - unimported: opts.CompleteUnimported, - documentation: opts.CompletionDocumentation && opts.HoverKind != source.NoDocumentation, - fullDocumentation: opts.HoverKind == source.FullDocumentation, - placeholders: opts.UsePlaceholders, - literal: opts.LiteralCompletions && opts.InsertTextFormat == protocol.SnippetTextFormat, - budget: opts.CompletionBudget, - snippets: opts.InsertTextFormat == protocol.SnippetTextFormat, - postfix: opts.ExperimentalPostfixCompletions, + matcher: opts.Matcher, + unimported: opts.CompleteUnimported, + documentation: opts.CompletionDocumentation && opts.HoverKind != source.NoDocumentation, + fullDocumentation: opts.HoverKind == source.FullDocumentation, + placeholders: opts.UsePlaceholders, + literal: opts.LiteralCompletions && opts.InsertTextFormat == protocol.SnippetTextFormat, + budget: opts.CompletionBudget, + snippets: opts.InsertTextFormat == protocol.SnippetTextFormat, + postfix: opts.ExperimentalPostfixCompletions, + completeFunctionCalls: opts.CompleteFunctionCalls, }, // default to a matcher that always matches matcher: prefixMatcher(""), diff --git a/gopls/internal/lsp/source/completion/snippet.go b/gopls/internal/lsp/source/completion/snippet.go index f4ea767e9dc..2be485f6d85 100644 --- a/gopls/internal/lsp/source/completion/snippet.go +++ b/gopls/internal/lsp/source/completion/snippet.go @@ -51,6 +51,11 @@ func (c *completer) structFieldSnippet(cand candidate, detail string, snip *snip // functionCallSnippet calculates the snippet for function calls. func (c *completer) functionCallSnippet(name string, tparams, params []string, snip *snippet.Builder) { + if !c.opts.completeFunctionCalls { + snip.WriteText(name) + return + } + // If there is no suffix then we need to reuse existing call parens // "()" if present. If there is an identifier suffix then we always // need to include "()" since we don't overwrite the suffix. diff --git a/gopls/internal/lsp/source/options.go b/gopls/internal/lsp/source/options.go index 0d610a42e30..ec544fc31b1 100644 --- a/gopls/internal/lsp/source/options.go +++ b/gopls/internal/lsp/source/options.go @@ -155,6 +155,7 @@ func DefaultOptions(overrides ...func(*Options)) *Options { Matcher: Fuzzy, CompletionBudget: 100 * time.Millisecond, ExperimentalPostfixCompletions: true, + CompleteFunctionCalls: true, }, Codelenses: map[string]bool{ string(command.Generate): true, @@ -388,6 +389,13 @@ type CompletionOptions struct { // ExperimentalPostfixCompletions enables artificial method snippets // such as "someSlice.sort!". ExperimentalPostfixCompletions bool `status:"experimental"` + + // CompleteFunctionCalls enables function call completion. + // + // When completing a statement, or when a function return type matches the + // expected of the expression being completed, completion may suggest call + // expressions (i.e. may include parentheses). + CompleteFunctionCalls bool } type DocumentationOptions struct { @@ -1186,6 +1194,8 @@ func (o *Options) set(name string, value interface{}, seen map[string]struct{}) " rebuild gopls with a more recent version of Go", result.Name, runtime.Version()) } } + case "completeFunctionCalls": + result.setBool(&o.CompleteFunctionCalls) case "semanticTokens": result.setBool(&o.SemanticTokens) From 08bdfeca001b5bbebfb77fd58afb03d477768464 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Wed, 27 Sep 2023 10:43:24 -0400 Subject: [PATCH 148/178] internal/refactor/inline: split up the big table It was prone to merge conflicts since every CL appends items to the table and they all have a similar prefix, which confuses diff. The new smaller tables are scoped by TestXYZ functions that impose a modicum of organizational sanity. No logical changes to test. Change-Id: I5f09477b915d3c62443c770fa87881ac0ad26c7a Reviewed-on: https://go-review.googlesource.com/c/tools/+/530600 LUCI-TryBot-Result: Go LUCI Reviewed-by: Robert Findley --- internal/refactor/inline/inline_test.go | 247 ++++++++++++++---------- 1 file changed, 148 insertions(+), 99 deletions(-) diff --git a/internal/refactor/inline/inline_test.go b/internal/refactor/inline/inline_test.go index c9cffb61a31..1ec72936db4 100644 --- a/internal/refactor/inline/inline_test.go +++ b/internal/refactor/inline/inline_test.go @@ -317,18 +317,27 @@ func findFuncByPosition(pkg *packages.Package, posn token.Position) (*ast.FuncDe return nil, fmt.Errorf("can't find FuncDecl at %v in package %q", posn, pkg.PkgPath) } -// TestTable is a table driven test, enabling more compact expression -// of single-package test cases than is possible with the txtar notation. -func TestTable(t *testing.T) { - // Each callee must declare a function or method named f, - // and each caller must call it. - const funcName = "f" +// Each callee must declare a function or method named f, +// and each caller must call it. +const funcName = "f" - var tests = []struct { - descr string - callee, caller string // Go source files (sans package decl) of caller, callee - want string // expected new portion of caller file, or "error: regexp" - }{ +// A testcase is an item in a table-driven test. +// +// The table-driven tests are less flexible, but enable more compact +// expression of single-package test cases than is possible with the +// txtar notation. +// +// TODO(adonovan): improve coverage of the cross product of each +// strategy with the checklist of concerns enumerated in the package +// doc comment. +type testcase struct { + descr string + callee, caller string // Go source files (sans package decl) of caller, callee + want string // expected new portion of caller file, or "error: regexp" +} + +func TestErrors(t *testing.T) { + runTests(t, []testcase{ { "Generic functions are not yet supported.", `func f[T any](x T) T { return x }`, @@ -341,6 +350,11 @@ func TestTable(t *testing.T) { `var _ = G[int]{}.f(0)`, `error: type parameters are not yet supported`, }, + }) +} + +func TestBasics(t *testing.T) { + runTests(t, []testcase{ { "Basic", `func f(x int) int { return x }`, @@ -359,6 +373,22 @@ func TestTable(t *testing.T) { `func _() { f(1, recover().(int), 3) }`, `func _() { _ = recover().(int) }`, }, + { + "Non-duplicable arguments are not substituted even if pure.", + `func f(s string, i int) { print(s, s, i, i) }`, + `func _() { f("", 0) }`, + `func _() { + { + var s string = "" + print(s, s, 0, 0) + } +}`, + }, + }) +} + +func TestTailCallStrategy(t *testing.T) { + runTests(t, []testcase{ { "Tail call.", `func f() int { return 1 }`, @@ -377,6 +407,11 @@ func TestTable(t *testing.T) { `func _() { f() }`, `func _() { func() { defer f(); println() }() }`, }, + }) +} + +func TestSpreadCalls(t *testing.T) { + runTests(t, []testcase{ { "Edge case: cannot literalize spread method call.", `type I int @@ -388,6 +423,11 @@ func TestTable(t *testing.T) { `func _() I { return recover().(I).f(g()) }`, `error: can't yet inline spread call to method`, }, + }) +} + +func TestVariadic(t *testing.T) { + runTests(t, []testcase{ { "Variadic cancellation (basic).", `func f(args ...any) { defer f(&args); println(args) }`, @@ -436,6 +476,11 @@ func TestTable(t *testing.T) { `func _() { f(g()) }`, `func _() { func(x, y int, rest ...int) { println(x, y, rest) }(g()) }`, }, + }) +} + +func TestEmbeddedFieldsJJJ(t *testing.T) { + runTests(t, []testcase{ { "IncDec counts as assignment.", `func f(x int) { x++ }`, @@ -488,7 +533,11 @@ func TestTable(t *testing.T) { `func _() { f(g(1), g(2), g(3)) }`, `func _() { func(int, y any, z int) { defer g(0); println(int, y, z) }(g(1), g(2), g(3)) }`, }, - // Embedded fields: + }) +} + +func TestEmbeddedFields(t *testing.T) { + runTests(t, []testcase{ { "Embedded fields in x.f method selection (direct).", `type T int; func (t T) f() { print(t) }; type U struct{ T }`, @@ -526,7 +575,11 @@ func TestTable(t *testing.T) { `func _(v V) { (*V).f(&v) }`, `func _(v V) { print(&(&v).U.T) }`, }, - // Parameter effect ordering. + }) +} + +func TestSubstitutionPreservesArgumentEffectOrder(t *testing.T) { + runTests(t, []testcase{ { "Arguments have effects, but parameters are evaluated in order.", `func f(a, b, c int) { print(a, b, c) }; func g(int) int`, @@ -578,59 +631,6 @@ func TestTable(t *testing.T) { } }`, }, - - // Treatment of named result vars across strategies: - { - "Stmt-context call to {return g()} that mentions named result.", - `func f() (x int) { return g(x) }; func g(int) int`, - `func _() { f() }`, - `func _() { - { - var x int - g(x) - } -}`, - }, - { - "Ditto, with binding decl again.", - `func f(y string) (x int) { return x+x+len(y+y) }`, - `func _() { f("") }`, - `func _() { - { - var ( - y string = "" - x int - ) - _ = x + x + len(y+y) - } -}`, - }, - - { - "Ditto, with binding decl (due to repeated y refs).", - `func f(y string) (x string) { return x+y+y }`, - `func _() { f("") }`, - `func _() { - { - var ( - y string = "" - x string - ) - _ = x + y + y - } -}`, - }, - { - "Stmt-context call to {return binary} that mentions named result.", - `func f() (x int) { return x+x }`, - `func _() { f() }`, - `func _() { - { - var x int - _ = x + x - } -}`, - }, { "Callee effects commute with pure arguments.", `func f(a, b, c int) { print(a, c, recover().(int), b) }; func g(int) int`, @@ -665,30 +665,6 @@ func TestTable(t *testing.T) { } }`, }, - { - "Tail call to {return expr} that mentions named result.", - `func f() (x int) { return x }`, - `func _() int { return f() }`, - `func _() int { return func() (x int) { return x }() }`, - }, - { - "Tail call to {return} that implicitly reads named result.", - `func f() (x int) { return }`, - `func _() int { return f() }`, - `func _() int { return func() (x int) { return }() }`, - }, - { - "Spread-context call to {return expr} that mentions named result.", - `func f() (x, y int) { return x, y }`, - `func _() { var _, _ = f() }`, - `func _() { var _, _ = func() (x, y int) { return x, y }() }`, - }, - { - "Shadowing in binding decl for named results => literalization.", - `func f(y string) (x y) { return x+x+len(y+y) }; type y = int`, - `func _() { f("") }`, - `func _() { func(y string) (x y) { return x + x + len(y+y) }("") }`, - }, { "[W1 R0 W2 W4 R3] -- test case for second iteration of effect loop", `func f(a, b, c, d, e int) { print(b, a, c, e, d) }; func g(int) int; var x, y int`, @@ -748,17 +724,91 @@ func TestTable(t *testing.T) { } }`, }, + }) +} + +func TestNamedResultVars(t *testing.T) { + runTests(t, []testcase{ { - "Non-duplicable arguments are not substituted even if pure.", - `func f(s string, i int) { print(s, s, i, i) }`, - `func _() { f("", 0) }`, + "Stmt-context call to {return g()} that mentions named result.", + `func f() (x int) { return g(x) }; func g(int) int`, + `func _() { f() }`, `func _() { { - var s string = "" - print(s, s, 0, 0) + var x int + g(x) + } +}`, + }, + { + "Ditto, with binding decl again.", + `func f(y string) (x int) { return x+x+len(y+y) }`, + `func _() { f("") }`, + `func _() { + { + var ( + y string = "" + x int + ) + _ = x + x + len(y+y) + } +}`, + }, + + { + "Ditto, with binding decl (due to repeated y refs).", + `func f(y string) (x string) { return x+y+y }`, + `func _() { f("") }`, + `func _() { + { + var ( + y string = "" + x string + ) + _ = x + y + y } }`, }, + { + "Stmt-context call to {return binary} that mentions named result.", + `func f() (x int) { return x+x }`, + `func _() { f() }`, + `func _() { + { + var x int + _ = x + x + } +}`, + }, + { + "Tail call to {return expr} that mentions named result.", + `func f() (x int) { return x }`, + `func _() int { return f() }`, + `func _() int { return func() (x int) { return x }() }`, + }, + { + "Tail call to {return} that implicitly reads named result.", + `func f() (x int) { return }`, + `func _() int { return f() }`, + `func _() int { return func() (x int) { return }() }`, + }, + { + "Spread-context call to {return expr} that mentions named result.", + `func f() (x, y int) { return x, y }`, + `func _() { var _, _ = f() }`, + `func _() { var _, _ = func() (x, y int) { return x, y }() }`, + }, + { + "Shadowing in binding decl for named results => literalization.", + `func f(y string) (x y) { return x+x+len(y+y) }; type y = int`, + `func _() { f("") }`, + `func _() { func(y string) (x y) { return x + x + len(y+y) }("") }`, + }, + }) +} + +func TestSubstitutionPreservesParameterType(t *testing.T) { + runTests(t, []testcase{ { "Substitution preserves argument type (#63193).", `func f(x int16) { y := x; _ = (*int16)(&y) }`, @@ -815,11 +865,10 @@ func TestTable(t *testing.T) { `func _() { type T bool; f(1) }`, `error: T.*shadowed.*by.*type`, }, + }) +} - // TODO(adonovan): improve coverage of the cross - // product of each strategy with the checklist of - // concerns enumerated in the package doc comment. - } +func runTests(t *testing.T, tests []testcase) { for _, test := range tests { test := test t.Run(test.descr, func(t *testing.T) { From 6ec9b0f07fb433132c19db3ba19020cda263fb1b Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Wed, 27 Sep 2023 11:25:29 -0400 Subject: [PATCH 149/178] internal/refactor/inline: refine "last ref to caller local" This change slightly refines the crude check for whether an argument might contain the last reference to a caller local var (and thus cannot safely be eliminated). It now checks that the variable is defined within the body of the caller (i.e. is not a parameter or result, or a global). Also, tests. Also, properly group the remaining test cases from the CL that split the test table. (Some had been left in a function called "JJJ", as a sign of unfinished business.) Change-Id: Id79bea69176f6da8991f194fa1c3d1813e92fcfc Reviewed-on: https://go-review.googlesource.com/c/tools/+/531415 LUCI-TryBot-Result: Go LUCI Reviewed-by: Robert Findley --- internal/refactor/inline/inline.go | 52 +++++++++++-------- internal/refactor/inline/inline_test.go | 31 ++++++++--- .../inline/testdata/import-shadow.txtar | 1 - .../refactor/inline/testdata/method.txtar | 37 +++++-------- 4 files changed, 66 insertions(+), 55 deletions(-) diff --git a/internal/refactor/inline/inline.go b/internal/refactor/inline/inline.go index 78677e30364..95d15bd10f2 100644 --- a/internal/refactor/inline/inline.go +++ b/internal/refactor/inline/inline.go @@ -34,7 +34,8 @@ type Caller struct { Call *ast.CallExpr Content []byte // source of file containing - path []ast.Node + path []ast.Node // path from call to root of file syntax tree + enclosingFunc *ast.FuncDecl // top-level function/method enclosing the call, if any } // Inline inlines the called function (callee) into the function call (caller) @@ -247,24 +248,28 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // -- analyze callee's free references in caller context -- - // syntax path enclosing Call, innermost first (Path[0]=Call) + // Compute syntax path enclosing Call, innermost first (Path[0]=Call), + // and outermost enclosing function, if any. caller.path, _ = astutil.PathEnclosingInterval(caller.File, caller.Call.Pos(), caller.Call.End()) + for _, n := range caller.path { + if decl, ok := n.(*ast.FuncDecl); ok { + caller.enclosingFunc = decl + break + } + } - // Find the outermost function enclosing the call site (if any). - // Analyze all its local vars for the "single assignment" property + // If call is within a function, analyze all its + // local vars for the "single assignment" property. // (Taking the address &v counts as a potential assignment.) var assign1 func(v *types.Var) bool // reports whether v a single-assignment local var { updatedLocals := make(map[*types.Var]bool) - for _, n := range caller.path { - if decl, ok := n.(*ast.FuncDecl); ok { - escape(caller.Info, decl.Body, func(v *types.Var, _ bool) { - updatedLocals[v] = true - }) - break - } + if caller.enclosingFunc != nil { + escape(caller.Info, caller.enclosingFunc, func(v *types.Var, _ bool) { + updatedLocals[v] = true + }) + logf("multiple-assignment vars: %v", updatedLocals) } - logf("multiple-assignment vars: %v", updatedLocals) assign1 = func(v *types.Var) bool { return !updatedLocals[v] } } @@ -1109,18 +1114,19 @@ next: continue } - // Eliminating an unreferenced parameter might + // If the caller is within a function body, + // eliminating an unreferenced parameter might // remove the last reference to a caller local var. - for free := range arg.freevars { - if v, ok := caller.lookup(free).(*types.Var); ok { - // TODO(adonovan): be more precise and check - // that v is defined within the body of the caller - // function (if any) and is indeed referenced - // only by the call. (See assign1 for analysis - // of enclosing func.) - logf("keeping param %q: arg contains perhaps the last reference to possible caller local %v @ %v", - param.info.Name, v, caller.Fset.PositionFor(v.Pos(), false)) - continue next + if caller.enclosingFunc != nil { + for free := range arg.freevars { + if v, ok := caller.lookup(free).(*types.Var); ok && within(v.Pos(), caller.enclosingFunc.Body) { + // TODO(adonovan): be more precise and check that v + // is indeed referenced only by call arguments. + // Better: proceed, but blank out its declaration as needed. + logf("keeping param %q: arg contains perhaps the last reference to possible caller local %v @ %v", + param.info.Name, v, caller.Fset.PositionFor(v.Pos(), false)) + continue next + } } } } diff --git a/internal/refactor/inline/inline_test.go b/internal/refactor/inline/inline_test.go index 1ec72936db4..2b0dd38eb90 100644 --- a/internal/refactor/inline/inline_test.go +++ b/internal/refactor/inline/inline_test.go @@ -387,6 +387,23 @@ func TestBasics(t *testing.T) { }) } +func TestSubstitution(t *testing.T) { + runTests(t, []testcase{ + { + "Arg to unref'd param can be eliminated if has no effects.", + `func f(x, y int) {}; var global int`, + `func _() { f(0, global) }`, + `func _() {}`, + }, + { + "But not if it may contain last reference to a caller local var.", + `func f(int) {}`, + `func _() { var local int; f(local) }`, + `func _() { var local int; _ = local }`, + }, + }) +} + func TestTailCallStrategy(t *testing.T) { runTests(t, []testcase{ { @@ -479,7 +496,7 @@ func TestVariadic(t *testing.T) { }) } -func TestEmbeddedFieldsJJJ(t *testing.T) { +func TestParameterBindingDecl(t *testing.T) { runTests(t, []testcase{ { "IncDec counts as assignment.", @@ -521,12 +538,6 @@ func TestEmbeddedFieldsJJJ(t *testing.T) { } }`, }, - { - "Defer f() evaluates f() before unknown effects", - `func f(int, y any, z int) { defer println(int, y, z) }; func g(int) int`, - `func _() { f(g(1), g(2), g(3)) }`, - `func _() { func() { defer println(any(g(1)), any(g(2)), g(3)) }() }`, - }, { "No binding decl due to shadowing of int", `func f(int, y any, z int) { defer g(0); println(int, y, z) }; func g(int) int`, @@ -724,6 +735,12 @@ func TestSubstitutionPreservesArgumentEffectOrder(t *testing.T) { } }`, }, + { + "Defer f() evaluates f() before unknown effects", + `func f(int, y any, z int) { defer println(int, y, z) }; func g(int) int`, + `func _() { f(g(1), g(2), g(3)) }`, + `func _() { func() { defer println(any(g(1)), any(g(2)), g(3)) }() }`, + }, }) } diff --git a/internal/refactor/inline/testdata/import-shadow.txtar b/internal/refactor/inline/testdata/import-shadow.txtar index c311ee69425..49aaf567208 100644 --- a/internal/refactor/inline/testdata/import-shadow.txtar +++ b/internal/refactor/inline/testdata/import-shadow.txtar @@ -91,7 +91,6 @@ var x b.T func A(b int) { { - var _ b0.T = x b0.One() b0.Two() } //@ inline(re"F", fresult) diff --git a/internal/refactor/inline/testdata/method.txtar b/internal/refactor/inline/testdata/method.txtar index 61c082eefda..fde9f366c12 100644 --- a/internal/refactor/inline/testdata/method.txtar +++ b/internal/refactor/inline/testdata/method.txtar @@ -15,7 +15,8 @@ go 1.12 package a type T int -func (T) f0() { println() } + +func (recv T) f0() { println(recv) } func _(x T) { x.f0() //@ inline(re"f0", f0) @@ -26,19 +27,16 @@ package a type T int -func (T) f0() { println() } +func (recv T) f0() { println(recv) } func _(x T) { - { - var _ T = x - println() - } //@ inline(re"f0", f0) + println(x) //@ inline(re"f0", f0) } -- a/g0.go -- package a -func (recv *T) g0() { println() } +func (recv *T) g0() { println(recv) } func _(x T) { x.g0() //@ inline(re"g0", g0) @@ -47,19 +45,16 @@ func _(x T) { -- g0 -- package a -func (recv *T) g0() { println() } +func (recv *T) g0() { println(recv) } func _(x T) { - { - var _ *T = &x - println() - } //@ inline(re"g0", g0) + println(&x) //@ inline(re"g0", g0) } -- a/f1.go -- package a -func (T) f1(int, int) { println() } +func (recv T) f1(int, int) { println(recv) } func _(x T) { x.f1(1, 2) //@ inline(re"f1", f1) @@ -68,19 +63,16 @@ func _(x T) { -- f1 -- package a -func (T) f1(int, int) { println() } +func (recv T) f1(int, int) { println(recv) } func _(x T) { - { - var _ T = x - println() - } //@ inline(re"f1", f1) + println(x) //@ inline(re"f1", f1) } -- a/g1.go -- package a -func (recv *T) g1(int, int) { println() } +func (recv *T) g1(int, int) { println(recv) } func _(x T) { x.g1(1, 2) //@ inline(re"g1", g1) @@ -89,13 +81,10 @@ func _(x T) { -- g1 -- package a -func (recv *T) g1(int, int) { println() } +func (recv *T) g1(int, int) { println(recv) } func _(x T) { - { - var _ *T = &x - println() - } //@ inline(re"g1", g1) + println(&x) //@ inline(re"g1", g1) } -- a/h.go -- From 4b34fbf5f10d5ba6b408e43181c50698b8243990 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Wed, 27 Sep 2023 12:38:23 -0400 Subject: [PATCH 150/178] internal/refactor/inline: fix bug discard receiver and spread This change fixes a bug described in a TODO comment in the logic for discarding the results of arguments evaluated for effect in a call that has both a receiver and a spread call. Also, a test. Change-Id: I9fae202a1eef8f7c130048f9cf655af91b1a67fe Reviewed-on: https://go-review.googlesource.com/c/tools/+/531416 Reviewed-by: Robert Findley LUCI-TryBot-Result: Go LUCI --- internal/refactor/inline/inline.go | 47 +++++++++++++++++-------- internal/refactor/inline/inline_test.go | 17 +++++++++ 2 files changed, 49 insertions(+), 15 deletions(-) diff --git a/internal/refactor/inline/inline.go b/internal/refactor/inline/inline.go index 95d15bd10f2..e61e3c10de6 100644 --- a/internal/refactor/inline/inline.go +++ b/internal/refactor/inline/inline.go @@ -588,27 +588,44 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu res.old = stmt if nargs := len(remainingArgs); nargs > 0 { // Emit "_, _ = args" to discard results. - // Make correction for spread calls - // f(g()) or x.f(g()) where g() is a tuple. - // - // TODO(adonovan): fix: it's not valid for a - // single AssignStmt to discard a receiver and - // a spread argument; use a var decl with two specs. - // + // TODO(adonovan): if args is the []T{a1, ..., an} // literal synthesized during variadic simplification, // consider unwrapping it to its (pure) elements. // Perhaps there's no harm doing this for any slice literal. - if last := last(args); last != nil { - if tuple, ok := last.typ.(*types.Tuple); ok { - nargs += tuple.Len() - 1 + + // Make correction for spread calls + // f(g()) or recv.f(g()) where g() is a tuple. + if last := last(args); last != nil && last.spread { + nspread := last.typ.(*types.Tuple).Len() + if len(args) > 1 { // [recv, g()] + // A single AssignStmt cannot discard both, so use a 2-spec var decl. + res.new = &ast.GenDecl{ + Tok: token.VAR, + Specs: []ast.Spec{ + &ast.ValueSpec{ + Names: []*ast.Ident{makeIdent("_")}, + Values: []ast.Expr{args[0].expr}, + }, + &ast.ValueSpec{ + Names: blanks[*ast.Ident](nspread), + Values: []ast.Expr{args[1].expr}, + }, + }, + } + return res, nil } + + // Sole argument is spread call. + nargs = nspread } + res.new = &ast.AssignStmt{ - Lhs: blanks(nargs), + Lhs: blanks[ast.Expr](nargs), Tok: token.ASSIGN, Rhs: remainingArgs, } + } else { // No remaining arguments: delete call statement entirely res.new = &ast.EmptyStmt{} @@ -674,7 +691,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // (f() or <-ch), explicitly discard the results: // Reduces to: _, _ = exprs discard := &ast.AssignStmt{ - Lhs: blanks(callee.NumResults), + Lhs: blanks[ast.Expr](callee.NumResults), Tok: token.ASSIGN, Rhs: results, } @@ -1825,13 +1842,13 @@ func assert(cond bool, msg string) { } // blanks returns a slice of n > 0 blank identifiers. -func blanks(n int) []ast.Expr { +func blanks[E ast.Expr](n int) []E { if n == 0 { panic("blanks(0)") } - res := make([]ast.Expr, n) + res := make([]E, n) for i := range res { - res[i] = makeIdent("_") + res[i] = ast.Expr(makeIdent("_")).(E) // ugh } return res } diff --git a/internal/refactor/inline/inline_test.go b/internal/refactor/inline/inline_test.go index 2b0dd38eb90..512632d8a41 100644 --- a/internal/refactor/inline/inline_test.go +++ b/internal/refactor/inline/inline_test.go @@ -440,6 +440,23 @@ func TestSpreadCalls(t *testing.T) { `func _() I { return recover().(I).f(g()) }`, `error: can't yet inline spread call to method`, }, + { + "Spread argument evaluated for effect.", + `func f(int, int) {}; func g() (int, int)`, + `func _() { f(g()) }`, + `func _() { _, _ = g() }`, + }, + { + "Edge case: receiver and spread argument, both evaluated for effect.", + `type T int; func (T) f(int, int) {}; func g() (int, int)`, + `func _() { T(0).f(g()) }`, + `func _() { + var ( + _ = T(0) + _, _ = g() + ) +}`, + }, }) } From 7577387a8364ea2ccd35fdefa9816b1f2889642d Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Wed, 27 Sep 2023 14:53:01 -0400 Subject: [PATCH 151/178] gopls/internal/lsp/source: don't complete to golang.org/toolchain Gopls was offering unimported completion to the synthetic golang.org/toolchain modules stored in the module cache. Suppress these results, as this is not a real module. Fixes golang/go#60062 Change-Id: I1fbdd34dc9f3ca110af3ef74eae9f6810505dbef Reviewed-on: https://go-review.googlesource.com/c/tools/+/531417 Auto-Submit: Robert Findley LUCI-TryBot-Result: Go LUCI Reviewed-by: Alan Donovan --- .../lsp/source/completion/completion.go | 13 +++++ .../regtest/completion/completion_test.go | 51 +++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/gopls/internal/lsp/source/completion/completion.go b/gopls/internal/lsp/source/completion/completion.go index b67b6772f79..937e3fb2874 100644 --- a/gopls/internal/lsp/source/completion/completion.go +++ b/gopls/internal/lsp/source/completion/completion.go @@ -1394,6 +1394,9 @@ func (c *completer) selector(ctx context.Context, sel *ast.SelectorExpr) error { ctx, cancel := context.WithCancel(ctx) var mu sync.Mutex add := func(pkgExport imports.PackageExport) { + if ignoreUnimportedCompletion(pkgExport.Fix) { + return + } mu.Lock() defer mu.Unlock() // TODO(adonovan): what if the actual package has a vendor/ prefix? @@ -1445,6 +1448,13 @@ func (c *completer) packageMembers(pkg *types.Package, score float64, imp *impor } } +// ignoreUnimportedCompletion reports whether an unimported completion +// resulting in the given import should be ignored. +func ignoreUnimportedCompletion(fix *imports.ImportFix) bool { + // golang/go#60062: don't add unimported completion to golang.org/toolchain. + return fix != nil && strings.HasPrefix(fix.StmtInfo.ImportPath, "golang.org/toolchain") +} + func (c *completer) methodsAndFields(typ types.Type, addressable bool, imp *importInfo, cb func(candidate)) { mset := c.methodSetCache[methodSetKey{typ, addressable}] if mset == nil { @@ -1757,6 +1767,9 @@ func (c *completer) unimportedPackages(ctx context.Context, seen map[string]stru var mu sync.Mutex add := func(pkg imports.ImportFix) { + if ignoreUnimportedCompletion(&pkg) { + return + } mu.Lock() defer mu.Unlock() if _, ok := seen[pkg.IdentName]; ok { diff --git a/gopls/internal/regtest/completion/completion_test.go b/gopls/internal/regtest/completion/completion_test.go index 3949578a597..81300eb07e0 100644 --- a/gopls/internal/regtest/completion/completion_test.go +++ b/gopls/internal/regtest/completion/completion_test.go @@ -952,3 +952,54 @@ func _() { } }) } + +// Fix for golang/go#60062: unimported completion included "golang.org/toolchain" results. +func TestToolchainCompletions(t *testing.T) { + const files = ` +-- go.mod -- +module foo.test/foo + +go 1.21 + +-- foo.go -- +package foo + +func _() { + os.Open +} + +func _() { + strings +} +` + + const proxy = ` +-- golang.org/toolchain@v0.0.1-go1.21.1.linux-amd64/go.mod -- +module golang.org/toolchain +-- golang.org/toolchain@v0.0.1-go1.21.1.linux-amd64/src/os/os.go -- +package os + +func Open() {} +-- golang.org/toolchain@v0.0.1-go1.21.1.linux-amd64/src/strings/strings.go -- +package strings + +func Join() {} +` + + WithOptions( + ProxyFiles(proxy), + ).Run(t, files, func(t *testing.T, env *Env) { + env.RunGoCommand("mod", "download", "golang.org/toolchain@v0.0.1-go1.21.1.linux-amd64") + env.OpenFile("foo.go") + + for _, pattern := range []string{"os.Open()", "string()"} { + loc := env.RegexpSearch("foo.go", pattern) + res := env.Completion(loc) + for _, item := range res.Items { + if strings.Contains(item.Detail, "golang.org/toolchain") { + t.Errorf("Completion(...) returned toolchain item %#v", item) + } + } + } + }) +} From 3d03fbd05d5330bd1b37f083923d2f026f0101f0 Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Wed, 27 Sep 2023 15:56:48 -0400 Subject: [PATCH 152/178] gopls/internal/lsp: use matcher score in ranking unimported candidates In the refactoring of CL 472183, we inadvertently dropped the fuzzy matching factor from the unimported completion item score. Fixes golang/go#62560 Change-Id: I35294c55d4229a885b781d3cdd0c6433e82f280b Reviewed-on: https://go-review.googlesource.com/c/tools/+/531418 LUCI-TryBot-Result: Go LUCI Auto-Submit: Robert Findley Reviewed-by: Alan Donovan --- gopls/internal/lsp/regtest/marker.go | 25 +++++++++++++++++-- .../lsp/source/completion/completion.go | 3 ++- .../marker/testdata/completion/issue62560.txt | 19 ++++++++++++++ 3 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 gopls/internal/regtest/marker/testdata/completion/issue62560.txt diff --git a/gopls/internal/lsp/regtest/marker.go b/gopls/internal/lsp/regtest/marker.go index f6d462aabbb..f6c1d4c6077 100644 --- a/gopls/internal/lsp/regtest/marker.go +++ b/gopls/internal/lsp/regtest/marker.go @@ -232,8 +232,11 @@ var update = flag.Bool("update", false, "if set, update test data during marker // request at the given location, and verifies that each expected // completion item occurs in the results, in the expected order. Other // unexpected completion items may occur in the results. -// TODO(rfindley): this should accept a slice of labels, rather than -// completion items. +// TODO(rfindley): this exists for compatibility with the old marker tests. +// Replace this with rankl, and rename. +// +// - rankl(location, ...label): like rank, but only cares about completion +// item labels. // // - refs(location, want ...location): executes a textDocument/references // request at the first location and asserts that the result is the set of @@ -712,6 +715,7 @@ var actionMarkerFuncs = map[string]func(marker){ "hover": actionMarkerFunc(hoverMarker), "implementation": actionMarkerFunc(implementationMarker), "rank": actionMarkerFunc(rankMarker), + "rankl": actionMarkerFunc(ranklMarker), "refs": actionMarkerFunc(refsMarker), "rename": actionMarkerFunc(renameMarker), "renameerr": actionMarkerFunc(renameErrMarker), @@ -1450,6 +1454,23 @@ func rankMarker(mark marker, src protocol.Location, items ...completionItem) { } } +func ranklMarker(mark marker, src protocol.Location, labels ...string) { + list := mark.run.env.Completion(src) + var got []string + // Collect results that are present in items, preserving their order. + for _, g := range list.Items { + for _, label := range labels { + if g.Label == label { + got = append(got, g.Label) + break + } + } + } + if diff := cmp.Diff(labels, got); diff != "" { + mark.errorf("completion rankings do not match (-want +got):\n%s", diff) + } +} + func snippetMarker(mark marker, src protocol.Location, item completionItem, want string) { list := mark.run.env.Completion(src) var ( diff --git a/gopls/internal/lsp/source/completion/completion.go b/gopls/internal/lsp/source/completion/completion.go index 937e3fb2874..bfaf6940d66 100644 --- a/gopls/internal/lsp/source/completion/completion.go +++ b/gopls/internal/lsp/source/completion/completion.go @@ -1296,7 +1296,7 @@ func (c *completer) selector(ctx context.Context, sel *ast.SelectorExpr) error { Label: id.Name, Detail: fmt.Sprintf("%s (from %q)", strings.ToLower(tok.String()), m.PkgPath), InsertText: id.Name, - Score: unimportedScore(relevances[path]), + Score: float64(score) * unimportedScore(relevances[path]), } switch tok { case token.FUNC: @@ -1397,6 +1397,7 @@ func (c *completer) selector(ctx context.Context, sel *ast.SelectorExpr) error { if ignoreUnimportedCompletion(pkgExport.Fix) { return } + mu.Lock() defer mu.Unlock() // TODO(adonovan): what if the actual package has a vendor/ prefix? diff --git a/gopls/internal/regtest/marker/testdata/completion/issue62560.txt b/gopls/internal/regtest/marker/testdata/completion/issue62560.txt new file mode 100644 index 00000000000..89763fe0221 --- /dev/null +++ b/gopls/internal/regtest/marker/testdata/completion/issue62560.txt @@ -0,0 +1,19 @@ +This test verifies that completion of package members in unimported packages +reflects their fuzzy score, even when those members are present in the +transitive import graph of the main module. (For technical reasons, this was +the nature of the regression in golang/go#62560.) + +-- go.mod -- +module mod.test + +-- foo/foo.go -- +package foo + +func _() { + json.U //@rankl(re"U()", "Unmarshal", "InvalidUTF8Error"), diag("json", re"(undefined|undeclared)") +} + +-- bar/bar.go -- +package bar + +import _ "encoding/json" From 7f23bc81dc216f83d56b5256abc053109bf5c58b Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Wed, 27 Sep 2023 16:59:40 -0400 Subject: [PATCH 153/178] gopls/internal/regtest/source/completion: reuse functionCallSnippet in unimported completion The unimported completion logic was using ad-hoc snippet logic that didn't correctly handle "usePlaceholders", and wasn't updated for the recently added "completeFunctionCalls" setting. Refactor to re-use the functionCallSnippet helper, and add a test. Fixes golang/go#62676 Change-Id: I08277cfc556455fbe17900ad56821e269990ead6 Reviewed-on: https://go-review.googlesource.com/c/tools/+/531458 Commit-Queue: Robert Findley LUCI-TryBot-Result: Go LUCI Reviewed-by: Alan Donovan --- .../lsp/source/completion/completion.go | 37 ++++------- .../marker/testdata/completion/issue62676.txt | 63 +++++++++++++++++++ 2 files changed, 75 insertions(+), 25 deletions(-) create mode 100644 gopls/internal/regtest/marker/testdata/completion/issue62676.txt diff --git a/gopls/internal/lsp/source/completion/completion.go b/gopls/internal/lsp/source/completion/completion.go index bfaf6940d66..4044d8446fd 100644 --- a/gopls/internal/lsp/source/completion/completion.go +++ b/gopls/internal/lsp/source/completion/completion.go @@ -1320,32 +1320,18 @@ func (c *completer) selector(ctx context.Context, sel *ast.SelectorExpr) error { // For functions, add a parameter snippet. if fn != nil { - var sn snippet.Builder - sn.WriteText(id.Name) - - paramList := func(open, close string, list *ast.FieldList) { + paramList := func(list *ast.FieldList) []string { + var params []string if list != nil { var cfg printer.Config // slight overkill - var nparams int param := func(name string, typ ast.Expr) { - if nparams > 0 { - sn.WriteText(", ") - } - nparams++ - if c.opts.placeholders { - sn.WritePlaceholder(func(b *snippet.Builder) { - var buf strings.Builder - buf.WriteString(name) - buf.WriteByte(' ') - cfg.Fprint(&buf, token.NewFileSet(), typ) - b.WriteText(buf.String()) - }) - } else { - sn.WriteText(name) - } + var buf strings.Builder + buf.WriteString(name) + buf.WriteByte(' ') + cfg.Fprint(&buf, token.NewFileSet(), typ) + params = append(params, buf.String()) } - sn.WriteText(open) for _, field := range list.List { if field.Names != nil { for _, name := range field.Names { @@ -1355,13 +1341,14 @@ func (c *completer) selector(ctx context.Context, sel *ast.SelectorExpr) error { param("_", field.Type) } } - sn.WriteText(close) } + return params } - paramList("[", "]", typeparams.ForFuncType(fn.Type)) - paramList("(", ")", fn.Type.Params) - + tparams := paramList(fn.Type.TypeParams) + params := paramList(fn.Type.Params) + var sn snippet.Builder + c.functionCallSnippet(id.Name, tparams, params, &sn) item.snippet = &sn } diff --git a/gopls/internal/regtest/marker/testdata/completion/issue62676.txt b/gopls/internal/regtest/marker/testdata/completion/issue62676.txt new file mode 100644 index 00000000000..af4c3b695ec --- /dev/null +++ b/gopls/internal/regtest/marker/testdata/completion/issue62676.txt @@ -0,0 +1,63 @@ +This test verifies that unimported completion respects the usePlaceholders setting. + +-- flags -- +-ignore_extra_diags + +-- settings.json -- +{ + "usePlaceholders": false +} + +-- go.mod -- +module mod.test + +go 1.21 + +-- foo/foo.go -- +package foo + +func _() { + // This uses goimports-based completion; TODO: this should insert snippets. + os.Open //@acceptcompletion(re"Open()", "Open", open) +} + +func _() { + // This uses metadata-based completion. + errors.New //@acceptcompletion(re"New()", "New", new) +} + +-- bar/bar.go -- +package bar + +import _ "errors" // important: doesn't transitively import os. + +-- @new/foo/foo.go -- +package foo + +import "errors" + +func _() { + // This uses goimports-based completion; TODO: this should insert snippets. + os.Open //@acceptcompletion(re"Open()", "Open", open) +} + +func _() { + // This uses metadata-based completion. + errors.New(${1:}) //@acceptcompletion(re"New()", "New", new) +} + +-- @open/foo/foo.go -- +package foo + +import "os" + +func _() { + // This uses goimports-based completion; TODO: this should insert snippets. + os.Open //@acceptcompletion(re"Open()", "Open", open) +} + +func _() { + // This uses metadata-based completion. + errors.New //@acceptcompletion(re"New()", "New", new) +} + From 6de34480a867b1198bb6c910cb77c147e89a6023 Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Wed, 27 Sep 2023 17:14:47 -0400 Subject: [PATCH 154/178] gopls/internal/lsp/cache: remove snapshot.typeCheckMu snapshot.typeCheckMu was added as a minor optimization for total CPU, but apparently has a significant impact on latency in some cases. Remove it. Fixes golang/go#62493 Change-Id: Ieff77556b2d9610b1049866aeaa8246fcfa1f8b5 Reviewed-on: https://go-review.googlesource.com/c/tools/+/531459 Reviewed-by: Pontus Leitzler Reviewed-by: Alan Donovan LUCI-TryBot-Result: Go LUCI --- gopls/internal/lsp/cache/check.go | 3 --- gopls/internal/lsp/cache/snapshot.go | 12 ------------ 2 files changed, 15 deletions(-) diff --git a/gopls/internal/lsp/cache/check.go b/gopls/internal/lsp/cache/check.go index 438da1604e6..e0be99b64f5 100644 --- a/gopls/internal/lsp/cache/check.go +++ b/gopls/internal/lsp/cache/check.go @@ -323,9 +323,6 @@ type ( // // Both pre and post may be called concurrently. func (s *snapshot) forEachPackage(ctx context.Context, ids []PackageID, pre preTypeCheck, post postTypeCheck) error { - s.typeCheckMu.Lock() - defer s.typeCheckMu.Unlock() - ctx, done := event.Start(ctx, "cache.forEachPackage", tag.PackageCount.Of(len(ids))) defer done() diff --git a/gopls/internal/lsp/cache/snapshot.go b/gopls/internal/lsp/cache/snapshot.go index 9d659f046dc..6fc69bdda3d 100644 --- a/gopls/internal/lsp/cache/snapshot.go +++ b/gopls/internal/lsp/cache/snapshot.go @@ -176,18 +176,6 @@ type snapshot struct { ignoreFilterOnce sync.Once ignoreFilter *ignoreFilter - // typeCheckMu guards type checking. - // - // Only one type checking pass should be running at a given time, for two reasons: - // 1. type checking batches are optimized to use all available processors. - // Generally speaking, running two type checking batches serially is about - // as fast as running them in parallel. - // 2. type checking produces cached artifacts that may be re-used by the - // next type-checking batch: the shared import graph and the set of - // active packages. Running type checking batches in parallel after an - // invalidation can cause redundant calculation of this shared state. - typeCheckMu sync.Mutex - // options holds the user configuration at the time this snapshot was // created. options *source.Options From 57ecf488ea5c4f041d609b6e5abb4084857ac20b Mon Sep 17 00:00:00 2001 From: Viktor Blomqvist Date: Thu, 21 Sep 2023 19:54:48 +0200 Subject: [PATCH 155/178] gopls/internal/lsp: hover over embed directives Enables hover on arguments of go:embed directives. The hover body lists files matching the argument pattern. Updates golang/go#50262 Change-Id: I523cec6ab633c90d6826cdbdd6d8fca6afd98935 Reviewed-on: https://go-review.googlesource.com/c/tools/+/530195 Reviewed-by: Robert Findley LUCI-TryBot-Result: Go LUCI Reviewed-by: Suzy Mueller --- gopls/internal/lsp/source/embeddirective.go | 139 ++++++++++++++++++++ gopls/internal/lsp/source/hover.go | 50 +++++++ gopls/internal/regtest/misc/hover_test.go | 45 +++++++ 3 files changed, 234 insertions(+) create mode 100644 gopls/internal/lsp/source/embeddirective.go diff --git a/gopls/internal/lsp/source/embeddirective.go b/gopls/internal/lsp/source/embeddirective.go new file mode 100644 index 00000000000..b1a0ff9d235 --- /dev/null +++ b/gopls/internal/lsp/source/embeddirective.go @@ -0,0 +1,139 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package source + +import ( + "fmt" + "strconv" + "strings" + "unicode" + "unicode/utf8" + + "golang.org/x/tools/gopls/internal/lsp/protocol" +) + +// parseEmbedDirective attempts to parse a go:embed directive argument at pos. +// If successful it return the directive argument and its range, else zero values are returned. +func parseEmbedDirective(m *protocol.Mapper, pos protocol.Position) (string, protocol.Range) { + lineStart, err := m.PositionOffset(protocol.Position{Line: pos.Line, Character: 0}) + if err != nil { + return "", protocol.Range{} + } + lineEnd, err := m.PositionOffset(protocol.Position{Line: pos.Line + 1, Character: 0}) + if err != nil { + return "", protocol.Range{} + } + + text := string(m.Content[lineStart:lineEnd]) + if !strings.HasPrefix(text, "//go:embed") { + return "", protocol.Range{} + } + text = text[len("//go:embed"):] + offset := lineStart + len("//go:embed") + + // Find the first pattern in text that covers the offset of the pos we are looking for. + findOffset, err := m.PositionOffset(pos) + if err != nil { + return "", protocol.Range{} + } + patterns, err := parseGoEmbed(text, offset) + if err != nil { + return "", protocol.Range{} + } + for _, p := range patterns { + if p.startOffset <= findOffset && findOffset <= p.endOffset { + // Found our match. + rng, err := m.OffsetRange(p.startOffset, p.endOffset) + if err != nil { + return "", protocol.Range{} + } + return p.pattern, rng + } + } + + return "", protocol.Range{} +} + +type fileEmbed struct { + pattern string + startOffset int + endOffset int +} + +// parseGoEmbed patterns that come after the directive. +// +// Copied and adapted from go/build/read.go. +// Replaced token.Position with start/end offset (including quotes if present). +func parseGoEmbed(args string, offset int) ([]fileEmbed, error) { + trimBytes := func(n int) { + offset += n + args = args[n:] + } + trimSpace := func() { + trim := strings.TrimLeftFunc(args, unicode.IsSpace) + trimBytes(len(args) - len(trim)) + } + + var list []fileEmbed + for trimSpace(); args != ""; trimSpace() { + var path string + pathOffset := offset + Switch: + switch args[0] { + default: + i := len(args) + for j, c := range args { + if unicode.IsSpace(c) { + i = j + break + } + } + path = args[:i] + trimBytes(i) + + case '`': + var ok bool + path, _, ok = strings.Cut(args[1:], "`") + if !ok { + return nil, fmt.Errorf("invalid quoted string in //go:embed: %s", args) + } + trimBytes(1 + len(path) + 1) + + case '"': + i := 1 + for ; i < len(args); i++ { + if args[i] == '\\' { + i++ + continue + } + if args[i] == '"' { + q, err := strconv.Unquote(args[:i+1]) + if err != nil { + return nil, fmt.Errorf("invalid quoted string in //go:embed: %s", args[:i+1]) + } + path = q + trimBytes(i + 1) + break Switch + } + } + if i >= len(args) { + return nil, fmt.Errorf("invalid quoted string in //go:embed: %s", args) + } + } + + if args != "" { + r, _ := utf8.DecodeRuneInString(args) + if !unicode.IsSpace(r) { + return nil, fmt.Errorf("invalid quoted string in //go:embed: %s", args) + } + } + list = append(list, fileEmbed{ + pattern: path, + startOffset: pathOffset, + endOffset: offset, + }) + } + return list, nil +} diff --git a/gopls/internal/lsp/source/hover.go b/gopls/internal/lsp/source/hover.go index a6830751a91..8578f13e2ca 100644 --- a/gopls/internal/lsp/source/hover.go +++ b/gopls/internal/lsp/source/hover.go @@ -14,6 +14,8 @@ import ( "go/format" "go/token" "go/types" + "io/fs" + "path/filepath" "strconv" "strings" "time" @@ -121,6 +123,12 @@ func hover(ctx context.Context, snapshot Snapshot, fh FileHandle, pp protocol.Po } } + // Handle hovering over embed directive argument. + pattern, embedRng := parseEmbedDirective(pgf.Mapper, pp) + if pattern != "" { + return hoverEmbed(fh, embedRng, pattern) + } + // Handle linkname directive by overriding what to look for. var linkedRange *protocol.Range // range referenced by linkname directive, or nil if pkgPath, name, offset := parseLinkname(ctx, snapshot, fh, pp); pkgPath != "" && name != "" { @@ -625,6 +633,48 @@ func hoverLit(pgf *ParsedGoFile, lit *ast.BasicLit, pos token.Pos) (protocol.Ran }, nil } +// hoverEmbed computes hover information for a filepath.Match pattern. +// Assumes that the pattern is relative to the location of fh. +func hoverEmbed(fh FileHandle, rng protocol.Range, pattern string) (protocol.Range, *HoverJSON, error) { + s := &strings.Builder{} + + dir := filepath.Dir(fh.URI().Filename()) + var matches []string + err := filepath.WalkDir(dir, func(abs string, _ fs.DirEntry, e error) error { + if e != nil { + return e + } + rel, err := filepath.Rel(dir, abs) + if err != nil { + return err + } + ok, err := filepath.Match(pattern, rel) + if err != nil { + return err + } + if ok { + matches = append(matches, rel) + } + return nil + }) + if err != nil { + return protocol.Range{}, nil, err + } + + for _, m := range matches { + // TODO: Renders each file as separate markdown paragraphs. + // If forcing (a single) newline is possible it might be more clear. + fmt.Fprintf(s, "%s\n\n", m) + } + + json := &HoverJSON{ + Signature: fmt.Sprintf("Embedding %q", pattern), + Synopsis: s.String(), + FullDocumentation: s.String(), + } + return rng, json, nil +} + // inferredSignatureString is a wrapper around the types.ObjectString function // that adds more information to inferred signatures. It will return an empty string // if the passed types.Object is not a signature. diff --git a/gopls/internal/regtest/misc/hover_test.go b/gopls/internal/regtest/misc/hover_test.go index 9b2d86ebb69..fadaf7f79bc 100644 --- a/gopls/internal/regtest/misc/hover_test.go +++ b/gopls/internal/regtest/misc/hover_test.go @@ -435,3 +435,48 @@ use ( _, _, _ = env.Editor.Hover(env.Ctx, env.RegexpSearch("go.work", "modb")) }) } + +const embedHover = ` +-- go.mod -- +module mod.com +go 1.19 +-- main.go -- +package main + +import "embed" + +//go:embed *.txt +var foo embed.FS + +func main() { +} +-- foo.txt -- +FOO +-- bar.txt -- +BAR +-- baz.txt -- +BAZ +-- other.sql -- +SKIPPED +` + +func TestHoverEmbedDirective(t *testing.T) { + testenv.NeedsGo1Point(t, 19) + Run(t, embedHover, func(t *testing.T, env *Env) { + env.OpenFile("main.go") + from := env.RegexpSearch("main.go", `\*.txt`) + + got, _ := env.Hover(from) + if got == nil { + t.Fatalf("hover over //go:embed arg not found") + } + content := got.Value + + wants := []string{"foo.txt", "bar.txt", "baz.txt"} + for _, want := range wants { + if !strings.Contains(content, want) { + t.Errorf("hover: %q does not contain: %q", content, want) + } + } + }) +} From 4cd12d6da09ca0625b319043ed9bc0b68fdc3f9c Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Thu, 28 Sep 2023 11:04:00 -0400 Subject: [PATCH 156/178] gopls/internal/lsp/fake: don't set a completion budget for tests We were already setting an arbitrarily high completion budget of 10s for tests, but this appears to be causing flakes in the heavily parallelized marker tests on slow builders (e.g. golang/go#60584). No reason to have any budget at all: any tests/benchmarks that depend on the budget should set it explicitly. For golang/go#60584 Change-Id: Ic5e84d483fdffb58b801aae7b814ff705e6ecee9 Reviewed-on: https://go-review.googlesource.com/c/tools/+/531556 Auto-Submit: Robert Findley Reviewed-by: Peter Weinberger LUCI-TryBot-Result: Go LUCI --- gopls/internal/lsp/fake/editor.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gopls/internal/lsp/fake/editor.go b/gopls/internal/lsp/fake/editor.go index 12d07a753e7..ccee51e6867 100644 --- a/gopls/internal/lsp/fake/editor.go +++ b/gopls/internal/lsp/fake/editor.go @@ -235,9 +235,9 @@ func makeSettings(sandbox *Sandbox, config EditorConfig) map[string]interface{} // asynchronous operations being completed (such as diagnosing a snapshot). "verboseWorkDoneProgress": true, - // Set a generous completion budget, so that tests don't flake because + // Set an unlimited completion budget, so that tests don't flake because // completions are too slow. - "completionBudget": "10s", + "completionBudget": "0s", } for k, v := range config.Settings { From 94162993020d79adb9a2316b4a64db7378c0dba4 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Wed, 27 Sep 2023 15:10:49 -0400 Subject: [PATCH 157/178] internal/refactor/inline: fix pkgname shadowing bug In cross-package inlining, when replacing free names with qualified identifiers, we failed to check for intervening names defined in the callee that might cast a shadow over the reference. This change re-uses the logic for shadow detection when replacing parameters with arguments. Plus tests. Fixes golang/go#62667 Change-Id: Ib6a620ed2cde313bf51d5fb8a0cd9363f9eadf6e Reviewed-on: https://go-review.googlesource.com/c/tools/+/531455 LUCI-TryBot-Result: Go LUCI Reviewed-by: Robert Findley --- internal/refactor/inline/callee.go | 65 ++++++++++++------- internal/refactor/inline/inline.go | 25 ++++--- .../inline/testdata/import-shadow.txtar | 42 ++++++++++-- .../refactor/inline/testdata/issue62667.txtar | 44 +++++++++++++ 4 files changed, 138 insertions(+), 38 deletions(-) create mode 100644 internal/refactor/inline/testdata/issue62667.txtar diff --git a/internal/refactor/inline/callee.go b/internal/refactor/inline/callee.go index 23ec98e8a52..4f1c4b5595b 100644 --- a/internal/refactor/inline/callee.go +++ b/internal/refactor/inline/callee.go @@ -58,10 +58,11 @@ type freeRef struct { // An object abstracts a free types.Object referenced by the callee. Gob-serializable. type object struct { - Name string // Object.Name() - Kind string // one of {var,func,const,type,pkgname,nil,builtin} - PkgPath string // pkgpath of object (or of imported package if kind="pkgname") - ValidPos bool // Object.Pos().IsValid() + Name string // Object.Name() + Kind string // one of {var,func,const,type,pkgname,nil,builtin} + PkgPath string // pkgpath of object (or of imported package if kind="pkgname") + ValidPos bool // Object.Pos().IsValid() + Shadow map[string]bool // names shadowed at one of the object's refs } // AnalyzeCallee analyzes a function that is a candidate for inlining @@ -119,7 +120,14 @@ func AnalyzeCallee(logf func(string, ...any), fset *token.FileSet, pkg *types.Pa ) var f func(n ast.Node) bool visit := func(n ast.Node) { ast.Inspect(n, f) } + var stack []ast.Node + stack = append(stack, decl.Type) // for scope of function itself f = func(n ast.Node) bool { + if n != nil { + stack = append(stack, n) // push + } else { + stack = stack[:len(stack)-1] // pop + } switch n := n.(type) { case *ast.SelectorExpr: // Check selections of free fields/methods. @@ -198,6 +206,8 @@ func AnalyzeCallee(logf func(string, ...any), fset *token.FileSet, pkg *types.Pa freeObjIndex[obj] = objidx } + freeObjs[objidx].Shadow = addShadows(freeObjs[objidx].Shadow, info, obj.Name(), stack) + freeRefs = append(freeRefs, freeRef{ Offset: int(n.Pos() - decl.Pos()), Object: objidx, @@ -417,6 +427,9 @@ func analyzeParams(logf func(string, ...any), fset *token.FileSet, info *types.I // Record locations of all references to parameters. // And record the set of intervening definitions for each parameter. + // + // TODO(adonovan): combine this traversal with the one that computes + // FreeRefs. The tricky part is that calleefx needs this one first. var stack []ast.Node stack = append(stack, decl.Type) // for scope of function itself ast.Inspect(decl.Body, func(n ast.Node) bool { @@ -429,26 +442,11 @@ func analyzeParams(logf func(string, ...any), fset *token.FileSet, info *types.I if id, ok := n.(*ast.Ident); ok { if v, ok := info.Uses[id].(*types.Var); ok { if pinfo, ok := paramInfos[v]; ok { - // Record location of ref to parameter/result. + // Record location of ref to parameter/result + // and any intervening (shadowing) names. offset := int(n.Pos() - decl.Pos()) pinfo.Refs = append(pinfo.Refs, offset) - - // Find set of names shadowed within body - // (excluding the parameter itself). - // If these names are free in the arg expression, - // we can't substitute the parameter. - for _, n := range stack { - if scope, ok := info.Scopes[n]; ok { - for _, name := range scope.Names() { - if name != pinfo.Name { - if pinfo.Shadow == nil { - pinfo.Shadow = make(map[string]bool) - } - pinfo.Shadow[name] = true - } - } - } - } + pinfo.Shadow = addShadows(pinfo.Shadow, info, pinfo.Name, stack) } } } @@ -465,6 +463,29 @@ func analyzeParams(logf func(string, ...any), fset *token.FileSet, info *types.I // -- callee helpers -- +// addShadows returns the shadows set augmented by the set of names +// locally shadowed at the location of the reference in the callee +// (identified by the stack). The name of the reference itself is +// excluded. +// +// These shadowed names may not be used in a replacement expression +// for the reference. +func addShadows(shadows map[string]bool, info *types.Info, exclude string, stack []ast.Node) map[string]bool { + for _, n := range stack { + if scope, ok := info.Scopes[n]; ok { + for _, name := range scope.Names() { + if name != exclude { + if shadows == nil { + shadows = make(map[string]bool) + } + shadows[name] = true + } + } + } + } + return shadows +} + // deref removes a pointer type constructor from the core type of t. func deref(t types.Type) types.Type { if ptr, ok := typeparams.CoreType(t).(*types.Pointer); ok { diff --git a/internal/refactor/inline/inline.go b/internal/refactor/inline/inline.go index e61e3c10de6..25bd5aa5eb5 100644 --- a/internal/refactor/inline/inline.go +++ b/internal/refactor/inline/inline.go @@ -291,14 +291,17 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // localImportName returns the local name for a given imported package path. var newImports []*ast.ImportSpec - localImportName := func(path string) string { + localImportName := func(path string, shadows map[string]bool) string { // Does an import exist? for _, name := range importMap[path] { // Check that either the import preexisted, - // or that it was newly added (no PkgName) but is not shadowed. - found := caller.lookup(name) - if is[*types.PkgName](found) || found == nil { - return name + // or that it was newly added (no PkgName) but is not shadowed, + // either in the callee (shadows) or caller (caller.lookup). + if !shadows[name] { + found := caller.lookup(name) + if is[*types.PkgName](found) || found == nil { + return name + } } } @@ -313,7 +316,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // use the package's declared name. base := pathpkg.Base(path) name := base - for n := 0; caller.lookup(name) != nil; n++ { + for n := 0; shadows[name] || caller.lookup(name) != nil; n++ { name = fmt.Sprintf("%s%d", base, n) } @@ -344,7 +347,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // => check not shadowed in caller. // - package-level var/func/const/types // => same package: check not shadowed in caller. - // => otherwise: import other package form a qualified identifier. + // => otherwise: import other package, form a qualified identifier. // (Unexported cross-package references were rejected already.) // - type parameter // => not yet supported @@ -353,10 +356,14 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // // There can be no free references to labels, fields, or methods. + // Note that we must consider potential shadowing both + // at the caller side (caller.lookup) and, when + // choosing new PkgNames, within the callee (obj.shadow). + var newName ast.Expr if obj.Kind == "pkgname" { // Use locally appropriate import, creating as needed. - newName = makeIdent(localImportName(obj.PkgPath)) // imported package + newName = makeIdent(localImportName(obj.PkgPath, obj.Shadow)) // imported package } else if !obj.ValidPos { // Built-in function, type, or value (e.g. nil, zero): @@ -396,7 +403,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // Form a qualified identifier, pkg.Name. if qualify { - pkgName := localImportName(obj.PkgPath) + pkgName := localImportName(obj.PkgPath, obj.Shadow) newName = &ast.SelectorExpr{ X: makeIdent(pkgName), Sel: makeIdent(obj.Name), diff --git a/internal/refactor/inline/testdata/import-shadow.txtar b/internal/refactor/inline/testdata/import-shadow.txtar index 49aaf567208..5d4f9243c18 100644 --- a/internal/refactor/inline/testdata/import-shadow.txtar +++ b/internal/refactor/inline/testdata/import-shadow.txtar @@ -2,13 +2,10 @@ Just because a package (e.g. log) is imported by the caller, and the name log is in scope, doesn't mean the name in scope refers to the package: it could be locally shadowed. -Two scenarios below: - -1. a second (renaming) import is added because the first import is - locally shadowed. - -2. a new import is added with a fresh name because the default - name is locally shadowed. +In all three scenarios below, renaming import with a fresh name is +added because the usual name is locally shadowed: in cases 1, 2 an +existing import is shadowed by (respectively) a local constant, +parameter; in case 3 there is no existing import. -- go.mod -- module testdata @@ -95,3 +92,34 @@ func A(b int) { b0.Two() } //@ inline(re"F", fresult) } + +-- d/d.go -- +package d + +import "testdata/e" + +func D() { + const log = "shadow" + e.E() //@ inline(re"E", eresult) +} + +-- e/e.go -- +package e + +import "log" + +func E() { + log.Printf("") +} + +-- eresult -- +package d + +import ( + log0 "log" +) + +func D() { + const log = "shadow" + log0.Printf("") //@ inline(re"E", eresult) +} diff --git a/internal/refactor/inline/testdata/issue62667.txtar b/internal/refactor/inline/testdata/issue62667.txtar new file mode 100644 index 00000000000..21420e21df4 --- /dev/null +++ b/internal/refactor/inline/testdata/issue62667.txtar @@ -0,0 +1,44 @@ +Regression test for #62667: the callee's reference to Split +was blindly qualified to path.Split even though the imported +PkgName path is shadowed by the parameter of the same name. + +The defer is to defeat reduction of the call and +substitution of the path parameter by g(). + +-- go.mod -- +module testdata +go 1.12 + +-- a/a.go -- +package a + +import "testdata/path" + +func A() { + path.Dir(g()) //@ inline(re"Dir", result) +} + +func g() string + +-- path/path.go -- +package path + +func Dir(path string) { + defer func(){}() + Split(path) +} + +func Split(string) {} + +-- result -- +package a + +import ( + path0 "testdata/path" +) + +func A() { + func(path string) { defer func() {}(); path0.Split(path) }(g()) //@ inline(re"Dir", result) +} + +func g() string \ No newline at end of file From 792f91f70441b10849bee72961c07a8a32923318 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Wed, 27 Sep 2023 15:53:59 -0400 Subject: [PATCH 158/178] internal/refactor/inline: tweak everything test for cgo We are down to one remaining call in all of x/tools that doesn't inline correctly (due to golang/go#62664). Change-Id: Ic39e60697323ede4565ce190bee69f670e627611 Reviewed-on: https://go-review.googlesource.com/c/tools/+/531456 LUCI-TryBot-Result: Go LUCI Reviewed-by: Robert Findley --- internal/refactor/inline/everything_test.go | 1 + internal/refactor/inline/inline.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/refactor/inline/everything_test.go b/internal/refactor/inline/everything_test.go index b3a6769063c..082d4e00c7e 100644 --- a/internal/refactor/inline/everything_test.go +++ b/internal/refactor/inline/everything_test.go @@ -153,6 +153,7 @@ func TestEverything(t *testing.T) { "has no body", "type parameters are not yet", "line directives", + "cgo-generated", } { if strings.Contains(err.Error(), ignore) { return diff --git a/internal/refactor/inline/inline.go b/internal/refactor/inline/inline.go index 25bd5aa5eb5..d7652e7fd0b 100644 --- a/internal/refactor/inline/inline.go +++ b/internal/refactor/inline/inline.go @@ -866,7 +866,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu return res, nil } - // TODO(adonovan): parameterless call to { stmt; return expr } + // TODO(adonovan): parameterless call to { stmts; return expr } // from one of these contexts: // x, y = f() // x, y := f() From 451716b51585d61afb394f6d6f0d604192705aa8 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Thu, 28 Sep 2023 16:08:18 -0400 Subject: [PATCH 159/178] internal/refactor/inline: consider "", 0.0, 1.0 duplicable There is a certain amount of arbitrariness, but these constants are common, fundamental, and short. Change-Id: I198e8dbb511d25d969e35a4308108dcf89f79bb0 Reviewed-on: https://go-review.googlesource.com/c/tools/+/531696 Reviewed-by: Robert Findley LUCI-TryBot-Result: Go LUCI Auto-Submit: Alan Donovan --- internal/refactor/inline/inline.go | 30 ++++++++++++++++++++++--- internal/refactor/inline/inline_test.go | 16 ++++++------- 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/internal/refactor/inline/inline.go b/internal/refactor/inline/inline.go index d7652e7fd0b..f3af6a3aa31 100644 --- a/internal/refactor/inline/inline.go +++ b/internal/refactor/inline/inline.go @@ -8,6 +8,7 @@ import ( "bytes" "fmt" "go/ast" + "go/constant" "go/format" "go/parser" "go/token" @@ -1816,17 +1817,30 @@ func duplicable(info *types.Info, e ast.Expr) bool { switch e := e.(type) { case *ast.ParenExpr: return duplicable(info, e.X) + case *ast.Ident: return true + case *ast.BasicLit: - return e.Kind == token.INT + v := info.Types[e].Value + switch e.Kind { + case token.INT: + return true // any int + case token.STRING: + return consteq(v, kZeroString) // only "" + case token.FLOAT: + return consteq(v, kZeroFloat) || consteq(v, kOneFloat) // only 0.0 or 1.0 + } + case *ast.UnaryExpr: // e.g. +1, -1 return (e.Op == token.ADD || e.Op == token.SUB) && duplicable(info, e.X) + case *ast.CallExpr: // Don't treat a conversion T(x) as duplicable even // if x is duplicable because it could duplicate // allocations. There may be cases to tease apart here. return false + case *ast.SelectorExpr: if sel, ok := info.Selections[e]; ok { // A field or method selection x.f is referentially @@ -1835,11 +1849,21 @@ func duplicable(info *types.Info, e ast.Expr) bool { } // A qualified identifier pkg.Name is referentially transparent. return true - default: - return false } + return false +} + +func consteq(x, y constant.Value) bool { + return constant.Compare(x, token.EQL, y) } +var ( + kZeroInt = constant.MakeInt64(0) + kZeroString = constant.MakeString("") + kZeroFloat = constant.MakeFloat64(0.0) + kOneFloat = constant.MakeFloat64(1.0) +) + // -- inline helpers -- func assert(cond bool, msg string) { diff --git a/internal/refactor/inline/inline_test.go b/internal/refactor/inline/inline_test.go index 512632d8a41..fa3913c9805 100644 --- a/internal/refactor/inline/inline_test.go +++ b/internal/refactor/inline/inline_test.go @@ -376,10 +376,10 @@ func TestBasics(t *testing.T) { { "Non-duplicable arguments are not substituted even if pure.", `func f(s string, i int) { print(s, s, i, i) }`, - `func _() { f("", 0) }`, + `func _() { f("hi", 0) }`, `func _() { { - var s string = "" + var s string = "hi" print(s, s, 0, 0) } }`, @@ -777,11 +777,11 @@ func TestNamedResultVars(t *testing.T) { { "Ditto, with binding decl again.", `func f(y string) (x int) { return x+x+len(y+y) }`, - `func _() { f("") }`, + `func _() { f(".") }`, `func _() { { var ( - y string = "" + y string = "." x int ) _ = x + x + len(y+y) @@ -792,11 +792,11 @@ func TestNamedResultVars(t *testing.T) { { "Ditto, with binding decl (due to repeated y refs).", `func f(y string) (x string) { return x+y+y }`, - `func _() { f("") }`, + `func _() { f(".") }`, `func _() { { var ( - y string = "" + y string = "." x string ) _ = x + y + y @@ -835,8 +835,8 @@ func TestNamedResultVars(t *testing.T) { { "Shadowing in binding decl for named results => literalization.", `func f(y string) (x y) { return x+x+len(y+y) }; type y = int`, - `func _() { f("") }`, - `func _() { func(y string) (x y) { return x + x + len(y+y) }("") }`, + `func _() { f(".") }`, + `func _() { func(y string) (x y) { return x + x + len(y+y) }(".") }`, }, }) } From 2ed429843552a1f927f6134925c678718a9ee42d Mon Sep 17 00:00:00 2001 From: Lasse Folger Date: Thu, 28 Sep 2023 15:13:30 +0200 Subject: [PATCH 160/178] go/analysis/analysistest: format golden files before comparing Change-Id: Ie8d68d38491f87fbbf3faff8eabd3783db1d0fd2 Reviewed-on: https://go-review.googlesource.com/c/tools/+/531615 LUCI-TryBot-Result: Go LUCI Reviewed-by: Alan Donovan --- go/analysis/analysistest/analysistest.go | 59 ++++++++++--------- go/analysis/analysistest/analysistest_test.go | 4 ++ 2 files changed, 34 insertions(+), 29 deletions(-) diff --git a/go/analysis/analysistest/analysistest.go b/go/analysis/analysistest/analysistest.go index 139c7587c52..0c066baa33d 100644 --- a/go/analysis/analysistest/analysistest.go +++ b/go/analysis/analysistest/analysistest.go @@ -211,24 +211,14 @@ func RunWithSuggestedFixes(t Testing, dir string, a *analysis.Analyzer, patterns for _, vf := range ar.Files { if vf.Name == sf { found = true - out, err := diff.ApplyBytes(orig, edits) - if err != nil { - t.Errorf("%s: error applying fixes: %v (see possible explanations at RunWithSuggestedFixes)", file.Name(), err) - continue - } // the file may contain multiple trailing // newlines if the user places empty lines // between files in the archive. normalize // this to a single newline. - want := string(bytes.TrimRight(vf.Data, "\n")) + "\n" - formatted, err := format.Source(out) - if err != nil { - t.Errorf("%s: error formatting edited source: %v\n%s", file.Name(), err, out) - continue - } - if got := string(formatted); got != want { - unified := diff.Unified(fmt.Sprintf("%s.golden [%s]", file.Name(), sf), "actual", want, got) - t.Errorf("suggested fixes failed for %s:\n%s", file.Name(), unified) + golden := append(bytes.TrimRight(vf.Data, "\n"), '\n') + + if err := applyDiffsAndCompare(orig, golden, edits, file.Name()); err != nil { + t.Errorf("%s", err) } break } @@ -245,21 +235,8 @@ func RunWithSuggestedFixes(t Testing, dir string, a *analysis.Analyzer, patterns catchallEdits = append(catchallEdits, edits...) } - out, err := diff.ApplyBytes(orig, catchallEdits) - if err != nil { - t.Errorf("%s: error applying fixes: %v (see possible explanations at RunWithSuggestedFixes)", file.Name(), err) - continue - } - want := string(ar.Comment) - - formatted, err := format.Source(out) - if err != nil { - t.Errorf("%s: error formatting resulting source: %v\n%s", file.Name(), err, out) - continue - } - if got := string(formatted); got != want { - unified := diff.Unified(file.Name()+".golden", "actual", want, got) - t.Errorf("suggested fixes failed for %s:\n%s", file.Name(), unified) + if err := applyDiffsAndCompare(orig, ar.Comment, catchallEdits, file.Name()); err != nil { + t.Errorf("%s", err) } } } @@ -267,6 +244,30 @@ func RunWithSuggestedFixes(t Testing, dir string, a *analysis.Analyzer, patterns return r } +// applyDiffsAndCompare applies edits to src and compares the results against +// golden after formatting both. fileName is use solely for error reporting. +func applyDiffsAndCompare(src, golden []byte, edits []diff.Edit, fileName string) error { + out, err := diff.ApplyBytes(src, edits) + if err != nil { + return fmt.Errorf("%s: error applying fixes: %v (see possible explanations at RunWithSuggestedFixes)", fileName, err) + } + wantRaw, err := format.Source(golden) + if err != nil { + return fmt.Errorf("%s.golden: error formatting golden file: %v\n%s", fileName, err, out) + } + want := string(wantRaw) + + formatted, err := format.Source(out) + if err != nil { + return fmt.Errorf("%s: error formatting resulting source: %v\n%s", fileName, err, out) + } + if got := string(formatted); got != want { + unified := diff.Unified(fileName+".golden", "actual", want, got) + return fmt.Errorf("suggested fixes failed for %s:\n%s", fileName, unified) + } + return nil +} + // Run applies an analysis to the packages denoted by the "go list" patterns. // // It loads the packages from the specified diff --git a/go/analysis/analysistest/analysistest_test.go b/go/analysis/analysistest/analysistest_test.go index 8c7ff7363c2..0b5f5ed524c 100644 --- a/go/analysis/analysistest/analysistest_test.go +++ b/go/analysis/analysistest/analysistest_test.go @@ -70,6 +70,8 @@ func main() { // OK (multiple expectations on same line) println(); println() // want "call of println(...)" "call of println(...)" + + // A Line that is not formatted correctly in the golden file. } // OK (facts and diagnostics on same line) @@ -109,6 +111,8 @@ func main() { // OK (multiple expectations on same line) println_TEST_() println_TEST_() // want "call of println(...)" "call of println(...)" + + // A Line that is not formatted correctly in the golden file. } // OK (facts and diagnostics on same line) From 0adbf9c67a49cc8525e31d2eb479fba3ec1473b8 Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Mon, 2 Oct 2023 09:38:40 -0400 Subject: [PATCH 161/178] gopls/internal/lsp: simplify the telemetry prompt Per discussion, update the telemetry prompt to reduce redundancy, and omit the upload date (which is not necessary for the opt-in model). Change-Id: I7e4db24076996f40a39e9653633e3e821a688f3b Reviewed-on: https://go-review.googlesource.com/c/tools/+/532095 Reviewed-by: Hyang-Ah Hana Kim LUCI-TryBot-Result: Go LUCI --- gopls/internal/lsp/prompt.go | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/gopls/internal/lsp/prompt.go b/gopls/internal/lsp/prompt.go index 1e63337d391..976f7c6e09f 100644 --- a/gopls/internal/lsp/prompt.go +++ b/gopls/internal/lsp/prompt.go @@ -198,8 +198,8 @@ Would you like to enable Go telemetry? if s.Options().LinkifyShowMessage { prompt = `Go telemetry helps us improve Go by periodically sending anonymous metrics and crash reports to the Go team. Learn more at [telemetry.go.dev/privacy](https://telemetry.go.dev/privacy). - Would you like to enable Go telemetry? - ` +Would you like to enable Go telemetry? +` } // TODO(rfindley): investigate a "tell me more" action in combination with ShowDocument. params := &protocol.ShowMessageRequestParams{ @@ -258,24 +258,15 @@ Would you like to enable Go telemetry? } func telemetryOnMessage(linkify bool) string { - reportDate := time.Now().AddDate(0, 0, 7).Format("2006-01-02") - format := `Telemetry uploading is now enabled and may be sent to https://telemetry.go.dev starting %s. Uploaded data is used to help improve the Go toolchain and related tools, and it will be published as part of a public dataset. - -For more details, see https://telemetry.go.dev/privacy. -This data is collected in accordance with the Google Privacy Policy (https://policies.google.com/privacy). + format := `Thank you. Telemetry uploading is now enabled. To disable telemetry uploading, run %s. ` + var runCmd = "`go run golang.org/x/telemetry/cmd/gotelemetry@latest off`" if linkify { - format = `Telemetry uploading is now enabled and may be sent to [telemetry.go.dev](https://telemetry.go.dev) starting %s. Uploaded data is used to help improve the Go toolchain and related tools, and it will be published as part of a public dataset. - -For more details, see [telemetry.go.dev/privacy](https://telemetry.go.dev/privacy). -This data is collected in accordance with the [Google Privacy Policy](https://policies.google.com/privacy). - -To disable telemetry uploading, run [%s](https://golang.org/x/telemetry/cmd/gotelemetry). -` + runCmd = "[gotelemetry off](https://golang.org/x/telemetry/cmd/gotelemetry)" } - return fmt.Sprintf(format, reportDate, "`go run golang.org/x/telemetry/cmd/gotelemetry@latest off`") + return fmt.Sprintf(format, runCmd) } // acquireLockFile attempts to "acquire a lock" for writing to path. From d8e94f20301c71a2043b08c565b379bc905dc362 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Fri, 29 Sep 2023 18:53:19 -0400 Subject: [PATCH 162/178] internal/refactor/inline: fix bug in shadow detection There's a kink in the scope tree at Func{Decl,Lit} that we forgot to straighten out in all cases. Fixed, with test. Change-Id: Iaed9636b4cc17537af6b607b153f8b4183ac7cb7 Reviewed-on: https://go-review.googlesource.com/c/tools/+/531699 LUCI-TryBot-Result: Go LUCI Reviewed-by: Robert Findley --- internal/refactor/inline/callee.go | 2 +- internal/refactor/inline/inline.go | 19 +++++++++++++------ internal/refactor/inline/inline_test.go | 11 +++++++++++ 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/internal/refactor/inline/callee.go b/internal/refactor/inline/callee.go index 4f1c4b5595b..27a7b284b5d 100644 --- a/internal/refactor/inline/callee.go +++ b/internal/refactor/inline/callee.go @@ -472,7 +472,7 @@ func analyzeParams(logf func(string, ...any), fset *token.FileSet, info *types.I // for the reference. func addShadows(shadows map[string]bool, info *types.Info, exclude string, stack []ast.Node) map[string]bool { for _, n := range stack { - if scope, ok := info.Scopes[n]; ok { + if scope := scopeFor(info, n); scope != nil { for _, name := range scope.Names() { if name != exclude { if shadows == nil { diff --git a/internal/refactor/inline/inline.go b/internal/refactor/inline/inline.go index f3af6a3aa31..0c9c1b87992 100644 --- a/internal/refactor/inline/inline.go +++ b/internal/refactor/inline/inline.go @@ -1536,12 +1536,7 @@ func createBindingDecl(logf func(string, ...any), caller *Caller, args []*argume func (caller *Caller) lookup(name string) types.Object { pos := caller.Call.Pos() for _, n := range caller.path { - // The function body scope (containing not just params) - // is associated with FuncDecl.Type, not FuncDecl.Body. - if decl, ok := n.(*ast.FuncDecl); ok { - n = decl.Type - } - if scope := caller.Info.Scopes[n]; scope != nil { + if scope := scopeFor(caller.Info, n); scope != nil { if _, obj := scope.LookupParent(name, pos); obj != nil { return obj } @@ -1550,6 +1545,18 @@ func (caller *Caller) lookup(name string) types.Object { return nil } +func scopeFor(info *types.Info, n ast.Node) *types.Scope { + // The function body scope (containing not just params) + // is associated with the function's type, not body. + switch fn := n.(type) { + case *ast.FuncDecl: + n = fn.Type + case *ast.FuncLit: + n = fn.Type + } + return info.Scopes[n] +} + // -- predicates over expressions -- // freeVars returns the names of all free identifiers of e: diff --git a/internal/refactor/inline/inline_test.go b/internal/refactor/inline/inline_test.go index fa3913c9805..eca5a914b8d 100644 --- a/internal/refactor/inline/inline_test.go +++ b/internal/refactor/inline/inline_test.go @@ -401,6 +401,17 @@ func TestSubstitution(t *testing.T) { `func _() { var local int; f(local) }`, `func _() { var local int; _ = local }`, }, + { + "Regression test for detection of shadowing in nested functions.", + `func f(x int) { _ = func() { y := 1; print(y); print(x) } }`, + `func _(y int) { f(y) } `, + `func _(y int) { + { + var x int = y + _ = func() { y := 1; print(y); print(x) } + } +}`, + }, }) } From 1058109b662941a394d08bbfbe728020129e5b78 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Sun, 1 Oct 2023 16:02:31 -0400 Subject: [PATCH 163/178] internal/refactor/inline: don't insert unnecessary parens This change causes parens to be inserted only if needed around the new node in the replacement. Parens are needed if the child is a unary or binary operation and either (a) the parent is a unary or binary of higher precedence or (b) the child is the operand of a postfix operator. Also, tests. Updates golang/go#63259 Change-Id: I12ee95ad79b4921755d9bc87952f6404cb166e2b Reviewed-on: https://go-review.googlesource.com/c/tools/+/532098 LUCI-TryBot-Result: Go LUCI Reviewed-by: Robert Findley Auto-Submit: Alan Donovan --- .../marker/testdata/codeaction/inline.txt | 2 +- internal/refactor/inline/inline.go | 121 ++++++++++++++---- internal/refactor/inline/inline_test.go | 49 ++++++- .../refactor/inline/testdata/basic-err.txtar | 2 +- .../inline/testdata/basic-reduce.txtar | 4 +- .../refactor/inline/testdata/comments.txtar | 2 +- .../inline/testdata/param-subst.txtar | 2 +- 7 files changed, 152 insertions(+), 30 deletions(-) diff --git a/gopls/internal/regtest/marker/testdata/codeaction/inline.txt b/gopls/internal/regtest/marker/testdata/codeaction/inline.txt index 134065f26b9..15d3cabfcc8 100644 --- a/gopls/internal/regtest/marker/testdata/codeaction/inline.txt +++ b/gopls/internal/regtest/marker/testdata/codeaction/inline.txt @@ -17,7 +17,7 @@ func add(x, y int) int { return x + y } package a func _() { - println((1 + 2)) //@codeaction("refactor.inline", "add", ")", inline) + println(1 + 2) //@codeaction("refactor.inline", "add", ")", inline) } func add(x, y int) int { return x + y } diff --git a/internal/refactor/inline/inline.go b/internal/refactor/inline/inline.go index 0c9c1b87992..e5fe3336122 100644 --- a/internal/refactor/inline/inline.go +++ b/internal/refactor/inline/inline.go @@ -73,6 +73,31 @@ func Inline(logf func(string, ...any), caller *Caller, callee *Callee) ([]byte, assert(res.old != nil, "old is nil") assert(res.new != nil, "new is nil") + // A single return operand inlined to a unary + // expression context may need parens. Otherwise: + // func two() int { return 1+1 } + // print(-two()) => print(-1+1) // oops! + // + // Usually it is not necessary to insert ParenExprs + // as the formatter is smart enough to insert them as + // needed by the context. But the res.{old,new} + // substitution is done by formatting res.new in isolation + // and then splicing its text over res.old, so the + // formatter doesn't see the parent node and cannot do + // the right thing. (One solution would be to always + // format the enclosing node of old, but that requires + // non-lossy comment handling, #20744.) + // + // So, we must analyze the call's context + // to see whether ambiguity is possible. + // For example, if the context is x[y:z], then + // the x subtree is subject to precedence ambiguity + // (replacing x by p+q would give p+q[y:z] which is wrong) + // but the y and z subtrees are safe. + if needsParens(caller.path, res.old, res.new) { + res.new = &ast.ParenExpr{X: res.new.(ast.Expr)} + } + // Don't call replaceNode(caller.File, res.old, res.new) // as it mutates the caller's syntax tree. // Instead, splice the file, replacing the extent of the "old" @@ -727,29 +752,8 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu if callee.NumResults == 1 { logf("strategy: reduce expr-context call to { return expr }") - // A single return operand inlined to a unary - // expression context may need parens. Otherwise: - // func two() int { return 1+1 } - // print(-two()) => print(-1+1) // oops! - // - // Usually it is not necessary to insert ParenExprs - // as the formatter is smart enough to insert them as - // needed by the context. But the res.{old,new} - // substitution is done by formatting res.new in isolation - // and then splicing its text over res.old, so the - // formatter doesn't see the parent node and cannot do - // the right thing. (One solution would be to always - // format the enclosing node of old, but that requires - // non-lossy comment handling, #20744.) - // - // TODO(adonovan): do better by analyzing 'context' - // to see whether ambiguity is possible. - // For example, if the context is x[y:z], then - // the x subtree is subject to precedence ambiguity - // (replacing x by p+q would give p+q[y:z] which is wrong) - // but the y and z subtrees are safe. res.old = caller.Call - res.new = &ast.ParenExpr{X: results[0]} + res.new = results[0] } else { logf("strategy: reduce spread-context call to { return expr }") @@ -2279,3 +2283,76 @@ func consistentOffsets(caller *Caller) bool { } return is[*ast.CallExpr](expr) } + +// needsParens reports whether parens are required to avoid ambiguity +// around the new node replacing the specified old node (which is some +// ancestor of the CallExpr identified by its PathEnclosingInterval). +func needsParens(callPath []ast.Node, old, new ast.Node) bool { + // Find enclosing old node and its parent. + // TODO(adonovan): Use index[ast.Node]() in go1.20. + i := -1 + for i = range callPath { + if callPath[i] == old { + break + } + } + if i == -1 { + panic("not found") + } + + // There is no precedence ambiguity when replacing + // (e.g.) a statement enclosing the call. + if !is[ast.Expr](old) { + return false + } + + // An expression beneath a non-expression + // has no precedence ambiguity. + parent, ok := callPath[i+1].(ast.Expr) + if !ok { + return false + } + + precedence := func(n ast.Node) int { + switch n := n.(type) { + case *ast.UnaryExpr, *ast.StarExpr: + return token.UnaryPrec + case *ast.BinaryExpr: + return n.Op.Precedence() + } + return -1 + } + + // Parens are not required if the new node + // is not unary or binary. + newprec := precedence(new) + if newprec < 0 { + return false + } + + // Parens are required if parent and child are both + // unary or binary and the parent has higher precedence. + if precedence(parent) > newprec { + return true + } + + // Was the old node the operand of a postfix operator? + // f().sel + // f()[i:j] + // f()[i] + // f().(T) + // f()(x) + switch parent := parent.(type) { + case *ast.SelectorExpr: + return parent.X == old + case *ast.IndexExpr: + return parent.X == old + case *ast.SliceExpr: + return parent.X == old + case *ast.TypeAssertExpr: + return parent.X == old + case *ast.CallExpr: + return parent.Fun == old + } + return false +} diff --git a/internal/refactor/inline/inline_test.go b/internal/refactor/inline/inline_test.go index eca5a914b8d..f0459464b26 100644 --- a/internal/refactor/inline/inline_test.go +++ b/internal/refactor/inline/inline_test.go @@ -359,7 +359,7 @@ func TestBasics(t *testing.T) { "Basic", `func f(x int) int { return x }`, `var _ = f(0)`, - `var _ = (0)`, + `var _ = 0`, }, { "Empty body, no arg effects.", @@ -387,6 +387,51 @@ func TestBasics(t *testing.T) { }) } +func TestPrecedenceParens(t *testing.T) { + // Ensure that parens are inserted when (and only when) necessary + // around the replacement for the call expression. (This is a special + // case in the way the inliner uses a combination of AST formatting + // for the call and text splicing for the rest of the file.) + runTests(t, []testcase{ + { + "Multiplication in addition context (no parens).", + `func f(x, y int) int { return x * y }`, + `func _() { _ = 1 + f(2, 3) }`, + `func _() { _ = 1 + 2*3 }`, + }, + { + "Addition in multiplication context (parens).", + `func f(x, y int) int { return x + y }`, + `func _() { _ = 1 * f(2, 3) }`, + `func _() { _ = 1 * (2 + 3) }`, + }, + { + "Addition in negation context (parens).", + `func f(x, y int) int { return x + y }`, + `func _() { _ = -f(1, 2) }`, + `func _() { _ = -(1 + 2) }`, + }, + { + "Addition in call context (no parens).", + `func f(x, y int) int { return x + y }`, + `func _() { println(f(1, 2)) }`, + `func _() { println(1 + 2) }`, + }, + { + "Addition in slice operand context (parens).", + `func f(x, y string) string { return x + y }`, + `func _() { _ = f("x", "y")[1:2] }`, + `func _() { _ = ("x" + "y")[1:2] }`, + }, + { + "String literal in slice operand context (no parens).", + `func f(x string) string { return x }`, + `func _() { _ = f("xy")[1:2] }`, + `func _() { _ = "xy"[1:2] }`, + }, + }) +} + func TestSubstitution(t *testing.T) { runTests(t, []testcase{ { @@ -421,7 +466,7 @@ func TestTailCallStrategy(t *testing.T) { "Tail call.", `func f() int { return 1 }`, `func _() int { return f() }`, - `func _() int { return (1) }`, + `func _() int { return 1 }`, }, { "Void tail call.", diff --git a/internal/refactor/inline/testdata/basic-err.txtar b/internal/refactor/inline/testdata/basic-err.txtar index c289e9bb544..4868b2cbfb1 100644 --- a/internal/refactor/inline/testdata/basic-err.txtar +++ b/internal/refactor/inline/testdata/basic-err.txtar @@ -19,6 +19,6 @@ package a import "io" -var _ = (io.EOF.Error()) //@ inline(re"getError", getError) +var _ = io.EOF.Error() //@ inline(re"getError", getError) func getError(err error) string { return err.Error() } diff --git a/internal/refactor/inline/testdata/basic-reduce.txtar b/internal/refactor/inline/testdata/basic-reduce.txtar index 46e44b77625..10aca5284ef 100644 --- a/internal/refactor/inline/testdata/basic-reduce.txtar +++ b/internal/refactor/inline/testdata/basic-reduce.txtar @@ -14,7 +14,7 @@ func zero() int { return 0 } -- zero -- package a -var _ = (0) //@ inline(re"zero", zero) +var _ = 0 //@ inline(re"zero", zero) func zero() int { return 0 } @@ -46,5 +46,5 @@ var _ = add(len(""), 2) //@ inline(re"add", add2) -- add2 -- package a -var _ = (len("") + 2) //@ inline(re"add", add2) +var _ = len("") + 2 //@ inline(re"add", add2) diff --git a/internal/refactor/inline/testdata/comments.txtar b/internal/refactor/inline/testdata/comments.txtar index 189e9a20e8d..76f64926b13 100644 --- a/internal/refactor/inline/testdata/comments.txtar +++ b/internal/refactor/inline/testdata/comments.txtar @@ -51,7 +51,7 @@ func g() int { return 1 /*hello*/ + /*there*/ 1 } package a func _() { - println((1 + 1)) //@ inline(re"g", g) + println(1 + 1) //@ inline(re"g", g) } func g() int { return 1 /*hello*/ + /*there*/ 1 } diff --git a/internal/refactor/inline/testdata/param-subst.txtar b/internal/refactor/inline/testdata/param-subst.txtar index 135313b865a..b6e462d7e71 100644 --- a/internal/refactor/inline/testdata/param-subst.txtar +++ b/internal/refactor/inline/testdata/param-subst.txtar @@ -14,6 +14,6 @@ func add(x, y int) int { return x + 2*y } -- add -- package a -var _ = (2 + 2*(1+1)) //@ inline(re"add", add) +var _ = 2 + 2*(1+1) //@ inline(re"add", add) func add(x, y int) int { return x + 2*y } \ No newline at end of file From c6d331deb4111249f9d2ed90829bc7d6e611b4af Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Fri, 29 Sep 2023 16:54:51 -0400 Subject: [PATCH 164/178] internal/refactor/inline: don't add same import PkgName twice The logic to choose a fresh names checked for conflicts in the caller and callee but not against the newly added imports. This change adds a fix and a test. Also, don't use "init" as a PkgName: it's reserved. Fixes golang/go#63298 Change-Id: I7670fb841d6a4f521d567d39fef86c88f06e5c98 Reviewed-on: https://go-review.googlesource.com/c/tools/+/531698 Reviewed-by: Robert Findley Auto-Submit: Alan Donovan LUCI-TryBot-Result: Go LUCI --- internal/refactor/inline/inline.go | 13 ++++- .../refactor/inline/testdata/issue63298.txtar | 52 +++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 internal/refactor/inline/testdata/issue63298.txtar diff --git a/internal/refactor/inline/inline.go b/internal/refactor/inline/inline.go index e5fe3336122..b38dbe85acb 100644 --- a/internal/refactor/inline/inline.go +++ b/internal/refactor/inline/inline.go @@ -331,18 +331,29 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu } } + newlyAdded := func(name string) bool { + for _, new := range newImports { + if new.Name.Name == name { + return true + } + } + return false + } + // import added by callee // // Choose local PkgName based on last segment of // package path plus, if needed, a numeric suffix to // ensure uniqueness. // + // "init" is not a legal PkgName. + // // TODO(adonovan): preserve the PkgName used // in the original source, or, for a dot import, // use the package's declared name. base := pathpkg.Base(path) name := base - for n := 0; shadows[name] || caller.lookup(name) != nil; n++ { + for n := 0; shadows[name] || caller.lookup(name) != nil || newlyAdded(name) || name == "init"; n++ { name = fmt.Sprintf("%s%d", base, n) } diff --git a/internal/refactor/inline/testdata/issue63298.txtar b/internal/refactor/inline/testdata/issue63298.txtar new file mode 100644 index 00000000000..ec113858585 --- /dev/null +++ b/internal/refactor/inline/testdata/issue63298.txtar @@ -0,0 +1,52 @@ +Regression test for #63298: inlining a function that +depends on two packages with the same name leads +to duplicate PkgNames. + +-- go.mod -- +module testdata +go 1.12 + +-- a/a.go -- +package a + +func _() { + a2() //@ inline(re"a2", result) +} + +-- a/a2.go -- +package a + +import "testdata/b" +import anotherb "testdata/another/b" + +func a2() { + b.B() + anotherb.B() +} + +-- b/b.go -- +package b + +func B() {} + +-- b/another/b.go -- +package b + +func B() {} + +-- result -- +package a + +import ( + b "testdata/b" + b0 "testdata/another/b" + + //@ inline(re"a2", result) +) + +func _() { + { + b.B() + b0.B() + } +} \ No newline at end of file From 6a38a5f6f17a406b87128735bec195e57be77487 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Mon, 2 Oct 2023 15:39:10 -0400 Subject: [PATCH 165/178] internal/refactor/inline: use default working directory Also, tweak comments. Change-Id: I5051eb902eb0cbd07fb10ae820b2e01bce2fe14e Reviewed-on: https://go-review.googlesource.com/c/tools/+/532275 Reviewed-by: Robert Findley Auto-Submit: Alan Donovan LUCI-TryBot-Result: Go LUCI --- internal/refactor/inline/everything_test.go | 29 +++++++++++++-------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/internal/refactor/inline/everything_test.go b/internal/refactor/inline/everything_test.go index 082d4e00c7e..7046780650c 100644 --- a/internal/refactor/inline/everything_test.go +++ b/internal/refactor/inline/everything_test.go @@ -22,21 +22,29 @@ import ( "golang.org/x/tools/internal/testenv" ) -// Run with this command: -// -// $ go test ./internal/refactor/inline/ -run=Everything -v -packages=./internal/... - -// TODO(adonovan): -// - report counters (number of attempts, failed AnalyzeCallee, failed -// Inline, etc.) -// - Make a pretty log of the entire output so that we can peruse it -// for opportunities for systematic improvement. - var packagesFlag = flag.String("packages", "", "set of packages for TestEverything") // TestEverything invokes the inliner on every single call site in a // given package. and checks that it produces either a reasonable // error, or output that parses and type-checks. +// +// It does nothing during ordinary testing, but may be used to find +// inlining bugs in large corpora. +// +// Use this command to inline everything in golang.org/x/tools: +// +// $ go test ./internal/refactor/inline/ -run=Everything -v -packages=../../../ +// +// And these commands to inline everything in the kubernetes repository: +// +// $ go build -o /tmp/everything ./internal/refactor/inline/ +// $ (cd kubernetes && /tmp/everything -run=Everything -v -packages=./...) +// +// TODO(adonovan): +// - report counters (number of attempts, failed AnalyzeCallee, failed +// Inline, etc.) +// - Make a pretty log of the entire output so that we can peruse it +// for opportunities for systematic improvement. func TestEverything(t *testing.T) { testenv.NeedsGoPackages(t) if testing.Short() { @@ -48,7 +56,6 @@ func TestEverything(t *testing.T) { // Load this package plus dependencies from typed syntax. cfg := &packages.Config{ - Dir: "../../..", // root of x/tools Mode: packages.LoadAllSyntax, Env: append(os.Environ(), "GO111MODULES=on", From ca34416d493f46e2967aba35dca9981eedc4736f Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Tue, 19 Sep 2023 15:07:52 -0400 Subject: [PATCH 166/178] internal/refactor/inline: fallible constant analysis This change adds an analysis of "fallible constant" (falcon) expressions, and uses it to reject substitution of parameters by constant arguments in the (rare) cases where it is not safe. When substituting a parameter by its argument expression, it is (perhaps surprisingly) not always safe to replace a variable by a constant expression, even of exactly the same type, as it may turn a dynamic error into a compile-time one, as in this example: func f(s string) { if len(s) > 0 { println(s[0]) } } f("") // inline The call is replaced by: if len("") > 0 { println(""[0]) // error: constant index out of range } In general, operations on constants (in particular -x, x+y, x[i], x[i:j], and f(x)) are subject to additional static checks that don't make sense for non-constant operands. This analysis scans the callee function body for all expressions that are not constant but would become constant if the parameter vars were redeclared as constants, and emits a constraint (a Go expression) for each one that has the property that it will not type-check (using types.CheckExpr) if the particular argument values are unsuitable. The substitution logic checks the constraints and falls back to a binding decl if they are not all satisfied. (More optimal solutions could be found, but this situation is very uncommon.) (It would be easy to detect these problems if we simply type-checked the transformed source, but, by design, we simply don't have the necessary type information for indirect dependencies when running in an environment like unitchecker.) Fixes golang/go#62664 Change-Id: I27d40adb681c469e9c711bf1ee6f7319b5725a2a Reviewed-on: https://go-review.googlesource.com/c/tools/+/531695 Reviewed-by: Robert Findley Auto-Submit: Alan Donovan LUCI-TryBot-Result: Go LUCI --- internal/refactor/inline/callee.go | 26 +- internal/refactor/inline/doc.go | 19 + internal/refactor/inline/falcon.go | 895 ++++++++++++++++++++++++ internal/refactor/inline/falcon_test.go | 421 +++++++++++ internal/refactor/inline/inline.go | 88 ++- internal/refactor/inline/inline_test.go | 2 +- 6 files changed, 1431 insertions(+), 20 deletions(-) create mode 100644 internal/refactor/inline/falcon.go create mode 100644 internal/refactor/inline/falcon_test.go diff --git a/internal/refactor/inline/callee.go b/internal/refactor/inline/callee.go index 27a7b284b5d..dc74eab4e1a 100644 --- a/internal/refactor/inline/callee.go +++ b/internal/refactor/inline/callee.go @@ -47,6 +47,7 @@ type gobCallee struct { TotalReturns int // number of return statements TrivialReturns int // number of return statements with trivial result conversions Labels []string // names of all control labels + Falcon falconResult // falcon constraint system } // A freeRef records a reference to a free object. Gob-serializable. @@ -331,7 +332,7 @@ func AnalyzeCallee(logf func(string, ...any), fset *token.FileSet, pkg *types.Pa return nil, err } - params, results, effects := analyzeParams(logf, fset, info, decl) + params, results, effects, falcon := analyzeParams(logf, fset, info, decl) return &Callee{gobCallee{ Content: content, PkgPath: pkg.Path(), @@ -349,6 +350,7 @@ func AnalyzeCallee(logf func(string, ...any), fset *token.FileSet, pkg *types.Pa TotalReturns: totalReturns, TrivialReturns: trivialReturns, Labels: labels, + Falcon: falcon, }}, nil } @@ -365,15 +367,15 @@ func parseCompact(content []byte) (*token.FileSet, *ast.FuncDecl, error) { } // A paramInfo records information about a callee receiver, parameter, or result variable. -// TODO(adonovan): rename to sigVarInfo or paramOrResultInfo? type paramInfo struct { - Name string // parameter name (may be blank, or even "") - Index int // index within signature - IsResult bool // false for receiver or parameter, true for result variable - Assigned bool // parameter appears on left side of an assignment statement - Escapes bool // parameter has its address taken - Refs []int // FuncDecl-relative byte offset of parameter ref within body - Shadow map[string]bool // names shadowed at one of the above refs + Name string // parameter name (may be blank, or even "") + Index int // index within signature + IsResult bool // false for receiver or parameter, true for result variable + Assigned bool // parameter appears on left side of an assignment statement + Escapes bool // parameter has its address taken + Refs []int // FuncDecl-relative byte offset of parameter ref within body + Shadow map[string]bool // names shadowed at one of the above refs + FalconType string // name of this parameter's type (if basic) in the falcon system } // analyzeParams computes information about parameters of function fn, @@ -383,7 +385,7 @@ type paramInfo struct { // the other of the result variables of function fn. // // The input must be well-typed. -func analyzeParams(logf func(string, ...any), fset *token.FileSet, info *types.Info, decl *ast.FuncDecl) (params, results []*paramInfo, effects []int) { +func analyzeParams(logf func(string, ...any), fset *token.FileSet, info *types.Info, decl *ast.FuncDecl) (params, results []*paramInfo, effects []int, _ falconResult) { fnobj, ok := info.Defs[decl.Name] if !ok { panic(fmt.Sprintf("%s: no func object for %q", @@ -458,7 +460,9 @@ func analyzeParams(logf func(string, ...any), fset *token.FileSet, info *types.I effects = calleefx(info, decl.Body, paramInfos) logf("effects list = %v", effects) - return params, results, effects + falcon := falcon(logf, fset, paramInfos, info, decl) + + return params, results, effects, falcon } // -- callee helpers -- diff --git a/internal/refactor/inline/doc.go b/internal/refactor/inline/doc.go index f83759ac0c3..b13241f1ec6 100644 --- a/internal/refactor/inline/doc.go +++ b/internal/refactor/inline/doc.go @@ -219,6 +219,18 @@ which they can be addressed. be prepared to eliminate the declaration too---this is where an iterative framework for simplification would really help). + - An expression such as s[i] may be valid if s and i are + variables but invalid if either or both of them are constants. + For example, a negative constant index s[-1] is always out of + bounds, and even a non-negative constant index may be out of + bounds depending on the particular string constant (e.g. + "abc"[4]). + + So, if a parameter participates in any expression that is + subject to additional compile-time checks when its operands are + constant, it may be unsafe to substitute that parameter by a + constant argument value (#62664). + More complex callee functions are inlinable with more elaborate and invasive changes to the statements surrounding the call expression. @@ -253,6 +265,13 @@ TODO(adonovan): future work: - Eliminate parens and braces inserted conservatively when they are redundant. + - Eliminate explicit conversions of "untyped" literals inserted + conservatively when they are redundant. For example, the + conversion int32(1) is redundant when this value is used only as a + slice index; but it may be crucial if it is used in x := int32(1) + as it changes the type of x, which may have further implications. + The conversions may also be important to the falcon analysis. + - Allow non-'go' build systems such as Bazel/Blaze a chance to decide whether an import is accessible using logic other than "/internal/" path segments. This could be achieved by returning diff --git a/internal/refactor/inline/falcon.go b/internal/refactor/inline/falcon.go new file mode 100644 index 00000000000..9ad57d727e4 --- /dev/null +++ b/internal/refactor/inline/falcon.go @@ -0,0 +1,895 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package inline + +// This file defines the callee side of the "fallible constant" analysis. + +import ( + "fmt" + "go/ast" + "go/constant" + "go/format" + "go/token" + "go/types" + "strconv" + "strings" + + "golang.org/x/tools/go/types/typeutil" + "golang.org/x/tools/internal/typeparams" +) + +// falconResult is the result of the analysis of the callee. +type falconResult struct { + Types []falconType // types for falcon constraint environment + Constraints []string // constraints (Go expressions) on values of fallible constants +} + +// A falconType specifies the name and underlying type of a synthetic +// defined type for use in falcon constraints. +// +// Unique types from callee code are bijectively mapped onto falcon +// types so that constraints are independent of callee type +// information but preserve type equivalence classes. +// +// Fresh names are deliberately obscure to avoid shadowing even if a +// callee parameter has a nanme like "int" or "any". +type falconType struct { + Name string + Kind types.BasicKind // string/number/bool +} + +// falcon identifies "fallible constant" expressions, which are +// expressions that may fail to compile if one or more of their +// operands is changed from non-constant to constant. +// +// Consider: +// +// func sub(s string, i, j int) string { return s[i:j] } +// +// If parameters are replaced by constants, the compiler is +// required to perform these additional checks: +// +// - if i is constant, 0 <= i. +// - if s and i are constant, i <= len(s). +// - ditto for j. +// - if i and j are constant, i <= j. +// +// s[i:j] is thus a "fallible constant" expression dependent on {s, i, +// j}. Each falcon creates a set of conditional constraints across one +// or more parameter variables. +// +// - When inlining a call such as sub("abc", -1, 2), the parameter i +// cannot be eliminated by substitution as its argument value is +// negative. +// +// - When inlining sub("", 2, 1), all three parameters cannot be be +// simultaneously eliminated by substitution without violating i +// <= len(s) and j <= len(s), but the parameters i and j could be +// safely eliminated without s. +// +// Parameters that cannot be eliminated must remain non-constant, +// either in the form of a binding declaration: +// +// { var i int = -1; return "abc"[i:2] } +// +// or a parameter of a literalization: +// +// func (i int) string { return "abc"[i:2] }(-1) +// +// These example expressions are obviously doomed to fail at run +// time, but in realistic cases such expressions are dominated by +// appropriate conditions that make them reachable only when safe: +// +// if 0 <= i && i <= j && j <= len(s) { _ = s[i:j] } +// +// (In principle a more sophisticated inliner could entirely eliminate +// such unreachable blocks based on the condition being always-false +// for the given parameter substitution, but this is tricky to do safely +// because the type-checker considers only a single configuration. +// Consider: if runtime.GOOS == "linux" { ... }.) +// +// We believe this is an exhaustive list of "fallible constant" operations: +// +// - switch z { case x: case y } // duplicate case values +// - s[i], s[i:j], s[i:j:k] // index out of bounds (0 <= i <= j <= k <= len(s)) +// - T{x: 0} // index out of bounds, duplicate index +// - x/y, x%y, x/=y, x%=y // integer division by zero; minint/-1 overflow +// - x+y, x-y, x*y // arithmetic overflow +// - x< 1 { + var elts []ast.Expr + for _, elem := range elems { + elts = append(elts, &ast.KeyValueExpr{ + Key: elem, + Value: makeIntLit(0), + }) + } + st.emit(&ast.CompositeLit{ + Type: typ, + Elts: elts, + }) + } +} + +// -- traversal -- + +// The traversal functions scan the callee body for expressions that +// are not constant but would become constant if the parameter vars +// were redeclared as constants, and emits for each one a constraint +// (a Go expression) with the property that it will not type-check +// (using types.CheckExpr) if the particular argument values are +// unsuitable. +// +// These constraints are checked by Inline with the actual +// constant argument values. Violations cause it to reject +// parameters as candidates for substitution. + +func (st *falconState) stmt(s ast.Stmt) { + ast.Inspect(s, func(n ast.Node) bool { + switch n := n.(type) { + case ast.Expr: + _ = st.expr(n) + return false // skip usual traversal + + case *ast.AssignStmt: + switch n.Tok { + case token.QUO_ASSIGN, token.REM_ASSIGN: + // x /= y + // Possible "integer division by zero" + // Emit constraint: 1/y. + _ = st.expr(n.Lhs[0]) + kY := st.expr(n.Rhs[0]) + if kY, ok := kY.(ast.Expr); ok { + op := token.QUO + if n.Tok == token.REM_ASSIGN { + op = token.REM + } + st.emit(&ast.BinaryExpr{ + Op: op, + X: makeIntLit(1), + Y: kY, + }) + } + return false // skip usual traversal + } + + case *ast.SwitchStmt: + if n.Init != nil { + st.stmt(n.Init) + } + tBool := types.Type(types.Typ[types.Bool]) + tagType := tBool // default: true + if n.Tag != nil { + st.expr(n.Tag) + tagType = st.info.TypeOf(n.Tag) + } + + // Possible "duplicate case value". + // Emit constraint map[T]int{v1: 0, ..., vN:0} + // to ensure all maybe-constant case values are unique + // (unless switch tag is boolean, which is relaxed). + var unique []ast.Expr + for _, clause := range n.Body.List { + clause := clause.(*ast.CaseClause) + for _, caseval := range clause.List { + if k := st.expr(caseval); k != nil { + unique = append(unique, st.toExpr(k)) + } + } + for _, stmt := range clause.Body { + st.stmt(stmt) + } + } + if unique != nil && !types.Identical(tagType.Underlying(), tBool) { + tname := st.any + if !types.IsInterface(tagType) { + tname = st.typename(tagType) + } + t := &ast.MapType{ + Key: makeIdent(tname), + Value: makeIdent(st.int), + } + st.emitUnique(t, unique) + } + } + return true + }) +} + +// fieldTypes visits the .Type of each field in the list. +func (st *falconState) fieldTypes(fields *ast.FieldList) { + if fields != nil { + for _, field := range fields.List { + _ = st.expr(field.Type) + } + } +} + +// expr visits the expression (or type) and returns a +// non-nil result if the expression is constant or would +// become constant if all suitable function parameters were +// redeclared as constants. +// +// If the expression is constant, st.expr returns its type +// and value (types.TypeAndValue). If the expression would +// become constant, st.expr returns an ast.Expr tree whose +// leaves are literals and parameter references, and whose +// interior nodes are operations that may become constant, +// such as -x, x+y, f(x), and T(x). We call these would-be +// constant expressions "fallible constants", since they may +// fail to type-check for some values of x, i, and j. (We +// refer to the non-nil cases collectively as "maybe +// constant", and the nil case as "definitely non-constant".) +// +// As a side effect, st.expr emits constraints for each +// fallible constant expression; this is its main purpose. +// +// Consequently, st.expr must visit the entire subtree so +// that all necessary constraints are emitted. It may not +// short-circuit the traversal when it encounters a constant +// subexpression as constants may contain arbitrary other +// syntax that may impose constraints. Consider (as always) +// this contrived but legal example of a type parameter (!) +// that contains statement syntax: +// +// func f[T [unsafe.Sizeof(func() { stmts })]int]() +// +// There is no need to emit constraints for (e.g.) s[i] when s +// and i are already constants, because we know the expression +// is sound, but it is sometimes easier to emit these +// redundant constraints than to avoid them. +func (st *falconState) expr(e ast.Expr) (res any) { // = types.TypeAndValue | ast.Expr + tv := st.info.Types[e] + if tv.Value != nil { + // A constant value overrides any other result. + defer func() { res = tv }() + } + + switch e := e.(type) { + case *ast.Ident: + if v, ok := st.info.Uses[e].(*types.Var); ok { + if _, ok := st.params[v]; ok && isBasic(v.Type(), types.IsConstType) { + return e // reference to constable parameter + } + } + // (References to *types.Const are handled by the defer.) + + case *ast.BasicLit: + // constant + + case *ast.ParenExpr: + return st.expr(e.X) + + case *ast.FuncLit: + _ = st.expr(e.Type) + st.stmt(e.Body) + // definitely non-constant + + case *ast.CompositeLit: + // T{k: v, ...}, where T ∈ {array,*array,slice,map}, + // imposes a constraint that all constant k are + // distinct and, for arrays [n]T, within range 0-n. + // + // Types matter, not just values. For example, + // an interface-keyed map may contain keys + // that are numerically equal so long as they + // are of distinct types. For example: + // + // type myint int + // map[any]bool{1: true, 1: true} // error: duplicate key + // map[any]bool{1: true, int16(1): true} // ok + // map[any]bool{1: true, myint(1): true} // ok + // + // This can be asserted by emitting a + // constraint of the form T{k1: 0, ..., kN: 0}. + if e.Type != nil { + _ = st.expr(e.Type) + } + t := deref(typeparams.CoreType(deref(tv.Type))) + var uniques []ast.Expr + for _, elt := range e.Elts { + if kv, ok := elt.(*ast.KeyValueExpr); ok { + if !is[*types.Struct](t) { + if k := st.expr(kv.Key); k != nil { + uniques = append(uniques, st.toExpr(k)) + } + } + _ = st.expr(kv.Value) + } else { + _ = st.expr(elt) + } + } + if uniques != nil { + // Inv: not a struct. + + // The type T in constraint T{...} depends on the CompLit: + // - for a basic-keyed map, use map[K]int; + // - for an interface-keyed map, use map[any]int; + // - for a slice, use []int; + // - for an array or *array, use [n]int. + // The last two entail progressively stronger index checks. + var ct ast.Expr // type syntax for constraint + switch t := t.(type) { + case *types.Map: + if types.IsInterface(t.Key()) { + ct = &ast.MapType{ + Key: makeIdent(st.any), + Value: makeIdent(st.int), + } + } else { + ct = &ast.MapType{ + Key: makeIdent(st.typename(t.Key())), + Value: makeIdent(st.int), + } + } + case *types.Array: // or *array + ct = &ast.ArrayType{ + Len: makeIntLit(t.Len()), + Elt: makeIdent(st.int), + } + default: + panic(t) + } + st.emitUnique(ct, uniques) + } + // definitely non-constant + + case *ast.SelectorExpr: + _ = st.expr(e.X) + _ = st.expr(e.Sel) + // The defer is sufficient to handle + // qualified identifiers (pkg.Const). + // All other cases are definitely non-constant. + + case *ast.IndexExpr: + if tv.IsType() { + // type C[T] + _ = st.expr(e.X) + _ = st.expr(e.Index) + } else { + // term x[i] + // + // Constraints (if x is slice/string/array/*array, not map): + // - i >= 0 + // if i is a fallible constant + // - i < len(x) + // if x is array/*array and + // i is a fallible constant; + // or if s is a string and both i, + // s are maybe-constants, + // but not both are constants. + kX := st.expr(e.X) + kI := st.expr(e.Index) + if kI != nil && !is[*types.Map](st.info.TypeOf(e.X).Underlying()) { + if kI, ok := kI.(ast.Expr); ok { + st.emitNonNegative(kI) + } + // Emit constraint to check indices against known length. + // TODO(adonovan): factor with SliceExpr logic. + var x ast.Expr + if kX != nil { + // string + x = st.toExpr(kX) + } else if arr, ok := deref(st.info.TypeOf(e.X).Underlying()).(*types.Array); ok { + // array, *array + x = &ast.CompositeLit{ + Type: &ast.ArrayType{ + Len: makeIntLit(arr.Len()), + Elt: makeIdent(st.int), + }, + } + } + if x != nil { + st.emit(&ast.IndexExpr{ + X: x, + Index: st.toExpr(kI), + }) + } + } + } + // definitely non-constant + + case *ast.SliceExpr: + // x[low:high:max] + // + // Emit non-negative constraints for each index, + // plus low <= high <= max <= len(x) + // for each pair that are maybe-constant + // but not definitely constant. + + kX := st.expr(e.X) + var kLow, kHigh, kMax any + if e.Low != nil { + kLow = st.expr(e.Low) + if kLow != nil { + if kLow, ok := kLow.(ast.Expr); ok { + st.emitNonNegative(kLow) + } + } + } + if e.High != nil { + kHigh = st.expr(e.High) + if kHigh != nil { + if kHigh, ok := kHigh.(ast.Expr); ok { + st.emitNonNegative(kHigh) + } + if kLow != nil { + st.emitMonotonic(st.toExpr(kLow), st.toExpr(kHigh)) + } + } + } + if e.Max != nil { + kMax = st.expr(e.Max) + if kMax != nil { + if kMax, ok := kMax.(ast.Expr); ok { + st.emitNonNegative(kMax) + } + if kHigh != nil { + st.emitMonotonic(st.toExpr(kHigh), st.toExpr(kMax)) + } + } + } + + // Emit constraint to check indices against known length. + var x ast.Expr + if kX != nil { + // string + x = st.toExpr(kX) + } else if arr, ok := deref(st.info.TypeOf(e.X).Underlying()).(*types.Array); ok { + // array, *array + x = &ast.CompositeLit{ + Type: &ast.ArrayType{ + Len: makeIntLit(arr.Len()), + Elt: makeIdent(st.int), + }, + } + } + if x != nil { + // Avoid slice[::max] if kHigh is nonconstant (nil). + high, max := st.toExpr(kHigh), st.toExpr(kMax) + if high == nil { + high = max // => slice[:max:max] + } + st.emit(&ast.SliceExpr{ + X: x, + Low: st.toExpr(kLow), + High: high, + Max: max, + }) + } + // definitely non-constant + + case *ast.TypeAssertExpr: + _ = st.expr(e.X) + if e.Type != nil { + _ = st.expr(e.Type) + } + + case *ast.CallExpr: + _ = st.expr(e.Fun) + if tv, ok := st.info.Types[e.Fun]; ok && tv.IsType() { + // conversion T(x) + // + // Possible "value out of range". + kX := st.expr(e.Args[0]) + if kX != nil && isBasic(tv.Type, types.IsConstType) { + conv := &ast.CallExpr{ + Fun: makeIdent(st.typename(tv.Type)), + Args: []ast.Expr{st.toExpr(kX)}, + } + if is[ast.Expr](kX) { + st.emit(conv) + } + return conv + } + return nil // definitely non-constant + } + + // call f(x) + + all := true // all args are possibly-constant + kArgs := make([]ast.Expr, len(e.Args)) + for i, arg := range e.Args { + if kArg := st.expr(arg); kArg != nil { + kArgs[i] = st.toExpr(kArg) + } else { + all = false + } + } + + // Calls to built-ins with fallibly constant arguments + // may become constant. All other calls are either + // constant or non-constant + if id, ok := e.Fun.(*ast.Ident); ok && all && tv.Value == nil { + if builtin, ok := st.info.Uses[id].(*types.Builtin); ok { + switch builtin.Name() { + case "len", "imag", "real", "complex", "min", "max": + return &ast.CallExpr{ + Fun: id, + Args: kArgs, + Ellipsis: e.Ellipsis, + } + } + } + } + + case *ast.StarExpr: // *T, *ptr + _ = st.expr(e.X) + + case *ast.UnaryExpr: + // + - ! ^ & <- ~ + // + // Possible "negation of minint". + // Emit constraint: -x + kX := st.expr(e.X) + if kX != nil && !is[types.TypeAndValue](kX) { + if e.Op == token.SUB { + st.emit(&ast.UnaryExpr{ + Op: e.Op, + X: st.toExpr(kX), + }) + } + + return &ast.UnaryExpr{ + Op: e.Op, + X: st.toExpr(kX), + } + } + + case *ast.BinaryExpr: + kX := st.expr(e.X) + kY := st.expr(e.Y) + switch e.Op { + case token.QUO, token.REM: + // x/y, x%y + // + // Possible "integer division by zero" or + // "minint / -1" overflow. + // Emit constraint: x/y or 1/y + if kY != nil { + if kX == nil { + kX = makeIntLit(1) + } + st.emit(&ast.BinaryExpr{ + Op: e.Op, + X: st.toExpr(kX), + Y: st.toExpr(kY), + }) + } + + case token.ADD, token.SUB, token.MUL: + // x+y, x-y, x*y + // + // Possible "arithmetic overflow". + // Emit constraint: x+y + if kX != nil && kY != nil { + st.emit(&ast.BinaryExpr{ + Op: e.Op, + X: st.toExpr(kX), + Y: st.toExpr(kY), + }) + } + + case token.SHL, token.SHR: + // x << y, x >> y + // + // Possible "constant shift too large". + // Either operand may be too large individually, + // and they may be too large together. + // Emit constraint: + // x << y (if both maybe-constant) + // x << 0 (if y is non-constant) + // 1 << y (if x is non-constant) + if kX != nil || kY != nil { + x := st.toExpr(kX) + if x == nil { + x = makeIntLit(1) + } + y := st.toExpr(kY) + if y == nil { + y = makeIntLit(0) + } + st.emit(&ast.BinaryExpr{ + Op: e.Op, + X: x, + Y: y, + }) + } + + case token.LSS, token.GTR, token.EQL, token.NEQ, token.LEQ, token.GEQ: + // < > == != <= <= + // + // A "x cmp y" expression with constant operands x, y is + // itself constant, but I can't see how a constant bool + // could be fallible: the compiler doesn't reject duplicate + // boolean cases in a switch, presumably because boolean + // switches are less like n-way branches and more like + // sequential if-else chains with possibly overlapping + // conditions; and there is (sadly) no way to convert a + // boolean constant to an int constant. + } + if kX != nil && kY != nil { + return &ast.BinaryExpr{ + Op: e.Op, + X: st.toExpr(kX), + Y: st.toExpr(kY), + } + } + + // types + // + // We need to visit types (and even type parameters) + // in order to reach all the places where things could go wrong: + // + // const ( + // s = "" + // i = 0 + // ) + // type C[T [unsafe.Sizeof(func() { _ = s[i] })]int] bool + + case *ast.IndexListExpr: + _ = st.expr(e.X) + for _, expr := range e.Indices { + _ = st.expr(expr) + } + + case *ast.Ellipsis: + if e.Elt != nil { + _ = st.expr(e.Elt) + } + + case *ast.ArrayType: + if e.Len != nil { + _ = st.expr(e.Len) + } + _ = st.expr(e.Elt) + + case *ast.StructType: + st.fieldTypes(e.Fields) + + case *ast.FuncType: + st.fieldTypes(e.TypeParams) + st.fieldTypes(e.Params) + st.fieldTypes(e.Results) + + case *ast.InterfaceType: + st.fieldTypes(e.Methods) + + case *ast.MapType: + _ = st.expr(e.Key) + _ = st.expr(e.Value) + + case *ast.ChanType: + _ = st.expr(e.Value) + } + return +} + +// toExpr converts the result of visitExpr to a falcon expression. +// (We don't do this in visitExpr as we first need to discriminate +// constants from maybe-constants.) +func (st *falconState) toExpr(x any) ast.Expr { + switch x := x.(type) { + case nil: + return nil + + case types.TypeAndValue: + lit := makeLiteral(x.Value) + if !isBasic(x.Type, types.IsUntyped) { + // convert to "typed" type + lit = &ast.CallExpr{ + Fun: makeIdent(st.typename(x.Type)), + Args: []ast.Expr{lit}, + } + } + return lit + + case ast.Expr: + return x + + default: + panic(x) + } +} + +func makeLiteral(v constant.Value) ast.Expr { + switch v.Kind() { + case constant.Bool: + // Rather than refer to the true or false built-ins, + // which could be shadowed by poorly chosen parameter + // names, we use 0 == 0 for true and 0 != 0 for false. + op := token.EQL + if !constant.BoolVal(v) { + op = token.NEQ + } + return &ast.BinaryExpr{ + Op: op, + X: makeIntLit(0), + Y: makeIntLit(0), + } + + case constant.String: + return &ast.BasicLit{ + Kind: token.STRING, + Value: v.ExactString(), + } + + case constant.Int: + return &ast.BasicLit{ + Kind: token.INT, + Value: v.ExactString(), + } + + case constant.Float: + return &ast.BasicLit{ + Kind: token.FLOAT, + Value: v.ExactString(), + } + + case constant.Complex: + // The components could be float or int. + y := makeLiteral(constant.Imag(v)) + y.(*ast.BasicLit).Value += "i" // ugh + if re := constant.Real(v); !consteq(re, kZeroInt) { + // complex: x + yi + y = &ast.BinaryExpr{ + Op: token.ADD, + X: makeLiteral(re), + Y: y, + } + } + return y + + default: + panic(v.Kind()) + } +} + +func makeIntLit(x int64) *ast.BasicLit { + return &ast.BasicLit{ + Kind: token.INT, + Value: strconv.FormatInt(x, 10), + } +} + +func isBasic(t types.Type, info types.BasicInfo) bool { + basic, ok := t.Underlying().(*types.Basic) + return ok && basic.Info()&info != 0 +} diff --git a/internal/refactor/inline/falcon_test.go b/internal/refactor/inline/falcon_test.go new file mode 100644 index 00000000000..c928440456f --- /dev/null +++ b/internal/refactor/inline/falcon_test.go @@ -0,0 +1,421 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package inline_test + +import "testing" + +// Testcases mostly come in pairs, of a success and a failure +// to substitute based on specific constant argument values. + +func TestFalconStringIndex(t *testing.T) { + runTests(t, []testcase{ + { + "Non-negative string index.", + `func f(i int) byte { return s[i] }; var s string`, + `func _() { f(0) }`, + `func _() { _ = s[0] }`, + }, + { + "Negative string index.", + `func f(i int) byte { return s[i] }; var s string`, + `func _() { f(-1) }`, + `func _() { + { + var i int = -1 + _ = s[i] + } +}`, + }, + { + "String index in range.", + `func f(s string, i int) byte { return s[i] }`, + `func _() { f("-", 0) }`, + `func _() { _ = "-"[0] }`, + }, + { + "String index out of range.", + `func f(s string, i int) byte { return s[i] }`, + `func _() { f("-", 1) }`, + `func _() { + { + var ( + s string = "-" + i int = 1 + ) + _ = s[i] + } +}`, + }, + { + "Remove known prefix (OK)", + `func f(s, prefix string) string { return s[:len(prefix)] }`, + `func _() { f("", "") }`, + `func _() { _ = ""[:len("")] }`, + }, + { + "Remove not-a-prefix (out of range)", + `func f(s, prefix string) string { return s[:len(prefix)] }`, + `func _() { f("", "pre") }`, + `func _() { + { + var s, prefix string = "", "pre" + _ = s[:len(prefix)] + } +}`, + }, + }) +} + +func TestFalconSliceIndices(t *testing.T) { + runTests(t, []testcase{ + { + "Monotonic (0<=i<=j) slice indices (len unknown).", + `func f(i, j int) []int { return s[i:j] }; var s []int`, + `func _() { f(0, 1) }`, + `func _() { _ = s[0:1] }`, + }, + { + "Non-monotonic slice indices (len unknown).", + `func f(i, j int) []int { return s[i:j] }; var s []int`, + `func _() { f(1, 0) }`, + `func _() { + { + var i, j int = 1, 0 + _ = s[i:j] + } +}`, + }, + { + "Negative slice index.", + `func f(i, j int) []int { return s[i:j] }; var s []int`, + `func _() { f(-1, 1) }`, + `func _() { + { + var i, j int = -1, 1 + _ = s[i:j] + } +}`, + }, + }) +} + +func TestFalconMapKeys(t *testing.T) { + runTests(t, []testcase{ + { + "Unique map keys (int)", + `func f(x int) { _ = map[int]bool{1: true, x: true} }`, + `func _() { f(2) }`, + `func _() { _ = map[int]bool{1: true, 2: true} }`, + }, + { + "Duplicate map keys (int)", + `func f(x int) { _ = map[int]bool{1: true, x: true} }`, + `func _() { f(1) }`, + `func _() { + { + var x int = 1 + _ = map[int]bool{1: true, x: true} + } +}`, + }, + { + "Unique map keys (varied built-in types)", + `func f(x int16) { _ = map[any]bool{1: true, x: true} }`, + `func _() { f(2) }`, + `func _() { _ = map[any]bool{1: true, int16(2): true} }`, + }, + { + "Duplicate map keys (varied built-in types)", + `func f(x int16) { _ = map[any]bool{1: true, x: true} }`, + `func _() { f(1) }`, + `func _() { _ = map[any]bool{1: true, int16(1): true} }`, + }, + { + "Unique map keys (varied user-defined types)", + `func f(x myint) { _ = map[any]bool{1: true, x: true} }; type myint int`, + `func _() { f(2) }`, + `func _() { _ = map[any]bool{1: true, myint(2): true} }`, + }, + { + "Duplicate map keys (varied user-defined types)", + `func f(x myint, y myint2) { _ = map[any]bool{x: true, y: true} }; type (myint int; myint2 int)`, + `func _() { f(1, 1) }`, + `func _() { + { + var ( + x myint = 1 + y myint2 = 1 + ) + _ = map[any]bool{x: true, y: true} + } +}`, + }, + { + "Duplicate map keys (user-defined alias to built-in)", + `func f(x myint, y int) { _ = map[any]bool{x: true, y: true} }; type myint = int`, + `func _() { f(1, 1) }`, + `func _() { + { + var ( + x myint = 1 + y int = 1 + ) + _ = map[any]bool{x: true, y: true} + } +}`, + }, + }) +} + +func TestFalconSwitchCases(t *testing.T) { + runTests(t, []testcase{ + { + "Unique switch cases (int).", + `func f(x int) { switch 0 { case x: case 1: } }`, + `func _() { f(2) }`, + `func _() { + switch 0 { + case 2: + case 1: + } +}`, + }, + { + "Duplicate switch cases (int).", + `func f(x int) { switch 0 { case x: case 1: } }`, + `func _() { f(1) }`, + `func _() { + { + var x int = 1 + switch 0 { + case x: + case 1: + } + } +}`, + }, + { + "Unique switch cases (varied built-in types).", + `func f(x int) { switch any(nil) { case x: case int16(1): } }`, + `func _() { f(2) }`, + `func _() { + switch any(nil) { + case 2: + case int16(1): + } +}`, + }, + { + "Duplicate switch cases (varied built-in types).", + `func f(x int) { switch any(nil) { case x: case int16(1): } }`, + `func _() { f(1) }`, + `func _() { + switch any(nil) { + case 1: + case int16(1): + } +}`, + }, + }) +} + +func TestFalconDivision(t *testing.T) { + runTests(t, []testcase{ + { + "Division by two.", + `func f(x, y int) int { return x / y }`, + `func _() { f(1, 2) }`, + `func _() { _ = 1 / 2 }`, + }, + { + "Division by zero.", + `func f(x, y int) int { return x / y }`, + `func _() { f(1, 0) }`, + `func _() { + { + var x, y int = 1, 0 + _ = x / y + } +}`, + }, + { + "Division by two (statement).", + `func f(x, y int) { x /= y }`, + `func _() { f(1, 2) }`, + `func _() { + { + var x int = 1 + x /= 2 + } +}`, + }, + { + "Division by zero (statement).", + `func f(x, y int) { x /= y }`, + `func _() { f(1, 0) }`, + `func _() { + { + var x, y int = 1, 0 + x /= y + } +}`, + }, + { + "Division of minint by two (ok).", + `func f(x, y int32) { _ = x / y }`, + `func _() { f(-0x80000000, 2) }`, + `func _() { _ = int32(-0x80000000) / int32(2) }`, + }, + { + "Division of minint by -1 (overflow).", + `func f(x, y int32) { _ = x / y }`, + `func _() { f(-0x80000000, -1) }`, + `func _() { + { + var x, y int32 = -0x80000000, -1 + _ = x / y + } +}`, + }, + }) +} + +func TestFalconMinusMinInt(t *testing.T) { + runTests(t, []testcase{ + { + "Negation of maxint.", + `func f(x int32) int32 { return -x }`, + `func _() { f(0x7fffffff) }`, + `func _() { _ = -int32(0x7fffffff) }`, + }, + { + "Negation of minint.", + `func f(x int32) int32 { return -x }`, + `func _() { f(-0x80000000) }`, + `func _() { + { + var x int32 = -0x80000000 + _ = -x + } +}`, + }, + }) +} + +func TestFalconArithmeticOverflow(t *testing.T) { + runTests(t, []testcase{ + { + "Addition without overflow.", + `func f(x, y int32) int32 { return x + y }`, + `func _() { f(100, 200) }`, + `func _() { _ = int32(100) + int32(200) }`, + }, + { + "Addition with overflow.", + `func f(x, y int32) int32 { return x + y }`, + `func _() { f(1<<30, 1<<30) }`, + `func _() { + { + var x, y int32 = 1 << 30, 1 << 30 + _ = x + y + } +}`, + }, + { + "Conversion in range.", + `func f(x int) int8 { return int8(x) }`, + `func _() { f(123) }`, + `func _() { _ = int8(123) }`, + }, + { + "Conversion out of range.", + `func f(x int) int8 { return int8(x) }`, + `func _() { f(456) }`, + `func _() { + { + var x int = 456 + _ = int8(x) + } +}`, + }, + }) +} + +func TestFalconComplex(t *testing.T) { + runTests(t, []testcase{ + { + "Complex arithmetic (good).", + `func f(re, im float64, z complex128) byte { return "x"[int(real(complex(re, im)*complex(re, -im)-z))] }`, + `func _() { f(1, 2, 5+0i) }`, + `func _() { _ = "x"[int(real(complex(float64(1), float64(2))*complex(float64(1), -float64(2))-(5+0i)))] }`, + }, + { + "Complex arithmetic (bad).", + `func f(re, im float64, z complex128) byte { return "x"[int(real(complex(re, im)*complex(re, -im)-z))] }`, + `func _() { f(1, 3, 5+0i) }`, + `func _() { + { + var ( + re, im float64 = 1, 3 + z complex128 = 5 + 0i + ) + _ = "x"[int(real(complex(re, im)*complex(re, -im)-z))] + } +}`, + }, + }) +} +func TestFalconMisc(t *testing.T) { + runTests(t, []testcase{ + { + "Compound constant expression (good).", + `func f(x, y string, i, j int) byte { return x[i*len(y)+j] }`, + `func _() { f("abc", "xy", 2, -3) }`, + `func _() { _ = "abc"[2*len("xy")+-3] }`, + }, + { + "Compound constant expression (index out of range).", + `func f(x, y string, i, j int) byte { return x[i*len(y)+j] }`, + `func _() { f("abc", "xy", 4, -3) }`, + `func _() { + { + var ( + x, y string = "abc", "xy" + i, j int = 4, -3 + ) + _ = x[i*len(y)+j] + } +}`, + }, + { + "Constraints within nested functions (good).", + `func f(x int) { _ = func() { _ = [1]int{}[x] } }`, + `func _() { f(0) }`, + `func _() { _ = func() { _ = [1]int{}[0] } }`, + }, + { + "Constraints within nested functions (bad).", + `func f(x int) { _ = func() { _ = [1]int{}[x] } }`, + `func _() { f(1) }`, + `func _() { + { + var x int = 1 + _ = func() { _ = [1]int{}[x] } + } +}`, + }, + { + "Falcon violation rejects only the constant arguments (x, z).", + `func f(x, y, z string) string { return x[:2] + y + z[:2] }; var b string`, + `func _() { f("a", b, "c") }`, + `func _() { + { + var x, z string = "a", "c" + _ = x[:2] + b + z[:2] + } +}`, + }, + }) +} diff --git a/internal/refactor/inline/inline.go b/internal/refactor/inline/inline.go index b38dbe85acb..ef2ce3aae87 100644 --- a/internal/refactor/inline/inline.go +++ b/internal/refactor/inline/inline.go @@ -60,7 +60,8 @@ func Inline(logf func(string, ...any), caller *Caller, callee *Callee) ([]byte, } // TODO(adonovan): use go1.21's ast.IsGenerated. - if bytes.Contains(caller.Content, []byte("// Code generated by cmd/cgo; DO NOT EDIT.")) { + // Break the string literal so we can use inlining in this file. :) + if bytes.Contains(caller.Content, []byte("// Code generated by "+"cmd/cgo; DO NOT EDIT.")) { return nil, fmt.Errorf("cannot inline calls from files that import \"C\"") } @@ -563,6 +564,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu Elts: elts, }, typ: lastParam.obj.Type(), + constant: nil, pure: pure, effects: effects, duplicable: false, @@ -585,7 +587,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // Perform parameter substitution. // May eliminate some elements of params/args. - substitute(logf, caller, params, args, callee.Effects, replaceCalleeID) + substitute(logf, caller, params, args, callee.Effects, callee.Falcon, replaceCalleeID) // Update the callee's signature syntax. updateCalleeParams(calleeDecl, params) @@ -933,6 +935,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu type argument struct { expr ast.Expr typ types.Type // may be tuple for sole non-receiver arg in spread call + constant constant.Value // value of argument if constant spread bool // final arg is call() assigned to multiple params pure bool // expr is pure (doesn't read variables) effects bool // expr has effects (updates variables) @@ -993,6 +996,7 @@ func arguments(caller *Caller, calleeDecl *ast.FuncDecl, assign1 func(*types.Var arg := &argument{ expr: recvArg, typ: caller.Info.TypeOf(recvArg), + constant: caller.Info.Types[recvArg].Value, pure: pure(caller.Info, assign1, recvArg), effects: effects(caller.Info, recvArg), duplicable: duplicable(caller.Info, recvArg), @@ -1041,11 +1045,12 @@ func arguments(caller *Caller, calleeDecl *ast.FuncDecl, assign1 func(*types.Var } } for _, expr := range callArgs { - typ := caller.Info.TypeOf(expr) + tv := caller.Info.Types[expr] args = append(args, &argument{ expr: expr, - typ: typ, - spread: is[*types.Tuple](typ), // => last + typ: tv.Type, + constant: tv.Value, + spread: is[*types.Tuple](tv.Type), // => last pure: pure(caller.Info, assign1, expr), effects: effects(caller.Info, expr), duplicable: duplicable(caller.Info, expr), @@ -1072,8 +1077,8 @@ func arguments(caller *Caller, calleeDecl *ast.FuncDecl, assign1 func(*types.Var // chan to <-chan, do not result in the type-checker imposing // the LHS type on the RHS value. for _, arg := range args { - if caller.Info.Types[arg.expr].Value == nil { - continue // not a constant + if arg.constant == nil { + continue } info := &types.Info{Types: make(map[ast.Expr]types.TypeAndValue)} if err := types.CheckExpr(caller.Fset, caller.Types, caller.Call.Pos(), arg.expr, info); err != nil { @@ -1111,7 +1116,7 @@ type parameter struct { // parameter, and is provided with its relative offset and replacement // expression (argument), and the corresponding elements of params and // args are replaced by nil. -func substitute(logf func(string, ...any), caller *Caller, params []*parameter, args []*argument, effects []int, replaceCalleeID func(offset int, repl ast.Expr)) { +func substitute(logf func(string, ...any), caller *Caller, params []*parameter, args []*argument, effects []int, falcon falconResult, replaceCalleeID func(offset int, repl ast.Expr)) { // Inv: // in calls to variadic, len(args) >= len(params)-1 // in spread calls to non-variadic, len(args) < len(params) @@ -1189,6 +1194,10 @@ next: arg.substitutable = true // may be substituted, if effects permit } + // Reject constant arguments as substitution candidates + // if they cause violation of falcon constraints. + checkFalconConstraints(logf, params, args, falcon) + // As a final step, introduce bindings to resolve any // evaluation order hazards. This must be done last, as // additional subsequent bindings could introduce new hazards. @@ -1232,6 +1241,69 @@ next: } } +// checkFalconConstraints checks whether constant arguments +// are safe to substitute (e.g. s[i] -> ""[0] is not safe.) +// +// Any failed constraint causes us to reject all constant arguments as +// substitution candidates (by clearing args[i].substitution=false). +// +// TODO(adonovan): we could obtain a finer result rejecting only the +// freevars of each failed constraint, and processing constraints in +// order of increasing arity, but failures are quite rare. +func checkFalconConstraints(logf func(string, ...any), params []*parameter, args []*argument, falcon falconResult) { + // Create a dummy package, as this is the only + // way to create an environment for CheckExpr. + pkg := types.NewPackage("falcon", "falcon") + + // Declare types used by constraints. + for _, typ := range falcon.Types { + logf("falcon env: type %s %s", typ.Name, types.Typ[typ.Kind]) + pkg.Scope().Insert(types.NewTypeName(token.NoPos, pkg, typ.Name, types.Typ[typ.Kind])) + } + + // Declared constants and variables for for parameters. + nconst := 0 + for i, param := range params { + name := param.info.Name + if name == "" { + continue // unreferenced + } + arg := args[i] + if arg.constant != nil && arg.substitutable && param.info.FalconType != "" { + t := pkg.Scope().Lookup(param.info.FalconType).Type() + pkg.Scope().Insert(types.NewConst(token.NoPos, pkg, name, t, arg.constant)) + logf("falcon env: const %s %s = %v", name, param.info.FalconType, arg.constant) + nconst++ + } else { + pkg.Scope().Insert(types.NewVar(token.NoPos, pkg, name, arg.typ)) + logf("falcon env: var %s %s", name, arg.typ) + } + } + if nconst == 0 { + return // nothing to do + } + + // Parse and evaluate the constraints in the environment. + fset := token.NewFileSet() + for _, falcon := range falcon.Constraints { + expr, err := parser.ParseExprFrom(fset, "falcon", falcon, 0) + if err != nil { + panic(fmt.Sprintf("failed to parse falcon constraint %s: %v", falcon, err)) + } + if err := types.CheckExpr(fset, pkg, token.NoPos, expr, nil); err != nil { + logf("falcon: constraint %s violated: %v", falcon, err) + for j, arg := range args { + if arg.constant != nil && arg.substitutable { + logf("keeping param %q due falcon violation", params[j].info.Name) + arg.substitutable = false + } + } + break + } + logf("falcon: constraint %s satisfied", falcon) + } +} + // resolveEffects marks arguments as non-substitutable to resolve // hazards resulting from the callee evaluation order described by the // effects list. diff --git a/internal/refactor/inline/inline_test.go b/internal/refactor/inline/inline_test.go index f0459464b26..b1e06aef5b1 100644 --- a/internal/refactor/inline/inline_test.go +++ b/internal/refactor/inline/inline_test.go @@ -1020,7 +1020,7 @@ func runTests(t *testing.T, tests []testcase) { conf := &types.Config{Error: func(err error) { t.Error(err) }} pkg, err := conf.Check("p", fset, []*ast.File{callerFile, calleeFile}, info) if err != nil { - t.Fatal(err) + t.Fatal("transformation introduced type errors") } // Analyze callee and inline call. From 586b21ae0964abe3f376aeb52d34f06ad27de00a Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Mon, 2 Oct 2023 12:32:29 -0400 Subject: [PATCH 167/178] internal/refactor/inline: elide redundant braces When replacing a CallExpr beneath an ExprStmt, a reduction strategy may return a BlockStmt containing zero or more statements. This change eliminates the braces for the block when it is safe to do so, in other words when these three conditions are met: (a) the parent of the ExprStmt is an unrestricted statement context e.g. a block or the body of a case of a switch or select, but not, say "if f(); cond {". (b) there are no forward gotos in the caller that may jump across a declaration. (Currently we check for any control labels at all in the caller.) (c) there are no conflicts between names declared in the callee block and in the caller block. Plus tests. Also, a fix and test for a latent bug allowing reduction in a restricted "if stmt; expr" context. Fixes golang/go#63259 Change-Id: I558c75d8306dfd0679768cb4b3dbf05f14b23c39 Reviewed-on: https://go-review.googlesource.com/c/tools/+/532099 LUCI-TryBot-Result: Go LUCI Auto-Submit: Alan Donovan Reviewed-by: Robert Findley --- internal/refactor/inline/inline.go | 176 +++++++++++++-- internal/refactor/inline/inline_test.go | 211 +++++++++++------- .../inline/testdata/import-shadow.txtar | 8 +- .../refactor/inline/testdata/method.txtar | 8 +- .../inline/testdata/multistmt-body.txtar | 8 +- .../refactor/inline/testdata/tailcall.txtar | 24 +- internal/refactor/inline/util.go | 13 ++ 7 files changed, 322 insertions(+), 126 deletions(-) diff --git a/internal/refactor/inline/inline.go b/internal/refactor/inline/inline.go index ef2ce3aae87..4fe918b7c1d 100644 --- a/internal/refactor/inline/inline.go +++ b/internal/refactor/inline/inline.go @@ -99,6 +99,40 @@ func Inline(logf func(string, ...any), caller *Caller, callee *Callee) ([]byte, res.new = &ast.ParenExpr{X: res.new.(ast.Expr)} } + // Some reduction strategies return a new block holding the + // callee's statements. The block's braces may be elided when + // there is no conflict between names declared in the block + // with those declared by the parent block, and no risk of + // a caller's goto jumping forward across a declaration. + // + // This elision is only safe when the ExprStmt is beneath a + // BlockStmt, CaseClause.Body, or CommClause.Body; + // (see "statement theory"). + elideBraces := false + if newBlock, ok := res.new.(*ast.BlockStmt); ok { + parent := caller.path[nodeIndex(caller.path, res.old)+1] + var body []ast.Stmt + switch parent := parent.(type) { + case *ast.BlockStmt: + body = parent.List + case *ast.CommClause: + body = parent.Body + case *ast.CaseClause: + body = parent.Body + } + if body != nil { + if len(callerLabels(caller.path)) > 0 { + // TODO(adonovan): be more precise and reject + // only forward gotos across the inlined block. + logf("keeping block braces: caller uses control labels") + } else if intersects(declares(newBlock.List), declares(body)) { + logf("keeping block braces: avoids name conflict") + } else { + elideBraces = true + } + } + } + // Don't call replaceNode(caller.File, res.old, res.new) // as it mutates the caller's syntax tree. // Instead, splice the file, replacing the extent of the "old" @@ -124,9 +158,16 @@ func Inline(logf func(string, ...any), caller *Caller, callee *Callee) ([]byte, // Precise comment handling would make this a // non-issue. Formatting wouldn't really need a // FileSet at all. + mark := out.Len() if err := format.Node(&out, caller.Fset, res.new); err != nil { return nil, err } + if elideBraces { + // Overwrite unnecessary {...} braces with spaces. + // TODO(adonovan): less hacky solution. + out.Bytes()[mark] = ' ' + out.Bytes()[out.Len()-1] = ' ' + } out.Write(caller.Content[end:]) const mode = parser.ParseComments | parser.SkipObjectResolution | parser.AllErrors f, err = parser.ParseFile(caller.Fset, "callee.go", &out, mode) @@ -630,7 +671,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu logf("strategy: reduce call to empty body") // Evaluate the arguments for effects and delete the call entirely. - stmt := callStmt(caller.path) // cannot fail + stmt := callStmt(caller.path, false) // cannot fail res.old = stmt if nargs := len(remainingArgs); nargs > 0 { // Emit "_, _ = args" to discard results. @@ -862,9 +903,10 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // - there is no label conflict between caller and callee // - all parameters and result vars can be eliminated // or replaced by a binding decl, + // - caller ExprStmt is in unrestricted statement context. // // If there is only a single statement, the braces are omitted. - if stmt := callStmt(caller.path); stmt != nil && + if stmt := callStmt(caller.path, true); stmt != nil && (!needBindingDecl || bindingDeclStmt != nil) && !callee.HasDefer && !hasLabelConflict(caller.path, callee.Labels) && @@ -876,7 +918,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu if needBindingDecl { body.List = prepend(bindingDeclStmt, body.List...) } - if len(body.List) == 1 { + if len(body.List) == 1 { // FIXME do this opt later repl = body.List[0] // singleton: omit braces } res.old = stmt @@ -2018,6 +2060,18 @@ func callContext(callPath []ast.Node) ast.Node { // enclosing the call (specified as a PathEnclosingInterval) // intersects with the set of callee labels. func hasLabelConflict(callPath []ast.Node, calleeLabels []string) bool { + labels := callerLabels(callPath) + for _, label := range calleeLabels { + if labels[label] { + return true // conflict + } + } + return false +} + +// callerLabels returns the set of control labels in the function (if +// any) enclosing the call (specified as a PathEnclosingInterval). +func callerLabels(callPath []ast.Node) map[string]bool { var callerBody *ast.BlockStmt switch f := callerFunc(callPath).(type) { case *ast.FuncDecl: @@ -2025,23 +2079,22 @@ func hasLabelConflict(callPath []ast.Node, calleeLabels []string) bool { case *ast.FuncLit: callerBody = f.Body } - conflict := false + var labels map[string]bool if callerBody != nil { ast.Inspect(callerBody, func(n ast.Node) bool { switch n := n.(type) { case *ast.FuncLit: return false // prune traversal case *ast.LabeledStmt: - for _, label := range calleeLabels { - if label == n.Label.Name { - conflict = true - } + if labels == nil { + labels = make(map[string]bool) } + labels[n.Label.Name] = true } return true }) } - return conflict + return labels } // callerFunc returns the innermost Func{Decl,Lit} node enclosing the @@ -2059,11 +2112,64 @@ func callerFunc(callPath []ast.Node) ast.Node { // callStmt reports whether the function call (specified // as a PathEnclosingInterval) appears within an ExprStmt, // and returns it if so. -func callStmt(callPath []ast.Node) *ast.ExprStmt { - stmt, _ := callContext(callPath).(*ast.ExprStmt) +// +// If unrestricted, callStmt returns nil if the ExprStmt f() appears +// in a restricted context (such as "if f(); cond {") where it cannot +// be replaced by an arbitrary statement. (See "statement theory".) +func callStmt(callPath []ast.Node, unrestricted bool) *ast.ExprStmt { + stmt, ok := callContext(callPath).(*ast.ExprStmt) + if ok && unrestricted { + switch callPath[nodeIndex(callPath, stmt)+1].(type) { + case *ast.LabeledStmt, + *ast.BlockStmt, + *ast.CaseClause, + *ast.CommClause: + // unrestricted + default: + // TODO(adonovan): handle restricted + // XYZStmt.Init contexts (but not ForStmt.Post) + // by creating a block around the if/for/switch: + // "if f(); cond {" -> "{ stmts; if cond {" + + return nil // restricted + } + } return stmt } +// Statement theory +// +// These are all the places a statement may appear in the AST: +// +// LabeledStmt.Stmt Stmt -- any +// BlockStmt.List []Stmt -- any (but see switch/select) +// IfStmt.Init Stmt? -- simple +// IfStmt.Body BlockStmt +// IfStmt.Else Stmt? -- IfStmt or BlockStmt +// CaseClause.Body []Stmt -- any +// SwitchStmt.Init Stmt? -- simple +// SwitchStmt.Body BlockStmt -- CaseClauses only +// TypeSwitchStmt.Init Stmt? -- simple +// TypeSwitchStmt.Assign Stmt -- AssignStmt(TypeAssertExpr) or ExprStmt(TypeAssertExpr) +// TypeSwitchStmt.Body BlockStmt -- CaseClauses only +// CommClause.Comm Stmt? -- SendStmt or ExprStmt(UnaryExpr) or AssignStmt(UnaryExpr) +// CommClause.Body []Stmt -- any +// SelectStmt.Body BlockStmt -- CommClauses only +// ForStmt.Init Stmt? -- simple +// ForStmt.Post Stmt? -- simple +// ForStmt.Body BlockStmt +// RangeStmt.Body BlockStmt +// +// simple = AssignStmt | SendStmt | IncDecStmt | ExprStmt. +// +// A BlockStmt cannot replace an ExprStmt in +// {If,Switch,TypeSwitch}Stmt.Init or ForStmt.Post. +// That is allowed only within: +// LabeledStmt.Stmt Stmt +// BlockStmt.List []Stmt +// CaseClause.Body []Stmt +// CommClause.Body []Stmt + // replaceNode performs a destructive update of the tree rooted at // root, replacing each occurrence of "from" with "to". If to is nil and // the element is within a slice, the slice element is removed. @@ -2372,13 +2478,7 @@ func consistentOffsets(caller *Caller) bool { // ancestor of the CallExpr identified by its PathEnclosingInterval). func needsParens(callPath []ast.Node, old, new ast.Node) bool { // Find enclosing old node and its parent. - // TODO(adonovan): Use index[ast.Node]() in go1.20. - i := -1 - for i = range callPath { - if callPath[i] == old { - break - } - } + i := nodeIndex(callPath, old) if i == -1 { panic("not found") } @@ -2439,3 +2539,43 @@ func needsParens(callPath []ast.Node, old, new ast.Node) bool { } return false } + +func nodeIndex(nodes []ast.Node, n ast.Node) int { + // TODO(adonovan): Use index[ast.Node]() in go1.20. + for i, node := range nodes { + if node == n { + return i + } + } + return -1 +} + +// declares returns the set of lexical names declared by a +// sequence of statements from the same block, excluding sub-blocks. +// (Lexical names do not include control labels.) +func declares(stmts []ast.Stmt) map[string]bool { + names := make(map[string]bool) + for _, stmt := range stmts { + switch stmt := stmt.(type) { + case *ast.DeclStmt: + for _, spec := range stmt.Decl.(*ast.GenDecl).Specs { + switch spec := spec.(type) { + case *ast.ValueSpec: + for _, id := range spec.Names { + names[id.Name] = true + } + case *ast.TypeSpec: + names[spec.Name.Name] = true + } + } + + case *ast.AssignStmt: + if stmt.Tok == token.DEFINE { + for _, lhs := range stmt.Lhs { + names[lhs.(*ast.Ident).Name] = true + } + } + } + } + return names +} diff --git a/internal/refactor/inline/inline_test.go b/internal/refactor/inline/inline_test.go index b1e06aef5b1..e47c07625ed 100644 --- a/internal/refactor/inline/inline_test.go +++ b/internal/refactor/inline/inline_test.go @@ -378,10 +378,91 @@ func TestBasics(t *testing.T) { `func f(s string, i int) { print(s, s, i, i) }`, `func _() { f("hi", 0) }`, `func _() { + var s string = "hi" + print(s, s, 0, 0) +}`, + }, + }) +} + +func TestExprStmtReduction(t *testing.T) { + runTests(t, []testcase{ + { + "A call in an unrestricted ExprStmt may be replaced by the body stmts.", + `func f() { var _ = len("") }`, + `func _() { f() }`, + `func _() { var _ = len("") }`, + }, + { + "ExprStmts in the body of a switch case are unrestricted.", + `func f() { x := 1; print(x) }`, + `func _() { switch { case true: f() } }`, + `func _() { + switch { + case true: + x := 1 + print(x) + } +}`, + }, + { + "ExprStmts in the body of a select case are unrestricted.", + `func f() { x := 1; print(x) }`, + `func _() { select { default: f() } }`, + `func _() { + select { + default: + x := 1 + print(x) + } +}`, + }, + { + "Some ExprStmt contexts are restricted to simple statements.", + `func f() { var _ = len("") }`, + `func _(cond bool) { if f(); cond {} }`, + `func _(cond bool) { + if func() { var _ = len("") }(); cond { + } +}`, + }, + { + "Braces must be preserved to avoid a name conflict (decl before).", + `func f() { x := 1; print(x) }`, + `func _() { x := 2; print(x); f() }`, + `func _() { + x := 2 + print(x) + { + x := 1 + print(x) + } +}`, + }, + { + "Braces must be preserved to avoid a name conflict (decl after).", + `func f() { x := 1; print(x) }`, + `func _() { f(); x := 2; print(x) }`, + `func _() { + { + x := 1 + print(x) + } + x := 2 + print(x) +}`, + }, + { + "Braces must be preserved to avoid a forward jump across a decl.", + `func f() { x := 1; print(x) }`, + `func _() { goto label; f(); label: }`, + `func _() { + goto label { - var s string = "hi" - print(s, s, 0, 0) + x := 1 + print(x) } +label: }`, }, }) @@ -576,10 +657,8 @@ func TestParameterBindingDecl(t *testing.T) { `func f(x int) { x++ }`, `func _() { f(1) }`, `func _() { - { - var x int = 1 - x++ - } + var x int = 1 + x++ }`, }, { @@ -587,10 +666,8 @@ func TestParameterBindingDecl(t *testing.T) { `func f(w, x, y any, z int) { println(w, y, z) }; func g(int) int`, `func _() { f(g(0), g(1), g(2), g(3)) }`, `func _() { - { - var w, _ any = g(0), g(1) - println(w, any(g(2)), g(3)) - } + var w, _ any = g(0), g(1) + println(w, any(g(2)), g(3)) }`, }, { @@ -605,10 +682,8 @@ func TestParameterBindingDecl(t *testing.T) { `func f(x int) int { return <-h(g(2), x) }; func g(int) int; func h(int, int) chan int`, `func _() { f(g(1)) }`, `func _() { - { - var x int = g(1) - <-h(g(2), x) - } + var x int = g(1) + <-h(g(2), x) }`, }, { @@ -675,10 +750,8 @@ func TestSubstitutionPreservesArgumentEffectOrder(t *testing.T) { `func f(a, b, c int) { print(a, c, b) }; func g(int) int`, `func _() { f(g(1), g(2), g(3)) }`, `func _() { - { - var a, b int = g(1), g(2) - print(a, g(3), b) - } + var a, b int = g(1), g(2) + print(a, g(3), b) }`, }, { @@ -698,10 +771,8 @@ func TestSubstitutionPreservesArgumentEffectOrder(t *testing.T) { `func f(a, b, c, d int) { print(a, c, b, d) }; func g(int) int; var x, y int`, `func _() { f(g(1), g(2), y, g(3)) }`, `func _() { - { - var a, b int = g(1), g(2) - print(a, y, b, g(3)) - } + var a, b int = g(1), g(2) + print(a, y, b, g(3)) }`, }, { @@ -709,10 +780,8 @@ func TestSubstitutionPreservesArgumentEffectOrder(t *testing.T) { `func f(a, b, c, d int) { print(a, c, b, d) }; func g(int) int; var x, y int`, `func _() { f(g(1), y, g(2), g(3)) }`, `func _() { - { - var a, b int = g(1), y - print(a, g(2), b, g(3)) - } + var a, b int = g(1), y + print(a, g(2), b, g(3)) }`, }, { @@ -732,10 +801,8 @@ func TestSubstitutionPreservesArgumentEffectOrder(t *testing.T) { `func f(a, b, c int) { print(a, b, recover().(int), c) }; var x, y, z int`, `func _() { f(x, y, z) }`, `func _() { - { - var c int = z - print(x, y, recover().(int), c) - } + var c int = z + print(x, y, recover().(int), c) }`, }, { @@ -743,10 +810,8 @@ func TestSubstitutionPreservesArgumentEffectOrder(t *testing.T) { `func f(a, b, c int) { print(a, b, recover().(int), c) }; func g(int) int; var x, y, z int`, `func _() { f(x, y, g(0)) }`, `func _() { - { - var a, b, c int = x, y, g(0) - print(a, b, recover().(int), c) - } + var a, b, c int = x, y, g(0) + print(a, b, recover().(int), c) }`, }, { @@ -754,10 +819,8 @@ func TestSubstitutionPreservesArgumentEffectOrder(t *testing.T) { `func f(a, b, c, d, e int) { print(b, a, c, e, d) }; func g(int) int; var x, y int`, `func _() { f(x, g(1), g(2), y, g(3)) }`, `func _() { - { - var a, b, c, d int = x, g(1), g(2), y - print(b, a, c, g(3), d) - } + var a, b, c, d int = x, g(1), g(2), y + print(b, a, c, g(3), d) }`, }, { @@ -788,10 +851,8 @@ func TestSubstitutionPreservesArgumentEffectOrder(t *testing.T) { `func f(x, y int) { _ = &y }; func g(int) int`, `func _() { f(g(1), g(2)) }`, `func _() { - { - var _, y int = g(1), g(2) - _ = &y - } + var _, y int = g(1), g(2) + _ = &y }`, }, { @@ -802,10 +863,8 @@ func TestSubstitutionPreservesArgumentEffectOrder(t *testing.T) { `func f(x, y int) { _ = x }; func g(int) int; var v int`, `func _() { f(v, g(2)) }`, `func _() { - { - var x, _ int = v, g(2) - _ = x - } + var x, _ int = v, g(2) + _ = x }`, }, { @@ -824,10 +883,8 @@ func TestNamedResultVars(t *testing.T) { `func f() (x int) { return g(x) }; func g(int) int`, `func _() { f() }`, `func _() { - { - var x int - g(x) - } + var x int + g(x) }`, }, { @@ -835,13 +892,11 @@ func TestNamedResultVars(t *testing.T) { `func f(y string) (x int) { return x+x+len(y+y) }`, `func _() { f(".") }`, `func _() { - { - var ( - y string = "." - x int - ) - _ = x + x + len(y+y) - } + var ( + y string = "." + x int + ) + _ = x + x + len(y+y) }`, }, @@ -850,13 +905,11 @@ func TestNamedResultVars(t *testing.T) { `func f(y string) (x string) { return x+y+y }`, `func _() { f(".") }`, `func _() { - { - var ( - y string = "." - x string - ) - _ = x + y + y - } + var ( + y string = "." + x string + ) + _ = x + y + y }`, }, { @@ -864,10 +917,8 @@ func TestNamedResultVars(t *testing.T) { `func f() (x int) { return x+x }`, `func _() { f() }`, `func _() { - { - var x int - _ = x + x - } + var x int + _ = x + x }`, }, { @@ -904,10 +955,8 @@ func TestSubstitutionPreservesParameterType(t *testing.T) { `func f(x int16) { y := x; _ = (*int16)(&y) }`, `func _() { f(1) }`, `func _() { - { - y := int16(1) - _ = (*int16)(&y) - } + y := int16(1) + _ = (*int16)(&y) }`, }, { @@ -915,10 +964,8 @@ func TestSubstitutionPreservesParameterType(t *testing.T) { `func f(x T) { y := x; _ = (*T)(&y) }; type T struct{}`, `func _() { f(struct{}{}) }`, `func _() { - { - y := T(struct{}{}) - _ = (*T)(&y) - } + y := T(struct{}{}) + _ = (*T)(&y) }`, }, { @@ -926,10 +973,8 @@ func TestSubstitutionPreservesParameterType(t *testing.T) { `func f(x T) { y := x; _ = (*T)(&y) }; type T = <-chan int; var ch chan int`, `func _() { f(ch) }`, `func _() { - { - y := T(ch) - _ = (*T)(&y) - } + y := T(ch) + _ = (*T)(&y) }`, }, { @@ -937,10 +982,8 @@ func TestSubstitutionPreservesParameterType(t *testing.T) { `func f(x *int) { y := x; _ = (**int)(&y) }`, `func _() { f(nil) }`, `func _() { - { - y := (*int)(nil) - _ = (**int)(&y) - } + y := (*int)(nil) + _ = (**int)(&y) }`, }, { diff --git a/internal/refactor/inline/testdata/import-shadow.txtar b/internal/refactor/inline/testdata/import-shadow.txtar index 5d4f9243c18..4188a52375d 100644 --- a/internal/refactor/inline/testdata/import-shadow.txtar +++ b/internal/refactor/inline/testdata/import-shadow.txtar @@ -87,10 +87,10 @@ import ( var x b.T func A(b int) { - { - b0.One() - b0.Two() - } //@ inline(re"F", fresult) + + b0.One() + b0.Two() + //@ inline(re"F", fresult) } -- d/d.go -- diff --git a/internal/refactor/inline/testdata/method.txtar b/internal/refactor/inline/testdata/method.txtar index fde9f366c12..b141b09d707 100644 --- a/internal/refactor/inline/testdata/method.txtar +++ b/internal/refactor/inline/testdata/method.txtar @@ -104,10 +104,10 @@ func (T) h() int { return 1 } func _() { var ptr *T - { - var _ T = *ptr - _ = 1 - } //@ inline(re"h", h) + + var _ T = *ptr + _ = 1 + //@ inline(re"h", h) } -- a/i.go -- diff --git a/internal/refactor/inline/testdata/multistmt-body.txtar b/internal/refactor/inline/testdata/multistmt-body.txtar index 1dc39c5c3b9..6bd0108e1fe 100644 --- a/internal/refactor/inline/testdata/multistmt-body.txtar +++ b/internal/refactor/inline/testdata/multistmt-body.txtar @@ -54,10 +54,10 @@ package a func _() { a := 1 - { - z := 1 - print(a + 2 + z) - } //@ inline(re"f", out2) + + z := 1 + print(a + 2 + z) + //@ inline(re"f", out2) } -- a/a3.go -- diff --git a/internal/refactor/inline/testdata/tailcall.txtar b/internal/refactor/inline/testdata/tailcall.txtar index 64f5f9735a0..53b6de367dd 100644 --- a/internal/refactor/inline/testdata/tailcall.txtar +++ b/internal/refactor/inline/testdata/tailcall.txtar @@ -36,19 +36,19 @@ start: package a func _() int { - { - total := 0 - start: - for i := 1; i <= 2; i++ { - total += i - if i == 6 { - goto start - } else if i == 7 { - return -1 - } + + total := 0 +start: + for i := 1; i <= 2; i++ { + total += i + if i == 6 { + goto start + } else if i == 7 { + return -1 } - return total - } //@ inline(re"sum", sum) + } + return total + //@ inline(re"sum", sum) } func sum(lo, hi int) int { diff --git a/internal/refactor/inline/util.go b/internal/refactor/inline/util.go index 82be0fb8ac6..6d8d3fac08f 100644 --- a/internal/refactor/inline/util.go +++ b/internal/refactor/inline/util.go @@ -90,3 +90,16 @@ func funcHasTypeParams(decl *ast.FuncDecl) bool { } return false } + +// intersects reports whether the maps' key sets intersect. +func intersects[K comparable, T1, T2 any](x map[K]T1, y map[K]T2) bool { + if len(x) > len(y) { + return intersects(y, x) + } + for k := range x { + if _, ok := y[k]; ok { + return true + } + } + return false +} From 197e2c43219fc0e43fa492c402e8129a33be5e00 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Tue, 3 Oct 2023 10:38:07 -0400 Subject: [PATCH 168/178] internal/refactor/inline: fix broken tests Two recent CLs had a semantic (but non-syntactic) conflict, so both passed CI independently but together tests fail. The fix is to remove the redundant braces from test expectations that the inliner no longer produces. Change-Id: I4e6c755e992c6755be36f2d767e77ed70c358dd7 Reviewed-on: https://go-review.googlesource.com/c/tools/+/532176 LUCI-TryBot-Result: Go LUCI Reviewed-by: Russ Cox Reviewed-by: Robert Findley --- internal/refactor/inline/falcon_test.go | 154 +++++++----------- internal/refactor/inline/inline_test.go | 6 +- .../refactor/inline/testdata/issue63298.txtar | 8 +- 3 files changed, 63 insertions(+), 105 deletions(-) diff --git a/internal/refactor/inline/falcon_test.go b/internal/refactor/inline/falcon_test.go index c928440456f..64a98f511b5 100644 --- a/internal/refactor/inline/falcon_test.go +++ b/internal/refactor/inline/falcon_test.go @@ -22,10 +22,8 @@ func TestFalconStringIndex(t *testing.T) { `func f(i int) byte { return s[i] }; var s string`, `func _() { f(-1) }`, `func _() { - { - var i int = -1 - _ = s[i] - } + var i int = -1 + _ = s[i] }`, }, { @@ -39,13 +37,11 @@ func TestFalconStringIndex(t *testing.T) { `func f(s string, i int) byte { return s[i] }`, `func _() { f("-", 1) }`, `func _() { - { - var ( - s string = "-" - i int = 1 - ) - _ = s[i] - } + var ( + s string = "-" + i int = 1 + ) + _ = s[i] }`, }, { @@ -59,10 +55,8 @@ func TestFalconStringIndex(t *testing.T) { `func f(s, prefix string) string { return s[:len(prefix)] }`, `func _() { f("", "pre") }`, `func _() { - { - var s, prefix string = "", "pre" - _ = s[:len(prefix)] - } + var s, prefix string = "", "pre" + _ = s[:len(prefix)] }`, }, }) @@ -81,10 +75,8 @@ func TestFalconSliceIndices(t *testing.T) { `func f(i, j int) []int { return s[i:j] }; var s []int`, `func _() { f(1, 0) }`, `func _() { - { - var i, j int = 1, 0 - _ = s[i:j] - } + var i, j int = 1, 0 + _ = s[i:j] }`, }, { @@ -92,10 +84,8 @@ func TestFalconSliceIndices(t *testing.T) { `func f(i, j int) []int { return s[i:j] }; var s []int`, `func _() { f(-1, 1) }`, `func _() { - { - var i, j int = -1, 1 - _ = s[i:j] - } + var i, j int = -1, 1 + _ = s[i:j] }`, }, }) @@ -114,10 +104,8 @@ func TestFalconMapKeys(t *testing.T) { `func f(x int) { _ = map[int]bool{1: true, x: true} }`, `func _() { f(1) }`, `func _() { - { - var x int = 1 - _ = map[int]bool{1: true, x: true} - } + var x int = 1 + _ = map[int]bool{1: true, x: true} }`, }, { @@ -143,13 +131,11 @@ func TestFalconMapKeys(t *testing.T) { `func f(x myint, y myint2) { _ = map[any]bool{x: true, y: true} }; type (myint int; myint2 int)`, `func _() { f(1, 1) }`, `func _() { - { - var ( - x myint = 1 - y myint2 = 1 - ) - _ = map[any]bool{x: true, y: true} - } + var ( + x myint = 1 + y myint2 = 1 + ) + _ = map[any]bool{x: true, y: true} }`, }, { @@ -157,13 +143,11 @@ func TestFalconMapKeys(t *testing.T) { `func f(x myint, y int) { _ = map[any]bool{x: true, y: true} }; type myint = int`, `func _() { f(1, 1) }`, `func _() { - { - var ( - x myint = 1 - y int = 1 - ) - _ = map[any]bool{x: true, y: true} - } + var ( + x myint = 1 + y int = 1 + ) + _ = map[any]bool{x: true, y: true} }`, }, }) @@ -187,12 +171,10 @@ func TestFalconSwitchCases(t *testing.T) { `func f(x int) { switch 0 { case x: case 1: } }`, `func _() { f(1) }`, `func _() { - { - var x int = 1 - switch 0 { - case x: - case 1: - } + var x int = 1 + switch 0 { + case x: + case 1: } }`, }, @@ -234,10 +216,8 @@ func TestFalconDivision(t *testing.T) { `func f(x, y int) int { return x / y }`, `func _() { f(1, 0) }`, `func _() { - { - var x, y int = 1, 0 - _ = x / y - } + var x, y int = 1, 0 + _ = x / y }`, }, { @@ -245,10 +225,8 @@ func TestFalconDivision(t *testing.T) { `func f(x, y int) { x /= y }`, `func _() { f(1, 2) }`, `func _() { - { - var x int = 1 - x /= 2 - } + var x int = 1 + x /= 2 }`, }, { @@ -256,10 +234,8 @@ func TestFalconDivision(t *testing.T) { `func f(x, y int) { x /= y }`, `func _() { f(1, 0) }`, `func _() { - { - var x, y int = 1, 0 - x /= y - } + var x, y int = 1, 0 + x /= y }`, }, { @@ -273,10 +249,8 @@ func TestFalconDivision(t *testing.T) { `func f(x, y int32) { _ = x / y }`, `func _() { f(-0x80000000, -1) }`, `func _() { - { - var x, y int32 = -0x80000000, -1 - _ = x / y - } + var x, y int32 = -0x80000000, -1 + _ = x / y }`, }, }) @@ -295,10 +269,8 @@ func TestFalconMinusMinInt(t *testing.T) { `func f(x int32) int32 { return -x }`, `func _() { f(-0x80000000) }`, `func _() { - { - var x int32 = -0x80000000 - _ = -x - } + var x int32 = -0x80000000 + _ = -x }`, }, }) @@ -317,10 +289,8 @@ func TestFalconArithmeticOverflow(t *testing.T) { `func f(x, y int32) int32 { return x + y }`, `func _() { f(1<<30, 1<<30) }`, `func _() { - { - var x, y int32 = 1 << 30, 1 << 30 - _ = x + y - } + var x, y int32 = 1 << 30, 1 << 30 + _ = x + y }`, }, { @@ -334,10 +304,8 @@ func TestFalconArithmeticOverflow(t *testing.T) { `func f(x int) int8 { return int8(x) }`, `func _() { f(456) }`, `func _() { - { - var x int = 456 - _ = int8(x) - } + var x int = 456 + _ = int8(x) }`, }, }) @@ -356,13 +324,11 @@ func TestFalconComplex(t *testing.T) { `func f(re, im float64, z complex128) byte { return "x"[int(real(complex(re, im)*complex(re, -im)-z))] }`, `func _() { f(1, 3, 5+0i) }`, `func _() { - { - var ( - re, im float64 = 1, 3 - z complex128 = 5 + 0i - ) - _ = "x"[int(real(complex(re, im)*complex(re, -im)-z))] - } + var ( + re, im float64 = 1, 3 + z complex128 = 5 + 0i + ) + _ = "x"[int(real(complex(re, im)*complex(re, -im)-z))] }`, }, }) @@ -380,13 +346,11 @@ func TestFalconMisc(t *testing.T) { `func f(x, y string, i, j int) byte { return x[i*len(y)+j] }`, `func _() { f("abc", "xy", 4, -3) }`, `func _() { - { - var ( - x, y string = "abc", "xy" - i, j int = 4, -3 - ) - _ = x[i*len(y)+j] - } + var ( + x, y string = "abc", "xy" + i, j int = 4, -3 + ) + _ = x[i*len(y)+j] }`, }, { @@ -400,10 +364,8 @@ func TestFalconMisc(t *testing.T) { `func f(x int) { _ = func() { _ = [1]int{}[x] } }`, `func _() { f(1) }`, `func _() { - { - var x int = 1 - _ = func() { _ = [1]int{}[x] } - } + var x int = 1 + _ = func() { _ = [1]int{}[x] } }`, }, { @@ -411,10 +373,8 @@ func TestFalconMisc(t *testing.T) { `func f(x, y, z string) string { return x[:2] + y + z[:2] }; var b string`, `func _() { f("a", b, "c") }`, `func _() { - { - var x, z string = "a", "c" - _ = x[:2] + b + z[:2] - } + var x, z string = "a", "c" + _ = x[:2] + b + z[:2] }`, }, }) diff --git a/internal/refactor/inline/inline_test.go b/internal/refactor/inline/inline_test.go index e47c07625ed..a70ddded34b 100644 --- a/internal/refactor/inline/inline_test.go +++ b/internal/refactor/inline/inline_test.go @@ -532,10 +532,8 @@ func TestSubstitution(t *testing.T) { `func f(x int) { _ = func() { y := 1; print(y); print(x) } }`, `func _(y int) { f(y) } `, `func _(y int) { - { - var x int = y - _ = func() { y := 1; print(y); print(x) } - } + var x int = y + _ = func() { y := 1; print(y); print(x) } }`, }, }) diff --git a/internal/refactor/inline/testdata/issue63298.txtar b/internal/refactor/inline/testdata/issue63298.txtar index ec113858585..e355e8e64d9 100644 --- a/internal/refactor/inline/testdata/issue63298.txtar +++ b/internal/refactor/inline/testdata/issue63298.txtar @@ -45,8 +45,8 @@ import ( ) func _() { - { - b.B() - b0.B() - } + + b.B() + b0.B() + } \ No newline at end of file From 102b64b540a5c70e954994db144d96a2b52b799c Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Tue, 3 Oct 2023 15:12:02 -0400 Subject: [PATCH 169/178] internal/refactor/inline: tweak everything-test docs again Change-Id: I5cdf4863af70c7dea446aed13015c619b0a75f57 Reviewed-on: https://go-review.googlesource.com/c/tools/+/532178 LUCI-TryBot-Result: Go LUCI Reviewed-by: Robert Findley Auto-Submit: Alan Donovan --- internal/refactor/inline/everything_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/refactor/inline/everything_test.go b/internal/refactor/inline/everything_test.go index 7046780650c..a1c5448d1d1 100644 --- a/internal/refactor/inline/everything_test.go +++ b/internal/refactor/inline/everything_test.go @@ -33,12 +33,12 @@ var packagesFlag = flag.String("packages", "", "set of packages for TestEverythi // // Use this command to inline everything in golang.org/x/tools: // -// $ go test ./internal/refactor/inline/ -run=Everything -v -packages=../../../ +// $ go test ./internal/refactor/inline/ -run=Everything -packages=../../../ // // And these commands to inline everything in the kubernetes repository: // -// $ go build -o /tmp/everything ./internal/refactor/inline/ -// $ (cd kubernetes && /tmp/everything -run=Everything -v -packages=./...) +// $ go build -c -o /tmp/everything ./internal/refactor/inline/ +// $ (cd kubernetes && /tmp/everything -test.run=Everything -packages=./...) // // TODO(adonovan): // - report counters (number of attempts, failed AnalyzeCallee, failed From a819c616c80e59828f9fea810e2e9b84d6023cce Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Tue, 3 Oct 2023 10:35:20 -0400 Subject: [PATCH 170/178] internal/refactor/inline: eliminate unnecessary binding decl In calls from one method to another with a pointer receiver, the inliner was inserting an unnecessary binding decl for the receiver var, because it relied on Selection.Indirect, but no actual indirection through the pointer is going on, so the receiver is pure. Instead, clear the purity flag only if there's a load through an embedded field. Also, fix a latent bug in the BlockStmt brace eliding logic: we forgot to include function params/results in scope when the block is the body of a function. Plus tests for both. Change-Id: I7e149f55fb344e5f733cdd6811d060ef0dc42883 Reviewed-on: https://go-review.googlesource.com/c/tools/+/532177 LUCI-TryBot-Result: Go LUCI Reviewed-by: Robert Findley --- internal/refactor/inline/inline.go | 35 +++++++++++++++++++++---- internal/refactor/inline/inline_test.go | 23 ++++++++++++++++ 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/internal/refactor/inline/inline.go b/internal/refactor/inline/inline.go index 4fe918b7c1d..08a4594ae27 100644 --- a/internal/refactor/inline/inline.go +++ b/internal/refactor/inline/inline.go @@ -110,7 +110,8 @@ func Inline(logf func(string, ...any), caller *Caller, callee *Callee) ([]byte, // (see "statement theory"). elideBraces := false if newBlock, ok := res.new.(*ast.BlockStmt); ok { - parent := caller.path[nodeIndex(caller.path, res.old)+1] + i := nodeIndex(caller.path, res.old) + parent := caller.path[i+1] var body []ast.Stmt switch parent := parent.(type) { case *ast.BlockStmt: @@ -121,11 +122,34 @@ func Inline(logf func(string, ...any), caller *Caller, callee *Callee) ([]byte, body = parent.Body } if body != nil { + callerNames := declares(body) + + // If BlockStmt is a function body, + // include its receiver, params, and results. + addFieldNames := func(fields *ast.FieldList) { + if fields != nil { + for _, field := range fields.List { + for _, id := range field.Names { + callerNames[id.Name] = true + } + } + } + } + switch f := caller.path[i+2].(type) { + case *ast.FuncDecl: + addFieldNames(f.Recv) + addFieldNames(f.Type.Params) + addFieldNames(f.Type.Results) + case *ast.FuncLit: + addFieldNames(f.Type.Params) + addFieldNames(f.Type.Results) + } + if len(callerLabels(caller.path)) > 0 { // TODO(adonovan): be more precise and reject // only forward gotos across the inlined block. logf("keeping block braces: caller uses control labels") - } else if intersects(declares(newBlock.List), declares(body)) { + } else if intersects(declares(newBlock.List), callerNames) { logf("keeping block braces: avoids name conflict") } else { elideBraces = true @@ -1060,6 +1084,9 @@ func arguments(caller *Caller, calleeDecl *ast.FuncDecl, assign1 func(*types.Var debugFormatNode(caller.Fset, caller.Call.Fun), fld.Name()) } + if is[*types.Pointer](arg.typ.Underlying()) { + arg.pure = false // implicit *ptr operation => impure + } arg.expr = &ast.SelectorExpr{ X: arg.expr, Sel: makeIdent(fld.Name()), @@ -1067,9 +1094,6 @@ func arguments(caller *Caller, calleeDecl *ast.FuncDecl, assign1 func(*types.Var arg.typ = fld.Type() arg.duplicable = false } - if seln.Indirect() { - arg.pure = false // one or more implicit *ptr operation => impure - } // Make * or & explicit. argIsPtr := arg.typ != deref(arg.typ) @@ -1083,6 +1107,7 @@ func arguments(caller *Caller, calleeDecl *ast.FuncDecl, assign1 func(*types.Var arg.expr = &ast.StarExpr{X: arg.expr} arg.typ = deref(arg.typ) arg.duplicable = false + arg.pure = false } } } diff --git a/internal/refactor/inline/inline_test.go b/internal/refactor/inline/inline_test.go index a70ddded34b..58abc1eda57 100644 --- a/internal/refactor/inline/inline_test.go +++ b/internal/refactor/inline/inline_test.go @@ -732,6 +732,29 @@ func TestEmbeddedFields(t *testing.T) { `func _(v V) { (*V).f(&v) }`, `func _(v V) { print(&(&v).U.T) }`, }, + { + // x is a single-assign var, and x.f does not load through a pointer + // (despite types.Selection.Indirect=true), so x is pure. + "No binding decl is required for recv in method-to-method calls.", + `type T struct{}; func (x *T) f() { g(); print(*x) }; func g()`, + `func (x *T) _() { x.f() }`, + `func (x *T) _() { + g() + print(*x) +}`, + }, + { + "Same, with implicit &recv.", + `type T struct{}; func (x *T) f() { g(); print(*x) }; func g()`, + `func (x T) _() { x.f() }`, + `func (x T) _() { + { + var x *T = &x + g() + print(*x) + } +}`, + }, }) } From e8722c010395d3c3528b2ea232476a2d15ac4af3 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Tue, 3 Oct 2023 11:16:17 -0400 Subject: [PATCH 171/178] go/types/internal/play: show types.Selection information Change-Id: I668b2fa0108d9f4cdea0f1075a4c97728435edf2 Reviewed-on: https://go-review.googlesource.com/c/tools/+/532395 Reviewed-by: Robert Findley LUCI-TryBot-Result: Go LUCI --- go/types/internal/play/play.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/go/types/internal/play/play.go b/go/types/internal/play/play.go index 80deff5090a..79cdc5afc7c 100644 --- a/go/types/internal/play/play.go +++ b/go/types/internal/play/play.go @@ -173,6 +173,26 @@ func handleSelectJSON(w http.ResponseWriter, req *http.Request) { } } + // selection x.f information (if cursor is over .f) + for _, n := range path[:2] { + if sel, ok := n.(*ast.SelectorExpr); ok { + seln, ok := pkg.TypesInfo.Selections[sel] + if ok { + fmt.Fprintf(out, "Selection: %s recv=%v obj=%v type=%v indirect=%t index=%d\n\n", + strings.Fields("FieldVal MethodVal MethodExpr")[seln.Kind()], + seln.Recv(), + seln.Obj(), + seln.Type(), + seln.Indirect(), + seln.Index()) + + } else { + fmt.Fprintf(out, "Selector is qualified identifier.\n\n") + } + break + } + } + // Object type information. switch n := path[0].(type) { case *ast.Ident: From c2725ad84073176c5c7b67f540af92eaf0e4b3da Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Wed, 4 Oct 2023 10:16:23 -0400 Subject: [PATCH 172/178] gopls: update x/telemetry dependency Update x/telemetry to pick up recent changes to upload logic. Change-Id: I4306cad40831ceda55a1d20687e96109aa72132b Reviewed-on: https://go-review.googlesource.com/c/tools/+/532575 LUCI-TryBot-Result: Go LUCI Reviewed-by: Peter Weinberger Auto-Submit: Robert Findley --- gopls/go.mod | 2 +- gopls/go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/gopls/go.mod b/gopls/go.mod index 64cd7cf2387..77a7d9fc0e1 100644 --- a/gopls/go.mod +++ b/gopls/go.mod @@ -10,7 +10,7 @@ require ( golang.org/x/mod v0.12.0 golang.org/x/sync v0.3.0 golang.org/x/sys v0.12.0 - golang.org/x/telemetry v0.0.0-20230923135512-f45a5404d02c + golang.org/x/telemetry v0.0.0-20231003223302-0168ef4ebbd3 golang.org/x/text v0.13.0 golang.org/x/tools v0.13.1-0.20230920233436-f9b8da7b22be golang.org/x/vuln v1.0.1 diff --git a/gopls/go.sum b/gopls/go.sum index e9ea5d282a7..b6720b61fce 100644 --- a/gopls/go.sum +++ b/gopls/go.sum @@ -48,6 +48,8 @@ golang.org/x/telemetry v0.0.0-20230919143304-dd3d43c296e4 h1:8UKgM6RV6tVUWPOKlSE golang.org/x/telemetry v0.0.0-20230919143304-dd3d43c296e4/go.mod h1:ppZ76JTkRgJC2GQEgtVY3fiuJR+N8FU2MAlp+gfN1E4= golang.org/x/telemetry v0.0.0-20230923135512-f45a5404d02c h1:az7Rs3XV7P68bKMPT50p2X4su02nhHqtDOi9T8f3IEw= golang.org/x/telemetry v0.0.0-20230923135512-f45a5404d02c/go.mod h1:ppZ76JTkRgJC2GQEgtVY3fiuJR+N8FU2MAlp+gfN1E4= +golang.org/x/telemetry v0.0.0-20231003223302-0168ef4ebbd3 h1:vxxQvncMbcRAtqHV5HsHGJkbya+BIOYIY+y6cdPZhzk= +golang.org/x/telemetry v0.0.0-20231003223302-0168ef4ebbd3/go.mod h1:ppZ76JTkRgJC2GQEgtVY3fiuJR+N8FU2MAlp+gfN1E4= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From 0ba9c8439eadd6c895b30ea434b398069e586c9b Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Fri, 29 Sep 2023 12:56:16 -0400 Subject: [PATCH 173/178] internal/fuzzy: several improvements for symbol matching Following the edge case discovered in golang/go#60201, take a more scientific approach to improving symbol match scoring: - Add a conformance test that compares Matcher with SymbolMatcher, querying all identifiers in x/tools. The two are not expected to agree in all cases, but this test helped find interesting ranking edge cases, which are added to the ranking test. - Don't count a capital letter in the middle of a sequence of capital letters (e.g. the M in YAML) as a word start. This was the inconsistency that led to golang/go#60201. - Compute the sequence bonus before role score; role score should take precedent. - Simplify the sequence scoring logic: a sequential character gets the same score as a word start, unless it is the final character in the pattern in which case we also adjust for whether it completes a word or segment. This feels like a reasonable heuristic. - Fix a bug in final-rune adjustment where we were checking the next input rune for a segment start, not a separator. Notably, the scoring improvements above were all derived from first principles, and happened to also improve the conformance rate in the new test. Additionally, make the following cleanup: - s/character/rune throughout, since that's what we mean - add debugging support for more easily understanding the match algorithm - add additional commentary - add benchmarks Fixes golang/go#60201 Change-Id: I838898c49cbb69af083a8cc837612da047778c40 Reviewed-on: https://go-review.googlesource.com/c/tools/+/531697 Reviewed-by: Alan Donovan Auto-Submit: Robert Findley LUCI-TryBot-Result: Go LUCI --- gopls/internal/lsp/command/interface.go | 1 + internal/fuzzy/self_test.go | 39 +++++ internal/fuzzy/symbol.go | 186 +++++++++++++++-------- internal/fuzzy/symbol_test.go | 192 +++++++++++++++++++++--- 4 files changed, 334 insertions(+), 84 deletions(-) create mode 100644 internal/fuzzy/self_test.go diff --git a/gopls/internal/lsp/command/interface.go b/gopls/internal/lsp/command/interface.go index 603e6121c8a..c3bd921fcf1 100644 --- a/gopls/internal/lsp/command/interface.go +++ b/gopls/internal/lsp/command/interface.go @@ -37,6 +37,7 @@ type Interface interface { // // Applies a fix to a region of source code. ApplyFix(context.Context, ApplyFixArgs) error + // Test: Run test(s) (legacy) // // Runs `go test` for a specific set of test or benchmark functions. diff --git a/internal/fuzzy/self_test.go b/internal/fuzzy/self_test.go new file mode 100644 index 00000000000..fae0aeae249 --- /dev/null +++ b/internal/fuzzy/self_test.go @@ -0,0 +1,39 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fuzzy_test + +import ( + "testing" + + . "golang.org/x/tools/internal/fuzzy" +) + +func BenchmarkSelf_Matcher(b *testing.B) { + idents := collectIdentifiers(b) + patterns := generatePatterns() + + for i := 0; i < b.N; i++ { + for _, pattern := range patterns { + sm := NewMatcher(pattern) + for _, ident := range idents { + _ = sm.Score(ident) + } + } + } +} + +func BenchmarkSelf_SymbolMatcher(b *testing.B) { + idents := collectIdentifiers(b) + patterns := generatePatterns() + + for i := 0; i < b.N; i++ { + for _, pattern := range patterns { + sm := NewSymbolMatcher(pattern) + for _, ident := range idents { + _, _ = sm.Match([]string{ident}) + } + } + } +} diff --git a/internal/fuzzy/symbol.go b/internal/fuzzy/symbol.go index bf93041521b..5fe2ce3e2a3 100644 --- a/internal/fuzzy/symbol.go +++ b/internal/fuzzy/symbol.go @@ -5,6 +5,9 @@ package fuzzy import ( + "bytes" + "fmt" + "log" "unicode" ) @@ -36,10 +39,12 @@ type SymbolMatcher struct { segments [256]uint8 // how many segments from the right is each rune } +// Rune roles. const ( - segmentStart uint32 = 1 << iota - wordStart - separator + segmentStart uint32 = 1 << iota // input rune starts a segment (i.e. follows '/' or '.') + wordStart // input rune starts a word, per camel-case naming rules + separator // input rune is a separator ('/' or '.') + upper // input rune is an upper case letter ) // NewSymbolMatcher creates a SymbolMatcher that may be used to match the given @@ -61,17 +66,17 @@ func NewSymbolMatcher(pattern string) *SymbolMatcher { return m } -// Match looks for the right-most match of the search pattern within the symbol -// represented by concatenating the given chunks, returning its offset and -// score. +// Match searches for the right-most match of the search pattern within the +// symbol represented by concatenating the given chunks. // -// If a match is found, the first return value will hold the absolute byte -// offset within all chunks for the start of the symbol. In other words, the -// index of the match within strings.Join(chunks, ""). If no match is found, -// the first return value will be -1. +// If a match is found, the first result holds the absolute byte offset within +// all chunks for the start of the symbol. In other words, the index of the +// match within strings.Join(chunks, ""). // // The second return value will be the score of the match, which is always // between 0 and 1, inclusive. A score of 0 indicates no match. +// +// If no match is found, Match returns (-1, 0). func (m *SymbolMatcher) Match(chunks []string) (int, float64) { // Explicit behavior for an empty pattern. // @@ -81,11 +86,25 @@ func (m *SymbolMatcher) Match(chunks []string) (int, float64) { return -1, 0 } - // First phase: populate the input buffer with lower-cased runes. + // Matching implements a heavily optimized linear scoring algorithm on the + // input. This is not guaranteed to produce the highest score, but works well + // enough, particularly due to the right-to-left significance of qualified + // symbols. + // + // Matching proceeds in three passes through the input: + // - The first pass populates the input buffer and collects rune roles. + // - The second pass proceeds right-to-left to find the right-most match. + // - The third pass proceeds left-to-right from the start of the right-most + // match, to find the most *compact* match, and computes the score of this + // match. + // + // See below for more details of each pass, as well as the scoring algorithm. + + // First pass: populate the input buffer out of the provided chunks + // (lower-casing in the process), and collect rune roles. // // We could also check for a forward match here, but since we'd have to write // the entire input anyway this has negligible impact on performance. - var ( inputLen = uint8(0) modifiers = wordStart | segmentStart @@ -107,7 +126,16 @@ input: l = unicode.ToLower(r) } if l != r { - modifiers |= wordStart + modifiers |= upper + + // If the current rune is capitalized *and the preceding rune was not*, + // mark this as a word start. This avoids spuriously high ranking of + // non-camelcase naming schemas, such as the + // yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE example of + // golang/go#60201. + if inputLen == 0 || m.roles[inputLen-1]&upper == 0 { + modifiers |= wordStart + } } m.inputBuffer[inputLen] = l m.roles[inputLen] = modifiers @@ -125,14 +153,13 @@ input: } } - // Second phase: find the right-most match, and count segments from the + // Second pass: find the right-most match, and count segments from the // right. - var ( pi = uint8(m.patternLen - 1) // pattern index p = m.pattern[pi] // pattern rune start = -1 // start offset of match - rseg = uint8(0) + rseg = uint8(0) // effective "depth" from the right of the current rune in consideration ) const maxSeg = 3 // maximum number of segments from the right to count, for scoring purposes. @@ -144,6 +171,8 @@ input: m.segments[ii] = rseg if p == r { if pi == 0 { + // TODO(rfindley): BUG: the docstring for Match says that it returns an + // absolute byte offset, but clearly it is returning a rune offset here. start = int(ii) break } @@ -161,85 +190,120 @@ input: return -1, 0 } - // Third phase: find the shortest match, and compute the score. + // Third pass: find the shortest match and compute the score. - // Score is the average score for each character. + // Score is the average score for each rune. // - // A character score is the multiple of: - // 1. 1.0 if the character starts a segment or is preceded by a matching - // character, 0.9 if the character starts a mid-segment word, else 0.6. + // A rune score is the multiple of: + // 1. The base score, which is 1.0 if the rune starts a segment, 0.9 if the + // rune starts a mid-segment word, else 0.6. // - // Note that characters preceded by a matching character get the max - // score of 1.0 so that sequential or exact matches are preferred, even - // if they don't start/end at a segment or word boundary. For example, a - // match for "func" in intfuncs should have a higher score than in - // ifunmatched. + // Runes preceded by a matching rune are treated the same as the start + // of a mid-segment word (with a 0.9 score), so that sequential or exact + // matches are preferred. We call this a sequential bonus. // - // For the final character match, the multiplier from (1) is reduced to - // 0.9 if the next character in the input is a mid-segment word, or 0.6 - // if the next character in the input is not a word or segment start. - // This ensures that we favor whole-word or whole-segment matches over - // prefix matches. + // For the final rune match, this sequential bonus is reduced to 0.8 if + // the next rune in the input is a mid-segment word, or 0.7 if the next + // rune in the input is not a word or segment start. This ensures that + // we favor whole-word or whole-segment matches over prefix matches. // - // 2. 1.0 if the character is part of the last segment, otherwise + // 2. 1.0 if the rune is part of the last segment, otherwise // 1.0-0.1*, with a max segment count of 3. // Notably 1.0-0.1*3 = 0.7 > 0.6, so that foo/_/_/_/_ (a match very - // early in a qualified symbol name) still scores higher than _f_o_o_ - // (a completely split match). + // early in a qualified symbol name) still scores higher than _f_o_o_ (a + // completely split match). // // This is a naive algorithm, but it is fast. There's lots of prior art here // that could be leveraged. For example, we could explicitly consider - // character distance, and exact matches of words or segments. + // rune distance, and exact matches of words or segments. // // Also note that this might not actually find the highest scoring match, as // doing so could require a non-linear algorithm, depending on how the score // is calculated. + // debugging support + const debug = false // enable to log debugging information + var ( + runeScores []float64 + runeIdxs []int + ) + pi = 0 p = m.pattern[pi] const ( - segStreak = 1.0 // start of segment or sequential match - wordStreak = 0.9 // start of word match - noStreak = 0.6 - perSegment = 0.1 // we count at most 3 segments above + segStartScore = 1.0 // base score of runes starting a segment + wordScore = 0.9 // base score of runes starting or continuing a word + noStreak = 0.6 + perSegment = 0.1 // we count at most 3 segments above ) - streakBonus := noStreak totScore := 0.0 + lastMatch := uint8(255) for ii := uint8(start); ii < inputLen; ii++ { r := m.inputBuffer[ii] if r == p { pi++ + finalRune := pi >= m.patternLen p = m.pattern[pi] - // Note: this could be optimized with some bit operations. + + baseScore := noStreak + + // Calculate the sequence bonus based on preceding matches. + // + // We do this first as it is overridden by role scoring below. + if lastMatch == ii-1 { + baseScore = wordScore + // Reduce the sequence bonus for the final rune of the pattern based on + // whether it borders a new segment or word. + if finalRune { + switch { + case ii == inputLen-1 || m.roles[ii+1]&separator != 0: + // Full segment: no reduction + case m.roles[ii+1]&wordStart != 0: + baseScore = wordScore - 0.1 + default: + baseScore = wordScore - 0.2 + } + } + } + lastMatch = ii + + // Calculate the rune's role score. If the rune starts a segment or word, + // this overrides the sequence score, as the rune starts a new sequence. switch { - case m.roles[ii]&segmentStart != 0 && segStreak > streakBonus: - streakBonus = segStreak - case m.roles[ii]&wordStart != 0 && wordStreak > streakBonus: - streakBonus = wordStreak + case m.roles[ii]&segmentStart != 0: + baseScore = segStartScore + case m.roles[ii]&wordStart != 0: + baseScore = wordScore } - finalChar := pi >= m.patternLen - // finalCost := 1.0 - if finalChar && streakBonus > noStreak { - switch { - case ii == inputLen-1 || m.roles[ii+1]&segmentStart != 0: - // Full segment: no reduction - case m.roles[ii+1]&wordStart != 0: - streakBonus = wordStreak - default: - streakBonus = noStreak - } + + // Apply the segment-depth penalty (segments from the right). + runeScore := baseScore * (1.0 - float64(m.segments[ii])*perSegment) + if debug { + runeScores = append(runeScores, runeScore) + runeIdxs = append(runeIdxs, int(ii)) } - totScore += streakBonus * (1.0 - float64(m.segments[ii])*perSegment) - if finalChar { + totScore += runeScore + if finalRune { break } - streakBonus = segStreak // see above: sequential characters get the max score - } else { - streakBonus = noStreak } } + if debug { + // Format rune roles and scores in line: + // fo[o:.52].[b:1]a[r:.6] + var summary bytes.Buffer + last := 0 + for i, idx := range runeIdxs { + summary.WriteString(string(m.inputBuffer[last:idx])) // encode runes + fmt.Fprintf(&summary, "[%s:%.2g]", string(m.inputBuffer[idx]), runeScores[i]) + last = idx + 1 + } + summary.WriteString(string(m.inputBuffer[last:inputLen])) // encode runes + log.Println(summary.String()) + } + return start, totScore / float64(m.patternLen) } diff --git a/internal/fuzzy/symbol_test.go b/internal/fuzzy/symbol_test.go index 2a9d9b663bd..43e629d51ef 100644 --- a/internal/fuzzy/symbol_test.go +++ b/internal/fuzzy/symbol_test.go @@ -5,8 +5,12 @@ package fuzzy_test import ( + "go/ast" + "go/token" + "sort" "testing" + "golang.org/x/tools/go/packages" . "golang.org/x/tools/internal/fuzzy" ) @@ -34,30 +38,173 @@ func TestSymbolMatchIndex(t *testing.T) { } func TestSymbolRanking(t *testing.T) { - matcher := NewSymbolMatcher("test") - // symbols to match, in ascending order of ranking. - symbols := []string{ - "this.is.better.than.most", - "test.foo.bar", - "thebest", - "test.foo", - "test.foo", - "atest", - "testage", - "tTest", - "foo.test", - "test", + // query -> symbols to match, in ascending order of score + queryRanks := map[string][]string{ + "test": { + "this.is.better.than.most", + "test.foo.bar", + "thebest", + "atest", + "test.foo", + "testage", + "tTest", + "foo.test", + }, + "parseside": { // golang/go#60201 + "yaml_PARSE_FLOW_SEQUENCE_ENTRY_MAPPING_END_STATE", + "parseContext.parse_sidebyside", + }, + "cvb": { + "filecache_test.testIPCValueB", + "cover.Boundary", + }, + "dho": { + "gocommand.DebugHangingGoCommands", + "protocol.DocumentHighlightOptions", + }, + "flg": { + "completion.FALLTHROUGH", + "main.flagGoCmd", + }, + "fvi": { + "godoc.fileIndexVersion", + "macho.FlagSubsectionsViaSymbols", + }, } - prev := 0.0 - for _, sym := range symbols { - _, score := matcher.Match([]string{sym}) - t.Logf("Match(%q) = %v", sym, score) - if score < prev { - t.Errorf("Match(%q) = _, %v, want > %v", sym, score, prev) + + for query, symbols := range queryRanks { + t.Run(query, func(t *testing.T) { + matcher := NewSymbolMatcher(query) + prev := 0.0 + for _, sym := range symbols { + _, score := matcher.Match([]string{sym}) + t.Logf("Match(%q) = %v", sym, score) + if score <= prev { + t.Errorf("Match(%q) = _, %v, want > %v", sym, score, prev) + } + prev = score + } + }) + } +} + +func TestMatcherSimilarities(t *testing.T) { + // This test compares the fuzzy matcher with the symbol matcher on a corpus + // of qualified identifiers extracted from x/tools. + // + // These two matchers are not expected to agree, but inspecting differences + // can be useful for finding interesting ranking edge cases. + t.Skip("unskip this test to compare matchers") + + idents := collectIdentifiers(t) + t.Logf("collected %d unique identifiers", len(idents)) + + // TODO: use go1.21 slices.MaxFunc. + topMatch := func(score func(string) float64) string { + top := "" + topScore := 0.0 + for _, cand := range idents { + if s := score(cand); s > topScore { + top = cand + topScore = s + } } - prev = score + return top } + + agreed := 0 + total := 0 + bad := 0 + patterns := generatePatterns() + for _, pattern := range patterns { + total++ + + fm := NewMatcher(pattern) + topFuzzy := topMatch(func(input string) float64 { + return float64(fm.Score(input)) + }) + sm := NewSymbolMatcher(pattern) + topSymbol := topMatch(func(input string) float64 { + _, score := sm.Match([]string{input}) + return score + }) + switch { + case topFuzzy == "" && topSymbol != "": + if false { + // The fuzzy matcher has a bug where it misses some matches; for this + // test we only care about the symbol matcher. + t.Logf("%q matched %q but no fuzzy match", pattern, topSymbol) + } + total-- + bad++ + case topFuzzy != "" && topSymbol == "": + t.Fatalf("%q matched %q but no symbol match", pattern, topFuzzy) + case topFuzzy == topSymbol: + agreed++ + default: + // Enable this log to see mismatches. + if false { + t.Logf("mismatch for %q: fuzzy: %q, symbol: %q", pattern, topFuzzy, topSymbol) + } + } + } + t.Logf("fuzzy matchers agreed on %d out of %d queries (%d bad)", agreed, total, bad) +} + +func collectIdentifiers(tb testing.TB) []string { + cfg := &packages.Config{ + Mode: packages.NeedName | packages.NeedSyntax | packages.NeedFiles, + Tests: true, + } + pkgs, err := packages.Load(cfg, "golang.org/x/tools/...") + if err != nil { + tb.Fatal(err) + } + uniqueIdents := make(map[string]bool) + decls := 0 + for _, pkg := range pkgs { + for _, f := range pkg.Syntax { + for _, decl := range f.Decls { + decls++ + switch decl := decl.(type) { + case *ast.GenDecl: + for _, spec := range decl.Specs { + switch decl.Tok { + case token.IMPORT: + case token.TYPE: + name := spec.(*ast.TypeSpec).Name.Name + qualified := pkg.Name + "." + name + uniqueIdents[qualified] = true + case token.CONST, token.VAR: + for _, n := range spec.(*ast.ValueSpec).Names { + qualified := pkg.Name + "." + n.Name + uniqueIdents[qualified] = true + } + } + } + } + } + } + } + var idents []string + for k := range uniqueIdents { + idents = append(idents, k) + } + sort.Strings(idents) + return idents +} + +func generatePatterns() []string { + var patterns []string + for x := 'a'; x <= 'z'; x++ { + for y := 'a'; y <= 'z'; y++ { + for z := 'a'; z <= 'z'; z++ { + patterns = append(patterns, string(x)+string(y)+string(z)) + } + } + } + return patterns } // Test that we strongly prefer exact matches. @@ -89,9 +236,8 @@ func TestSymbolRanking_Issue60027(t *testing.T) { func TestChunkedMatch(t *testing.T) { matcher := NewSymbolMatcher("test") - + _, want := matcher.Match([]string{"test"}) chunked := [][]string{ - {"test"}, {"", "test"}, {"test", ""}, {"te", "st"}, @@ -99,7 +245,7 @@ func TestChunkedMatch(t *testing.T) { for _, chunks := range chunked { offset, score := matcher.Match(chunks) - if offset != 0 || score != 1.0 { + if offset != 0 || score != want { t.Errorf("Match(%v) = %v, %v, want 0, 1.0", chunks, offset, score) } } From 2be977ecc58df8383c28bc0587b6db6af20638d5 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Wed, 4 Oct 2023 11:06:14 -0400 Subject: [PATCH 174/178] internal/refactor/inline: work around channel type misformatting This change adds parens around the type in T(x) conversions where T is a receive-only channel type, as previously it would be misformatted as a receive of a receive. Updates golang/go#63362 Change-Id: I935b5598d4bc3ea57dd52964e8b02005f5e6ef72 Reviewed-on: https://go-review.googlesource.com/c/tools/+/532576 Reviewed-by: Robert Findley LUCI-TryBot-Result: Go LUCI --- internal/refactor/inline/falcon.go | 5 +---- internal/refactor/inline/inline.go | 5 +---- internal/refactor/inline/inline_test.go | 6 ++++++ internal/refactor/inline/util.go | 14 ++++++++++++++ 4 files changed, 22 insertions(+), 8 deletions(-) diff --git a/internal/refactor/inline/falcon.go b/internal/refactor/inline/falcon.go index 9ad57d727e4..9863e8dbcfb 100644 --- a/internal/refactor/inline/falcon.go +++ b/internal/refactor/inline/falcon.go @@ -610,10 +610,7 @@ func (st *falconState) expr(e ast.Expr) (res any) { // = types.TypeAndValue | as // Possible "value out of range". kX := st.expr(e.Args[0]) if kX != nil && isBasic(tv.Type, types.IsConstType) { - conv := &ast.CallExpr{ - Fun: makeIdent(st.typename(tv.Type)), - Args: []ast.Expr{st.toExpr(kX)}, - } + conv := convert(makeIdent(st.typename(tv.Type)), st.toExpr(kX)) if is[ast.Expr](kX) { st.emit(conv) } diff --git a/internal/refactor/inline/inline.go b/internal/refactor/inline/inline.go index 08a4594ae27..f3bc4dbf6e5 100644 --- a/internal/refactor/inline/inline.go +++ b/internal/refactor/inline/inline.go @@ -1284,10 +1284,7 @@ next: // a binding decl or when using the literalization // strategy. if len(param.info.Refs) > 0 && !trivialConversion(args[i].typ, params[i].obj) { - arg.expr = &ast.CallExpr{ - Fun: params[i].fieldType, // formatter adds parens as needed - Args: []ast.Expr{arg.expr}, - } + arg.expr = convert(params[i].fieldType, arg.expr) logf("param %q: adding explicit %s -> %s conversion around argument", param.info.Name, args[i].typ, params[i].obj.Type()) } diff --git a/internal/refactor/inline/inline_test.go b/internal/refactor/inline/inline_test.go index 58abc1eda57..891897704b2 100644 --- a/internal/refactor/inline/inline_test.go +++ b/internal/refactor/inline/inline_test.go @@ -382,6 +382,12 @@ func TestBasics(t *testing.T) { print(s, s, 0, 0) }`, }, + { + "Workaround for T(x) misformatting (#63362).", + `func f(ch <-chan int) { <-ch }`, + `func _(ch chan int) { f(ch) }`, + `func _(ch chan int) { <-(<-chan int)(ch) }`, + }, }) } diff --git a/internal/refactor/inline/util.go b/internal/refactor/inline/util.go index 6d8d3fac08f..98d654eeb51 100644 --- a/internal/refactor/inline/util.go +++ b/internal/refactor/inline/util.go @@ -103,3 +103,17 @@ func intersects[K comparable, T1, T2 any](x map[K]T1, y map[K]T2) bool { } return false } + +// convert returns syntax for the conversion T(x). +func convert(T, x ast.Expr) *ast.CallExpr { + // The formatter generally adds parens as needed, + // but before go1.22 it had a bug (#63362) for + // channel types that requires this workaround. + if ch, ok := T.(*ast.ChanType); ok && ch.Dir == ast.RECV { + T = &ast.ParenExpr{X: T} + } + return &ast.CallExpr{ + Fun: T, + Args: []ast.Expr{x}, + } +} From db1d1e0d3387b47f53b696e1ba08fc6349611854 Mon Sep 17 00:00:00 2001 From: Viktor Blomqvist Date: Fri, 29 Sep 2023 17:25:34 +0200 Subject: [PATCH 175/178] gopls/internal/lsp: go to definition from embed directive Enable jump to definition on //go:embed directive arguments. If multiple files match the pattern one is picked at random. Improve the pattern matching for both definition and hover to exclude directories since they are not embeddable in themselves. Updates golang/go#50262 Change-Id: I09da40f195e8edfe661acaacd99f62d9f577e9ea Reviewed-on: https://go-review.googlesource.com/c/tools/+/531775 Reviewed-by: Robert Findley Reviewed-by: Suzy Mueller LUCI-TryBot-Result: Go LUCI --- gopls/internal/lsp/definition.go | 6 -- gopls/internal/lsp/source/definition.go | 13 +++++ gopls/internal/lsp/source/embeddirective.go | 56 +++++++++++++++++++ gopls/internal/lsp/source/hover.go | 4 +- .../internal/regtest/misc/definition_test.go | 40 +++++++++++++ gopls/internal/regtest/misc/hover_test.go | 11 ++++ 6 files changed, 122 insertions(+), 8 deletions(-) diff --git a/gopls/internal/lsp/definition.go b/gopls/internal/lsp/definition.go index a438bebc8d3..892e48d6377 100644 --- a/gopls/internal/lsp/definition.go +++ b/gopls/internal/lsp/definition.go @@ -6,7 +6,6 @@ package lsp import ( "context" - "errors" "fmt" "golang.org/x/tools/gopls/internal/lsp/protocol" @@ -36,11 +35,6 @@ func (s *Server) definition(ctx context.Context, params *protocol.DefinitionPara case source.Tmpl: return template.Definition(snapshot, fh, params.Position) case source.Go: - // Partial support for jumping from linkname directive (position at 2nd argument). - locations, err := source.LinknameDefinition(ctx, snapshot, fh, params.Position) - if !errors.Is(err, source.ErrNoLinkname) { - return locations, err - } return source.Definition(ctx, snapshot, fh, params.Position) default: return nil, fmt.Errorf("can't find definitions for file type %s", kind) diff --git a/gopls/internal/lsp/source/definition.go b/gopls/internal/lsp/source/definition.go index 90a432966b8..dd3feda70a2 100644 --- a/gopls/internal/lsp/source/definition.go +++ b/gopls/internal/lsp/source/definition.go @@ -6,6 +6,7 @@ package source import ( "context" + "errors" "fmt" "go/ast" "go/token" @@ -58,6 +59,18 @@ func Definition(ctx context.Context, snapshot Snapshot, fh FileHandle, position return []protocol.Location{loc}, nil } + // Handle the case where the cursor is in a linkname directive. + locations, err := LinknameDefinition(ctx, snapshot, fh, position) + if !errors.Is(err, ErrNoLinkname) { + return locations, err + } + + // Handle the case where the cursor is in an embed directive. + locations, err = EmbedDefinition(pgf.Mapper, position) + if !errors.Is(err, ErrNoEmbed) { + return locations, err + } + // The general case: the cursor is on an identifier. _, obj, _ := referencedObject(pkg, pgf, pos) if obj == nil { diff --git a/gopls/internal/lsp/source/embeddirective.go b/gopls/internal/lsp/source/embeddirective.go index b1a0ff9d235..d4e85d7add2 100644 --- a/gopls/internal/lsp/source/embeddirective.go +++ b/gopls/internal/lsp/source/embeddirective.go @@ -5,7 +5,10 @@ package source import ( + "errors" "fmt" + "io/fs" + "path/filepath" "strconv" "strings" "unicode" @@ -14,6 +17,59 @@ import ( "golang.org/x/tools/gopls/internal/lsp/protocol" ) +// ErrNoEmbed is returned by EmbedDefinition when no embed +// directive is found at a particular position. +// As such it indicates that other definitions could be worth checking. +var ErrNoEmbed = errors.New("no embed directive found") + +var errStopWalk = errors.New("stop walk") + +// EmbedDefinition finds a file matching the embed directive at pos in the mapped file. +// If there is no embed directive at pos, returns ErrNoEmbed. +// If multiple files match the embed pattern, one is picked at random. +func EmbedDefinition(m *protocol.Mapper, pos protocol.Position) ([]protocol.Location, error) { + pattern, _ := parseEmbedDirective(m, pos) + if pattern == "" { + return nil, ErrNoEmbed + } + + // Find the first matching file. + var match string + dir := filepath.Dir(m.URI.Filename()) + err := filepath.WalkDir(dir, func(abs string, d fs.DirEntry, e error) error { + if e != nil { + return e + } + rel, err := filepath.Rel(dir, abs) + if err != nil { + return err + } + ok, err := filepath.Match(pattern, rel) + if err != nil { + return err + } + if ok && !d.IsDir() { + match = abs + return errStopWalk + } + return nil + }) + if err != nil && !errors.Is(err, errStopWalk) { + return nil, err + } + if match == "" { + return nil, fmt.Errorf("%q does not match any files in %q", pattern, dir) + } + + loc := protocol.Location{ + URI: protocol.URIFromPath(match), + Range: protocol.Range{ + Start: protocol.Position{Line: 0, Character: 0}, + }, + } + return []protocol.Location{loc}, nil +} + // parseEmbedDirective attempts to parse a go:embed directive argument at pos. // If successful it return the directive argument and its range, else zero values are returned. func parseEmbedDirective(m *protocol.Mapper, pos protocol.Position) (string, protocol.Range) { diff --git a/gopls/internal/lsp/source/hover.go b/gopls/internal/lsp/source/hover.go index 8578f13e2ca..95317833489 100644 --- a/gopls/internal/lsp/source/hover.go +++ b/gopls/internal/lsp/source/hover.go @@ -640,7 +640,7 @@ func hoverEmbed(fh FileHandle, rng protocol.Range, pattern string) (protocol.Ran dir := filepath.Dir(fh.URI().Filename()) var matches []string - err := filepath.WalkDir(dir, func(abs string, _ fs.DirEntry, e error) error { + err := filepath.WalkDir(dir, func(abs string, d fs.DirEntry, e error) error { if e != nil { return e } @@ -652,7 +652,7 @@ func hoverEmbed(fh FileHandle, rng protocol.Range, pattern string) (protocol.Ran if err != nil { return err } - if ok { + if ok && !d.IsDir() { matches = append(matches, rel) } return nil diff --git a/gopls/internal/regtest/misc/definition_test.go b/gopls/internal/regtest/misc/definition_test.go index f11b2073292..d16539f0dbb 100644 --- a/gopls/internal/regtest/misc/definition_test.go +++ b/gopls/internal/regtest/misc/definition_test.go @@ -529,3 +529,43 @@ const _ = b.K } }) } + +const embedDefinition = ` +-- go.mod -- +module mod.com + +-- main.go -- +package main + +import ( + "embed" +) + +//go:embed *.txt +var foo embed.FS + +func main() {} + +-- skip.sql -- +SKIP + +-- foo.txt -- +FOO + +-- skip.bat -- +SKIP +` + +func TestGoToEmbedDefinition(t *testing.T) { + Run(t, embedDefinition, func(t *testing.T, env *Env) { + env.OpenFile("main.go") + + start := env.RegexpSearch("main.go", `\*.txt`) + loc := env.GoToDefinition(start) + + name := env.Sandbox.Workdir.URIToPath(loc.URI) + if want := "foo.txt"; name != want { + t.Errorf("GoToDefinition: got file %q, want %q", name, want) + } + }) +} diff --git a/gopls/internal/regtest/misc/hover_test.go b/gopls/internal/regtest/misc/hover_test.go index fadaf7f79bc..7b84f8aa871 100644 --- a/gopls/internal/regtest/misc/hover_test.go +++ b/gopls/internal/regtest/misc/hover_test.go @@ -458,6 +458,8 @@ BAR BAZ -- other.sql -- SKIPPED +-- dir.txt/skip.txt -- +SKIPPED ` func TestHoverEmbedDirective(t *testing.T) { @@ -478,5 +480,14 @@ func TestHoverEmbedDirective(t *testing.T) { t.Errorf("hover: %q does not contain: %q", content, want) } } + + // A directory should never be matched, even if it happens to have a matching name. + // Content in subdirectories should not match on only one asterisk. + skips := []string{"other.sql", "dir.txt", "skip.txt"} + for _, skip := range skips { + if strings.Contains(content, skip) { + t.Errorf("hover: %q should not contain: %q", content, skip) + } + } }) } From ee20ddf1f71a4f7eb6e7b5dff2d13125f81ce480 Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Thu, 5 Oct 2023 11:14:41 -0400 Subject: [PATCH 176/178] internal/refactor/inline: permit return conversions in tailcall Previously, the tail-call strategies required that the callee's implicit return conversions must be trivial. That meant returning a nil error (for example) defeated these strategies, even though it is common in tail-call situations for the caller function to have identical result types. For example: func callee() error { return nil } // nontrivial conversion func caller() error { return callee() } // identical result types This change permits the tail-call strategies when the callee and caller's results tuples are identical. Fixes golang/go#63336 Change-Id: I57d62213023861a2cfebed25b01ec28921efe441 Reviewed-on: https://go-review.googlesource.com/c/tools/+/533075 Reviewed-by: Robert Findley LUCI-TryBot-Result: Go LUCI --- internal/refactor/inline/inline.go | 39 +++++++++++++++++++++++-- internal/refactor/inline/inline_test.go | 19 ++++++++++++ 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/internal/refactor/inline/inline.go b/internal/refactor/inline/inline.go index f3bc4dbf6e5..9615ab43c59 100644 --- a/internal/refactor/inline/inline.go +++ b/internal/refactor/inline/inline.go @@ -760,6 +760,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // // If: // - the body is just "return expr" with trivial implicit conversions, + // or the caller's return type matches the callee's, // - all parameters and result vars can be eliminated // or replaced by a binding decl, // then the call expression can be replaced by the @@ -767,7 +768,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu if len(calleeDecl.Body.List) == 1 && is[*ast.ReturnStmt](calleeDecl.Body.List[0]) && len(calleeDecl.Body.List[0].(*ast.ReturnStmt).Results) > 0 && // not a bare return - callee.TrivialReturns == callee.TotalReturns { + safeReturn(caller, calleeSymbol, callee) { results := calleeDecl.Body.List[0].(*ast.ReturnStmt).Results context := callContext(caller.path) @@ -880,7 +881,8 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // so long as: // - all parameters can be eliminated or replaced by a binding decl, // - call is a tail-call; - // - all returns in body have trivial result conversions; + // - all returns in body have trivial result conversions, + // or the caller's return type matches the callee's, // - there is no label conflict; // - no result variable is referenced by name, // or implicitly by a bare return. @@ -896,7 +898,7 @@ func inline(logf func(string, ...any), caller *Caller, callee *gobCallee) (*resu // or implicit) return. if ret, ok := callContext(caller.path).(*ast.ReturnStmt); ok && len(ret.Results) == 1 && - callee.TrivialReturns == callee.TotalReturns && + safeReturn(caller, calleeSymbol, callee) && !callee.HasBareReturn && (!needBindingDecl || bindingDeclStmt != nil) && !hasLabelConflict(caller.path, callee.Labels) && @@ -2601,3 +2603,34 @@ func declares(stmts []ast.Stmt) map[string]bool { } return names } + +// safeReturn reports whether the callee's return statements may be safely +// used to return from the function enclosing the caller (which must exist). +func safeReturn(caller *Caller, calleeSymbol *types.Func, callee *gobCallee) bool { + // It is safe if all callee returns involve only trivial conversions. + if callee.TrivialReturns == callee.TotalReturns { + return true + } + + var callerType types.Type + // Find type of innermost function enclosing call. + // (Beware: Caller.enclosingFunc is the outermost.) +loop: + for _, n := range caller.path { + switch f := n.(type) { + case *ast.FuncDecl: + callerType = caller.Info.ObjectOf(f.Name).Type() + break loop + case *ast.FuncLit: + callerType = caller.Info.TypeOf(f) + break loop + } + } + + // Non-trivial return conversions in the callee are permitted + // if the same non-trivial conversion would occur after inlining, + // i.e. if the caller and callee results tuples are identical. + callerResults := callerType.(*types.Signature).Results() + calleeResults := calleeSymbol.Type().(*types.Signature).Results() + return types.Identical(callerResults, calleeResults) +} diff --git a/internal/refactor/inline/inline_test.go b/internal/refactor/inline/inline_test.go index 891897704b2..8362e445b66 100644 --- a/internal/refactor/inline/inline_test.go +++ b/internal/refactor/inline/inline_test.go @@ -565,6 +565,25 @@ func TestTailCallStrategy(t *testing.T) { `func _() { f() }`, `func _() { func() { defer f(); println() }() }`, }, + // Tests for issue #63336: + { + "Tail call with non-trivial return conversion (caller.sig = callee.sig).", + `func f() error { if true { return nil } else { return e } }; var e struct{error}`, + `func _() error { return f() }`, + `func _() error { + if true { + return nil + } else { + return e + } +}`, + }, + { + "Tail call with non-trivial return conversion (caller.sig != callee.sig).", + `func f() error { return E{} }; type E struct{error}`, + `func _() any { return f() }`, + `func _() any { return func() error { return E{} }() }`, + }, }) } From 1e4ce7c30c4154090d22d1c2b9bd490e91a7796b Mon Sep 17 00:00:00 2001 From: Alan Donovan Date: Thu, 5 Oct 2023 13:14:29 -0400 Subject: [PATCH 177/178] internal/refactor/inline: yet more tweaks to everything test Change-Id: I052a9de860abbad199329b1c9ef52507988a59dc Reviewed-on: https://go-review.googlesource.com/c/tools/+/533175 Auto-Submit: Alan Donovan Reviewed-by: Robert Findley LUCI-TryBot-Result: Go LUCI --- internal/refactor/inline/everything_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/refactor/inline/everything_test.go b/internal/refactor/inline/everything_test.go index a1c5448d1d1..f6111495a48 100644 --- a/internal/refactor/inline/everything_test.go +++ b/internal/refactor/inline/everything_test.go @@ -10,6 +10,7 @@ import ( "go/ast" "go/parser" "go/types" + "log" "os" "path/filepath" "strings" @@ -37,7 +38,7 @@ var packagesFlag = flag.String("packages", "", "set of packages for TestEverythi // // And these commands to inline everything in the kubernetes repository: // -// $ go build -c -o /tmp/everything ./internal/refactor/inline/ +// $ go test -c -o /tmp/everything ./internal/refactor/inline/ // $ (cd kubernetes && /tmp/everything -test.run=Everything -packages=./...) // // TODO(adonovan): @@ -226,7 +227,7 @@ func TestEverything(t *testing.T) { noMutCheck() } } - t.Errorf("Analyzed %d packages", len(pkgs)) + log.Printf("Analyzed %d packages", len(pkgs)) } type importerFunc func(path string) (*types.Package, error) From 3f4194ee29d7db9b59757dfff729ef55cf89661c Mon Sep 17 00:00:00 2001 From: Gopher Robot Date: Thu, 5 Oct 2023 23:58:20 +0000 Subject: [PATCH 178/178] go.mod: update golang.org/x dependencies Update golang.org/x dependencies to their latest tagged versions. Change-Id: I705c0d75bcc979d8c45266ba2e43b97cbf98e701 Reviewed-on: https://go-review.googlesource.com/c/tools/+/533240 LUCI-TryBot-Result: Go LUCI Reviewed-by: Benny Siegert Reviewed-by: Dmitri Shuralyov Auto-Submit: Gopher Robot --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- gopls/go.mod | 6 +++--- gopls/go.sum | 22 +++++++++------------- 4 files changed, 24 insertions(+), 28 deletions(-) diff --git a/go.mod b/go.mod index 4a01af35e72..9688b9dae25 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,9 @@ go 1.18 require ( github.com/yuin/goldmark v1.4.13 - golang.org/x/mod v0.12.0 - golang.org/x/net v0.15.0 - golang.org/x/sys v0.12.0 + golang.org/x/mod v0.13.0 + golang.org/x/net v0.16.0 + golang.org/x/sys v0.13.0 ) -require golang.org/x/sync v0.3.0 +require golang.org/x/sync v0.4.0 diff --git a/go.sum b/go.sum index 3f717edf725..78a350fe890 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,10 @@ github.com/yuin/goldmark v1.4.13 h1:fVcFKWvrslecOb/tg+Cc05dkeYx540o0FuFt3nUVDoE= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= -golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= +golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos= +golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/gopls/go.mod b/gopls/go.mod index 77a7d9fc0e1..866fb259581 100644 --- a/gopls/go.mod +++ b/gopls/go.mod @@ -7,9 +7,9 @@ require ( github.com/jba/printsrc v0.2.2 github.com/jba/templatecheck v0.6.0 github.com/sergi/go-diff v1.1.0 - golang.org/x/mod v0.12.0 - golang.org/x/sync v0.3.0 - golang.org/x/sys v0.12.0 + golang.org/x/mod v0.13.0 + golang.org/x/sync v0.4.0 + golang.org/x/sys v0.13.0 golang.org/x/telemetry v0.0.0-20231003223302-0168ef4ebbd3 golang.org/x/text v0.13.0 golang.org/x/tools v0.13.1-0.20230920233436-f9b8da7b22be diff --git a/gopls/go.sum b/gopls/go.sum index b6720b61fce..365fd0c1a4f 100644 --- a/gopls/go.sum +++ b/gopls/go.sum @@ -29,29 +29,25 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp/typeparams v0.0.0-20221212164502-fae10dda9338 h1:2O2DON6y3XMJiQRAS1UWU+54aec2uopH3x7MAiqGW6Y= golang.org/x/exp/typeparams v0.0.0-20221212164502-fae10dda9338/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= +golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/telemetry v0.0.0-20230919143304-dd3d43c296e4 h1:8UKgM6RV6tVUWPOKlSEunJpIzYW3oy9aV//8cJz0UE4= -golang.org/x/telemetry v0.0.0-20230919143304-dd3d43c296e4/go.mod h1:ppZ76JTkRgJC2GQEgtVY3fiuJR+N8FU2MAlp+gfN1E4= -golang.org/x/telemetry v0.0.0-20230923135512-f45a5404d02c h1:az7Rs3XV7P68bKMPT50p2X4su02nhHqtDOi9T8f3IEw= -golang.org/x/telemetry v0.0.0-20230923135512-f45a5404d02c/go.mod h1:ppZ76JTkRgJC2GQEgtVY3fiuJR+N8FU2MAlp+gfN1E4= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/telemetry v0.0.0-20231003223302-0168ef4ebbd3 h1:vxxQvncMbcRAtqHV5HsHGJkbya+BIOYIY+y6cdPZhzk= golang.org/x/telemetry v0.0.0-20231003223302-0168ef4ebbd3/go.mod h1:ppZ76JTkRgJC2GQEgtVY3fiuJR+N8FU2MAlp+gfN1E4= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=