Merge pull request #15979 from github/mbg/go/deal-with-incorrect-versions

Author: mbg

go/extractor/cli/go-autobuilder/go-autobuilder.go

@@ -417,7 +417,7 @@ func installDependencies(workspace project.GoWorkspace) {
 	} else {
 		if workspace.Modules == nil {
 			project.InitGoModForLegacyProject(workspace.BaseDir)
-			workspace.Modules = project.LoadGoModules([]string{filepath.Join(workspace.BaseDir, "go.mod")})
+			workspace.Modules = project.LoadGoModules(true, []string{filepath.Join(workspace.BaseDir, "go.mod")})
 		}
 
 		// get dependencies for all modules

go/extractor/diagnostics/diagnostics.go

@@ -497,7 +497,7 @@ func EmitNewerSystemGoRequired(requiredVersion string) {
 func EmitExtractionFailedForProjects(path []string) {
 	emitDiagnostic(
 		"go/autobuilder/extraction-failed-for-project",
-		fmt.Sprintf("Unable to extract %d Go projects", len(path)),
+		"Unable to extract some Go projects",
 		fmt.Sprintf(
 			"The following %d Go project%s could not be extracted successfully:\n\n`%s`\n",
 			len(path),
@@ -508,3 +508,18 @@ func EmitExtractionFailedForProjects(path []string) {
 		noLocation,
 	)
 }
+
+func EmitInvalidToolchainVersion(goModPath string, version string) {
+	emitDiagnostic(
+		"go/autobuilder/invalid-go-toolchain-version",
+		"Invalid Go toolchain version",
+		strings.Join([]string{
+			"As of Go 1.21, toolchain versions [must use the 1.N.P syntax](https://go.dev/doc/toolchain#version).",
+			fmt.Sprintf("`%s` in `%s` does not match this syntax and there is no additional `toolchain` directive, which may cause some `go` commands to fail.", version, goModPath),
+		},
+			"\n\n"),
+		severityWarning,
+		fullVisibility,
+		&locationStruct{File: goModPath},
+	)
+}

go/extractor/project/project.go

@@ -176,10 +176,13 @@ func findGoModFiles(root string) []string {
 	return util.FindAllFilesWithName(root, "go.mod", "vendor")
 }
 
+// A regular expression for the Go toolchain version syntax.
+var toolchainVersionRe *regexp.Regexp = regexp.MustCompile(`(?m)^([0-9]+\.[0-9]+\.[0-9]+)$`)
+
 // Given a list of `go.mod` file paths, try to parse them all. The resulting array of `GoModule` objects
 // will be the same length as the input array and the objects will contain at least the `go.mod` path.
 // If parsing the corresponding file is successful, then the parsed contents will also be available.
-func LoadGoModules(goModFilePaths []string) []*GoModule {
+func LoadGoModules(emitDiagnostics bool, goModFilePaths []string) []*GoModule {
 	results := make([]*GoModule, len(goModFilePaths))
 
 	for i, goModFilePath := range goModFilePaths {
@@ -201,6 +204,25 @@ func LoadGoModules(goModFilePaths []string) []*GoModule {
 		}
 
 		results[i].Module = modFile
+
+		// If this `go.mod` file specifies a Go language version, that version is `1.21` or greater, and
+		// there is no `toolchain` directive, check that it is a valid Go toolchain version. Otherwise,
+		// `go` commands which try to download the right version of the Go toolchain will fail. We detect
+		// this situation and emit a diagnostic.
+		if modFile.Toolchain == nil && modFile.Go != nil &&
+			!toolchainVersionRe.Match([]byte(modFile.Go.Version)) && semver.Compare("v"+modFile.Go.Version, "v1.21.0") >= 0 {
+			diagnostics.EmitInvalidToolchainVersion(goModFilePath, modFile.Go.Version)
+
+			modPath := filepath.Dir(goModFilePath)
+
+			log.Printf(
+				"`%s` is not a valid toolchain version, trying to install it explicitly using the canonical representation in `%s`.",
+				modFile.Go.Version,
+				modPath,
+			)
+
+			toolchain.InstallVersion(modPath, modFile.Go.Version)
+		}
 	}
 
 	return results
@@ -209,7 +231,7 @@ func LoadGoModules(goModFilePaths []string) []*GoModule {
 // Given a path to a `go.work` file, this function attempts to parse the `go.work` file. If unsuccessful,
 // we attempt to discover `go.mod` files within subdirectories of the directory containing the `go.work`
 // file ourselves.
-func discoverWorkspace(workFilePath string) GoWorkspace {
+func discoverWorkspace(emitDiagnostics bool, workFilePath string) GoWorkspace {
 	log.Printf("Loading %s...\n", workFilePath)
 	baseDir := filepath.Dir(workFilePath)
 	workFileSrc, err := os.ReadFile(workFilePath)
@@ -223,7 +245,7 @@ func discoverWorkspace(workFilePath string) GoWorkspace {
 
 		return GoWorkspace{
 			BaseDir: baseDir,
-			Modules: LoadGoModules(goModFilePaths),
+			Modules: LoadGoModules(emitDiagnostics, goModFilePaths),
 			DepMode: GoGetWithModules,
 			ModMode: getModMode(GoGetWithModules, baseDir),
 		}
@@ -240,7 +262,7 @@ func discoverWorkspace(workFilePath string) GoWorkspace {
 
 		return GoWorkspace{
 			BaseDir: baseDir,
-			Modules: LoadGoModules(goModFilePaths),
+			Modules: LoadGoModules(emitDiagnostics, goModFilePaths),
 			DepMode: GoGetWithModules,
 			ModMode: getModMode(GoGetWithModules, baseDir),
 		}
@@ -263,7 +285,7 @@ func discoverWorkspace(workFilePath string) GoWorkspace {
 	return GoWorkspace{
 		BaseDir:       baseDir,
 		WorkspaceFile: workFile,
-		Modules:       LoadGoModules(goModFilePaths),
+		Modules:       LoadGoModules(emitDiagnostics, goModFilePaths),
 		DepMode:       GoGetWithModules,
 		ModMode:       ModReadonly, // Workspaces only support "readonly"
 	}
@@ -286,7 +308,7 @@ func discoverWorkspaces(emitDiagnostics bool) []GoWorkspace {
 		for i, goModFile := range goModFiles {
 			results[i] = GoWorkspace{
 				BaseDir: filepath.Dir(goModFile),
-				Modules: LoadGoModules([]string{goModFile}),
+				Modules: LoadGoModules(emitDiagnostics, []string{goModFile}),
 				DepMode: GoGetWithModules,
 				ModMode: getModMode(GoGetWithModules, filepath.Dir(goModFile)),
 			}
@@ -303,7 +325,7 @@ func discoverWorkspaces(emitDiagnostics bool) []GoWorkspace {
 
 		results := make([]GoWorkspace, len(goWorkFiles))
 		for i, workFilePath := range goWorkFiles {
-			results[i] = discoverWorkspace(workFilePath)
+			results[i] = discoverWorkspace(emitDiagnostics, workFilePath)
 		}
 
 		// Add all stray `go.mod` files (i.e. those not referenced by `go.work` files)
@@ -335,7 +357,7 @@ func discoverWorkspaces(emitDiagnostics bool) []GoWorkspace {
 				log.Printf("Module %s is not referenced by any go.work file; adding it separately.\n", goModFile)
 				results = append(results, GoWorkspace{
 					BaseDir: filepath.Dir(goModFile),
-					Modules: LoadGoModules([]string{goModFile}),
+					Modules: LoadGoModules(emitDiagnostics, []string{goModFile}),
 					DepMode: GoGetWithModules,
 					ModMode: getModMode(GoGetWithModules, filepath.Dir(goModFile)),
 				})

go/extractor/toolchain/toolchain.go

@@ -18,7 +18,15 @@ func IsInstalled() bool {
 	return err == nil
 }
 
+// The default Go version that is available on a system and a set of all versions
+// that we know are installed on the system.
 var goVersion = ""
+var goVersions = map[string]struct{}{}
+
+// Adds an entry to the set of installed Go versions for the normalised `version` number.
+func addGoVersion(version string) {
+	goVersions[semver.Canonical("v"+version)] = struct{}{}
+}
 
 // Returns the current Go version as returned by 'go version', e.g. go1.14.4
 func GetEnvGoVersion() string {
@@ -27,7 +35,7 @@ func GetEnvGoVersion() string {
 		// download the version of Go specified in there. That may either fail or result in us just
 		// being told what's already in 'go.mod'. Setting 'GOTOOLCHAIN' to 'local' will force it
 		// to use the local Go toolchain instead.
-		cmd := exec.Command("go", "version")
+		cmd := Version()
 		cmd.Env = append(os.Environ(), "GOTOOLCHAIN=local")
 		out, err := cmd.CombinedOutput()
 
@@ -36,10 +44,59 @@ func GetEnvGoVersion() string {
 		}
 
 		goVersion = parseGoVersion(string(out))
+		addGoVersion(goVersion[2:])
 	}
 	return goVersion
 }
 
+// Determines whether, to our knowledge, `version` is available on the current system.
+func HasGoVersion(version string) bool {
+	_, found := goVersions[semver.Canonical("v"+version)]
+	return found
+}
+
+// Attempts to install the Go toolchain `version`.
+func InstallVersion(workingDir string, version string) bool {
+	// No need to install it if we know that it is already installed.
+	if HasGoVersion(version) {
+		return true
+	}
+
+	// Construct a command to invoke `go version` with `GOTOOLCHAIN=go1.N.0` to give
+	// Go a valid toolchain version to download the toolchain we need; subsequent commands
+	// should then work even with an invalid version that's still in `go.mod`
+	toolchainArg := "GOTOOLCHAIN=go" + semver.Canonical("v" + version)[1:]
+	versionCmd := Version()
+	versionCmd.Dir = workingDir
+	versionCmd.Env = append(os.Environ(), toolchainArg)
+	versionCmd.Stdout = os.Stdout
+	versionCmd.Stderr = os.Stderr
+
+	log.Printf(
+		"Trying to install Go %s using its canonical representation in `%s`.",
+		version,
+		workingDir,
+	)
+
+	// Run the command. If something goes wrong, report it to the log and signal failure
+	// to the caller.
+	if versionErr := versionCmd.Run(); versionErr != nil {
+		log.Printf(
+			"Failed to invoke `%s go version` in %s: %s\n",
+			toolchainArg,
+			versionCmd.Dir,
+			versionErr.Error(),
+		)
+
+		return false
+	}
+
+	// Add the version to the set of versions that we know are installed and signal
+	// success to the caller.
+	addGoVersion(version)
+	return true
+}
+
 // Returns the current Go version in semver format, e.g. v1.14.4
 func GetEnvGoSemVer() string {
 	goVersion := GetEnvGoVersion()
@@ -107,3 +164,9 @@ func VendorModule(path string) *exec.Cmd {
 	modVendor.Dir = path
 	return modVendor
 }
+
+// Constructs a command to run `go version`.
+func Version() *exec.Cmd {
+	version := exec.Command("go", "version")
+	return version
+}

go/extractor/toolchain/toolchain_test.go

@@ -14,3 +14,9 @@ func TestParseGoVersion(t *testing.T) {
 		}
 	}
 }
+
+func TestHasGoVersion(t *testing.T) {
+	if HasGoVersion("1.21") {
+		t.Error("Expected HasGoVersion(\"1.21\") to be false, but got true")
+	}
+}

go/ql/integration-tests/all-platforms/go/diagnostics/invalid-toolchain-version/diagnostics.expected

@@ -0,0 +1,31 @@
+{
+  "location": {
+    "file": "go.mod"
+  },
+  "markdownMessage": "As of Go 1.21, toolchain versions [must use the 1.N.P syntax](https://go.dev/doc/toolchain#version).\n\n`1.21` in `go.mod` does not match this syntax and there is no additional `toolchain` directive, which may cause some `go` commands to fail.",
+  "severity": "warning",
+  "source": {
+    "extractorName": "go",
+    "id": "go/autobuilder/invalid-go-toolchain-version",
+    "name": "Invalid Go toolchain version"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}
+{
+  "markdownMessage": "A single `go.mod` file was found.\n\n`go.mod`",
+  "severity": "note",
+  "source": {
+    "extractorName": "go",
+    "id": "go/autobuilder/single-root-go-mod-found",
+    "name": "A single `go.mod` file was found in the root"
+  },
+  "visibility": {
+    "cliSummaryTable": false,
+    "statusPage": false,
+    "telemetry": true
+  }
+}

go/ql/integration-tests/all-platforms/go/diagnostics/invalid-toolchain-version/src/go.mod

@@ -0,0 +1,3 @@
+go 1.21
+
+module example

go/ql/integration-tests/all-platforms/go/diagnostics/invalid-toolchain-version/src/main.go

@@ -0,0 +1,5 @@
+package main
+
+func main() {
+
+}

go/ql/integration-tests/all-platforms/go/diagnostics/invalid-toolchain-version/test.py

@@ -0,0 +1,19 @@
+import os
+import subprocess
+
+from create_database_utils import *
+from diagnostics_test_utils import *
+
+# Set up a GOPATH relative to this test's root directory;
+# we set os.environ instead of using extra_env because we
+# need it to be set for the call to "go clean -modcache" later
+goPath = os.path.join(os.path.abspath(os.getcwd()), ".go")
+os.environ['GOPATH'] = goPath
+os.environ['LGTM_INDEX_IMPORT_PATH'] = "test"
+run_codeql_database_create([], lang="go", source="src")
+
+check_diagnostics()
+
+# Clean up the temporary GOPATH to prevent Bazel failures next
+# time the tests are run; see https://github.com/golang/go/issues/27161
+subprocess.call(["go", "clean", "-modcache"])

go/ql/integration-tests/all-platforms/go/go-version-bump/diagnostics.expected

@@ -0,0 +1,31 @@
+{
+  "location": {
+    "file": "go.mod"
+  },
+  "markdownMessage": "As of Go 1.21, toolchain versions [must use the 1.N.P syntax](https://go.dev/doc/toolchain#version).\n\n`1.21` in `go.mod` does not match this syntax and there is no additional `toolchain` directive, which may cause some `go` commands to fail.",
+  "severity": "warning",
+  "source": {
+    "extractorName": "go",
+    "id": "go/autobuilder/invalid-go-toolchain-version",
+    "name": "Invalid Go toolchain version"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}
+{
+  "markdownMessage": "A single `go.mod` file was found.\n\n`go.mod`",
+  "severity": "note",
+  "source": {
+    "extractorName": "go",
+    "id": "go/autobuilder/single-root-go-mod-found",
+    "name": "A single `go.mod` file was found in the root"
+  },
+  "visibility": {
+    "cliSummaryTable": false,
+    "statusPage": false,
+    "telemetry": true
+  }
+}

go/ql/integration-tests/all-platforms/go/go-version-bump/src/go.mod

@@ -0,0 +1,3 @@
+go 1.21
+
+module test

go/ql/integration-tests/all-platforms/go/go-version-bump/src/main.go

@@ -0,0 +1,3 @@
+package main
+
+func main() {}

go/ql/integration-tests/all-platforms/go/go-version-bump/test.py

@@ -0,0 +1,18 @@
+import os
+import subprocess
+
+from create_database_utils import *
+from diagnostics_test_utils import *
+
+# Set up a GOPATH relative to this test's root directory;
+# we set os.environ instead of using extra_env because we
+# need it to be set for the call to "go clean -modcache" later
+goPath = os.path.join(os.path.abspath(os.getcwd()), ".go")
+os.environ['GOPATH'] = goPath
+run_codeql_database_create([], lang="go", source="src")
+
+check_diagnostics()
+
+# Clean up the temporary GOPATH to prevent Bazel failures next
+# time the tests are run; see https://github.com/golang/go/issues/27161
+subprocess.call(["go", "clean", "-modcache"])

go/ql/integration-tests/all-platforms/go/two-go-mods-one-failure/diagnostics.expected

@@ -18,7 +18,7 @@
   "source": {
     "extractorName": "go",
     "id": "go/autobuilder/extraction-failed-for-project",
-    "name": "Unable to extract 1 Go projects"
+    "name": "Unable to extract some Go projects"
   },
   "visibility": {
     "cliSummaryTable": true,