Go: Improve diagnostics when no packages are extracted

go/extractor/cli/go-autobuilder/go-autobuilder.go

@@ -620,6 +620,16 @@ func installDependenciesAndBuild() {
 	} else {
 		log.Printf("Success: extraction succeeded for all %d discovered project(s).\n", len(workspaces))
 	}
+
+	// Check whether we have been able to extract any packages. Emit an error-level diagnostic if not.
+	// Each extractor run should have stored information about the outcome in a file in the database
+	// scratch directory, which are retrieving this information from.
+	extractionResults := make(util.ExtractionResults)
+	util.ReadExtractionResults(&extractionResults)
+
+	if extractionResults.HasSources() && extractionResults.TotalPackageCount() == 0 {
+		diagnostics.EmitGoFilesFoundButNotProcessed()
+	}
 }
 
 func main() {

go/extractor/diagnostics/diagnostics.go

@@ -190,6 +190,20 @@ func EmitGoFilesFoundButNotProcessed() {
 	)
 }
 
+func EmitGoFilesFoundButNotProcessedForDirectory(wd string) {
+	emitDiagnostic(
+		"go/autobuilder/go-files-found-but-not-processed-for-directory",
+		"Go files were found but not processed for a directory",
+		fmt.Sprintf(
+			"CodeQL was unable to extract any Go files in %s. If this is unexpected, [specify a custom build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages) that includes one or more `go build` commands to build the `.go` files to be analyzed.",
+			wd,
+		),
+		severityWarning,
+		fullVisibility,
+		noLocation,
+	)
+}
+
 func EmitRelativeImportPaths() {
 	emitDiagnostic(
 		"go/autobuilder/relative-import-paths",

go/extractor/extractor.go

@@ -124,15 +124,30 @@ func ExtractWithFlags(buildFlags []string, patterns []string, extractTests bool)
 	}
 	log.Println("Done running packages.Load.")
 
-	if len(pkgs) == 0 {
-		log.Println("No packages found.")
+	// Determine the working directory for this run of the extractor.
+	wd, err := os.Getwd()
+	if err != nil {
+		log.Printf("Warning: failed to get working directory: %s\n", err.Error())
+	} else {
+		// Determine whether there are Go source files in the working directory.
+		hasSources := util.FindGoFiles(wd)
 
-		wd, err := os.Getwd()
-		if err != nil {
-			log.Printf("Warning: failed to get working directory: %s\n", err.Error())
-		} else if util.FindGoFiles(wd) {
-			diagnostics.EmitGoFilesFoundButNotProcessed()
+		// Determine how many packages we were able to load.
+		pkgCount := len(pkgs)
+
+		if pkgCount == 0 {
+			log.Println("No packages found.")
+
+			if hasSources {
+				diagnostics.EmitGoFilesFoundButNotProcessedForDirectory(wd)
+			}
 		}
+
+		// Write the number of packages to a file that can be inspected by the autobuilder.
+		util.WriteExtractionResult(wd, util.ExtractionResult{
+			PackageCount: pkgCount,
+			HasSources:   hasSources,
+		})
 	}
 
 	log.Println("Extracting universe scope.")

go/extractor/util/extractor.go

@@ -0,0 +1,101 @@
+package util
+
+import (
+	"encoding/json"
+	"log"
+	"os"
+	"path/filepath"
+)
+
+// Gets the path of the JSON file in the database scratch directory that is used
+// to store extraction results.
+func extractionResultsPath() string {
+	return filepath.Join(ScratchDir(), "extraction.json")
+}
+
+// Represents results of an extractor run that are of interest to the autobuilder.
+type ExtractionResult struct {
+	// The number of packages that were extracted.
+	PackageCount int `json:"packageCount"`
+	// Indicates whether there are Go sources for this project.
+	HasSources bool `json:"hasSources"`
+}
+
+// Represents a mapping of module roots to extraction results.
+type ExtractionResults map[string]ExtractionResult
+
+/* Returns the total number of packages extracted */
+func (results ExtractionResults) TotalPackageCount() int {
+	result := 0
+	for _, v := range results {
+		result += v.PackageCount
+	}
+	return result
+}
+
+/* Returns a value indicating whether any Go source files were found */
+func (results ExtractionResults) HasSources() bool {
+	for _, v := range results {
+		if v.HasSources {
+			return true
+		}
+	}
+	return false
+}
+
+// Reads extraction results produced by the extractor from a well-known location in the
+// database scratch directory and stores them in `results`. Returns `nil` if successful
+// or an error if not. Note that if the file does not exist, `results` are not modified
+// and `nil` is returned. If it matters whether the file was created by an extractor
+// run, then this should be checked explicitly.
+func ReadExtractionResults(results *ExtractionResults) error {
+	path := extractionResultsPath()
+
+	if FileExists(path) {
+		contents, err := os.ReadFile(path)
+
+		if err != nil {
+			log.Printf("Found %s, but could not read it: %s\n", path, err)
+			return err
+		}
+
+		if err = json.Unmarshal(contents, results); err != nil {
+			log.Printf("Failed to unmarshal JSON from %s: %s\n", path, err)
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Writes an extraction `result` for the module at root `wd` to a well-known location in the
+// database scratch directory. If the file with extraction results exists already, it is updated.
+// Note: this assumes that multiple copies of the extractor are not run concurrently.
+func WriteExtractionResult(wd string, result ExtractionResult) {
+	path := extractionResultsPath()
+	results := make(ExtractionResults)
+
+	// Create the scratch directory, if needed.
+	if !DirExists(ScratchDir()) {
+		os.Mkdir(ScratchDir(), 0755)
+	}
+
+	// Read existing extraction results, if there are any.
+	ReadExtractionResults(&results)
+
+	// Store the new extraction result.
+	results[wd] = result
+
+	// Write all results to the file as JSON.
+	bytes, err := json.Marshal(results)
+
+	if err != nil {
+		log.Printf("Unable to marshal: %s\n", err)
+	}
+
+	err = os.WriteFile(path, bytes, 0644)
+
+	if err != nil {
+		log.Printf("Failed to write to %s: %s\n", path, err)
+	}
+}

go/extractor/util/util.go

@@ -34,6 +34,11 @@ func Getenv(key string, aliases ...string) string {
 	return ""
 }
 
+// Retrieves the path of the scratch directory in the database.
+func ScratchDir() string {
+	return os.Getenv("CODEQL_EXTRACTOR_GO_SCRATCH_DIR")
+}
+
 // FileExists tests whether the file at `filename` exists and is not a directory.
 func FileExists(filename string) bool {
 	info, err := os.Stat(filename)

go/ql/integration-tests/diagnostics/go-files-found-not-processed/diagnostics.expected

@@ -13,12 +13,12 @@
   }
 }
 {
-  "markdownMessage": "[Specify a custom build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages) that includes one or more `go build` commands to build the `.go` files to be analyzed.",
-  "severity": "error",
+  "markdownMessage": "CodeQL was unable to extract any Go files in <test-root-directory>/work. If this is unexpected, [specify a custom build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages) that includes one or more `go build` commands to build the `.go` files to be analyzed.",
+  "severity": "warning",
   "source": {
     "extractorName": "go",
-    "id": "go/autobuilder/go-files-found-but-not-processed",
-    "name": "Go files were found but not processed"
+    "id": "go/autobuilder/go-files-found-but-not-processed-for-directory",
+    "name": "Go files were found but not processed for a directory"
   },
   "visibility": {
     "cliSummaryTable": true,

go/ql/integration-tests/mixed-layout/diagnostics.expected

@@ -13,7 +13,21 @@
   }
 }
 {
-  "markdownMessage": "Go files were found outside of the Go modules corresponding to these `go.mod` files.\n\n`workspace/subdir/go.mod`, `module/go.mod`",
+  "markdownMessage": "CodeQL was unable to extract any Go files in <test-root-directory>/src/no-packages. If this is unexpected, [specify a custom build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages) that includes one or more `go build` commands to build the `.go` files to be analyzed.",
+  "severity": "warning",
+  "source": {
+    "extractorName": "go",
+    "id": "go/autobuilder/go-files-found-but-not-processed-for-directory",
+    "name": "Go files were found but not processed for a directory"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}
+{
+  "markdownMessage": "Go files were found outside of the Go modules corresponding to these `go.mod` files.\n\n`workspace/subdir/go.mod`, `module/go.mod`, `no-packages/go.mod`",
   "severity": "note",
   "source": {
     "extractorName": "go",

go/ql/integration-tests/mixed-layout/src/no-packages/README.md

@@ -0,0 +1,3 @@
+This folder contains a Go module and a Go source file, but no packages will be extracted due to build constraints.
+
+Overall extraction should still succeed, because there are other Go modules in this workspace for which we are able to extract files.

go/ql/integration-tests/mixed-layout/src/no-packages/go.mod

@@ -0,0 +1,5 @@
+go 1.22
+
+toolchain go1.22.0
+
+module module

go/ql/integration-tests/mixed-layout/src/no-packages/package_test.go

@@ -0,0 +1,11 @@
+//go:build unit
+
+package subdir
+
+import (
+	"fmt"
+)
+
+func test() {
+	fmt.Print("Hello world")
+}

go/ql/integration-tests/no-packages-in-any-module/build_environment.expected

@@ -0,0 +1,5 @@
+{
+  "configuration" : {
+    "go" : { }
+  }
+}

go/ql/integration-tests/no-packages-in-any-module/diagnostics.expected

@@ -0,0 +1,56 @@
+{
+  "markdownMessage": "2 `go.mod` files were found:\n\n`module/go.mod`, `no-packages/go.mod`",
+  "severity": "note",
+  "source": {
+    "extractorName": "go",
+    "id": "go/autobuilder/multiple-go-mod-found-not-nested",
+    "name": "Multiple `go.mod` files found, not all nested under one root `go.mod` file"
+  },
+  "visibility": {
+    "cliSummaryTable": false,
+    "statusPage": false,
+    "telemetry": true
+  }
+}
+{
+  "markdownMessage": "CodeQL was unable to extract any Go files in <test-root-directory>/src/module. If this is unexpected, [specify a custom build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages) that includes one or more `go build` commands to build the `.go` files to be analyzed.",
+  "severity": "warning",
+  "source": {
+    "extractorName": "go",
+    "id": "go/autobuilder/go-files-found-but-not-processed-for-directory",
+    "name": "Go files were found but not processed for a directory"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}
+{
+  "markdownMessage": "CodeQL was unable to extract any Go files in <test-root-directory>/src/no-packages. If this is unexpected, [specify a custom build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages) that includes one or more `go build` commands to build the `.go` files to be analyzed.",
+  "severity": "warning",
+  "source": {
+    "extractorName": "go",
+    "id": "go/autobuilder/go-files-found-but-not-processed-for-directory",
+    "name": "Go files were found but not processed for a directory"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}
+{
+  "markdownMessage": "[Specify a custom build command](https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-the-codeql-workflow-for-compiled-languages) that includes one or more `go build` commands to build the `.go` files to be analyzed.",
+  "severity": "error",
+  "source": {
+    "extractorName": "go",
+    "id": "go/autobuilder/go-files-found-but-not-processed",
+    "name": "Go files were found but not processed"
+  },
+  "visibility": {
+    "cliSummaryTable": true,
+    "statusPage": true,
+    "telemetry": true
+  }
+}

go/ql/integration-tests/no-packages-in-any-module/src/module/go.mod

@@ -0,0 +1,5 @@
+go 1.14
+
+require golang.org/x/net v0.23.0
+
+module module

go/ql/integration-tests/no-packages-in-any-module/src/module/go.sum

@@ -0,0 +1,45 @@
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
+golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

go/ql/integration-tests/no-packages-in-any-module/src/module/test.go

@@ -0,0 +1,15 @@
+//go:build unit
+
+package subdir
+
+import (
+	"fmt"
+
+	"golang.org/x/net/ipv4"
+)
+
+func test() {
+
+	header := ipv4.Header{}
+	fmt.Print(header.String())
+}

go/ql/integration-tests/no-packages-in-any-module/src/no-packages/go.mod

@@ -0,0 +1,5 @@
+go 1.22
+
+toolchain go1.22.0
+
+module module