From ded27bcee45adf55f1c97a2f1b36af4c02fa1201 Mon Sep 17 00:00:00 2001 From: "Michael B. Gale" Date: Mon, 7 Apr 2025 12:50:19 +0100 Subject: [PATCH 1/3] Go: Replace `exec.Command("go"` with `toolchain.GoCommand(` --- go/extractor/cli/go-autobuilder/go-autobuilder.go | 6 +++--- go/extractor/toolchain/toolchain.go | 15 ++++++++++----- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/go/extractor/cli/go-autobuilder/go-autobuilder.go b/go/extractor/cli/go-autobuilder/go-autobuilder.go index aeff4f0bd6e1..23e0cb7fe0db 100644 --- a/go/extractor/cli/go-autobuilder/go-autobuilder.go +++ b/go/extractor/cli/go-autobuilder/go-autobuilder.go @@ -67,13 +67,13 @@ func restoreRepoLayout(fromDir string, dirEntries []string, scratchDirName strin // addVersionToMod add a go version directive, e.g. `go 1.14` to a `go.mod` file. func addVersionToMod(version string) bool { - cmd := exec.Command("go", "mod", "edit", "-go="+version) + cmd := toolchain.GoCommand("mod", "edit", "-go="+version) return util.RunCmd(cmd) } // checkVendor tests to see whether a vendor directory is inconsistent according to the go frontend func checkVendor() bool { - vendorCheckCmd := exec.Command("go", "list", "-mod=vendor", "./...") + vendorCheckCmd := toolchain.GoCommand("list", "-mod=vendor", "./...") outp, err := vendorCheckCmd.CombinedOutput() if err != nil { badVendorRe := regexp.MustCompile(`(?m)^go: inconsistent vendoring in .*:$`) @@ -438,7 +438,7 @@ func installDependencies(workspace project.GoWorkspace) { util.RunCmd(vendor) } - install = exec.Command("go", "get", "-v", "./...") + install = toolchain.GoCommand("get", "-v", "./...") install.Dir = path log.Printf("Installing dependencies using `go get -v ./...` in `%s`.\n", path) util.RunCmd(install) diff --git a/go/extractor/toolchain/toolchain.go b/go/extractor/toolchain/toolchain.go index 119e3782f6f6..47df9b11ad09 100644 --- a/go/extractor/toolchain/toolchain.go +++ b/go/extractor/toolchain/toolchain.go @@ -137,9 +137,14 @@ func SupportsWorkspaces() bool { return GetEnvGoSemVer().IsAtLeast(V1_18) } +// Constructs a `*exec.Cmd` for `go` with the specified arguments. +func GoCommand(arg ...string) *exec.Cmd { + return exec.Command("go", arg...) +} + // Run `go mod tidy -e` in the directory given by `path`. func TidyModule(path string) *exec.Cmd { - cmd := exec.Command("go", "mod", "tidy", "-e") + cmd := GoCommand("mod", "tidy", "-e") cmd.Dir = path return cmd } @@ -159,21 +164,21 @@ func InitModule(path string) *exec.Cmd { } } - modInit := exec.Command("go", "mod", "init", moduleName) + modInit := GoCommand("mod", "init", moduleName) modInit.Dir = path return modInit } // Constructs a command to run `go mod vendor -e` in the directory given by `path`. func VendorModule(path string) *exec.Cmd { - modVendor := exec.Command("go", "mod", "vendor", "-e") + modVendor := GoCommand("mod", "vendor", "-e") modVendor.Dir = path return modVendor } // Constructs a command to run `go version`. func Version() *exec.Cmd { - version := exec.Command("go", "version") + version := GoCommand("version") return version } @@ -209,7 +214,7 @@ func RunListWithEnv(format string, patterns []string, additionalEnv []string, fl func ListWithEnv(format string, patterns []string, additionalEnv []string, flags ...string) *exec.Cmd { args := append([]string{"list", "-e", "-f", format}, flags...) args = append(args, patterns...) - cmd := exec.Command("go", args...) + cmd := GoCommand(args...) cmd.Env = append(os.Environ(), additionalEnv...) return cmd } From 0f78e11376315a15a62375e535313851e16cd66e Mon Sep 17 00:00:00 2001 From: "Michael B. Gale" Date: Tue, 8 Apr 2025 11:18:40 +0100 Subject: [PATCH 2/3] Go: Detect and apply proxy settings (WIP) --- go/extractor/toolchain/toolchain.go | 4 +- go/extractor/util/BUILD.bazel | 2 + go/extractor/util/registryproxy.go | 114 ++++++++++++++++++++++++ go/extractor/util/registryproxy_test.go | 49 ++++++++++ 4 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 go/extractor/util/registryproxy.go create mode 100644 go/extractor/util/registryproxy_test.go diff --git a/go/extractor/toolchain/toolchain.go b/go/extractor/toolchain/toolchain.go index 47df9b11ad09..43581b921005 100644 --- a/go/extractor/toolchain/toolchain.go +++ b/go/extractor/toolchain/toolchain.go @@ -139,7 +139,9 @@ func SupportsWorkspaces() bool { // Constructs a `*exec.Cmd` for `go` with the specified arguments. func GoCommand(arg ...string) *exec.Cmd { - return exec.Command("go", arg...) + cmd := exec.Command("go", arg...) + util.ApplyProxyEnvVars(cmd) + return cmd } // Run `go mod tidy -e` in the directory given by `path`. diff --git a/go/extractor/util/BUILD.bazel b/go/extractor/util/BUILD.bazel index b7a7783aa799..dd294a63a9c1 100644 --- a/go/extractor/util/BUILD.bazel +++ b/go/extractor/util/BUILD.bazel @@ -6,6 +6,7 @@ go_library( name = "util", srcs = [ "extractvendordirs.go", + "registryproxy.go", "semver.go", "util.go", ], @@ -17,6 +18,7 @@ go_library( go_test( name = "util_test", srcs = [ + "registryproxy_test.go", "semver_test.go", "util_test.go", ], diff --git a/go/extractor/util/registryproxy.go b/go/extractor/util/registryproxy.go new file mode 100644 index 000000000000..8f833c30d90b --- /dev/null +++ b/go/extractor/util/registryproxy.go @@ -0,0 +1,114 @@ +package util + +import ( + "encoding/json" + "fmt" + "log/slog" + "os" + "os/exec" + "strings" +) + +const PROXY_HOST = "CODEQL_PROXY_HOST" +const PROXY_PORT = "CODEQL_PROXY_PORT" +const PROXY_CA_CERTIFICATE = "CODEQL_PROXY_CA_CERTIFICATE" +const PROXY_URLS = "CODEQL_PROXY_URLS" +const GOPROXY_SERVER = "goproxy_server" + +type RegistryConfig struct { + Type string `json:"type"` + URL string `json:"url"` +} + +var proxy_address string +var proxy_cert_file string +var proxy_configs []RegistryConfig + +// Tries to parse the given string value into an array of RegistryConfig values. +func parseRegistryConfigs(str string) ([]RegistryConfig, error) { + var configs []RegistryConfig + if err := json.Unmarshal([]byte(str), &configs); err != nil { + return nil, err + } + + return configs, nil +} + +func checkEnvVars() { + if proxy_host, proxy_host_set := os.LookupEnv(PROXY_HOST); proxy_host_set { + if proxy_port, proxy_port_set := os.LookupEnv(PROXY_PORT); proxy_port_set { + proxy_address = fmt.Sprintf("http://%s:%s", proxy_host, proxy_port) + slog.Info("Found private registry proxy", slog.String("proxy_address", proxy_address)) + } + } + + if proxy_cert, proxy_cert_set := os.LookupEnv(PROXY_CA_CERTIFICATE); proxy_cert_set { + // Write the certificate to a temporary file + slog.Info("Found certificate") + + f, err := os.CreateTemp("", "codeql-proxy.crt") + if err != nil { + slog.Error("Unable to create temporary file for the proxy certificate", slog.String("error", err.Error())) + } + + _, err = f.WriteString(proxy_cert) + if err != nil { + slog.Error("Failed to write to the temporary certificate file", slog.String("error", err.Error())) + } + + err = f.Close() + if err != nil { + slog.Error("Failed to close the temporary certificate file", slog.String("error", err.Error())) + } else { + proxy_cert_file = f.Name() + } + } + + if proxy_urls, proxy_urls_set := os.LookupEnv(PROXY_URLS); proxy_urls_set { + val, err := parseRegistryConfigs(proxy_urls) + if err != nil { + slog.Error("Unable to parse proxy configurations", slog.String("error", err.Error())) + } else { + // We only care about private registry configurations that are relevant to Go and + // filter others out at this point. + proxy_configs = make([]RegistryConfig, 0) + for _, cfg := range val { + if cfg.Type == GOPROXY_SERVER { + proxy_configs = append(proxy_configs, cfg) + slog.Info("Found GOPROXY server", slog.String("url", cfg.URL)) + } + } + } + } +} + +// Applies private package proxy related environment variables to `cmd`. +func ApplyProxyEnvVars(cmd *exec.Cmd) { + slog.Info( + "Applying private registry proxy environment variables", + slog.String("cmd_args", strings.Join(cmd.Args, " ")), + ) + + checkEnvVars() + + if proxy_address != "" { + cmd.Env = append(cmd.Env, fmt.Sprintf("HTTP_PROXY=%s", proxy_address)) + cmd.Env = append(cmd.Env, fmt.Sprintf("HTTPS_PROXY=%s", proxy_address)) + } + if proxy_cert_file != "" { + slog.Info("Setting SSL_CERT_FILE", slog.String("proxy_cert_file", proxy_cert_file)) + cmd.Env = append(cmd.Env, fmt.Sprintf("SSL_CERT_FILE=%s", proxy_cert_file)) + } + + if len(proxy_configs) > 0 { + goproxy_val := "https://proxy.golang.org,direct" + + for _, cfg := range proxy_configs { + goproxy_val = cfg.URL + "," + goproxy_val + } + + cmd.Env = append(cmd.Env, fmt.Sprintf("GOPROXY=%s", goproxy_val)) + cmd.Env = append(cmd.Env, "GOPRIVATE=") + cmd.Env = append(cmd.Env, "GONOPROXY=") + } +} diff --git a/go/extractor/util/registryproxy_test.go b/go/extractor/util/registryproxy_test.go new file mode 100644 index 000000000000..b5fde50347f2 --- /dev/null +++ b/go/extractor/util/registryproxy_test.go @@ -0,0 +1,49 @@ +package util + +import ( + "testing" +) + +func parseRegistryConfigsFail(t *testing.T, str string) { + _, err := parseRegistryConfigs(str) + + if err == nil { + t.Fatal("Expected `parseRegistryConfigs` to fail, but it succeeded.") + } +} + +func parseRegistryConfigsSuccess(t *testing.T, str string) []RegistryConfig { + val, err := parseRegistryConfigs(str) + + if err != nil { + t.Fatalf("Expected `parseRegistryConfigs` to succeed, but it failed: %s", err.Error()) + } + + return val +} + +func TestParseRegistryConfigs(t *testing.T) { + // parseRegistryConfigsFail(t, "") + + empty := parseRegistryConfigsSuccess(t, "[]") + + if len(empty) != 0 { + t.Fatal("Expected `parseRegistryConfigs(\"[]\")` to return no configurations, but got some.") + } + + single := parseRegistryConfigsSuccess(t, "[{ \"type\": \"goproxy_server\", \"url\": \"https://proxy.example.com/mod\" }]") + + if len(single) != 1 { + t.Fatalf("Expected `parseRegistryConfigs` to return one configuration, but got %d.", len(single)) + } + + first := single[0] + + if first.Type != "goproxy_server" { + t.Fatalf("Expected `Type` to be `goproxy_server`, but got `%s`", first.Type) + } + + if first.URL != "https://proxy.example.com/mod" { + t.Fatalf("Expected `URL` to be `https://proxy.example.com/mod`, but got `%s`", first.URL) + } +} From e210be7bb29b79f1005821ce832dd789ac5fd458 Mon Sep 17 00:00:00 2001 From: "Michael B. Gale" Date: Tue, 8 Apr 2025 12:38:38 +0100 Subject: [PATCH 3/3] Go: Preserve environment variables in `ApplyProxyEnvVars` --- go/extractor/util/registryproxy.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/go/extractor/util/registryproxy.go b/go/extractor/util/registryproxy.go index 8f833c30d90b..71071e838146 100644 --- a/go/extractor/util/registryproxy.go +++ b/go/extractor/util/registryproxy.go @@ -91,6 +91,9 @@ func ApplyProxyEnvVars(cmd *exec.Cmd) { checkEnvVars() + // Preserve environment variables + cmd.Env = os.Environ() + if proxy_address != "" { cmd.Env = append(cmd.Env, fmt.Sprintf("HTTP_PROXY=%s", proxy_address)) cmd.Env = append(cmd.Env, fmt.Sprintf("HTTPS_PROXY=%s", proxy_address))