Skip to content

Commit 987af4c

Browse files
authored
Merge pull request #19248 from github/mbg/go/private-registries
Go: Support private registries via `GOPROXY`
2 parents 46fb041 + 7592ce4 commit 987af4c

File tree

5 files changed

+200
-8
lines changed

5 files changed

+200
-8
lines changed

go/extractor/cli/go-autobuilder/go-autobuilder.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -67,13 +67,13 @@ func restoreRepoLayout(fromDir string, dirEntries []string, scratchDirName strin
6767

6868
// addVersionToMod add a go version directive, e.g. `go 1.14` to a `go.mod` file.
6969
func addVersionToMod(version string) bool {
70-
cmd := exec.Command("go", "mod", "edit", "-go="+version)
70+
cmd := toolchain.GoCommand("mod", "edit", "-go="+version)
7171
return util.RunCmd(cmd)
7272
}
7373

7474
// checkVendor tests to see whether a vendor directory is inconsistent according to the go frontend
7575
func checkVendor() bool {
76-
vendorCheckCmd := exec.Command("go", "list", "-mod=vendor", "./...")
76+
vendorCheckCmd := toolchain.GoCommand("list", "-mod=vendor", "./...")
7777
outp, err := vendorCheckCmd.CombinedOutput()
7878
if err != nil {
7979
badVendorRe := regexp.MustCompile(`(?m)^go: inconsistent vendoring in .*:$`)
@@ -438,7 +438,7 @@ func installDependencies(workspace project.GoWorkspace) {
438438
util.RunCmd(vendor)
439439
}
440440

441-
install = exec.Command("go", "get", "-v", "./...")
441+
install = toolchain.GoCommand("get", "-v", "./...")
442442
install.Dir = path
443443
log.Printf("Installing dependencies using `go get -v ./...` in `%s`.\n", path)
444444
util.RunCmd(install)

go/extractor/toolchain/toolchain.go

+12-5
Original file line numberDiff line numberDiff line change
@@ -137,9 +137,16 @@ func SupportsWorkspaces() bool {
137137
return GetEnvGoSemVer().IsAtLeast(V1_18)
138138
}
139139

140+
// Constructs a `*exec.Cmd` for `go` with the specified arguments.
141+
func GoCommand(arg ...string) *exec.Cmd {
142+
cmd := exec.Command("go", arg...)
143+
util.ApplyProxyEnvVars(cmd)
144+
return cmd
145+
}
146+
140147
// Run `go mod tidy -e` in the directory given by `path`.
141148
func TidyModule(path string) *exec.Cmd {
142-
cmd := exec.Command("go", "mod", "tidy", "-e")
149+
cmd := GoCommand("mod", "tidy", "-e")
143150
cmd.Dir = path
144151
return cmd
145152
}
@@ -159,21 +166,21 @@ func InitModule(path string) *exec.Cmd {
159166
}
160167
}
161168

162-
modInit := exec.Command("go", "mod", "init", moduleName)
169+
modInit := GoCommand("mod", "init", moduleName)
163170
modInit.Dir = path
164171
return modInit
165172
}
166173

167174
// Constructs a command to run `go mod vendor -e` in the directory given by `path`.
168175
func VendorModule(path string) *exec.Cmd {
169-
modVendor := exec.Command("go", "mod", "vendor", "-e")
176+
modVendor := GoCommand("mod", "vendor", "-e")
170177
modVendor.Dir = path
171178
return modVendor
172179
}
173180

174181
// Constructs a command to run `go version`.
175182
func Version() *exec.Cmd {
176-
version := exec.Command("go", "version")
183+
version := GoCommand("version")
177184
return version
178185
}
179186

@@ -209,7 +216,7 @@ func RunListWithEnv(format string, patterns []string, additionalEnv []string, fl
209216
func ListWithEnv(format string, patterns []string, additionalEnv []string, flags ...string) *exec.Cmd {
210217
args := append([]string{"list", "-e", "-f", format}, flags...)
211218
args = append(args, patterns...)
212-
cmd := exec.Command("go", args...)
219+
cmd := GoCommand(args...)
213220
cmd.Env = append(os.Environ(), additionalEnv...)
214221
return cmd
215222
}

go/extractor/util/BUILD.bazel

+2
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

go/extractor/util/registryproxy.go

+134
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
package util
2+
3+
import (
4+
"encoding/json"
5+
"fmt"
6+
"log/slog"
7+
"os"
8+
"os/exec"
9+
"strings"
10+
)
11+
12+
const PROXY_HOST = "CODEQL_PROXY_HOST"
13+
const PROXY_PORT = "CODEQL_PROXY_PORT"
14+
const PROXY_CA_CERTIFICATE = "CODEQL_PROXY_CA_CERTIFICATE"
15+
const PROXY_URLS = "CODEQL_PROXY_URLS"
16+
const GOPROXY_SERVER = "goproxy_server"
17+
18+
type RegistryConfig struct {
19+
Type string `json:"type"`
20+
URL string `json:"url"`
21+
}
22+
23+
// The address of the proxy including protocol and port (e.g. http://localhost:1234)
24+
var proxy_address string
25+
26+
// The path to the temporary file that stores the proxy certificate, if any.
27+
var proxy_cert_file string
28+
29+
// An array of registry configurations that are relevant to Go.
30+
// This excludes other registry configurations that may be available, but are not relevant to Go.
31+
var proxy_configs []RegistryConfig
32+
33+
// Stores the environment variables that we wish to pass on to `go` commands.
34+
var proxy_vars []string = nil
35+
36+
// Keeps track of whether we have inspected the proxy environment variables.
37+
// Needed since `proxy_vars` may be nil either way.
38+
var proxy_vars_checked bool = false
39+
40+
// Tries to parse the given string value into an array of RegistryConfig values.
41+
func parseRegistryConfigs(str string) ([]RegistryConfig, error) {
42+
var configs []RegistryConfig
43+
if err := json.Unmarshal([]byte(str), &configs); err != nil {
44+
return nil, err
45+
}
46+
47+
return configs, nil
48+
}
49+
50+
func getEnvVars() []string {
51+
var result []string
52+
53+
if proxy_host, proxy_host_set := os.LookupEnv(PROXY_HOST); proxy_host_set {
54+
if proxy_port, proxy_port_set := os.LookupEnv(PROXY_PORT); proxy_port_set {
55+
proxy_address = fmt.Sprintf("http://%s:%s", proxy_host, proxy_port)
56+
result = append(result, fmt.Sprintf("HTTP_PROXY=%s", proxy_address), fmt.Sprintf("HTTPS_PROXY=%s", proxy_address))
57+
58+
slog.Info("Found private registry proxy", slog.String("proxy_address", proxy_address))
59+
}
60+
}
61+
62+
if proxy_cert, proxy_cert_set := os.LookupEnv(PROXY_CA_CERTIFICATE); proxy_cert_set {
63+
// Write the certificate to a temporary file
64+
slog.Info("Found certificate")
65+
66+
f, err := os.CreateTemp("", "codeql-proxy.crt")
67+
if err != nil {
68+
slog.Error("Failed to create temporary file for the proxy certificate", slog.String("error", err.Error()))
69+
}
70+
71+
_, err = f.WriteString(proxy_cert)
72+
if err != nil {
73+
slog.Error("Failed to write to the temporary certificate file", slog.String("error", err.Error()))
74+
}
75+
76+
err = f.Close()
77+
if err != nil {
78+
slog.Error("Failed to close the temporary certificate file", slog.String("error", err.Error()))
79+
} else {
80+
proxy_cert_file = f.Name()
81+
result = append(result, fmt.Sprintf("SSL_CERT_FILE=%s", proxy_cert_file))
82+
}
83+
}
84+
85+
if proxy_urls, proxy_urls_set := os.LookupEnv(PROXY_URLS); proxy_urls_set {
86+
val, err := parseRegistryConfigs(proxy_urls)
87+
if err != nil {
88+
slog.Error("Unable to parse proxy configurations", slog.String("error", err.Error()))
89+
} else {
90+
// We only care about private registry configurations that are relevant to Go and
91+
// filter others out at this point.
92+
for _, cfg := range val {
93+
if cfg.Type == GOPROXY_SERVER {
94+
proxy_configs = append(proxy_configs, cfg)
95+
slog.Info("Found GOPROXY server", slog.String("url", cfg.URL))
96+
}
97+
}
98+
99+
if len(proxy_configs) > 0 {
100+
goproxy_val := "https://proxy.golang.org,direct"
101+
102+
for _, cfg := range proxy_configs {
103+
goproxy_val = cfg.URL + "," + goproxy_val
104+
}
105+
106+
result = append(result, fmt.Sprintf("GOPROXY=%s", goproxy_val), "GOPRIVATE=", "GONOPROXY=")
107+
}
108+
}
109+
}
110+
111+
return result
112+
}
113+
114+
// Applies private package proxy related environment variables to `cmd`.
115+
func ApplyProxyEnvVars(cmd *exec.Cmd) {
116+
slog.Debug(
117+
"Applying private registry proxy environment variables",
118+
slog.String("cmd_args", strings.Join(cmd.Args, " ")),
119+
)
120+
121+
// If we haven't done so yet, check whether the proxy environment variables are set
122+
// and extract information from them.
123+
if !proxy_vars_checked {
124+
proxy_vars = getEnvVars()
125+
proxy_vars_checked = true
126+
}
127+
128+
// If the proxy is configured, `proxy_vars` will be not `nil`. We append those
129+
// variables to the existing environment to preserve those environment variables.
130+
// If `cmd.Env` is not changed, then the existing environment is also preserved.
131+
if proxy_vars != nil {
132+
cmd.Env = append(os.Environ(), proxy_vars...)
133+
}
134+
}
+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
package util
2+
3+
import (
4+
"testing"
5+
)
6+
7+
func parseRegistryConfigsFail(t *testing.T, str string) {
8+
_, err := parseRegistryConfigs(str)
9+
10+
if err == nil {
11+
t.Fatal("Expected `parseRegistryConfigs` to fail, but it succeeded.")
12+
}
13+
}
14+
15+
func parseRegistryConfigsSuccess(t *testing.T, str string) []RegistryConfig {
16+
val, err := parseRegistryConfigs(str)
17+
18+
if err != nil {
19+
t.Fatalf("Expected `parseRegistryConfigs` to succeed, but it failed: %s", err.Error())
20+
}
21+
22+
return val
23+
}
24+
25+
func TestParseRegistryConfigs(t *testing.T) {
26+
parseRegistryConfigsFail(t, "")
27+
28+
empty := parseRegistryConfigsSuccess(t, "[]")
29+
30+
if len(empty) != 0 {
31+
t.Fatal("Expected `parseRegistryConfigs(\"[]\")` to return no configurations, but got some.")
32+
}
33+
34+
single := parseRegistryConfigsSuccess(t, "[{ \"type\": \"goproxy_server\", \"url\": \"https://proxy.example.com/mod\" }]")
35+
36+
if len(single) != 1 {
37+
t.Fatalf("Expected `parseRegistryConfigs` to return one configuration, but got %d.", len(single))
38+
}
39+
40+
first := single[0]
41+
42+
if first.Type != "goproxy_server" {
43+
t.Fatalf("Expected `Type` to be `goproxy_server`, but got `%s`", first.Type)
44+
}
45+
46+
if first.URL != "https://proxy.example.com/mod" {
47+
t.Fatalf("Expected `URL` to be `https://proxy.example.com/mod`, but got `%s`", first.URL)
48+
}
49+
}

0 commit comments

Comments
 (0)