diff --git a/cmd/kosli/root.go b/cmd/kosli/root.go index c69458649..db0a779a8 100644 --- a/cmd/kosli/root.go +++ b/cmd/kosli/root.go @@ -560,6 +560,8 @@ func newRootCmd(out, errOut io.Writer, args []string) (*cobra.Command, error) { global.DryRun = true } + kosliClient.SetUsageHeaders(global.Host, usageHeaders(cmd)) + // A value that starts with a dash is the next flag, swallowed as this // flag's value: "--description --sha256 ..." leaves --description // holding "--sha256". Cobra's own error for what follows describes diff --git a/cmd/kosli/usageHeaders.go b/cmd/kosli/usageHeaders.go new file mode 100644 index 000000000..e7651a305 --- /dev/null +++ b/cmd/kosli/usageHeaders.go @@ -0,0 +1,89 @@ +package main + +import ( + "os" + "sort" + "strconv" + "strings" + + "github.com/google/uuid" + "github.com/kosli-dev/cli/internal/version" + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +const maxCliFlagsHeaderBytes = 1024 + +// invocationID is shared by every request this process makes, so the server +// can group the requests of one CLI run. +var invocationID = uuid.NewString() + +// connectionFlags say where to send a request rather than what the command +// does, and several of them carry credentials or customer hostnames. +var connectionFlags = map[string]bool{ + "api-token": true, + "host": true, + "org": true, + "config-file": true, + "http-proxy": true, +} + +var ciVendors = map[string]string{ + github: "github", + gitlab: "gitlab", + bitbucket: "bitbucket", + azureDevops: "azure-devops", + circleci: "circleci", + teamcity: "teamcity", + codeBuild: "codebuild", + jenkins: "jenkins", +} + +// usageHeaders describes the running command to the Kosli server. Values never +// include flag values or positional arguments, since many flags carry secrets. +func usageHeaders(cmd *cobra.Command) map[string]string { + return map[string]string{ + "X-Kosli-Client": "cli/" + version.GetVersion(), + "X-Kosli-Operation": strings.TrimPrefix(strings.TrimPrefix(cmd.CommandPath(), cmd.Root().Name()), " "), + "X-Kosli-Invocation-Id": invocationID, + "X-Kosli-Cli-Flags": flagNamesInEffect(cmd), + "X-Kosli-Ci": ciVendor(), + } +} + +// flagNamesInEffect lists the flags given on the command line, in the +// environment or in the config file: bindFlags sets the latter two on the flag, +// which marks them changed. The list is cut at a flag boundary to fit +// maxCliFlagsHeaderBytes. +func flagNamesInEffect(cmd *cobra.Command) string { + var names []string + cmd.Flags().Visit(func(f *pflag.Flag) { + if !connectionFlags[f.Name] { + names = append(names, f.Name) + } + }) + sort.Strings(names) + + value := "" + for _, name := range names { + next := name + if value != "" { + next = value + "," + name + } + if len(next) > maxCliFlagsHeaderBytes { + break + } + value = next + } + return value +} + +func ciVendor() string { + if vendor, ok := ciVendors[WhichCI()]; ok { + return vendor + } + if isCI, _ := strconv.ParseBool(os.Getenv("CI")); isCI { + return "other-ci" + } + return "none" +} diff --git a/cmd/kosli/usageHeaders_test.go b/cmd/kosli/usageHeaders_test.go new file mode 100644 index 000000000..44c480556 --- /dev/null +++ b/cmd/kosli/usageHeaders_test.go @@ -0,0 +1,161 @@ +package main + +import ( + "fmt" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + + "github.com/google/uuid" + "github.com/kosli-dev/cli/internal/testHelpers" + "github.com/kosli-dev/cli/internal/version" + "github.com/spf13/cobra" + "github.com/stretchr/testify/require" +) + +var usageHeaderNames = []string{ + "X-Kosli-Client", + "X-Kosli-Operation", + "X-Kosli-Invocation-Id", + "X-Kosli-Cli-Flags", + "X-Kosli-Ci", +} + +var ciEnvVars = []string{ + "BITBUCKET_BUILD_NUMBER", "GITHUB_RUN_NUMBER", "TEAMCITY_VERSION", "GITLAB_CI", + "TF_BUILD", "CIRCLECI", "CODEBUILD_CI", "JENKINS_URL", "CI", +} + +// clearCIEnv unsets the variables WhichCI and ciVendor read, so a test run in +// CI sees the environment it declares. +func clearCIEnv(t *testing.T) { + t.Helper() + for _, name := range ciEnvVars { + t.Setenv(name, "") + require.NoError(t, os.Unsetenv(name)) + } +} + +func TestUsageHeadersDescribeTheCommandOnlyToTheKosliHost(t *testing.T) { + clearCIEnv(t) + const ( + apiToken = "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpZCI6ImNkNzg4OTg5In0.e8i_lA_QrEhFncb05Xw6E_tkCHU9QfcY4OLTVUCHffY" + jiraToken = "jira-secret-7f3a91" + orgName = "usage-headers-org" + flowName = "flow-from-environment" + ) + t.Setenv("KOSLI_FLOW", flowName) + + var jiraHeaders, kosliHeaders []http.Header + jiraServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + jiraHeaders = append(jiraHeaders, r.Header.Clone()) + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"errorMessages":["Issue does not exist"]}`)) + })) + defer jiraServer.Close() + kosliServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + kosliHeaders = append(kosliHeaders, r.Header.Clone()) + _, _ = w.Write([]byte(`{}`)) + })) + defer kosliServer.Close() + + tmpDir := t.TempDir() + _, workTree, fs, err := testHelpers.InitializeGitRepo(tmpDir) + require.NoError(t, err) + commitSha, err := testHelpers.CommitToRepo(workTree, fs, "EX-1 fix the thing") + require.NoError(t, err) + + _, _, _, _, err = executeCommandC(fmt.Sprintf(`attest jira --name foo --trail trail + --org %s --host %s --api-token %s --repo-root %s --commit %s + --jira-base-url %s --jira-username user@example.com --jira-api-token %s`, + orgName, kosliServer.URL, apiToken, tmpDir, commitSha, jiraServer.URL, jiraToken)) + require.NoError(t, err) + + require.NotEmpty(t, kosliHeaders) + for _, headers := range kosliHeaders { + require.Equal(t, "cli/"+version.GetVersion(), headers.Get("X-Kosli-Client")) + require.Equal(t, "attest jira", headers.Get("X-Kosli-Operation")) + require.Equal(t, invocationID, headers.Get("X-Kosli-Invocation-Id")) + require.Equal(t, "commit,flow,jira-api-token,jira-base-url,jira-username,name,repo-root,trail", + headers.Get("X-Kosli-Cli-Flags")) + require.Equal(t, "none", headers.Get("X-Kosli-Ci")) + require.Equal(t, "Kosli/"+version.GetVersion(), headers.Get("User-Agent")) + + for _, name := range usageHeaderNames { + for _, value := range []string{apiToken, jiraToken, orgName, flowName, kosliServer.URL, jiraServer.URL, commitSha, tmpDir} { + require.NotContains(t, headers.Get(name), value, name) + } + } + } + _, err = uuid.Parse(invocationID) + require.NoError(t, err) + + require.NotEmpty(t, jiraHeaders, "attest jira should have called the Jira server") + for _, headers := range jiraHeaders { + for _, name := range usageHeaderNames { + require.Empty(t, headers.Values(name), name) + } + } +} + +func TestCIVendor(t *testing.T) { + for _, tt := range []struct { + env map[string]string + want string + }{ + {map[string]string{"GITHUB_RUN_NUMBER": "12"}, "github"}, + {map[string]string{"GITLAB_CI": "true"}, "gitlab"}, + {map[string]string{"BITBUCKET_BUILD_NUMBER": "3"}, "bitbucket"}, + {map[string]string{"TF_BUILD": "True"}, "azure-devops"}, + {map[string]string{"CIRCLECI": "true"}, "circleci"}, + {map[string]string{"TEAMCITY_VERSION": "2024.1"}, "teamcity"}, + {map[string]string{"CODEBUILD_CI": "true"}, "codebuild"}, + {map[string]string{"JENKINS_URL": "https://jenkins.example.com/"}, "jenkins"}, + {map[string]string{"GITHUB_RUN_NUMBER": "12", "CI": "true"}, "github"}, + {map[string]string{"CI": "true"}, "other-ci"}, + {map[string]string{"CI": "false"}, "none"}, + {map[string]string{}, "none"}, + } { + t.Run(fmt.Sprintf("%v", tt.env), func(t *testing.T) { + clearCIEnv(t) + for k, v := range tt.env { + t.Setenv(k, v) + } + require.Equal(t, tt.want, ciVendor()) + }) + } +} + +func TestFlagNamesInEffectExcludeConnectionFlagsAndValues(t *testing.T) { + cmd := &cobra.Command{Use: "probe"} + for _, name := range []string{"api-token", "host", "org", "config-file", "http-proxy", "github-token", "debug"} { + cmd.Flags().String(name, "", "") + } + cmd.Flags().String("unset", "", "") + for _, name := range []string{"api-token", "host", "org", "config-file", "http-proxy", "github-token", "debug"} { + require.NoError(t, cmd.Flags().Set(name, "value-of-"+name)) + } + + require.Equal(t, "debug,github-token", flagNamesInEffect(cmd)) +} + +func TestFlagNamesInEffectIsCutAtAFlagBoundary(t *testing.T) { + cmd := &cobra.Command{Use: "probe"} + names := map[string]bool{} + for i := range 60 { + name := fmt.Sprintf("a-rather-long-flag-name-number-%02d", i) + names[name] = true + cmd.Flags().String(name, "", "") + require.NoError(t, cmd.Flags().Set(name, "x")) + } + + value := flagNamesInEffect(cmd) + + require.LessOrEqual(t, len(value), maxCliFlagsHeaderBytes) + require.Greater(t, len(value), maxCliFlagsHeaderBytes-len("a-rather-long-flag-name-number-00,")) + for _, name := range strings.Split(value, ",") { + require.True(t, names[name], "%q is not a whole flag name", name) + } +} diff --git a/go.mod b/go.mod index 0a9512a2c..e2427da65 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/go-git/go-git/v5 v5.19.3 github.com/go-playground/validator/v10 v10.30.5 github.com/google/go-github/v42 v42.0.0 + github.com/google/uuid v1.6.0 github.com/googleapis/gax-go/v2 v2.27.0 github.com/hashicorp/go-retryablehttp v0.7.8 github.com/joshdk/go-junit v1.0.0 @@ -168,7 +169,6 @@ require ( github.com/google/go-querystring v1.2.0 // indirect github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 // indirect github.com/google/s2a-go v0.1.11 // indirect - github.com/google/uuid v1.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.23 // indirect github.com/gorilla/mux v1.8.1 // indirect github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect diff --git a/internal/requests/requests.go b/internal/requests/requests.go index 4b70fe589..bc60ec893 100644 --- a/internal/requests/requests.go +++ b/internal/requests/requests.go @@ -66,6 +66,65 @@ type Client struct { Debug bool Logger *logger.Logger HttpClient *http.Client + usageHost *url.URL + usageHeaders map[string]string +} + +// SetUsageHeaders adds headers to every request this client sends to +// kosliHost. The same client also calls registries, VCS providers and storage +// buckets, and those requests never carry them. An unparseable kosliHost +// disables the headers rather than failing the command. +func (c *Client) SetUsageHeaders(kosliHost string, headers map[string]string) { + host, err := url.Parse(kosliHost) + if err != nil || host.Hostname() == "" { + if c.Logger != nil { + c.Logger.Debug("usage headers disabled: [%s] is not a URL with a host", kosliHost) + } + c.usageHost, c.usageHeaders = nil, nil + return + } + c.usageHost, c.usageHeaders = host, headers +} + +// usageHeaderTransport applies the usage headers on every hop, because +// http.Client copies a request's headers onto a redirect to another host. +type usageHeaderTransport struct { + next http.RoundTripper + client *Client +} + +func (t *usageHeaderTransport) RoundTrip(req *http.Request) (*http.Response, error) { + c := t.client + if len(c.usageHeaders) == 0 { + return t.next.RoundTrip(req) + } + req = req.Clone(req.Context()) + toKosli := sameHost(req.URL, c.usageHost) + for k, v := range c.usageHeaders { + if toKosli { + req.Header.Set(k, v) + } else { + req.Header.Del(k) + } + } + return t.next.RoundTrip(req) +} + +func sameHost(a, b *url.URL) bool { + return strings.EqualFold(a.Hostname(), b.Hostname()) && effectivePort(a) == effectivePort(b) +} + +func effectivePort(u *url.URL) string { + if port := u.Port(); port != "" { + return port + } + switch strings.ToLower(u.Scheme) { + case "http": + return "80" + case "https": + return "443" + } + return "" } // CustomLogger wraps log.Logger and implements the Printf method @@ -109,12 +168,16 @@ func NewKosliClient(httpProxyURL string, maxAPIRetries int, debug bool, logger * client.Transport.(*retryablehttp.RoundTripper).Client.HTTPClient.Transport.(*http.Transport).Proxy = http.ProxyURL(proxyURL) } - return &Client{ + c := &Client{ MaxAPIRetries: maxAPIRetries, Debug: debug, Logger: logger, HttpClient: client, - }, nil + } + // Wrapped below the retry client, since its inner http.Client is the one + // that follows redirects. + retryClient.HTTPClient.Transport = &usageHeaderTransport{next: retryClient.HTTPClient.Transport, client: c} + return c, nil } type RequestParams struct { diff --git a/internal/requests/usageHeaders_test.go b/internal/requests/usageHeaders_test.go new file mode 100644 index 000000000..fccc5d78d --- /dev/null +++ b/internal/requests/usageHeaders_test.go @@ -0,0 +1,160 @@ +package requests + +import ( + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/kosli-dev/cli/internal/logger" + "github.com/stretchr/testify/require" +) + +func recordingServer(t *testing.T) (*httptest.Server, *[]http.Header) { + t.Helper() + var seen []http.Header + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + seen = append(seen, r.Header.Clone()) + _, _ = w.Write([]byte(`{}`)) + })) + t.Cleanup(srv.Close) + return srv, &seen +} + +func newTestClient(t *testing.T) *Client { + t.Helper() + client, err := NewKosliClient("", 0, false, logger.NewStandardLogger()) + require.NoError(t, err) + return client +} + +var testUsageHeaders = map[string]string{ + "X-Kosli-Client": "cli/v0.0.0-test", + "X-Kosli-Operation": "attest generic", + "X-Kosli-Invocation-Id": "0b6c2b5e-9a8e-4c43-8d2b-3f0c3f6a1f11", + "X-Kosli-Cli-Flags": "flow,name", + "X-Kosli-Ci": "github", +} + +func TestUsageHeadersAreSentToTheKosliHost(t *testing.T) { + kosli, seen := recordingServer(t) + client := newTestClient(t) + client.SetUsageHeaders(kosli.URL, testUsageHeaders) + + for range 2 { + _, err := client.Do(&RequestParams{Method: http.MethodGet, URL: kosli.URL + "/api/v2/flows/acme"}) + require.NoError(t, err) + } + + require.Len(t, *seen, 2) + for _, headers := range *seen { + for name, want := range testUsageHeaders { + require.Equal(t, want, headers.Get(name), name) + } + require.Contains(t, headers.Get("User-Agent"), "Kosli/") + } +} + +func TestUsageHeadersAreNotSentToAnotherHost(t *testing.T) { + kosli, _ := recordingServer(t) + other, seen := recordingServer(t) + client := newTestClient(t) + client.SetUsageHeaders(kosli.URL, testUsageHeaders) + + _, err := client.Do(&RequestParams{Method: http.MethodGet, URL: other.URL + "/v2/registry/manifests/latest"}) + require.NoError(t, err) + + require.Len(t, *seen, 1) + for name := range testUsageHeaders { + require.Empty(t, (*seen)[0].Values(name), name) + } +} + +func TestUsageHeadersAreNotSentWhenNeverSet(t *testing.T) { + kosli, seen := recordingServer(t) + client := newTestClient(t) + + _, err := client.Do(&RequestParams{Method: http.MethodGet, URL: kosli.URL}) + require.NoError(t, err) + + for name := range testUsageHeaders { + require.Empty(t, (*seen)[0].Values(name), name) + } +} + +func TestUsageHeadersAreDisabledByAnUnparseableKosliHost(t *testing.T) { + kosli, seen := recordingServer(t) + client := newTestClient(t) + client.SetUsageHeaders(kosli.URL, testUsageHeaders) + client.SetUsageHeaders("://not a url", testUsageHeaders) + + _, err := client.Do(&RequestParams{Method: http.MethodGet, URL: kosli.URL}) + require.NoError(t, err) + + for name := range testUsageHeaders { + require.Empty(t, (*seen)[0].Values(name), name) + } +} + +func TestSameHost(t *testing.T) { + for _, tt := range []struct { + a, b string + want bool + }{ + {"https://app.kosli.com", "https://app.kosli.com/api/v2/flows/acme", true}, + {"https://APP.kosli.com", "https://app.kosli.com/api", true}, + {"https://app.kosli.com", "https://app.kosli.com:443/api", true}, + {"http://localhost:8001", "http://localhost:8001/api", true}, + {"http://localhost:8001", "http://localhost:8002/api", false}, + {"https://app.kosli.com", "http://app.kosli.com/api", false}, + {"https://app.kosli.com", "https://app.us.kosli.com/api", false}, + {"https://app.kosli.com", "https://bucket.s3.amazonaws.com/app.kosli.com", false}, + } { + t.Run(tt.a+" vs "+tt.b, func(t *testing.T) { + a, err := url.Parse(tt.a) + require.NoError(t, err) + b, err := url.Parse(tt.b) + require.NoError(t, err) + require.Equal(t, tt.want, sameHost(a, b)) + }) + } +} + +func TestUsageHeadersAreNotForwardedOnARedirectToAnotherHost(t *testing.T) { + other, otherSeen := recordingServer(t) + var kosliSeen []http.Header + kosli := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + kosliSeen = append(kosliSeen, r.Header.Clone()) + http.Redirect(w, r, other.URL+"/bucket/evidence.tgz", http.StatusFound) + })) + t.Cleanup(kosli.Close) + client := newTestClient(t) + client.SetUsageHeaders(kosli.URL, testUsageHeaders) + + _, err := client.Do(&RequestParams{Method: http.MethodGet, URL: kosli.URL + "/api/v2/evidence/acme/download"}) + require.NoError(t, err) + + require.Len(t, kosliSeen, 1) + for name, want := range testUsageHeaders { + require.Equal(t, want, kosliSeen[0].Get(name), name) + } + require.Len(t, *otherSeen, 1) + for name := range testUsageHeaders { + require.Empty(t, (*otherSeen)[0].Values(name), name) + } +} + +func TestUsageHeadersAreSentThroughTheHTTPProxy(t *testing.T) { + proxy, seen := recordingServer(t) + client, err := NewKosliClient(proxy.URL, 0, false, logger.NewStandardLogger()) + require.NoError(t, err) + client.SetUsageHeaders("http://kosli.example", testUsageHeaders) + + _, err = client.Do(&RequestParams{Method: http.MethodGet, URL: "http://kosli.example/api/v2/flows/acme"}) + require.NoError(t, err) + + require.Len(t, *seen, 1) + for name, want := range testUsageHeaders { + require.Equal(t, want, (*seen)[0].Get(name), name) + } +}