feat: send first-party usage headers to the Kosli host

cmd/kosli/root.go

@@ -560,6 +560,8 @@ func newRootCmd(out, errOut io.Writer, args []string) (*cobra.Command, error) {
 				global.DryRun = true
 			}
 
+			kosliClient.SetUsageHeaders(global.Host, usageHeaders(cmd))
+
 			// A value that starts with a dash is the next flag, swallowed as this
 			// flag's value: "--description --sha256 ..." leaves --description
 			// holding "--sha256". Cobra's own error for what follows describes

cmd/kosli/usageHeaders.go

@@ -0,0 +1,89 @@
+package main
+
+import (
+	"os"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/google/uuid"
+	"github.com/kosli-dev/cli/internal/version"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+)
+
+const maxCliFlagsHeaderBytes = 1024
+
+// invocationID is shared by every request this process makes, so the server
+// can group the requests of one CLI run.
+var invocationID = uuid.NewString()
+
+// connectionFlags say where to send a request rather than what the command
+// does, and several of them carry credentials or customer hostnames.
+var connectionFlags = map[string]bool{
+	"api-token":   true,
+	"host":        true,
+	"org":         true,
+	"config-file": true,
+	"http-proxy":  true,
+}
+
+var ciVendors = map[string]string{
+	github:      "github",
+	gitlab:      "gitlab",
+	bitbucket:   "bitbucket",
+	azureDevops: "azure-devops",
+	circleci:    "circleci",
+	teamcity:    "teamcity",
+	codeBuild:   "codebuild",
+	jenkins:     "jenkins",
+}
+
+// usageHeaders describes the running command to the Kosli server. Values never
+// include flag values or positional arguments, since many flags carry secrets.
+func usageHeaders(cmd *cobra.Command) map[string]string {
+	return map[string]string{
+		"X-Kosli-Client":        "cli/" + version.GetVersion(),
+		"X-Kosli-Operation":     strings.TrimPrefix(strings.TrimPrefix(cmd.CommandPath(), cmd.Root().Name()), " "),
+		"X-Kosli-Invocation-Id": invocationID,
+		"X-Kosli-Cli-Flags":     flagNamesInEffect(cmd),
+		"X-Kosli-Ci":            ciVendor(),
+	}
+}
+
+// flagNamesInEffect lists the flags given on the command line, in the
+// environment or in the config file: bindFlags sets the latter two on the flag,
+// which marks them changed. The list is cut at a flag boundary to fit
+// maxCliFlagsHeaderBytes.
+func flagNamesInEffect(cmd *cobra.Command) string {
+	var names []string
+	cmd.Flags().Visit(func(f *pflag.Flag) {
+		if !connectionFlags[f.Name] {
+			names = append(names, f.Name)
+		}
+	})
+	sort.Strings(names)
+
+	value := ""
+	for _, name := range names {
+		next := name
+		if value != "" {
+			next = value + "," + name
+		}
+		if len(next) > maxCliFlagsHeaderBytes {
+			break
+		}
+		value = next
+	}
+	return value
+}
+
+func ciVendor() string {
+	if vendor, ok := ciVendors[WhichCI()]; ok {
+		return vendor
+	}
+	if isCI, _ := strconv.ParseBool(os.Getenv("CI")); isCI {
+		return "other-ci"
+	}
+	return "none"
+}

cmd/kosli/usageHeaders_test.go

@@ -0,0 +1,161 @@
+package main
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/google/uuid"
+	"github.com/kosli-dev/cli/internal/testHelpers"
+	"github.com/kosli-dev/cli/internal/version"
+	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/require"
+)
+
+var usageHeaderNames = []string{
+	"X-Kosli-Client",
+	"X-Kosli-Operation",
+	"X-Kosli-Invocation-Id",
+	"X-Kosli-Cli-Flags",
+	"X-Kosli-Ci",
+}
+
+var ciEnvVars = []string{
+	"BITBUCKET_BUILD_NUMBER", "GITHUB_RUN_NUMBER", "TEAMCITY_VERSION", "GITLAB_CI",
+	"TF_BUILD", "CIRCLECI", "CODEBUILD_CI", "JENKINS_URL", "CI",
+}
+
+// clearCIEnv unsets the variables WhichCI and ciVendor read, so a test run in
+// CI sees the environment it declares.
+func clearCIEnv(t *testing.T) {
+	t.Helper()
+	for _, name := range ciEnvVars {
+		t.Setenv(name, "")
+		require.NoError(t, os.Unsetenv(name))
+	}
+}
+
+func TestUsageHeadersDescribeTheCommandOnlyToTheKosliHost(t *testing.T) {
+	clearCIEnv(t)
+	const (
+		apiToken  = "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpZCI6ImNkNzg4OTg5In0.e8i_lA_QrEhFncb05Xw6E_tkCHU9QfcY4OLTVUCHffY"
+		jiraToken = "jira-secret-7f3a91"
+		orgName   = "usage-headers-org"
+		flowName  = "flow-from-environment"
+	)
+	t.Setenv("KOSLI_FLOW", flowName)
+
+	var jiraHeaders, kosliHeaders []http.Header
+	jiraServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		jiraHeaders = append(jiraHeaders, r.Header.Clone())
+		w.WriteHeader(http.StatusNotFound)
+		_, _ = w.Write([]byte(`{"errorMessages":["Issue does not exist"]}`))
+	}))
+	defer jiraServer.Close()
+	kosliServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		kosliHeaders = append(kosliHeaders, r.Header.Clone())
+		_, _ = w.Write([]byte(`{}`))
+	}))
+	defer kosliServer.Close()
+
+	tmpDir := t.TempDir()
+	_, workTree, fs, err := testHelpers.InitializeGitRepo(tmpDir)
+	require.NoError(t, err)
+	commitSha, err := testHelpers.CommitToRepo(workTree, fs, "EX-1 fix the thing")
+	require.NoError(t, err)
+
+	_, _, _, _, err = executeCommandC(fmt.Sprintf(`attest jira --name foo --trail trail
+		--org %s --host %s --api-token %s --repo-root %s --commit %s
+		--jira-base-url %s --jira-username user@example.com --jira-api-token %s`,
+		orgName, kosliServer.URL, apiToken, tmpDir, commitSha, jiraServer.URL, jiraToken))
+	require.NoError(t, err)
+
+	require.NotEmpty(t, kosliHeaders)
+	for _, headers := range kosliHeaders {
+		require.Equal(t, "cli/"+version.GetVersion(), headers.Get("X-Kosli-Client"))
+		require.Equal(t, "attest jira", headers.Get("X-Kosli-Operation"))
+		require.Equal(t, invocationID, headers.Get("X-Kosli-Invocation-Id"))
+		require.Equal(t, "commit,flow,jira-api-token,jira-base-url,jira-username,name,repo-root,trail",
+			headers.Get("X-Kosli-Cli-Flags"))
+		require.Equal(t, "none", headers.Get("X-Kosli-Ci"))
+		require.Equal(t, "Kosli/"+version.GetVersion(), headers.Get("User-Agent"))
+
+		for _, name := range usageHeaderNames {
+			for _, value := range []string{apiToken, jiraToken, orgName, flowName, kosliServer.URL, jiraServer.URL, commitSha, tmpDir} {
+				require.NotContains(t, headers.Get(name), value, name)
+			}
+		}
+	}
+	_, err = uuid.Parse(invocationID)
+	require.NoError(t, err)
+
+	require.NotEmpty(t, jiraHeaders, "attest jira should have called the Jira server")
+	for _, headers := range jiraHeaders {
+		for _, name := range usageHeaderNames {
+			require.Empty(t, headers.Values(name), name)
+		}
+	}
+}
+
+func TestCIVendor(t *testing.T) {
+	for _, tt := range []struct {
+		env  map[string]string
+		want string
+	}{
+		{map[string]string{"GITHUB_RUN_NUMBER": "12"}, "github"},
+		{map[string]string{"GITLAB_CI": "true"}, "gitlab"},
+		{map[string]string{"BITBUCKET_BUILD_NUMBER": "3"}, "bitbucket"},
+		{map[string]string{"TF_BUILD": "True"}, "azure-devops"},
+		{map[string]string{"CIRCLECI": "true"}, "circleci"},
+		{map[string]string{"TEAMCITY_VERSION": "2024.1"}, "teamcity"},
+		{map[string]string{"CODEBUILD_CI": "true"}, "codebuild"},
+		{map[string]string{"JENKINS_URL": "https://jenkins.example.com/"}, "jenkins"},
+		{map[string]string{"GITHUB_RUN_NUMBER": "12", "CI": "true"}, "github"},
+		{map[string]string{"CI": "true"}, "other-ci"},
+		{map[string]string{"CI": "false"}, "none"},
+		{map[string]string{}, "none"},
+	} {
+		t.Run(fmt.Sprintf("%v", tt.env), func(t *testing.T) {
+			clearCIEnv(t)
+			for k, v := range tt.env {
+				t.Setenv(k, v)
+			}
+			require.Equal(t, tt.want, ciVendor())
+		})
+	}
+}
+
+func TestFlagNamesInEffectExcludeConnectionFlagsAndValues(t *testing.T) {
+	cmd := &cobra.Command{Use: "probe"}
+	for _, name := range []string{"api-token", "host", "org", "config-file", "http-proxy", "github-token", "debug"} {
+		cmd.Flags().String(name, "", "")
+	}
+	cmd.Flags().String("unset", "", "")
+	for _, name := range []string{"api-token", "host", "org", "config-file", "http-proxy", "github-token", "debug"} {
+		require.NoError(t, cmd.Flags().Set(name, "value-of-"+name))
+	}
+
+	require.Equal(t, "debug,github-token", flagNamesInEffect(cmd))
+}
+
+func TestFlagNamesInEffectIsCutAtAFlagBoundary(t *testing.T) {
+	cmd := &cobra.Command{Use: "probe"}
+	names := map[string]bool{}
+	for i := range 60 {
+		name := fmt.Sprintf("a-rather-long-flag-name-number-%02d", i)
+		names[name] = true
+		cmd.Flags().String(name, "", "")
+		require.NoError(t, cmd.Flags().Set(name, "x"))
+	}
+
+	value := flagNamesInEffect(cmd)
+
+	require.LessOrEqual(t, len(value), maxCliFlagsHeaderBytes)
+	require.Greater(t, len(value), maxCliFlagsHeaderBytes-len("a-rather-long-flag-name-number-00,"))
+	for _, name := range strings.Split(value, ",") {
+		require.True(t, names[name], "%q is not a whole flag name", name)
+	}
+}

go.mod

@@ -26,6 +26,7 @@ require (
 	github.com/go-git/go-git/v5 v5.19.3
 	github.com/go-playground/validator/v10 v10.30.5
 	github.com/google/go-github/v42 v42.0.0
+	github.com/google/uuid v1.6.0
 	github.com/googleapis/gax-go/v2 v2.27.0
 	github.com/hashicorp/go-retryablehttp v0.7.8
 	github.com/joshdk/go-junit v1.0.0
@@ -168,7 +169,6 @@ require (
 	github.com/google/go-querystring v1.2.0 // indirect
 	github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 // indirect
 	github.com/google/s2a-go v0.1.11 // indirect
-	github.com/google/uuid v1.6.0 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.23 // indirect
 	github.com/gorilla/mux v1.8.1 // indirect
 	github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect

internal/requests/requests.go

@@ -66,6 +66,65 @@ type Client struct {
 	Debug         bool
 	Logger        *logger.Logger
 	HttpClient    *http.Client
+	usageHost     *url.URL
+	usageHeaders  map[string]string
+}
+
+// SetUsageHeaders adds headers to every request this client sends to
+// kosliHost. The same client also calls registries, VCS providers and storage
+// buckets, and those requests never carry them. An unparseable kosliHost
+// disables the headers rather than failing the command.
+func (c *Client) SetUsageHeaders(kosliHost string, headers map[string]string) {
+	host, err := url.Parse(kosliHost)
+	if err != nil || host.Hostname() == "" {
+		if c.Logger != nil {
+			c.Logger.Debug("usage headers disabled: [%s] is not a URL with a host", kosliHost)
+		}
+		c.usageHost, c.usageHeaders = nil, nil
+		return
+	}
+	c.usageHost, c.usageHeaders = host, headers
+}
+
+// usageHeaderTransport applies the usage headers on every hop, because
+// http.Client copies a request's headers onto a redirect to another host.
+type usageHeaderTransport struct {
+	next   http.RoundTripper
+	client *Client
+}
+
+func (t *usageHeaderTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	c := t.client
+	if len(c.usageHeaders) == 0 {
+		return t.next.RoundTrip(req)
+	}
+	req = req.Clone(req.Context())
+	toKosli := sameHost(req.URL, c.usageHost)
+	for k, v := range c.usageHeaders {
+		if toKosli {
+			req.Header.Set(k, v)
+		} else {
+			req.Header.Del(k)
+		}
+	}
+	return t.next.RoundTrip(req)
+}
+
+func sameHost(a, b *url.URL) bool {
+	return strings.EqualFold(a.Hostname(), b.Hostname()) && effectivePort(a) == effectivePort(b)
+}
+
+func effectivePort(u *url.URL) string {
+	if port := u.Port(); port != "" {
+		return port
+	}
+	switch strings.ToLower(u.Scheme) {
+	case "http":
+		return "80"
+	case "https":
+		return "443"
+	}
+	return ""
 }
 
 // CustomLogger wraps log.Logger and implements the Printf method
@@ -109,12 +168,16 @@ func NewKosliClient(httpProxyURL string, maxAPIRetries int, debug bool, logger *
 		client.Transport.(*retryablehttp.RoundTripper).Client.HTTPClient.Transport.(*http.Transport).Proxy = http.ProxyURL(proxyURL)
 	}
 
-	return &Client{
+	c := &Client{
 		MaxAPIRetries: maxAPIRetries,
 		Debug:         debug,
 		Logger:        logger,
 		HttpClient:    client,
-	}, nil
+	}
+	// Wrapped below the retry client, since its inner http.Client is the one
+	// that follows redirects.
+	retryClient.HTTPClient.Transport = &usageHeaderTransport{next: retryClient.HTTPClient.Transport, client: c}
+	return c, nil
 }
 
 type RequestParams struct {