From 24b023022a2ffa762b420a718f59ecaecba1156d Mon Sep 17 00:00:00 2001 From: Brian Flad Date: Thu, 19 Nov 2026 12:32:07 -0500 Subject: [PATCH 1/4] feat: add a partition setting to Speakeasy processes https://linear.app/speakeasy/issue/AGE-3680/feat-add-a-partition-setting-to-gram-processes The server binary, pystreams and the tunnel gateway read GRAM_PARTITION (us or eu, default us) and fail at startup on an unknown or empty value. The server binary takes it as a global flag, so every command tags its logs with gram.partition, and o11y.SetupOTelSDK adds it to the trace and metric resource and rejects an empty partition. pystreams adds it to its log attributes and OTel resource. The tunnel gateway adds it to its logs. US runs with the default, so nothing else changes. The setting that lists other partitions' site URLs moved to AGE-3684, its only consumer. --- .changeset/partition-setting.md | 6 ++ pystreams/src/pystreams/attr/__init__.py | 3 + pystreams/src/pystreams/cmd/flags_service.py | 32 +++++++ pystreams/src/pystreams/cmd/multi.py | 3 + pystreams/src/pystreams/deps/otel.py | 2 + pystreams/src/pystreams/partition.py | 19 +++++ pystreams/tests/test_multi.py | 2 + pystreams/tests/test_otel.py | 2 + pystreams/tests/test_partition.py | 60 +++++++++++++ server/cmd/gram/admin.go | 1 + server/cmd/gram/flags_partition.go | 57 +++++++++++++ server/cmd/gram/flags_partition_test.go | 89 ++++++++++++++++++++ server/cmd/gram/mcp.go | 1 + server/cmd/gram/netingress_attestor.go | 1 + server/cmd/gram/network_ingress_worker.go | 2 +- server/cmd/gram/private_ingress.go | 2 +- server/cmd/gram/root.go | 8 ++ server/cmd/gram/start.go | 1 + server/cmd/gram/streams.go | 1 + server/cmd/gram/tailnet_gateway.go | 2 +- server/cmd/gram/worker.go | 1 + server/internal/attr/conventions.go | 12 +++ server/internal/o11y/setup.go | 19 ++++- server/internal/o11y/setup_resource_test.go | 49 +++++++++++ server/internal/o11y/setup_test.go | 15 ++++ server/internal/partition/partition.go | 59 +++++++++++++ server/internal/partition/partition_test.go | 51 +++++++++++ tunnel/cmd/tunnel-gateway/main.go | 30 +++++++ tunnel/cmd/tunnel-gateway/main_test.go | 22 +++++ 29 files changed, 547 insertions(+), 5 deletions(-) create mode 100644 .changeset/partition-setting.md create mode 100644 pystreams/src/pystreams/partition.py create mode 100644 pystreams/tests/test_partition.py create mode 100644 server/cmd/gram/flags_partition.go create mode 100644 server/cmd/gram/flags_partition_test.go create mode 100644 server/internal/o11y/setup_resource_test.go create mode 100644 server/internal/partition/partition.go create mode 100644 server/internal/partition/partition_test.go diff --git a/.changeset/partition-setting.md b/.changeset/partition-setting.md new file mode 100644 index 00000000000..ba1359144f1 --- /dev/null +++ b/.changeset/partition-setting.md @@ -0,0 +1,6 @@ +--- +"server": minor +"tunnel": minor +--- + +The Speakeasy server binary (every command, including mcp, worker, streams and admin), pystreams and the tunnel gateway read a `GRAM_PARTITION` setting (`us` or `eu`, default `us`) naming the partition they run in. Each rejects an unknown or empty value at startup and tags its logs with `gram.partition`; the server and pystreams also put it on their traces and metrics. Nothing else changes yet: existing deployments run as `us`. diff --git a/pystreams/src/pystreams/attr/__init__.py b/pystreams/src/pystreams/attr/__init__.py index 401f38a25a2..3b09a38430f 100644 --- a/pystreams/src/pystreams/attr/__init__.py +++ b/pystreams/src/pystreams/attr/__init__.py @@ -12,6 +12,9 @@ DATADOG_GIT_REPOSITORY_URL: Final = "git.repository_url" COMPONENT: Final = "gram.component" +# The Speakeasy partition (an independent production deployment, such as us or +# eu) that emitted the telemetry. Mirrors attr.PartitionKey in the Go server. +PARTITION: Final = "gram.partition" ORGANIZATION_ID: Final = "gram.org.id" ORGANIZATION_SLUG: Final = "gram.org.slug" PROJECT_ID: Final = "gram.project.id" diff --git a/pystreams/src/pystreams/cmd/flags_service.py b/pystreams/src/pystreams/cmd/flags_service.py index c81ec05bdf2..76a390e8314 100644 --- a/pystreams/src/pystreams/cmd/flags_service.py +++ b/pystreams/src/pystreams/cmd/flags_service.py @@ -1,7 +1,28 @@ +import os from collections.abc import Sequence import click +from pystreams import partition + + +def _reject_empty_partition_env( + _ctx: click.Context, _param: click.Parameter, value: str +) -> str: + """Fail on a GRAM_PARTITION that is set but empty. + + Click treats an empty environment variable as unset and falls back to the + default, while the Go server rejects it. Rejecting it here too keeps a + misrendered deployment from running pystreams as ``us`` while the Go + processes refuse to start. + """ + if os.environ.get("GRAM_PARTITION") == "": + raise click.BadParameter( + "GRAM_PARTITION is set but empty; set it to one of: " + + ", ".join(partition.PARTITIONS) + ) + return value + def service_options() -> Sequence[click.Option]: return [ @@ -17,6 +38,17 @@ def service_options() -> Sequence[click.Option]: envvar="GRAM_ENVIRONMENT", help="Set the environment.", ), + click.Option( + ["--partition"], + type=click.Choice(partition.PARTITIONS, case_sensitive=True), + default=partition.DEFAULT, + show_default=True, + envvar="GRAM_PARTITION", + callback=_reject_empty_partition_env, + help="Partition this deployment runs in. A partition is an " + "independent production deployment with its own data stores and " + "hosts.", + ), click.Option( ["--git-sha"], type=str, diff --git a/pystreams/src/pystreams/cmd/multi.py b/pystreams/src/pystreams/cmd/multi.py index 862409ec6d0..a51367861f7 100644 --- a/pystreams/src/pystreams/cmd/multi.py +++ b/pystreams/src/pystreams/cmd/multi.py @@ -87,6 +87,7 @@ async def multi( # Service options service_version: str | None, environment: str | None, + partition: str, git_sha: str | None, log_level: str, pretty_log: bool, @@ -121,6 +122,7 @@ async def multi( attr.SERVICE_NAME: "gram-pystreams", attr.SERVICE_VERSION: service_version, attr.SERVICE_ENVIRONMENT: environment, + attr.PARTITION: partition, }, ) logger: structlog.stdlib.BoundLogger = structlog.get_logger() @@ -144,6 +146,7 @@ async def multi( service_version=service_version, git_sha=git_sha, environment=environment, + partition=partition, enable_tracing=enable_tracing, enable_metrics=enable_metrics, ) diff --git a/pystreams/src/pystreams/deps/otel.py b/pystreams/src/pystreams/deps/otel.py index a29df5e4c2c..b8d72c6dada 100644 --- a/pystreams/src/pystreams/deps/otel.py +++ b/pystreams/src/pystreams/deps/otel.py @@ -83,6 +83,7 @@ class OTelOptions: service_version: str | None git_sha: str | None environment: str | None + partition: str enable_tracing: bool enable_metrics: bool @@ -186,6 +187,7 @@ def _build_resource(options: OTelOptions) -> Resource: attr.SERVICE_NAME: options.service_name, attr.SERVICE_VERSION: options.service_version, attr.SERVICE_ENVIRONMENT: options.environment, + attr.PARTITION: options.partition, attr.DATADOG_GIT_COMMIT_SHA: options.git_sha, attr.DATADOG_GIT_REPOSITORY_URL: _REPOSITORY_URL, } diff --git a/pystreams/src/pystreams/partition.py b/pystreams/src/pystreams/partition.py new file mode 100644 index 00000000000..4376bde6d6b --- /dev/null +++ b/pystreams/src/pystreams/partition.py @@ -0,0 +1,19 @@ +"""The Speakeasy partition a process runs in. + +A partition is an independent production deployment with its own data stores +and hosts. The set is closed and mirrors ``server/internal/partition`` in the Go +server: adding a partition is a code change in both, so a typo in configuration +fails at startup instead of silently naming a new partition. +""" + +from typing import Final + +US: Final = "us" +EU: Final = "eu" + +# Every known partition, in the same order as ``partition.All`` in Go. +PARTITIONS: Final = (US, EU) + +# The partition a process runs in when none is configured. Every deployment +# that predates partitions is the US one. +DEFAULT: Final = US diff --git a/pystreams/tests/test_multi.py b/pystreams/tests/test_multi.py index 88b8fb67ae9..186b10bd6a0 100644 --- a/pystreams/tests/test_multi.py +++ b/pystreams/tests/test_multi.py @@ -56,6 +56,7 @@ def fake_activate(*_args, **_kwargs): await multi_mod.multi( service_version=None, environment="local", + partition="us", git_sha=None, log_level="INFO", pretty_log=False, @@ -155,6 +156,7 @@ async def test_enforcement_role_requires_redis_and_pepper( await multi_mod.multi( service_version=None, environment="local", + partition="us", git_sha=None, log_level="INFO", pretty_log=False, diff --git a/pystreams/tests/test_otel.py b/pystreams/tests/test_otel.py index 99259b3327f..f0e69fc6004 100644 --- a/pystreams/tests/test_otel.py +++ b/pystreams/tests/test_otel.py @@ -21,6 +21,7 @@ def _options(**overrides) -> otel.OTelOptions: "service_version": "1.2.3", "git_sha": "abc123", "environment": "test", + "partition": "eu", "enable_tracing": False, "enable_metrics": False, } @@ -50,6 +51,7 @@ def test_build_resource_includes_git_and_service_attrs(): assert resource.attributes[attr.SERVICE_NAME] == "gram-pystreams" assert resource.attributes[attr.SERVICE_VERSION] == "1.2.3" assert resource.attributes[attr.SERVICE_ENVIRONMENT] == "test" + assert resource.attributes[attr.PARTITION] == "eu" assert resource.attributes[attr.DATADOG_GIT_COMMIT_SHA] == "abc123" assert ( resource.attributes[attr.DATADOG_GIT_REPOSITORY_URL] diff --git a/pystreams/tests/test_partition.py b/pystreams/tests/test_partition.py new file mode 100644 index 00000000000..659ae2c675b --- /dev/null +++ b/pystreams/tests/test_partition.py @@ -0,0 +1,60 @@ +"""The partition option and its closed set of values. + +The set mirrors ``server/internal/partition`` in the Go server. The golden +values below must change together with ``partition.All`` there, so the two +services never disagree about which partitions exist. +""" + +import click +import pytest +from click.testing import CliRunner + +from pystreams import partition +from pystreams.cmd.flags_service import service_options + + +@pytest.fixture +def parse(): + """Parse argv/env through a command carrying only the service options.""" + + def _parse(args: list[str], env: dict[str, str] | None = None) -> dict: + captured: dict = {} + + def _record(**kwargs) -> None: + captured.update(kwargs) + + command = click.Command( + "probe", params=list(service_options()), callback=_record + ) + CliRunner().invoke( + command, args, env=env, standalone_mode=False, catch_exceptions=False + ) + return captured + + return _parse + + +def test_partitions_match_go_server(): + # Golden values from server/internal/partition (partition.All, Default). + assert partition.PARTITIONS == ("us", "eu") + assert partition.DEFAULT == "us" + + +def test_partition_defaults_to_us(parse): + assert parse([])["partition"] == "us" + + +def test_partition_reads_environment_variable(parse): + assert parse([], env={"GRAM_PARTITION": "eu"})["partition"] == "eu" + + +def test_empty_partition_environment_variable_is_rejected(parse): + # Matches the Go server, which rejects a set-but-empty GRAM_PARTITION. + with pytest.raises(click.BadParameter, match="set but empty"): + parse([], env={"GRAM_PARTITION": ""}) + + +@pytest.mark.parametrize("bad", ["apac", "US", ""]) +def test_unknown_partition_is_rejected(parse, bad): + with pytest.raises(click.UsageError): + parse(["--partition", bad]) diff --git a/server/cmd/gram/admin.go b/server/cmd/gram/admin.go index 7eae921b6b6..f4eb52230a7 100644 --- a/server/cmd/gram/admin.go +++ b/server/cmd/gram/admin.go @@ -375,6 +375,7 @@ func newAdminCommand() *cli.Command { ServiceName: serviceName, ServiceVersion: shortGitSHA(), GitSHA: GitSHA, + Partition: PullPartition(ctx), EnableTracing: c.Bool("with-otel-tracing"), EnableMetrics: c.Bool("with-otel-metrics"), }) diff --git a/server/cmd/gram/flags_partition.go b/server/cmd/gram/flags_partition.go new file mode 100644 index 00000000000..e6041fac38e --- /dev/null +++ b/server/cmd/gram/flags_partition.go @@ -0,0 +1,57 @@ +package gram + +import ( + "context" + "fmt" + "strings" + + "github.com/urfave/cli/v2" + + "github.com/speakeasy-api/gram/server/internal/partition" +) + +const partitionFlag = "partition" + +// partitionCLIFlag selects the partition every command of this binary runs +// in. It is a global flag, so a command added later gets the partition on its +// logs without registering the flag; commands that set up OpenTelemetry still +// pass PullPartition to o11y.SetupOTelSDK, which rejects an empty partition. +func partitionCLIFlag() cli.Flag { + return &cli.StringFlag{ + Name: partitionFlag, + Value: partition.Default.String(), + Usage: fmt.Sprintf("Partition this deployment runs in, one of: %s. A partition is an independent production deployment with its own data stores and hosts. As a global flag it goes before the command name.", strings.Join(partition.Names(), ", ")), + EnvVars: []string{"GRAM_PARTITION"}, + } +} + +// partitionFromCLI parses the global partition flag. A GRAM_PARTITION that is +// set but empty is rejected rather than treated as the default. +func partitionFromCLI(c *cli.Context) (partition.Partition, error) { + p, err := partition.Parse(c.String(partitionFlag)) + if err != nil { + return "", fmt.Errorf("invalid --%s: %w", partitionFlag, err) + } + + return p, nil +} + +type partitionKey struct{} + +// PushPartition records the partition the root command parsed, so commands +// read the validated value instead of parsing the flag again. +func PushPartition(ctx context.Context, p partition.Partition) context.Context { + return context.WithValue(ctx, partitionKey{}, p) +} + +// PullPartition returns the partition recorded by PushPartition. It panics +// when none was recorded, because the root command always records one before +// any command runs. +func PullPartition(ctx context.Context) partition.Partition { + p, ok := ctx.Value(partitionKey{}).(partition.Partition) + if !ok { + panic("PullPartition: partition not found in context") + } + + return p +} diff --git a/server/cmd/gram/flags_partition_test.go b/server/cmd/gram/flags_partition_test.go new file mode 100644 index 00000000000..174a2005985 --- /dev/null +++ b/server/cmd/gram/flags_partition_test.go @@ -0,0 +1,89 @@ +package gram + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" + "github.com/urfave/cli/v2" + + "github.com/speakeasy-api/gram/server/internal/partition" +) + +// runPartitionApp runs a minimal app with the global partition flag and +// returns the partition the root command recorded in the context. +func runPartitionApp(t *testing.T, args ...string) (partition.Partition, error) { + t.Helper() + + var got partition.Partition + app := &cli.App{ + Name: "gram", + Flags: []cli.Flag{partitionCLIFlag()}, + Before: func(c *cli.Context) error { + p, err := partitionFromCLI(c) + if err != nil { + return err + } + c.Context = PushPartition(c.Context, p) + return nil + }, + Commands: []*cli.Command{{ + Name: "probe", + Action: func(c *cli.Context) error { + got = PullPartition(c.Context) + return nil + }, + }}, + } + + if err := app.RunContext(t.Context(), append([]string{"gram"}, args...)); err != nil { + return "", fmt.Errorf("run app: %w", err) + } + + return got, nil +} + +func TestPartitionFlag_DefaultsToUS(t *testing.T) { //nolint:paralleltest // app.Run mutates urfave's shared help command + got, err := runPartitionApp(t, "probe") + require.NoError(t, err) + require.Equal(t, partition.US, got) +} + +func TestPartitionFlag_ReachesSubcommand(t *testing.T) { //nolint:paralleltest // app.Run mutates urfave's shared help command + got, err := runPartitionApp(t, "--partition", "eu", "probe") + require.NoError(t, err) + require.Equal(t, partition.EU, got) +} + +func TestPartitionFlag_ReadsEnvironmentVariable(t *testing.T) { + t.Setenv("GRAM_PARTITION", "eu") + + got, err := runPartitionApp(t, "probe") + require.NoError(t, err) + require.Equal(t, partition.EU, got) +} + +func TestPartitionFlag_RejectsEmptyEnvironmentVariable(t *testing.T) { + t.Setenv("GRAM_PARTITION", "") + + _, err := runPartitionApp(t, "probe") + require.ErrorContains(t, err, `invalid --partition: unknown partition ""`) +} + +func TestPartitionFlag_RejectsUnknownPartition(t *testing.T) { //nolint:paralleltest // app.Run mutates urfave's shared help command + _, err := runPartitionApp(t, "--partition", "apac", "probe") + require.ErrorContains(t, err, `invalid --partition: unknown partition "apac"`) +} + +func TestApp_RejectsUnknownPartitionBeforeAnyCommand(t *testing.T) { //nolint:paralleltest // app.Run mutates urfave's shared help command + err := newApp().RunContext(t.Context(), []string{"gram", "--partition", "apac", "version"}) + require.ErrorContains(t, err, `invalid --partition: unknown partition "apac"`) +} + +func TestPullPartition_PanicsWithoutRootCommand(t *testing.T) { + t.Parallel() + + require.PanicsWithValue(t, "PullPartition: partition not found in context", func() { + PullPartition(t.Context()) + }) +} diff --git a/server/cmd/gram/mcp.go b/server/cmd/gram/mcp.go index f0bf1ef4d63..94ae2615a38 100644 --- a/server/cmd/gram/mcp.go +++ b/server/cmd/gram/mcp.go @@ -149,6 +149,7 @@ func runMCPServer(c *cli.Context, shutdown *mcpServerShutdown) error { ServiceName: serviceName, ServiceVersion: shortGitSHA(), GitSHA: GitSHA, + Partition: PullPartition(ctx), EnableTracing: c.Bool("with-otel-tracing"), EnableMetrics: c.Bool("with-otel-metrics"), }) diff --git a/server/cmd/gram/netingress_attestor.go b/server/cmd/gram/netingress_attestor.go index be0279a4d15..a4f564d4b12 100644 --- a/server/cmd/gram/netingress_attestor.go +++ b/server/cmd/gram/netingress_attestor.go @@ -87,6 +87,7 @@ func newNetingressAttestorCommand() *cli.Command { ServiceName: serviceName, ServiceVersion: shortGitSHA(), GitSHA: GitSHA, + Partition: PullPartition(ctx), EnableTracing: c.Bool("with-otel-tracing"), EnableMetrics: c.Bool("with-otel-metrics"), }) diff --git a/server/cmd/gram/network_ingress_worker.go b/server/cmd/gram/network_ingress_worker.go index 191886327a2..2f5c8b1fa24 100644 --- a/server/cmd/gram/network_ingress_worker.go +++ b/server/cmd/gram/network_ingress_worker.go @@ -145,7 +145,7 @@ func newNetworkIngressWorkerCommand() *cli.Command { ctx, cancel := context.WithCancel(c.Context) defer cancel() shutdownOTel, err := o11y.SetupOTelSDK(ctx, logger, o11y.SetupOTelSDKOptions{ - ServiceName: serviceName, ServiceVersion: shortGitSHA(), GitSHA: GitSHA, + ServiceName: serviceName, ServiceVersion: shortGitSHA(), GitSHA: GitSHA, Partition: PullPartition(ctx), EnableTracing: c.Bool("with-otel-tracing"), EnableMetrics: c.Bool("with-otel-metrics"), }) if err != nil { diff --git a/server/cmd/gram/private_ingress.go b/server/cmd/gram/private_ingress.go index 21dcb32969f..48c50d2103f 100644 --- a/server/cmd/gram/private_ingress.go +++ b/server/cmd/gram/private_ingress.go @@ -83,7 +83,7 @@ func newNetworkIngressServerCommand() *cli.Command { o11y.PullAppInfo(ctx).Command = "network-ingress-server" slog.SetDefault(logger) shutdown, err := o11y.SetupOTelSDK(ctx, logger, o11y.SetupOTelSDKOptions{ - ServiceName: "gram-network-ingress-server", ServiceVersion: shortGitSHA(), GitSHA: GitSHA, + ServiceName: "gram-network-ingress-server", ServiceVersion: shortGitSHA(), GitSHA: GitSHA, Partition: PullPartition(ctx), EnableTracing: c.Bool("with-otel-tracing"), EnableMetrics: c.Bool("with-otel-metrics"), }) if err != nil { diff --git a/server/cmd/gram/root.go b/server/cmd/gram/root.go index e71763ccb27..0247fc34f90 100644 --- a/server/cmd/gram/root.go +++ b/server/cmd/gram/root.go @@ -37,6 +37,7 @@ func newApp() *cli.App { Usage: "Enable pretty logging", EnvVars: []string{"GRAM_LOG_PRETTY"}, }, + partitionCLIFlag(), }, Commands: []*cli.Command{ newStartCommand(), @@ -61,6 +62,12 @@ func newApp() *cli.App { newVersionCommand(), }, Before: func(c *cli.Context) error { + p, err := partitionFromCLI(c) + if err != nil { + return err + } + c.Context = PushPartition(c.Context, p) + c.Context = o11y.PushAppInfo(c.Context, &o11y.AppInfo{ Name: "gram", Command: "", @@ -74,6 +81,7 @@ func newApp() *cli.App { })).With( attr.SlogDataDogGitCommitSHA(GitSHA), attr.SlogDataDogGitRepoURL("github.com/speakeasy-api/gram"), + attr.SlogPartition(p), ) // Sets `GOMEMLIMIT` to 90% of cgroup's memory limit. diff --git a/server/cmd/gram/start.go b/server/cmd/gram/start.go index edf553adb57..ce244098786 100644 --- a/server/cmd/gram/start.go +++ b/server/cmd/gram/start.go @@ -778,6 +778,7 @@ func newStartCommand() *cli.Command { ServiceName: serviceName, ServiceVersion: shortGitSHA(), GitSHA: GitSHA, + Partition: PullPartition(ctx), EnableTracing: c.Bool("with-otel-tracing"), EnableMetrics: c.Bool("with-otel-metrics"), }) diff --git a/server/cmd/gram/streams.go b/server/cmd/gram/streams.go index 5cd598deb5e..0a080d0a67d 100644 --- a/server/cmd/gram/streams.go +++ b/server/cmd/gram/streams.go @@ -389,6 +389,7 @@ func newStreamsCommand() *cli.Command { ServiceName: serviceName, ServiceVersion: shortGitSHA(), GitSHA: GitSHA, + Partition: PullPartition(ctx), EnableTracing: c.Bool("with-otel-tracing"), EnableMetrics: c.Bool("with-otel-metrics"), }) diff --git a/server/cmd/gram/tailnet_gateway.go b/server/cmd/gram/tailnet_gateway.go index f4b13536ea3..9dd5b9fe9ef 100644 --- a/server/cmd/gram/tailnet_gateway.go +++ b/server/cmd/gram/tailnet_gateway.go @@ -178,7 +178,7 @@ func newTailnetGatewayCommand() *cli.Command { defer cancel() shutdownOTel, err := o11y.SetupOTelSDK(ctx, logger, o11y.SetupOTelSDKOptions{ - ServiceName: serviceName, ServiceVersion: shortGitSHA(), GitSHA: GitSHA, + ServiceName: serviceName, ServiceVersion: shortGitSHA(), GitSHA: GitSHA, Partition: PullPartition(ctx), EnableTracing: c.Bool("with-otel-tracing"), EnableMetrics: c.Bool("with-otel-metrics"), }) if err != nil { diff --git a/server/cmd/gram/worker.go b/server/cmd/gram/worker.go index a437bff5269..8c3fbaa5ca5 100644 --- a/server/cmd/gram/worker.go +++ b/server/cmd/gram/worker.go @@ -415,6 +415,7 @@ func newWorkerCommand() *cli.Command { ServiceName: serviceName, ServiceVersion: shortGitSHA(), GitSHA: GitSHA, + Partition: PullPartition(ctx), EnableTracing: c.Bool("with-otel-tracing"), EnableMetrics: c.Bool("with-otel-metrics"), }) diff --git a/server/internal/attr/conventions.go b/server/internal/attr/conventions.go index aad8a99f123..a3bdcb147a4 100644 --- a/server/internal/attr/conventions.go +++ b/server/internal/attr/conventions.go @@ -702,6 +702,10 @@ const ( // told apart without revealing the key. EncryptionKeyFingerprintKey = attribute.Key("gram.encryption.key_fingerprint") + // PartitionKey is the Speakeasy partition (an independent production + // deployment, such as us or eu) that emitted the telemetry. + PartitionKey = attribute.Key("gram.partition") + // OrgEncryptionUseCaseKey is the organization encryption use case a value // or key belongs to, such as oauth-session-tokens. OrgEncryptionUseCaseKey = attribute.Key("gram.orgencryption.use_case") @@ -2759,6 +2763,14 @@ func SlogEncryptionKeyFingerprint(v string) slog.Attr { return slog.String(string(EncryptionKeyFingerprintKey), v) } +func Partition[V ~string](v V) attribute.KeyValue { + return PartitionKey.String(string(v)) +} + +func SlogPartition[V ~string](v V) slog.Attr { + return slog.String(string(PartitionKey), string(v)) +} + func OrgEncryptionUseCase[V ~string](v V) attribute.KeyValue { return OrgEncryptionUseCaseKey.String(string(v)) } diff --git a/server/internal/o11y/setup.go b/server/internal/o11y/setup.go index 8e897737868..b28949ddc8e 100644 --- a/server/internal/o11y/setup.go +++ b/server/internal/o11y/setup.go @@ -24,19 +24,31 @@ import ( "goa.design/clue/clue" "github.com/speakeasy-api/gram/server/internal/attr" + "github.com/speakeasy-api/gram/server/internal/partition" ) type SetupOTelSDKOptions struct { ServiceName string ServiceVersion string GitSHA string - EnableTracing bool - EnableMetrics bool + + // Partition is the Speakeasy partition the process runs in. It is added + // to the resource so every span and metric carries it. SetupOTelSDK + // rejects a partition that partition.Parse does not accept, including + // the empty zero value. + Partition partition.Partition + + EnableTracing bool + EnableMetrics bool } // SetupOTelSDK bootstraps the OpenTelemetry pipeline. // If it does not return an error, make sure to call shutdown for proper cleanup. func SetupOTelSDK(ctx context.Context, logger *slog.Logger, options SetupOTelSDKOptions) (shutdown func(context.Context) error, err error) { + if _, err := partition.Parse(options.Partition.String()); err != nil { + return nil, fmt.Errorf("otel resource partition: %w", err) + } + logger = logger.With(attr.SlogComponent("otel")) var shutdownFuncs []func(context.Context) error @@ -114,6 +126,7 @@ func SetupOTelSDK(ctx context.Context, logger *slog.Logger, options SetupOTelSDK options.ServiceName, options.ServiceVersion, options.GitSHA, + options.Partition, metricExporter, spanExporter, prop, @@ -145,6 +158,7 @@ func newClueConfig( svcName string, svcVersion string, gitSHA string, + part partition.Partition, metricExporter sdkmetric.Exporter, spanExporter sdktrace.SpanExporter, propagators propagation.TextMapPropagator, @@ -159,6 +173,7 @@ func newClueConfig( semconv.ServiceVersionKey.String(svcVersion), attr.DataDogGitCommitSHA(gitSHA), attr.DataDogGitRepoURL("github.com/speakeasy-api/gram"), + attr.Partition(part), )) if err != nil { return nil, fmt.Errorf("create resource: %w", err) diff --git a/server/internal/o11y/setup_resource_test.go b/server/internal/o11y/setup_resource_test.go new file mode 100644 index 00000000000..f3d5ae51573 --- /dev/null +++ b/server/internal/o11y/setup_resource_test.go @@ -0,0 +1,49 @@ +package o11y + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + + "github.com/speakeasy-api/gram/server/internal/attr" + "github.com/speakeasy-api/gram/server/internal/partition" +) + +func TestNewClueConfig_ResourceCarriesPartition(t *testing.T) { + t.Parallel() + + exporter := tracetest.NewInMemoryExporter() + cfg, err := newClueConfig( + t.Context(), + "gram-test", + "test", + "deadbeef", + partition.EU, + nil, + exporter, + propagation.TraceContext{}, + sdktrace.AlwaysSample(), + otel.ErrorHandlerFunc(func(err error) { require.NoError(t, err) }), + ) + require.NoError(t, err) + + provider, ok := cfg.TracerProvider.(*sdktrace.TracerProvider) + require.True(t, ok) + t.Cleanup(func() { require.NoError(t, provider.Shutdown(context.Background())) }) + + _, span := provider.Tracer("test").Start(t.Context(), "probe") + span.End() + require.NoError(t, provider.ForceFlush(t.Context())) + + spans := exporter.GetSpans() + require.Len(t, spans, 1) + + value, found := spans[0].Resource.Set().Value(attr.PartitionKey) + require.True(t, found) + require.Equal(t, "eu", value.AsString()) +} diff --git a/server/internal/o11y/setup_test.go b/server/internal/o11y/setup_test.go index 6386e8eb30d..95b1b6ad0c0 100644 --- a/server/internal/o11y/setup_test.go +++ b/server/internal/o11y/setup_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/speakeasy-api/gram/server/internal/o11y" + "github.com/speakeasy-api/gram/server/internal/partition" "github.com/speakeasy-api/gram/server/internal/testenv" "github.com/stretchr/testify/require" ) @@ -15,6 +16,7 @@ func TestSetupOTelSDK(t *testing.T) { shutdown, err := o11y.SetupOTelSDK(t.Context(), logger, o11y.SetupOTelSDKOptions{ ServiceName: "gram-test", ServiceVersion: "0.1.0", + Partition: partition.US, EnableTracing: true, EnableMetrics: true, }) @@ -24,3 +26,16 @@ func TestSetupOTelSDK(t *testing.T) { err = shutdown(t.Context()) require.NoError(t, err) } + +func TestSetupOTelSDK_RejectsMissingPartition(t *testing.T) { + t.Parallel() + + shutdown, err := o11y.SetupOTelSDK(t.Context(), testenv.NewLogger(t), o11y.SetupOTelSDKOptions{ + ServiceName: "gram-test", + ServiceVersion: "0.1.0", + EnableTracing: false, + EnableMetrics: false, + }) + require.ErrorContains(t, err, `otel resource partition: unknown partition ""`) + require.Nil(t, shutdown) +} diff --git a/server/internal/partition/partition.go b/server/internal/partition/partition.go new file mode 100644 index 00000000000..7b11cf894a5 --- /dev/null +++ b/server/internal/partition/partition.go @@ -0,0 +1,59 @@ +// Package partition identifies the Speakeasy partition a process runs in. A +// partition is an independent production deployment with its own data stores +// and hosts; nothing at runtime crosses between partitions. Organizations are +// assigned to exactly one partition. +package partition + +import ( + "fmt" + "slices" + "strings" +) + +// Partition names one partition. The set is closed: adding a partition is a +// code change, so a typo in configuration fails at startup instead of +// silently creating a new partition. +type Partition string + +const ( + // US is the original production deployment. Organizations without a + // recorded partition belong to it. + US Partition = "us" + + // EU is the deployment that keeps customer data in EU locations. + EU Partition = "eu" +) + +// Default is the partition a process runs in when none is configured. It is +// US because every deployment that predates partitions is the US one. +const Default = US + +// All lists every known partition in a stable order. +func All() []Partition { + return []Partition{US, EU} +} + +// Parse returns the partition named by raw. Names are case sensitive and must +// be one of All. +func Parse(raw string) (Partition, error) { + p := Partition(raw) + if !slices.Contains(All(), p) { + return "", fmt.Errorf("unknown partition %q: must be one of %s", raw, strings.Join(Names(), ", ")) + } + + return p, nil +} + +func (p Partition) String() string { + return string(p) +} + +// Names lists the name of every known partition, in the order of All. +func Names() []string { + names := make([]string, 0, len(All())) + for _, p := range All() { + names = append(names, p.String()) + } + + return names +} diff --git a/server/internal/partition/partition_test.go b/server/internal/partition/partition_test.go new file mode 100644 index 00000000000..70a6391bd6c --- /dev/null +++ b/server/internal/partition/partition_test.go @@ -0,0 +1,51 @@ +package partition_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/speakeasy-api/gram/server/internal/partition" +) + +func TestParse_KnownPartitions(t *testing.T) { + t.Parallel() + + for _, want := range partition.All() { + t.Run(want.String(), func(t *testing.T) { + t.Parallel() + + got, err := partition.Parse(want.String()) + require.NoError(t, err) + require.Equal(t, want, got) + }) + } +} + +func TestParse_RejectsUnknownNames(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + raw string + }{ + {name: "empty", raw: ""}, + {name: "unknown", raw: "apac"}, + {name: "uppercase", raw: "US"}, + {name: "padded", raw: " us"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + _, err := partition.Parse(tc.raw) + require.ErrorContains(t, err, "must be one of us, eu") + }) + } +} + +func TestDefault_IsUS(t *testing.T) { + t.Parallel() + + require.Equal(t, partition.US, partition.Default) +} diff --git a/tunnel/cmd/tunnel-gateway/main.go b/tunnel/cmd/tunnel-gateway/main.go index 49cccba2eb5..2777d9c2e46 100644 --- a/tunnel/cmd/tunnel-gateway/main.go +++ b/tunnel/cmd/tunnel-gateway/main.go @@ -4,10 +4,12 @@ package main import ( "context" "errors" + "fmt" "log/slog" "net/http" "os" "os/signal" + "slices" "strconv" "strings" "sync" @@ -22,9 +24,25 @@ import ( "github.com/speakeasy-api/gram/tunnel/route" ) +// partitionAttrKey matches the gram.partition attribute the Speakeasy server +// puts on its logs, so the gateway's logs filter the same way. +const partitionAttrKey = "gram.partition" + +// knownPartitions mirrors the closed set in server/internal/partition, which +// this module cannot import. Its first entry is the default. +var knownPartitions = []string{"us", "eu"} + func main() { logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo})) + rawPartition, partitionSet := os.LookupEnv("GRAM_PARTITION") + partition, err := parsePartition(rawPartition, partitionSet) + if err != nil { + logger.ErrorContext(context.Background(), "invalid GRAM_PARTITION", slog.Any("error", err)) + os.Exit(2) + } + logger = logger.With(slog.String(partitionAttrKey, partition)) + publicListenAddr := envOr("TUNNEL_GATEWAY_PUBLIC_ADDR", envOr("TUNNEL_GATEWAY_ADDR", ":8090")) forwardListenAddr := envOr("TUNNEL_GATEWAY_FORWARD_ADDR", ":8091") // AdvertiseAddr is the internal gram-server -> gateway address, not the public agent URL. @@ -198,3 +216,15 @@ func parseMaxSessions(raw string) (int, error) { } return parsed, nil } + +// parsePartition returns the partition named by GRAM_PARTITION. Unset selects +// the default (us); set but empty or unknown is an error, as in the server. +func parsePartition(raw string, set bool) (string, error) { + if !set { + return knownPartitions[0], nil + } + if !slices.Contains(knownPartitions, raw) { + return "", fmt.Errorf("unknown partition %q: must be one of %s", raw, strings.Join(knownPartitions, ", ")) + } + return raw, nil +} diff --git a/tunnel/cmd/tunnel-gateway/main_test.go b/tunnel/cmd/tunnel-gateway/main_test.go index a1806351e4d..5f42b138e7d 100644 --- a/tunnel/cmd/tunnel-gateway/main_test.go +++ b/tunnel/cmd/tunnel-gateway/main_test.go @@ -29,3 +29,25 @@ func TestParseMaxSessionsRejectsInvalidValue(t *testing.T) { _, err = parseMaxSessions("-1") require.Error(t, err) } + +func TestParsePartitionDefaultsToUSWhenUnset(t *testing.T) { + got, err := parsePartition("", false) + require.NoError(t, err) + require.Equal(t, "us", got) +} + +func TestParsePartitionAcceptsKnownPartitions(t *testing.T) { + for _, want := range []string{"us", "eu"} { + got, err := parsePartition(want, true) + require.NoError(t, err) + require.Equal(t, want, got) + } +} + +func TestParsePartitionRejectsEmptyOrUnknown(t *testing.T) { + _, err := parsePartition("", true) + require.ErrorContains(t, err, `unknown partition ""`) + + _, err = parsePartition("apac", true) + require.ErrorContains(t, err, `unknown partition "apac"`) +} From bb8d95f22939d1e5bcb47e14df1c9f563bd60301 Mon Sep 17 00:00:00 2001 From: Brian Flad Date: Thu, 19 Nov 2026 14:10:32 -0500 Subject: [PATCH 2/4] fix: address review feedback on the partition setting https://linear.app/speakeasy/issue/AGE-3680/feat-add-a-partition-setting-to-gram-processes An explicit --partition now wins over an empty GRAM_PARTITION in pystreams, as it does in Go. The pystreams tests clear any ambient GRAM_PARTITION, the tunnel gateway names its default partition explicitly, and its rejection tests are table driven. --- pystreams/src/pystreams/cmd/flags_service.py | 12 +++++++++--- pystreams/tests/test_partition.py | 15 +++++++++++++-- tunnel/cmd/tunnel-gateway/main.go | 11 ++++++++--- tunnel/cmd/tunnel-gateway/main_test.go | 19 ++++++++++++++----- 4 files changed, 44 insertions(+), 13 deletions(-) diff --git a/pystreams/src/pystreams/cmd/flags_service.py b/pystreams/src/pystreams/cmd/flags_service.py index 76a390e8314..60b86b7d8c7 100644 --- a/pystreams/src/pystreams/cmd/flags_service.py +++ b/pystreams/src/pystreams/cmd/flags_service.py @@ -7,16 +7,22 @@ def _reject_empty_partition_env( - _ctx: click.Context, _param: click.Parameter, value: str + ctx: click.Context, param: click.Parameter, value: str ) -> str: """Fail on a GRAM_PARTITION that is set but empty. Click treats an empty environment variable as unset and falls back to the default, while the Go server rejects it. Rejecting it here too keeps a misrendered deployment from running pystreams as ``us`` while the Go - processes refuse to start. + processes refuse to start. An explicit ``--partition`` wins over the + environment, as it does in Go, so the check only applies when the value + came from the default. """ - if os.environ.get("GRAM_PARTITION") == "": + source = ctx.get_parameter_source(param.name) if param.name else None + if ( + source is click.core.ParameterSource.DEFAULT + and os.environ.get("GRAM_PARTITION") == "" + ): raise click.BadParameter( "GRAM_PARTITION is set but empty; set it to one of: " + ", ".join(partition.PARTITIONS) diff --git a/pystreams/tests/test_partition.py b/pystreams/tests/test_partition.py index 659ae2c675b..e9e826e4857 100644 --- a/pystreams/tests/test_partition.py +++ b/pystreams/tests/test_partition.py @@ -14,8 +14,13 @@ @pytest.fixture -def parse(): - """Parse argv/env through a command carrying only the service options.""" +def parse(monkeypatch: pytest.MonkeyPatch): + """Parse argv/env through a command carrying only the service options. + + Clears any ambient GRAM_PARTITION so a value exported in the shell cannot + change the outcome; tests pass the environment they need explicitly. + """ + monkeypatch.delenv("GRAM_PARTITION", raising=False) def _parse(args: list[str], env: dict[str, str] | None = None) -> dict: captured: dict = {} @@ -54,6 +59,12 @@ def test_empty_partition_environment_variable_is_rejected(parse): parse([], env={"GRAM_PARTITION": ""}) +def test_explicit_partition_wins_over_empty_environment_variable(parse): + # Matches the Go server, where a --partition flag takes precedence over + # the environment variable. + assert parse(["--partition", "eu"], env={"GRAM_PARTITION": ""})["partition"] == "eu" + + @pytest.mark.parametrize("bad", ["apac", "US", ""]) def test_unknown_partition_is_rejected(parse, bad): with pytest.raises(click.UsageError): diff --git a/tunnel/cmd/tunnel-gateway/main.go b/tunnel/cmd/tunnel-gateway/main.go index 2777d9c2e46..d2a024f60e7 100644 --- a/tunnel/cmd/tunnel-gateway/main.go +++ b/tunnel/cmd/tunnel-gateway/main.go @@ -28,10 +28,15 @@ import ( // puts on its logs, so the gateway's logs filter the same way. const partitionAttrKey = "gram.partition" -// knownPartitions mirrors the closed set in server/internal/partition, which -// this module cannot import. Its first entry is the default. +// knownPartitions mirrors the closed set in server/internal/partition. Go's +// internal visibility rule keeps this package from importing it, so the list +// is repeated here and the tests pin it. var knownPartitions = []string{"us", "eu"} +// defaultPartition is the partition the gateway runs in when GRAM_PARTITION is +// unset, matching partition.Default in the server. +const defaultPartition = "us" + func main() { logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo})) @@ -221,7 +226,7 @@ func parseMaxSessions(raw string) (int, error) { // the default (us); set but empty or unknown is an error, as in the server. func parsePartition(raw string, set bool) (string, error) { if !set { - return knownPartitions[0], nil + return defaultPartition, nil } if !slices.Contains(knownPartitions, raw) { return "", fmt.Errorf("unknown partition %q: must be one of %s", raw, strings.Join(knownPartitions, ", ")) diff --git a/tunnel/cmd/tunnel-gateway/main_test.go b/tunnel/cmd/tunnel-gateway/main_test.go index 5f42b138e7d..3ae4c18cf20 100644 --- a/tunnel/cmd/tunnel-gateway/main_test.go +++ b/tunnel/cmd/tunnel-gateway/main_test.go @@ -45,9 +45,18 @@ func TestParsePartitionAcceptsKnownPartitions(t *testing.T) { } func TestParsePartitionRejectsEmptyOrUnknown(t *testing.T) { - _, err := parsePartition("", true) - require.ErrorContains(t, err, `unknown partition ""`) - - _, err = parsePartition("apac", true) - require.ErrorContains(t, err, `unknown partition "apac"`) + tests := []struct { + name string + raw string + wantErr string + }{ + {name: "empty", raw: "", wantErr: `unknown partition ""`}, + {name: "unknown", raw: "apac", wantErr: `unknown partition "apac"`}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, err := parsePartition(tc.raw, true) + require.ErrorContains(t, err, tc.wantErr) + }) + } } From e9517c1a397331b050ebdcad15e097b14254b068 Mon Sep 17 00:00:00 2001 From: Brian Flad Date: Thu, 19 Nov 2026 15:04:14 -0500 Subject: [PATCH 3/4] feat: require the partition setting with no default https://linear.app/speakeasy/issue/AGE-3680/feat-add-a-partition-setting-to-gram-processes The server binary, pystreams and the tunnel gateway now refuse to start when GRAM_PARTITION is missing, instead of falling back to us, so a deployment that forgets it cannot silently run as the wrong partition. gram version and help output still work without it. Local development sets GRAM_PARTITION=us in mise.toml, and every deployed workload already receives it from the gram-infra Helm chart. --- .changeset/partition-setting.md | 2 +- mise.toml | 3 + pystreams/src/pystreams/cmd/flags_service.py | 34 ++------ pystreams/src/pystreams/partition.py | 4 - pystreams/tests/test_partition.py | 10 +-- server/cmd/gram/flags_partition.go | 47 ++++++++-- server/cmd/gram/flags_partition_test.go | 91 ++++++++++++++++++-- server/cmd/gram/root.go | 8 +- server/internal/partition/partition.go | 4 - server/internal/partition/partition_test.go | 6 -- tunnel/cmd/tunnel-gateway/main.go | 18 ++-- tunnel/cmd/tunnel-gateway/main_test.go | 14 +-- 12 files changed, 154 insertions(+), 87 deletions(-) diff --git a/.changeset/partition-setting.md b/.changeset/partition-setting.md index ba1359144f1..3c2e1ebdf98 100644 --- a/.changeset/partition-setting.md +++ b/.changeset/partition-setting.md @@ -3,4 +3,4 @@ "tunnel": minor --- -The Speakeasy server binary (every command, including mcp, worker, streams and admin), pystreams and the tunnel gateway read a `GRAM_PARTITION` setting (`us` or `eu`, default `us`) naming the partition they run in. Each rejects an unknown or empty value at startup and tags its logs with `gram.partition`; the server and pystreams also put it on their traces and metrics. Nothing else changes yet: existing deployments run as `us`. +The Speakeasy server binary (every command, including mcp, worker, streams and admin), pystreams and the tunnel gateway require a `GRAM_PARTITION` setting (`us` or `eu`) naming the partition they run in. There is no default: each refuses to start when it is missing, empty or unknown, so a deployment cannot silently run as the wrong partition. Each tags its logs with `gram.partition`; the server and pystreams also put it on their traces and metrics. `gram version` and help output work without it. Local development sets `GRAM_PARTITION=us` in `mise.toml`. diff --git a/mise.toml b/mise.toml index 37bcff22027..57a7d8a6397 100644 --- a/mise.toml +++ b/mise.toml @@ -136,6 +136,9 @@ GRAM_RISK_LLM_API_KEY = "" ############################################### ## The following variable can be one of: local, test, prod GRAM_ENVIRONMENT = "local" +## Partition the processes run in: us or eu. Required, with no default in the +## server, pystreams or tunnel gateway, so local development sets it here. +GRAM_PARTITION = "us" GRAM_HOST = "localhost" # Only needed for local development. In production, you should set GRAM_SERVER_URL and GRAM_SITE_URL directly. ## Public listener hosts accept IPv4 or bracketed IPv6 literals (e.g. [::1]). ## Empty uses wildcard public listeners; health/control listeners use loopback. diff --git a/pystreams/src/pystreams/cmd/flags_service.py b/pystreams/src/pystreams/cmd/flags_service.py index 60b86b7d8c7..3b8c63f818e 100644 --- a/pystreams/src/pystreams/cmd/flags_service.py +++ b/pystreams/src/pystreams/cmd/flags_service.py @@ -1,4 +1,3 @@ -import os from collections.abc import Sequence import click @@ -6,30 +5,6 @@ from pystreams import partition -def _reject_empty_partition_env( - ctx: click.Context, param: click.Parameter, value: str -) -> str: - """Fail on a GRAM_PARTITION that is set but empty. - - Click treats an empty environment variable as unset and falls back to the - default, while the Go server rejects it. Rejecting it here too keeps a - misrendered deployment from running pystreams as ``us`` while the Go - processes refuse to start. An explicit ``--partition`` wins over the - environment, as it does in Go, so the check only applies when the value - came from the default. - """ - source = ctx.get_parameter_source(param.name) if param.name else None - if ( - source is click.core.ParameterSource.DEFAULT - and os.environ.get("GRAM_PARTITION") == "" - ): - raise click.BadParameter( - "GRAM_PARTITION is set but empty; set it to one of: " - + ", ".join(partition.PARTITIONS) - ) - return value - - def service_options() -> Sequence[click.Option]: return [ click.Option( @@ -47,10 +22,13 @@ def service_options() -> Sequence[click.Option]: click.Option( ["--partition"], type=click.Choice(partition.PARTITIONS, case_sensitive=True), - default=partition.DEFAULT, - show_default=True, + # No default: a deployment that forgets the partition fails at + # startup instead of running as the wrong one. Click treats a set + # but empty GRAM_PARTITION as unset, so it fails the same way, + # matching the Go server. + required=True, envvar="GRAM_PARTITION", - callback=_reject_empty_partition_env, + show_envvar=True, help="Partition this deployment runs in. A partition is an " "independent production deployment with its own data stores and " "hosts.", diff --git a/pystreams/src/pystreams/partition.py b/pystreams/src/pystreams/partition.py index 4376bde6d6b..dde334e739b 100644 --- a/pystreams/src/pystreams/partition.py +++ b/pystreams/src/pystreams/partition.py @@ -13,7 +13,3 @@ # Every known partition, in the same order as ``partition.All`` in Go. PARTITIONS: Final = (US, EU) - -# The partition a process runs in when none is configured. Every deployment -# that predates partitions is the US one. -DEFAULT: Final = US diff --git a/pystreams/tests/test_partition.py b/pystreams/tests/test_partition.py index e9e826e4857..21476bf8d98 100644 --- a/pystreams/tests/test_partition.py +++ b/pystreams/tests/test_partition.py @@ -40,13 +40,13 @@ def _record(**kwargs) -> None: def test_partitions_match_go_server(): - # Golden values from server/internal/partition (partition.All, Default). + # Golden values from server/internal/partition (partition.All). assert partition.PARTITIONS == ("us", "eu") - assert partition.DEFAULT == "us" -def test_partition_defaults_to_us(parse): - assert parse([])["partition"] == "us" +def test_partition_is_required(parse): + with pytest.raises(click.MissingParameter, match="partition"): + parse([]) def test_partition_reads_environment_variable(parse): @@ -55,7 +55,7 @@ def test_partition_reads_environment_variable(parse): def test_empty_partition_environment_variable_is_rejected(parse): # Matches the Go server, which rejects a set-but-empty GRAM_PARTITION. - with pytest.raises(click.BadParameter, match="set but empty"): + with pytest.raises(click.MissingParameter, match="partition"): parse([], env={"GRAM_PARTITION": ""}) diff --git a/server/cmd/gram/flags_partition.go b/server/cmd/gram/flags_partition.go index e6041fac38e..d027e0b425f 100644 --- a/server/cmd/gram/flags_partition.go +++ b/server/cmd/gram/flags_partition.go @@ -3,6 +3,7 @@ package gram import ( "context" "fmt" + "slices" "strings" "github.com/urfave/cli/v2" @@ -13,21 +14,31 @@ import ( const partitionFlag = "partition" // partitionCLIFlag selects the partition every command of this binary runs -// in. It is a global flag, so a command added later gets the partition on its -// logs without registering the flag; commands that set up OpenTelemetry still -// pass PullPartition to o11y.SetupOTelSDK, which rejects an empty partition. +// in. It has no default: a deployment that forgets it fails at startup rather +// than running as the wrong partition. It is a global flag, so a command added +// later requires it and gets the partition on its logs without registering the +// flag; commands that set up OpenTelemetry still pass PullPartition to +// o11y.SetupOTelSDK, which rejects an empty partition. func partitionCLIFlag() cli.Flag { return &cli.StringFlag{ Name: partitionFlag, - Value: partition.Default.String(), - Usage: fmt.Sprintf("Partition this deployment runs in, one of: %s. A partition is an independent production deployment with its own data stores and hosts. As a global flag it goes before the command name.", strings.Join(partition.Names(), ", ")), + Usage: fmt.Sprintf("Partition this deployment runs in, one of: %s. Required. A partition is an independent production deployment with its own data stores and hosts. As a global flag it goes before the command name.", strings.Join(partition.Names(), ", ")), EnvVars: []string{"GRAM_PARTITION"}, } } -// partitionFromCLI parses the global partition flag. A GRAM_PARTITION that is -// set but empty is rejected rather than treated as the default. +// partitionFromCLI parses the global partition flag. It returns an empty +// partition and no error only for invocations that print help or the version, +// which must work without deployment configuration. A GRAM_PARTITION that is +// set but empty is rejected. func partitionFromCLI(c *cli.Context) (partition.Partition, error) { + if !c.IsSet(partitionFlag) { + if printsHelpOrVersion(c.Args().Slice()) { + return "", nil + } + return "", fmt.Errorf("--%s (GRAM_PARTITION) is required: set it to one of %s", partitionFlag, strings.Join(partition.Names(), ", ")) + } + p, err := partition.Parse(c.String(partitionFlag)) if err != nil { return "", fmt.Errorf("invalid --%s: %w", partitionFlag, err) @@ -36,6 +47,24 @@ func partitionFromCLI(c *cli.Context) (partition.Partition, error) { return p, nil } +// printsHelpOrVersion reports whether the arguments after the global flags +// only print help or the version: no command, the help or version command, or +// a help flag anywhere. +func printsHelpOrVersion(args []string) bool { + if len(args) == 0 { + return true + } + + switch args[0] { + case "help", "h", "version": + return true + } + + return slices.ContainsFunc(args, func(arg string) bool { + return arg == "--help" || arg == "-h" + }) +} + type partitionKey struct{} // PushPartition records the partition the root command parsed, so commands @@ -45,8 +74,8 @@ func PushPartition(ctx context.Context, p partition.Partition) context.Context { } // PullPartition returns the partition recorded by PushPartition. It panics -// when none was recorded, because the root command always records one before -// any command runs. +// when none was recorded, because the root command records one before any +// command other than help or version runs. func PullPartition(ctx context.Context) partition.Partition { p, ok := ctx.Value(partitionKey{}).(partition.Partition) if !ok { diff --git a/server/cmd/gram/flags_partition_test.go b/server/cmd/gram/flags_partition_test.go index 174a2005985..7df0a72dbf4 100644 --- a/server/cmd/gram/flags_partition_test.go +++ b/server/cmd/gram/flags_partition_test.go @@ -2,6 +2,7 @@ package gram import ( "fmt" + "os" "testing" "github.com/stretchr/testify/require" @@ -43,13 +44,22 @@ func runPartitionApp(t *testing.T, args ...string) (partition.Partition, error) return got, nil } -func TestPartitionFlag_DefaultsToUS(t *testing.T) { //nolint:paralleltest // app.Run mutates urfave's shared help command - got, err := runPartitionApp(t, "probe") - require.NoError(t, err) - require.Equal(t, partition.US, got) +func TestPartitionFlag_RequiredWhenUnset(t *testing.T) { + // Unset rather than empty: a value from the shell or mise.toml must not + // change the outcome. t.Setenv restores the original when the test ends. + t.Setenv("GRAM_PARTITION", "") + require.NoError(t, os.Unsetenv("GRAM_PARTITION")) + + _, err := runPartitionApp(t, "probe") + require.ErrorContains(t, err, "--partition (GRAM_PARTITION) is required: set it to one of us, eu") } -func TestPartitionFlag_ReachesSubcommand(t *testing.T) { //nolint:paralleltest // app.Run mutates urfave's shared help command +func TestPartitionFlag_ReachesSubcommand(t *testing.T) { + // Unset rather than empty: a value from the shell or mise.toml must not + // change the outcome. t.Setenv restores the original when the test ends. + t.Setenv("GRAM_PARTITION", "") + require.NoError(t, os.Unsetenv("GRAM_PARTITION")) + got, err := runPartitionApp(t, "--partition", "eu", "probe") require.NoError(t, err) require.Equal(t, partition.EU, got) @@ -70,16 +80,83 @@ func TestPartitionFlag_RejectsEmptyEnvironmentVariable(t *testing.T) { require.ErrorContains(t, err, `invalid --partition: unknown partition ""`) } -func TestPartitionFlag_RejectsUnknownPartition(t *testing.T) { //nolint:paralleltest // app.Run mutates urfave's shared help command +func TestPartitionFlag_RejectsUnknownPartition(t *testing.T) { + // Unset rather than empty: a value from the shell or mise.toml must not + // change the outcome. t.Setenv restores the original when the test ends. + t.Setenv("GRAM_PARTITION", "") + require.NoError(t, os.Unsetenv("GRAM_PARTITION")) + _, err := runPartitionApp(t, "--partition", "apac", "probe") require.ErrorContains(t, err, `invalid --partition: unknown partition "apac"`) } -func TestApp_RejectsUnknownPartitionBeforeAnyCommand(t *testing.T) { //nolint:paralleltest // app.Run mutates urfave's shared help command +func TestApp_RequiresPartitionBeforeAnyCommand(t *testing.T) { + // Unset rather than empty: a value from the shell or mise.toml must not + // change the outcome. t.Setenv restores the original when the test ends. + t.Setenv("GRAM_PARTITION", "") + require.NoError(t, os.Unsetenv("GRAM_PARTITION")) + + err := newApp().RunContext(t.Context(), []string{"gram", "start"}) + require.ErrorContains(t, err, "--partition (GRAM_PARTITION) is required") +} + +func TestApp_RejectsUnknownPartitionBeforeAnyCommand(t *testing.T) { + // Unset rather than empty: a value from the shell or mise.toml must not + // change the outcome. t.Setenv restores the original when the test ends. + t.Setenv("GRAM_PARTITION", "") + require.NoError(t, os.Unsetenv("GRAM_PARTITION")) + err := newApp().RunContext(t.Context(), []string{"gram", "--partition", "apac", "version"}) require.ErrorContains(t, err, `invalid --partition: unknown partition "apac"`) } +func TestApp_HelpAndVersionRunWithoutPartition(t *testing.T) { + // Unset rather than empty: a value from the shell or mise.toml must not + // change the outcome. t.Setenv restores the original when the test ends. + t.Setenv("GRAM_PARTITION", "") + require.NoError(t, os.Unsetenv("GRAM_PARTITION")) + + tests := []struct { + name string + args []string + }{ + {name: "version", args: []string{"gram", "version"}}, + {name: "help command", args: []string{"gram", "help"}}, + {name: "root help flag", args: []string{"gram", "--help"}}, + {name: "command help flag", args: []string{"gram", "start", "--help"}}, + } + for _, tc := range tests { //nolint:paralleltest // subtests share the parent's environment and urfave's help command + t.Run(tc.name, func(t *testing.T) { + require.NoError(t, newApp().RunContext(t.Context(), tc.args)) + }) + } +} + +func TestPrintsHelpOrVersion(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + args []string + want bool + }{ + {name: "no command", args: nil, want: true}, + {name: "version", args: []string{"version"}, want: true}, + {name: "help command", args: []string{"help", "start"}, want: true}, + {name: "help flag after command", args: []string{"start", "--help"}, want: true}, + {name: "short help flag", args: []string{"worker", "-h"}, want: true}, + {name: "service command", args: []string{"start"}, want: false}, + {name: "service command with flags", args: []string{"worker", "--environment", "prod"}, want: false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + require.Equal(t, tc.want, printsHelpOrVersion(tc.args)) + }) + } +} + func TestPullPartition_PanicsWithoutRootCommand(t *testing.T) { t.Parallel() diff --git a/server/cmd/gram/root.go b/server/cmd/gram/root.go index 0247fc34f90..222aea23016 100644 --- a/server/cmd/gram/root.go +++ b/server/cmd/gram/root.go @@ -66,7 +66,6 @@ func newApp() *cli.App { if err != nil { return err } - c.Context = PushPartition(c.Context, p) c.Context = o11y.PushAppInfo(c.Context, &o11y.AppInfo{ Name: "gram", @@ -81,8 +80,13 @@ func newApp() *cli.App { })).With( attr.SlogDataDogGitCommitSHA(GitSHA), attr.SlogDataDogGitRepoURL("github.com/speakeasy-api/gram"), - attr.SlogPartition(p), ) + // Help and version run without a partition; every other command + // has one and carries it on every log line. + if p != "" { + c.Context = PushPartition(c.Context, p) + logger = logger.With(attr.SlogPartition(p)) + } // Sets `GOMEMLIMIT` to 90% of cgroup's memory limit. // Cgroups are Linux-only. Keep all real Linux errors visible. diff --git a/server/internal/partition/partition.go b/server/internal/partition/partition.go index 7b11cf894a5..0b6999b9708 100644 --- a/server/internal/partition/partition.go +++ b/server/internal/partition/partition.go @@ -24,10 +24,6 @@ const ( EU Partition = "eu" ) -// Default is the partition a process runs in when none is configured. It is -// US because every deployment that predates partitions is the US one. -const Default = US - // All lists every known partition in a stable order. func All() []Partition { return []Partition{US, EU} diff --git a/server/internal/partition/partition_test.go b/server/internal/partition/partition_test.go index 70a6391bd6c..f88c7fc3031 100644 --- a/server/internal/partition/partition_test.go +++ b/server/internal/partition/partition_test.go @@ -43,9 +43,3 @@ func TestParse_RejectsUnknownNames(t *testing.T) { }) } } - -func TestDefault_IsUS(t *testing.T) { - t.Parallel() - - require.Equal(t, partition.US, partition.Default) -} diff --git a/tunnel/cmd/tunnel-gateway/main.go b/tunnel/cmd/tunnel-gateway/main.go index d2a024f60e7..cabe8bf31e2 100644 --- a/tunnel/cmd/tunnel-gateway/main.go +++ b/tunnel/cmd/tunnel-gateway/main.go @@ -33,15 +33,10 @@ const partitionAttrKey = "gram.partition" // is repeated here and the tests pin it. var knownPartitions = []string{"us", "eu"} -// defaultPartition is the partition the gateway runs in when GRAM_PARTITION is -// unset, matching partition.Default in the server. -const defaultPartition = "us" - func main() { logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo})) - rawPartition, partitionSet := os.LookupEnv("GRAM_PARTITION") - partition, err := parsePartition(rawPartition, partitionSet) + partition, err := parsePartition(os.Getenv("GRAM_PARTITION")) if err != nil { logger.ErrorContext(context.Background(), "invalid GRAM_PARTITION", slog.Any("error", err)) os.Exit(2) @@ -222,11 +217,12 @@ func parseMaxSessions(raw string) (int, error) { return parsed, nil } -// parsePartition returns the partition named by GRAM_PARTITION. Unset selects -// the default (us); set but empty or unknown is an error, as in the server. -func parsePartition(raw string, set bool) (string, error) { - if !set { - return defaultPartition, nil +// parsePartition returns the partition named by GRAM_PARTITION. It has no +// default: unset, empty or unknown is an error, as in the server, so a +// deployment that forgets it fails at startup. +func parsePartition(raw string) (string, error) { + if raw == "" { + return "", fmt.Errorf("GRAM_PARTITION is required: set it to one of %s", strings.Join(knownPartitions, ", ")) } if !slices.Contains(knownPartitions, raw) { return "", fmt.Errorf("unknown partition %q: must be one of %s", raw, strings.Join(knownPartitions, ", ")) diff --git a/tunnel/cmd/tunnel-gateway/main_test.go b/tunnel/cmd/tunnel-gateway/main_test.go index 3ae4c18cf20..23a1c7f7d65 100644 --- a/tunnel/cmd/tunnel-gateway/main_test.go +++ b/tunnel/cmd/tunnel-gateway/main_test.go @@ -30,32 +30,26 @@ func TestParseMaxSessionsRejectsInvalidValue(t *testing.T) { require.Error(t, err) } -func TestParsePartitionDefaultsToUSWhenUnset(t *testing.T) { - got, err := parsePartition("", false) - require.NoError(t, err) - require.Equal(t, "us", got) -} - func TestParsePartitionAcceptsKnownPartitions(t *testing.T) { for _, want := range []string{"us", "eu"} { - got, err := parsePartition(want, true) + got, err := parsePartition(want) require.NoError(t, err) require.Equal(t, want, got) } } -func TestParsePartitionRejectsEmptyOrUnknown(t *testing.T) { +func TestParsePartitionRejectsMissingOrUnknown(t *testing.T) { tests := []struct { name string raw string wantErr string }{ - {name: "empty", raw: "", wantErr: `unknown partition ""`}, + {name: "missing", raw: "", wantErr: "GRAM_PARTITION is required: set it to one of us, eu"}, {name: "unknown", raw: "apac", wantErr: `unknown partition "apac"`}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - _, err := parsePartition(tc.raw, true) + _, err := parsePartition(tc.raw) require.ErrorContains(t, err, tc.wantErr) }) } From c63f858598517325d69b36ba8fbc8576a5f09784 Mon Sep 17 00:00:00 2001 From: Brian Flad Date: Thu, 19 Nov 2026 15:28:17 -0500 Subject: [PATCH 4/4] fix: require the partition when --help follows the argument terminator https://linear.app/speakeasy/issue/AGE-3680/feat-add-a-partition-setting-to-gram-processes Arguments after "--" are positional, so a --help or -h there no longer exempts a command from requiring GRAM_PARTITION. --- server/cmd/gram/flags_partition.go | 7 ++++++- server/cmd/gram/flags_partition_test.go | 13 +++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/server/cmd/gram/flags_partition.go b/server/cmd/gram/flags_partition.go index d027e0b425f..9a846d16c46 100644 --- a/server/cmd/gram/flags_partition.go +++ b/server/cmd/gram/flags_partition.go @@ -49,7 +49,8 @@ func partitionFromCLI(c *cli.Context) (partition.Partition, error) { // printsHelpOrVersion reports whether the arguments after the global flags // only print help or the version: no command, the help or version command, or -// a help flag anywhere. +// a help flag. Arguments after a "--" terminator are positional, never flags, +// so a --help there does not exempt a command from requiring a partition. func printsHelpOrVersion(args []string) bool { if len(args) == 0 { return true @@ -60,6 +61,10 @@ func printsHelpOrVersion(args []string) bool { return true } + if end := slices.Index(args, "--"); end >= 0 { + args = args[:end] + } + return slices.ContainsFunc(args, func(arg string) bool { return arg == "--help" || arg == "-h" }) diff --git a/server/cmd/gram/flags_partition_test.go b/server/cmd/gram/flags_partition_test.go index 7df0a72dbf4..28ba0b009e3 100644 --- a/server/cmd/gram/flags_partition_test.go +++ b/server/cmd/gram/flags_partition_test.go @@ -100,6 +100,16 @@ func TestApp_RequiresPartitionBeforeAnyCommand(t *testing.T) { require.ErrorContains(t, err, "--partition (GRAM_PARTITION) is required") } +func TestApp_RequiresPartitionWithHelpFlagAfterTerminator(t *testing.T) { + // Unset rather than empty: a value from the shell or mise.toml must not + // change the outcome. t.Setenv restores the original when the test ends. + t.Setenv("GRAM_PARTITION", "") + require.NoError(t, os.Unsetenv("GRAM_PARTITION")) + + err := newApp().RunContext(t.Context(), []string{"gram", "start", "--", "--help"}) + require.ErrorContains(t, err, "--partition (GRAM_PARTITION) is required") +} + func TestApp_RejectsUnknownPartitionBeforeAnyCommand(t *testing.T) { // Unset rather than empty: a value from the shell or mise.toml must not // change the outcome. t.Setenv restores the original when the test ends. @@ -145,6 +155,9 @@ func TestPrintsHelpOrVersion(t *testing.T) { {name: "help command", args: []string{"help", "start"}, want: true}, {name: "help flag after command", args: []string{"start", "--help"}, want: true}, {name: "short help flag", args: []string{"worker", "-h"}, want: true}, + {name: "help flag before terminator", args: []string{"start", "--help", "--", "x"}, want: true}, + {name: "help flag after terminator", args: []string{"start", "--", "--help"}, want: false}, + {name: "short help flag after terminator", args: []string{"demo-seed", "--", "-h"}, want: false}, {name: "service command", args: []string{"start"}, want: false}, {name: "service command with flags", args: []string{"worker", "--environment", "prod"}, want: false}, }