diff --git a/.sampo/changesets/experiment-exposure.md b/.sampo/changesets/experiment-exposure.md new file mode 100644 index 000000000..66e9d70c1 --- /dev/null +++ b/.sampo/changesets/experiment-exposure.md @@ -0,0 +1,5 @@ +--- +pypi/posthog: minor +--- + +Send `$experiment_exposure` for config version 2 experiment splits, and send the reason code and rule attribution on `$feature_flag_called` for config version 2 flags in place of `$feature_flag_reason`. diff --git a/posthog/args.py b/posthog/args.py index 42083b923..aeaf5a1d8 100644 --- a/posthog/args.py +++ b/posthog/args.py @@ -5,6 +5,9 @@ Any, Dict, FrozenSet, + List, + Mapping, + NamedTuple, Union, Tuple, Type, @@ -65,6 +68,27 @@ class OptionalCaptureArgs(TypedDict): _property_allowlist: NotRequired[ Optional[FrozenSet[str]] ] # Internal: strict allowlist applied to the fully-enriched event properties. Used by minimal $feature_flag_called events. + _pinned_properties: NotRequired[ + Optional["_PinnedProperties"] + ] # Internal: properties fixed after super properties are merged. Used by config version 2 flag events. + _before_send_drops: NotRequired[ + Optional[List[str]] + ] # Internal: gets the event name when before_send drops the event. Used by flag event dedupe. + + +class _PinnedProperties(NamedTuple): + """Internal: event properties that super properties cannot change. + + Applied to the fully-enriched properties, after super properties are merged and + before any allowlist: ``removed`` names are dropped, then ``values`` are set. + """ + + removed: FrozenSet[str] + values: Mapping[str, Any] + + def apply(self, properties: Dict[str, Any]) -> Dict[str, Any]: + kept = {k: v for k, v in properties.items() if k not in self.removed} + return {**kept, **self.values} class OptionalSetArgs(TypedDict): diff --git a/posthog/async_client.py b/posthog/async_client.py index 48de0abd6..7e5c3b3ff 100644 --- a/posthog/async_client.py +++ b/posthog/async_client.py @@ -38,8 +38,8 @@ from .capture_mode import CaptureMode, _resolve_capture_mode from .client import ( MAX_DICT_SIZE as _MAX_DICT_SIZE, - _MINIMAL_FLAG_CALLED_EVENT_PROPERTIES, Client as _SyncClient, + _capture_flag_events, add_context_tags as _add_context_tags, get_identity_state as _get_identity_state, stringify_id as _stringify_id, @@ -77,6 +77,7 @@ ) from .release_id import _resolve_release_id from .request import QuotaLimitError, determine_server_host, normalize_host +from .types import FeatureFlag as _FeatureFlag from .types import ( FlagValue, _has_untyped_flag_records, @@ -196,6 +197,9 @@ def __init__( 0, feature_flags_request_max_retries ) self.distinct_ids_feature_flags_reported = SizeLimitedDict(_MAX_DICT_SIZE, set) + # $experiment_exposure dedupe, kept apart from $feature_flag_called's so that a + # reported call never suppresses its exposure. Same bound and eviction. + self._experiment_exposures_reported = SizeLimitedDict(_MAX_DICT_SIZE, set) self._feature_flag_evaluations_host_cache: Optional[ _FeatureFlagEvaluationsHost ] = None @@ -468,6 +472,7 @@ def _prepare_event( msg: dict[str, Any], disable_geoip: Optional[bool], property_allowlist=None, + pinned_properties=None, ) -> tuple[Optional[dict[str, Any]], Optional[str]]: if self.disabled or not self._accepting: return None, None @@ -499,6 +504,10 @@ def _prepare_event( msg["properties"].setdefault("$release_id", self._release_id) if self.is_server: msg["properties"]["$is_server"] = True + # After every enrichment step, so a super property never changes or adds + # config version 2 flag attribution. + if pinned_properties is not None: + msg["properties"] = pinned_properties.apply(msg["properties"]) if property_allowlist is not None: msg["properties"] = { key: value @@ -531,7 +540,7 @@ async def _process_event(self, msg: dict[str, Any]) -> Optional[dict[str, Any]]: def _build_capture_event( self, event: str, kwargs: OptionalCaptureArgs - ) -> tuple[dict[str, Any], Optional[bool], Any]: + ) -> tuple[dict[str, Any], Optional[bool], Any, Any]: properties = {**(kwargs.get("properties") or {}), **system_context()} if self.capture_trace_context: properties = {**_get_current_otel_span_properties(), **properties} @@ -567,6 +576,7 @@ def _build_capture_event( }, kwargs.get("disable_geoip"), kwargs.get("_property_allowlist"), + kwargs.get("_pinned_properties"), ) def capture( @@ -574,11 +584,11 @@ def capture( ) -> Optional[str]: """Queue an event without blocking for network delivery.""" try: - msg, disable_geoip, property_allowlist = self._build_capture_event( - event, kwargs + msg, disable_geoip, property_allowlist, pinned_properties = ( + self._build_capture_event(event, kwargs) ) prepared, sent_uuid = self._prepare_event( - msg, disable_geoip, property_allowlist + msg, disable_geoip, property_allowlist, pinned_properties ) if prepared is None or sent_uuid is None: return None @@ -622,11 +632,11 @@ async def capture_immediate( self._immediate_callers[current] = self._immediate_callers.get(current, 0) + 1 error_batch: list[dict[str, Any]] = [] try: - msg, disable_geoip, property_allowlist = self._build_capture_event( - event, kwargs + msg, disable_geoip, property_allowlist, pinned_properties = ( + self._build_capture_event(event, kwargs) ) prepared, sent_uuid = self._prepare_event( - msg, disable_geoip, property_allowlist + msg, disable_geoip, property_allowlist, pinned_properties ) if prepared is None or sent_uuid is None: return None @@ -1090,38 +1100,29 @@ def _capture_feature_flag_called_if_needed( disable_geoip: Optional[bool] = None, has_experiment: Optional[bool] = None, minimal_flag_called_events: bool = False, + details: Optional[_FeatureFlag] = None, + value_from_details: bool = False, ) -> None: - groups_key = ( - tuple(sorted((str(k), str(v)) for k, v in groups.items())) if groups else () - ) - reported_key = (key, response, groups_key) - reported_flags = self.distinct_ids_feature_flags_reported.get(distinct_id) - if reported_flags is None: - reported_flags = set() - self.distinct_ids_feature_flags_reported[distinct_id] = reported_flags - if reported_key in reported_flags: - return - - if has_experiment is not None: - properties["$feature_flag_has_experiment"] = has_experiment - capture_kwargs: dict[str, Any] = {} - if minimal_flag_called_events and has_experiment is False: - capture_kwargs["_property_allowlist"] = ( - _MINIMAL_FLAG_CALLED_EVENT_PROPERTIES - ) capture_groups = { str(group_type): _stringify_id(group_key) for group_type, group_key in (groups or {}).items() } - self.capture( - "$feature_flag_called", + _capture_flag_events( + self.capture, + self.distinct_ids_feature_flags_reported, + self._experiment_exposures_reported, distinct_id=distinct_id, + key=key, + response=response, properties=properties, - groups=capture_groups, + groups=groups, + capture_groups=capture_groups, disable_geoip=disable_geoip, - **capture_kwargs, + has_experiment=has_experiment, + minimal_flag_called_events=minimal_flag_called_events, + details=details, + value_from_details=value_from_details, ) - reported_flags.add(reported_key) def _pending_queue_items(self) -> int: return int(getattr(self._queue, "_unfinished_tasks", self._queue.qsize())) @@ -1283,6 +1284,7 @@ async def shutdown(self) -> None: errors.append(error) self.distinct_ids_feature_flags_reported.clear() + self._experiment_exposures_reported.clear() self._closed = True if errors and self.debug: raise errors[0] diff --git a/posthog/client.py b/posthog/client.py index 66c161866..a95944c32 100644 --- a/posthog/client.py +++ b/posthog/client.py @@ -4,6 +4,7 @@ import json import logging import os +import re import sys import threading import time @@ -11,14 +12,33 @@ import weakref from contextvars import ContextVar from datetime import datetime, timedelta, timezone -from typing import Any, Callable, Dict, List, Mapping, Optional, Tuple, Union, cast +from typing import ( + Any, + Callable, + Dict, + FrozenSet, + List, + Mapping, + NamedTuple, + Optional, + Set, + Tuple, + Union, + cast, +) from uuid import UUID, uuid4 from typing_extensions import Unpack from posthog._async_utils import _BackgroundEventLoopRunner from posthog._disabled_lane_queue import _DisabledLaneQueue -from posthog.args import ID_TYPES, ExceptionArg, OptionalCaptureArgs, OptionalSetArgs +from posthog.args import ( + ID_TYPES, + ExceptionArg, + OptionalCaptureArgs, + OptionalSetArgs, + _PinnedProperties, +) from posthog.metrics_capture import ( _DEFAULT_FLUSH_INTERVAL_SECONDS, PostHogMetrics, @@ -353,6 +373,20 @@ def wrapper(self, *args, **kwargs): } ) +# A config version 2 event also keeps its attribution when minimized. Only those +# events use this wider allowlist, so a config version 1 event is trimmed as before, +# even when super properties happen to use these names. +_MINIMAL_V2_FLAG_CALLED_EVENT_PROPERTIES: frozenset[str] = ( + _MINIMAL_FLAG_CALLED_EVENT_PROPERTIES + | { + "$feature_flag_config_version", + "$feature_flag_reason_code", + "$feature_flag_rule_id", + "$feature_flag_rule_type", + "$feature_flag_variant_key", + } +) + def _parse_has_experiment(value: Any) -> Optional[bool]: """Server-reported experiment linkage; anything but an explicit bool means unknown.""" @@ -365,6 +399,401 @@ def _metadata_has_experiment(metadata: Any) -> Optional[bool]: return metadata.has_experiment if isinstance(metadata, FlagMetadata) else None +# The reason code and rule type of the one outcome that sends ``$experiment_exposure``: +# a variant assigned by an experiment rule. +_EXPERIMENT_SPLIT_REASON = "experiment_split" +_EXPERIMENT_RULE_TYPE = "experiment" + +# The reason codes of a config version 2 record that report the rule that produced +# the result, and the rule types each one allows. Any other reason code, and any +# failed record, reports no rule, even if the record names one. +_RULE_TYPES_BY_REASON_CODE: Dict[str, FrozenSet[str]] = { + "targeting_match": frozenset({"targeted_release", "percentage_rollout"}), + _EXPERIMENT_SPLIT_REASON: frozenset({_EXPERIMENT_RULE_TYPE}), + "rollout_miss": frozenset({"percentage_rollout", _EXPERIMENT_RULE_TYPE}), + "experiment_paused": frozenset({_EXPERIMENT_RULE_TYPE}), + "holdout": frozenset({_EXPERIMENT_RULE_TYPE}), +} + +# Properties a super property must not add to, or change on, a config version 2 +# ``$feature_flag_called``: the attribution, which is set again from the record +# after super properties are merged, and the fields such an event never carries. +# The flag identity of the read is set again with the attribution. +_V2_FLAG_CALLED_PINNED = frozenset( + { + "$feature_flag_config_version", + "$feature_flag_reason_code", + "$feature_flag_rule_id", + "$feature_flag_rule_type", + "$feature_flag_variant_key", + "$feature_flag_reason", + "$feature_flag_forced_variant", + "$feature_flag_experiment_id", + } +) + +# Properties an ``$experiment_exposure`` never carries, even from a super property. +_EXPOSURE_FORBIDDEN = frozenset( + { + "$feature_flag_config_version", + "$feature_flag_reason_code", + "$feature_flag_rule_type", + "$experiment_exposure_source", + "$feature_flag_experiment_id", + "$feature_flag_forced_variant", + } +) + +# ``$experiment_exposure`` copies these from the ``$feature_flag_called`` of the same +# read, when it has them. +_EXPOSURE_PROPERTIES_FROM_FLAG_CALLED = ( + "$feature_flag_id", + "$feature_flag_version", + "$feature_flag_request_id", + "$feature_flag_evaluated_at", +) + + +def _v4_field(record: Any, name: str) -> Any: + """Read a ``/flags?v=4`` field of a parsed flag record or its metadata, such as + ``failed`` or ``config_version``. + + These fields stay private, as ``_failed``, ``_config_version`` and so on, until + the typed flag accessors make them public under their wire names. Either spelling + is read, so the two changes can land in either order. + """ + return getattr(record, name, getattr(record, f"_{name}", None)) + + +# The rule ID's wire format: a hyphenated UUID, as the contract's event schemas +# require. +_UUID_PATTERN = re.compile( + r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}" +) + + +def _is_uuid(value: Any) -> bool: + return isinstance(value, str) and _UUID_PATTERN.fullmatch(value) is not None + + +class _RuleAttribution(NamedTuple): + """What a config version 2 flag record reports on its flag events.""" + + reason_code: Optional[str] + rule_id: Optional[str] + rule_type: Optional[str] + # Set only for a complete experiment split, the outcome that sends an exposure. + variant_key: Optional[str] + + def flag_called_properties(self) -> Dict[str, Any]: + properties: Dict[str, Any] = {"$feature_flag_config_version": 2} + if self.reason_code is not None: + properties["$feature_flag_reason_code"] = self.reason_code + if self.rule_id is not None: + properties["$feature_flag_rule_id"] = self.rule_id + if self.rule_type is not None: + properties["$feature_flag_rule_type"] = self.rule_type + if self.variant_key is not None: + properties["$feature_flag_variant_key"] = self.variant_key + return properties + + +def _rule_attribution(details: Optional[FeatureFlag]) -> Optional[_RuleAttribution]: + """Read the event attribution of a config version 2 flag record. + + Returns ``None`` for anything else: no record, an unreadable record, a config + version 1 record, or a record from a server without typed values. Only the + evaluation details are read, never the value a getter returned. + + The config version and reason code are always reported. The rule ID and rule + type are reported together or not at all, only for a targeting match, a rollout + miss, a paused experiment, a holdout and a complete experiment split, and only + with a rule type the reason code allows (``_RULE_TYPES_BY_REASON_CODE``). A + record with no rule ID, no rule type or another rule type, such as a targeting + match that names an experiment rule, reports neither. A rule ID that is not a + UUID counts as missing. So does a failed record or any other reason code, + including one this SDK does not know, and a record with no reason code. The + variant key is reported only for a complete experiment split: reason + ``experiment_split``, rule type ``experiment``, a rule ID and a variant key (any + string, including an empty one). A split without all of them is malformed. A + record that reports no rule keeps its reason, reports no rule ID, rule type or + variant, and sends no exposure. Its identity is never guessed from the value, + the key or an earlier result. + """ + if details is None or not isinstance(details.metadata, FlagMetadata): + return None + metadata = details.metadata + if _v4_field(metadata, "config_version") != 2: + return None + reason_code = (details.reason.code if details.reason else None) or None + allowed_rule_types = _RULE_TYPES_BY_REASON_CODE.get(reason_code or "") + if _v4_field(details, "failed") is True or allowed_rule_types is None: + return _RuleAttribution(reason_code, None, None, None) + rule_id = _v4_field(metadata, "rule_id") + rule_type = _v4_field(metadata, "rule_type") + if not _is_uuid(rule_id) or rule_type not in allowed_rule_types: + return _RuleAttribution(reason_code, None, None, None) + if reason_code != _EXPERIMENT_SPLIT_REASON: + # Only an experiment split assigns a variant. + return _RuleAttribution(reason_code, rule_id, rule_type, None) + variant_key = _v4_field(metadata, "variant_key") + if not isinstance(variant_key, str): + return _RuleAttribution(reason_code, None, None, None) + return _RuleAttribution(reason_code, rule_id, rule_type, variant_key) + + +_flag_events_log = logging.getLogger("posthog") + +# Guards the check and the reservation of config version 2 flag event dedupe keys on +# both clients. It is held only for the dict and set operations, never while an event +# is captured. +_flag_event_keys_lock = threading.Lock() + + +def _reinit_flag_event_keys_lock_after_fork() -> None: + global _flag_event_keys_lock + # A parent thread can hold the lock at fork time. Replace it, never acquire it. + _flag_event_keys_lock = threading.Lock() + + +if hasattr(os, "register_at_fork"): + os.register_at_fork(after_in_child=_reinit_flag_event_keys_lock_after_fork) + + +def _reserve_dedupe_key( + reported_by_distinct_id: Dict[Any, Any], distinct_id: Any, dedupe_key: Any +) -> Optional[Set[Any]]: + """Reserve ``dedupe_key`` for ``distinct_id`` unless a read already holds it. + + The check and the reservation are one step, so when concurrent reads have the + same key, only one of them gets it. Returns the set that now holds the key, for + ``_capture_once``. Returns ``None`` when the key is already there: the event was + reported, or another read is capturing it. + """ + with _flag_event_keys_lock: + reported = reported_by_distinct_id.get(distinct_id) + if reported is None: + reported = set() + reported_by_distinct_id[distinct_id] = reported + if dedupe_key in reported: + return None + reported.add(dedupe_key) + return reported + + +def _capture_once( + capture: Callable[..., Any], + reported: Set[Any], + dedupe_key: Any, + event: str, + **kwargs: Any, +) -> None: + """Capture one flag event whose key ``_reserve_dedupe_key`` reserved, and keep + the key unless capture failed. + + The key stays reserved while capture runs, so concurrent reads do not send the + same event twice. It is released when capture raises, or returns ``None`` + because the event could not be queued (a full or closed queue, opt-out or a + disabled client), so a later read tries again. An event that ``before_send`` + drops keeps its key: the application chose to drop it, and every later read + would be dropped the same way. A failure is logged and never reaches the read. + """ + before_send_drops: List[str] = [] + try: + accepted = capture( + event, _before_send_drops=before_send_drops, **kwargs + ) is not None or bool(before_send_drops) + except Exception: + _flag_events_log.exception("[FEATURE FLAGS] Failed to capture %s", event) + accepted = False + if not accepted: + with _flag_event_keys_lock: + reported.discard(dedupe_key) + + +def _capture_flag_events( + capture: Callable[..., Any], + flag_calls_reported: Dict[Any, Any], + exposures_reported: Dict[Any, Any], + *, + distinct_id: ID_TYPES, + key: str, + response: Optional[FlagValue], + properties: Dict[str, Any], + groups: Optional[Mapping[str, Union[str, int]]], + capture_groups: Mapping[str, Any], + disable_geoip: Optional[bool], + has_experiment: Optional[bool], + minimal_flag_called_events: bool, + details: Optional[FeatureFlag], + value_from_details: bool, +) -> None: + """Send the flag events of one flag read: ``$feature_flag_called`` and, for an + experiment split, ``$experiment_exposure``. + + Every read that sends flag events comes here, on both clients: the single-flag + getters and the reads of an ``evaluate_flags()`` snapshot. Both events are built + from ``details``, the parsed ``/flags`` record that supplied the value, and the + call's ``properties``; the flag is never evaluated again. + + ``$feature_flag_called`` is sent if the (distinct_id, flag, response, groups) + tuple hasn't already been reported on this client. Group context is included so + that group-scoped flags fire a separate event for each group a user is evaluated + under. A config version 2 record that supplied the reported value + (``value_from_details``) adds its attribution to the event and to that tuple, so + equal-valued A/A arms, successive experiments of one flag, and holdouts are each + reported. Its reason code replaces the human-readable ``$feature_flag_reason``. + A config version 1 event and tuple are unchanged, and so are those of a read + whose value the record did not supply. + + ``has_experiment`` is the server-reported signal for whether the flag is linked + to an experiment (``None`` when the server did not report it). When the + server-controlled gate is on and the flag is known non-experiment, the event is + trimmed to a strict allowlist; any missing signal sends the full legacy shape, + and experiment-linked flags keep the full set for exposure analysis. + + ``minimal_flag_called_events`` is the gate as observed by the evaluation that + produced ``response``, not a fresh read of client-wide state: both callers + resolve it themselves (the snapshot pins it at construction; the single-flag + path reads it from the specific local/remote source that produced the value) so + a concurrent poller refresh or another ``/flags`` call can't reshape an event + after the fact. + + ``$experiment_exposure`` is sent for a complete config version 2 experiment + split whose record supplied the reported value (``value_from_details``), once + per distinct ID, groups, flag, rule and variant. It has its own dedupe, so a + ``$feature_flag_called`` already reported never suppresses it. + + Both config version 2 events are attempted independently: a failed capture of + one does not stop the other, and keeps no dedupe key, so a later read tries + again. An event that ``before_send`` drops keeps its key. Each key is reserved + before its capture, in the same step as the check, so concurrent reads of one + outcome send each event once. Their attribution is pinned after super properties + are merged, so a super property can neither change it nor add a field the event + never carries. The flag key, the response and the evaluation source are pinned + the same way on both events, so the pair always reports the same read. + Config version 1 events are sent and deduped as before. + """ + groups_key = ( + tuple(sorted((str(k), str(v)) for k, v in groups.items())) if groups else () + ) + # Only a record that supplied the reported value attributes it. A caller's + # match_value and a stale cached value are reported as before, with no config + # version 2 properties, even when a config version 2 record came with them. + attribution = _rule_attribution(details) if value_from_details else None + # What both config version 2 events report about the read itself. Each event + # pins it, so a super property cannot make the pair disagree. + flag_identity = { + "$feature_flag": key, + "$feature_flag_response": response, + "locally_evaluated": properties.get("locally_evaluated") is True, + } + feature_flag_reported_key: Tuple[Any, ...] = (key, response, groups_key) + if attribution is not None: + feature_flag_reported_key += (2, *attribution) + + # The set this read records its $feature_flag_called key in, or None when the + # read sends no $feature_flag_called. + reported_flags: Optional[Set[Any]] + if attribution is None: + reported_flags = flag_calls_reported.get(distinct_id) + if reported_flags is None: + reported_flags = set() + flag_calls_reported[distinct_id] = reported_flags + if feature_flag_reported_key in reported_flags: + reported_flags = None + else: + reported_flags = _reserve_dedupe_key( + flag_calls_reported, distinct_id, feature_flag_reported_key + ) + + if reported_flags is not None: + if attribution is not None: + properties.pop("$feature_flag_reason", None) + properties.update(attribution.flag_called_properties()) + # Record the server's experiment signal when known, so minimization's impact + # can be measured by segmenting on it. + if has_experiment is not None: + properties["$feature_flag_has_experiment"] = has_experiment + + # Minimize iff the server-controlled gate is on AND the flag is known to have + # no linked experiment. Any missing signal (gate absent, has_experiment + # missing) fails safe to the full legacy shape. + should_minimize = minimal_flag_called_events and has_experiment is False + # Only thread the internal allowlist through when minimizing, so the + # full-property path's capture() call signature stays unchanged. + extra_capture_kwargs: Dict[str, Any] = {} + if should_minimize: + extra_capture_kwargs["_property_allowlist"] = ( + _MINIMAL_FLAG_CALLED_EVENT_PROPERTIES + if attribution is None + else _MINIMAL_V2_FLAG_CALLED_EVENT_PROPERTIES + ) + + if attribution is None: + # Config version 1 events are sent and deduped as before. + capture( + "$feature_flag_called", + distinct_id=distinct_id, + properties=properties, + groups=capture_groups, + disable_geoip=disable_geoip, + **extra_capture_kwargs, + ) + reported_flags.add(feature_flag_reported_key) + else: + _capture_once( + capture, + reported_flags, + feature_flag_reported_key, + "$feature_flag_called", + distinct_id=distinct_id, + properties=properties, + groups=capture_groups, + disable_geoip=disable_geoip, + _pinned_properties=_PinnedProperties( + _V2_FLAG_CALLED_PINNED, + {**attribution.flag_called_properties(), **flag_identity}, + ), + **extra_capture_kwargs, + ) + + if attribution is None or attribution.variant_key is None: + return + + # The value, request ID, evaluation time and evaluation source are left out, so + # one assignment is one exposure however often it is read. + exposure_key = (key, groups_key, 2, attribution.rule_id, attribution.variant_key) + reported_exposures = _reserve_dedupe_key( + exposures_reported, str(distinct_id), exposure_key + ) + if reported_exposures is None: + return + + exposure_properties: Dict[str, Any] = { + name: properties[name] + for name in _EXPOSURE_PROPERTIES_FROM_FLAG_CALLED + if name in properties + } + exposure_identity = { + **flag_identity, + "$feature_flag_variant_key": attribution.variant_key, + "$feature_flag_rule_id": attribution.rule_id, + } + exposure_properties.update(exposure_identity) + _capture_once( + capture, + reported_exposures, + exposure_key, + "$experiment_exposure", + distinct_id=distinct_id, + properties=exposure_properties, + groups=capture_groups, + disable_geoip=disable_geoip, + _pinned_properties=_PinnedProperties(_EXPOSURE_FORBIDDEN, exposure_identity), + ) + + class _Lane: """A capture lane: a queue drained by its own consumer pool, posting to one endpoint. @@ -944,6 +1373,9 @@ def __init__( ) self.poller: Optional[Poller] = None self.distinct_ids_feature_flags_reported = SizeLimitedDict(MAX_DICT_SIZE, set) + # $experiment_exposure dedupe, kept apart from $feature_flag_called's so that a + # reported call never suppresses its exposure. Same bound and eviction. + self._experiment_exposures_reported = SizeLimitedDict(MAX_DICT_SIZE, set) self.flag_fallback_cache_url = flag_fallback_cache_url self.flag_cache = self._initialize_flag_cache(flag_fallback_cache_url) self.flag_definition_version = 0 @@ -1844,6 +2276,8 @@ def _capture( # Internal, set for minimal $feature_flag_called events: a strict allowlist # applied to the fully-enriched properties dict just before enqueueing. property_allowlist = kwargs.get("_property_allowlist", None) + pinned_properties = kwargs.get("_pinned_properties", None) + before_send_drops = kwargs.get("_before_send_drops", None) properties = {**(properties or {}), **system_context()} @@ -1961,7 +2395,12 @@ def _capture( msg["properties"] = properties return self._enqueue( - msg, disable_geoip, lane, property_allowlist=property_allowlist + msg, + disable_geoip, + lane, + property_allowlist=property_allowlist, + pinned_properties=pinned_properties, + before_send_drops=before_send_drops, ) def _parse_send_feature_flags(self, send_feature_flags) -> SendFeatureFlagsOptions: @@ -2570,7 +3009,15 @@ def _normalize_event_uuid(self, msg): # Always send a uuid, so we can always return one msg["uuid"] = stringify_id(uuid4()) - def _enqueue(self, msg, disable_geoip, lane=None, property_allowlist=None): + def _enqueue( + self, + msg, + disable_geoip, + lane=None, + property_allowlist=None, + pinned_properties=None, + before_send_drops=None, + ): # type: (...) -> Optional[str] """Push a new `msg` onto a lane's queue (analytics when unspecified), return the event uuid or None.""" @@ -2622,6 +3069,11 @@ def _enqueue(self, msg, disable_geoip, lane=None, property_allowlist=None): # Applied after every enrichment step (system context, context tags, super # properties, $lib/$lib_version) so the final event shape is exactly the # allowlist regardless of where a property came from. + # After every enrichment step, so a super property never changes or adds + # config version 2 flag attribution. + if pinned_properties is not None: + msg["properties"] = pinned_properties.apply(msg["properties"]) + if property_allowlist is not None: msg["properties"] = { k: v for k, v in msg["properties"].items() if k in property_allowlist @@ -2636,6 +3088,8 @@ def _enqueue(self, msg, disable_geoip, lane=None, property_allowlist=None): modified_msg = self.before_send(msg) if modified_msg is None: self.log.debug("Event dropped by before_send callback") + if before_send_drops is not None: + before_send_drops.append(msg["event"]) return None if not isinstance(modified_msg, dict): raise TypeError("before_send must return a dict or None") @@ -3233,6 +3687,11 @@ def _shutdown_once(self, errors: list[Exception]) -> None: self.distinct_ids_feature_flags_reported.clear, errors, ) + self._run_lifecycle_cleanup( + "Failed to clear experiment exposure deduplication state on shutdown", + self._experiment_exposures_reported.clear, + errors, + ) if self.exception_capture: self._run_lifecycle_cleanup( @@ -4103,6 +4562,9 @@ def _get_feature_flag_result( flag_result = None flag_details = None + # Whether flag_details supplied the value this read reports. Only then may the + # read report config version 2 attribution or send $experiment_exposure. + value_from_details = False request_id = None evaluated_at = None feature_flag_error: Optional[str] = None @@ -4216,6 +4678,11 @@ def _get_feature_flag_result( self.log.debug( f"Successfully computed flag remotely: #{key} -> #{flag_result}" ) + # A payload lookup for a caller's match_value reports that value, not + # the record's. A failure above falls back to a stale cached value. + value_from_details = ( + flag_details is not None and override_match_value is None + ) except QuotaLimitError as e: self.log.warning(f"[FEATURE FLAGS] Quota limit exceeded: {e}") feature_flag_error = FeatureFlagError.QUOTA_LIMITED @@ -4277,6 +4744,7 @@ def _get_feature_flag_result( minimal_flag_called_events, local_flag_id=local_flag_id, local_flag_version=local_flag_version, + value_from_details=value_from_details, ) return flag_result @@ -4298,6 +4766,9 @@ def get_feature_flag_result( Get a FeatureFlagResult object which contains the flag result and payload for a key by evaluating locally or remotely depending on whether local evaluation is enabled and the flag can be locally evaluated. This also captures the `$feature_flag_called` event unless `send_feature_flag_events` is `False`. + When the flag was evaluated remotely and its config version 2 record is a complete experiment + split (an experiment rule with a rule ID and a variant key), it also captures the + `$experiment_exposure` event, once per distinct ID, groups, flag, rule and variant. Examples: ```python @@ -4568,6 +5039,11 @@ def _capture_feature_flag_called( minimal_flag_called_events: bool = False, local_flag_id: Optional[int] = None, local_flag_version: Optional[int] = None, + *, + # Keyword-only, and false by default: a caller that does not say the record + # supplied the value reports no config version 2 attribution and sends no + # $experiment_exposure. + value_from_details: bool = False, ): properties: dict[str, Any] = { "$feature_flag": key, @@ -4608,6 +5084,8 @@ def _capture_feature_flag_called( disable_geoip=disable_geoip, has_experiment=has_experiment, minimal_flag_called_events=minimal_flag_called_events, + details=flag_details, + value_from_details=value_from_details, ) def _capture_feature_flag_called_if_needed( @@ -4621,66 +5099,35 @@ def _capture_feature_flag_called_if_needed( disable_geoip: Optional[bool] = None, has_experiment: Optional[bool] = None, minimal_flag_called_events: bool = False, + details: Optional[FeatureFlag] = None, + value_from_details: bool = False, ) -> None: - """Fire a ``$feature_flag_called`` event if the (distinct_id, flag, response, - groups) tuple hasn't already been reported on this client. Group context is - included so that group-scoped flags fire a separate event for each group a user - is evaluated under. Shared by the single-flag evaluation path and - ``FeatureFlagEvaluations.is_enabled() / get_flag()`` so both paths dedupe - identically. - - ``has_experiment`` is the server-reported signal for whether the flag is linked - to an experiment (``None`` when the server did not report it). When the - server-controlled gate is on and the flag is known non-experiment, the event is - trimmed to a strict allowlist; any missing signal sends the full legacy shape, - and experiment-linked flags keep the full set for exposure analysis. - - ``minimal_flag_called_events`` is the gate as observed by the evaluation that - produced ``response``, not a fresh read of client-wide state: both callers - resolve it themselves (the snapshot pins it at construction; the single-flag - path reads it from the specific local/remote source that produced the value) - so a concurrent poller refresh or another ``/flags`` call can't reshape an - event after the fact. + """Send the flag events of a flag read: ``$feature_flag_called`` and, for an + experiment split, ``$experiment_exposure``. See ``_capture_flag_events``. + + Shared by the single-flag evaluation path and the reads of a + ``FeatureFlagEvaluations`` snapshot, so every read reports and dedupes + identically. ``details`` is the parsed ``/flags`` record of the read, or + ``None`` for a locally evaluated flag. ``value_from_details`` says whether that + record supplied the reported value; only then does the read report config + version 2 attribution or send an exposure. """ - groups_key = ( - tuple(sorted((str(k), str(v)) for k, v in groups.items())) if groups else () - ) - feature_flag_reported_key = (key, response, groups_key) - - reported_flags = self.distinct_ids_feature_flags_reported.get(distinct_id) - if reported_flags is None: - reported_flags = set() - self.distinct_ids_feature_flags_reported[distinct_id] = reported_flags - - if feature_flag_reported_key in reported_flags: - return - - # Record the server's experiment signal when known, so minimization's impact - # can be measured by segmenting on it. - if has_experiment is not None: - properties["$feature_flag_has_experiment"] = has_experiment - - # Minimize iff the server-controlled gate is on AND the flag is known to have - # no linked experiment. Any missing signal (gate absent, has_experiment - # missing) fails safe to the full legacy shape. - should_minimize = minimal_flag_called_events and has_experiment is False - # Only thread the internal allowlist through when minimizing, so the - # full-property path's capture() call signature stays unchanged. - extra_capture_kwargs: dict[str, Any] = {} - if should_minimize: - extra_capture_kwargs["_property_allowlist"] = ( - _MINIMAL_FLAG_CALLED_EVENT_PROPERTIES - ) - - self.capture( - "$feature_flag_called", + _capture_flag_events( + self.capture, + self.distinct_ids_feature_flags_reported, + self._experiment_exposures_reported, distinct_id=distinct_id, + key=key, + response=response, properties=properties, - groups=groups or {}, + groups=groups, + capture_groups=groups or {}, disable_geoip=disable_geoip, - **extra_capture_kwargs, + has_experiment=has_experiment, + minimal_flag_called_events=minimal_flag_called_events, + details=details, + value_from_details=value_from_details, ) - reported_flags.add(feature_flag_reported_key) def get_remote_config_payload(self, key: str): """ diff --git a/posthog/feature_flag_evaluations.py b/posthog/feature_flag_evaluations.py index 28e5bf31e..ee179d4bc 100644 --- a/posthog/feature_flag_evaluations.py +++ b/posthog/feature_flag_evaluations.py @@ -125,6 +125,13 @@ class FeatureFlagEvaluations: To narrow the set of flags that get attached to a captured event, use the in-memory helpers :meth:`only` and :meth:`only_accessed`. To narrow the set of flags requested from the server in the first place, pass ``flag_keys`` to :meth:`Client.evaluate_flags`. + + Reading a flag sends ``$feature_flag_called``, deduplicated as :meth:`is_enabled` + describes. When the flag was evaluated remotely and its config version 2 record is + a complete experiment split (an experiment rule with a rule ID and a variant key), + the read also sends ``$experiment_exposure``. The SDK sends one exposure per + distinct ID, groups, flag, rule and variant. ``send_feature_flag_events=False`` + suppresses both events. """ def __init__( @@ -387,4 +394,8 @@ def _record_access(self, key: str, send_feature_flag_events: bool) -> None: properties=properties, has_experiment=flag.has_experiment if flag else None, minimal_flag_called_events=self._minimal_flag_called_events, + # The record that supplied the value. A remotely evaluated experiment + # split also sends $experiment_exposure, whatever this read returned. + details=flag.details if isinstance(flag, _RemoteFlagRecord) else None, + value_from_details=True, ) diff --git a/posthog/test/fixtures/feature_flag_rules_v2/SOURCE.json b/posthog/test/fixtures/feature_flag_rules_v2/SOURCE.json index df7f1c7e0..8274761b1 100644 --- a/posthog/test/fixtures/feature_flag_rules_v2/SOURCE.json +++ b/posthog/test/fixtures/feature_flag_rules_v2/SOURCE.json @@ -1,14 +1,14 @@ { "description": "Copied unchanged from the released feature flag rules v2 contract in PostHog/posthog-sdk-test-harness. Do not edit these files; copy a newer release instead.", "repository": "https://github.com/PostHog/posthog-sdk-test-harness", - "release": "1.22.0", - "revision": "17d861f8e230fe943934d70e3eebf5cff6bf3aa4", + "release": "1.30.0", + "revision": "6414114501396d96aadf2412fdd8054a03076782", "contract_directory": "contracts/feature_flag_rules_v2", - "contract_version": "7.0.0", + "contract_version": "10.0.0", "files": { "readers.json": { "source_path": "fixtures/wire/readers.json", - "sha256": "fd8edd89f827b732c4659ce7246eab534b0106b96a465a4a2d98cfa0a17d7827" + "sha256": "e75abb9bf6656989a147dda4c6e5232e7b544b657d7d540164dce65688641501" }, "responses.json": { "source_path": "fixtures/wire/responses.json", @@ -21,6 +21,14 @@ "v2_value_evaluation.json": { "source_path": "corpus/v2_value_evaluation.json", "sha256": "aaddd05e47fc5a4ae07ea42ada5a659798f4b5fc8d0c0827f6a68e4daf68c128" + }, + "feature_flag_called_context.schema.json": { + "source_path": "schemas/feature_flag_called_context.schema.json", + "sha256": "8a475f5ab56ba4dba76501c4a4e23f26b62c2c2f8037f23d21d0ea7650bd2cf8" + }, + "experiment_exposure_properties.schema.json": { + "source_path": "schemas/experiment_exposure_properties.schema.json", + "sha256": "4b1af8fc98b6f6cb49f974970eb289ca85a78a978e1e74fc4d417eee9adee449" } } } diff --git a/posthog/test/fixtures/feature_flag_rules_v2/experiment_exposure_properties.schema.json b/posthog/test/fixtures/feature_flag_rules_v2/experiment_exposure_properties.schema.json new file mode 100644 index 000000000..ee8604e38 --- /dev/null +++ b/posthog/test/fixtures/feature_flag_rules_v2/experiment_exposure_properties.schema.json @@ -0,0 +1,50 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://posthog.com/contracts/feature_flag_rules_v2/experiment_exposure_properties.schema.json", + "title": "SDK-sent $experiment_exposure properties", + "type": "object", + "additionalProperties": false, + "required": [ + "$feature_flag", + "$feature_flag_response", + "$feature_flag_variant_key", + "$feature_flag_rule_id", + "locally_evaluated" + ], + "properties": { + "$feature_flag": { + "type": "string", + "minLength": 1 + }, + "$feature_flag_response": { + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "string", + "minLength": 1 + } + ] + }, + "$feature_flag_variant_key": { + "type": "string" + }, + "$feature_flag_rule_id": { + "type": "string", + "format": "uuid" + }, + "locally_evaluated": { + "type": "boolean" + }, + "$feature_flag_forced_variant": { + "const": true + } + }, + "description": "Closed projection of the properties of an SDK-sent exposure. The SDK sends it only for a config version 2 experiment split, so it carries no config version, rule type, reason code or source marker; the rule ID marks it as SDK-sent. Experiment ID and forced-variant context are forbidden. Capture identity and existing optional transport properties are outside this projection. Scan the entire event recursively for assignment seeds.", + "not": { + "required": [ + "$feature_flag_forced_variant" + ] + } +} diff --git a/posthog/test/fixtures/feature_flag_rules_v2/feature_flag_called_context.schema.json b/posthog/test/fixtures/feature_flag_rules_v2/feature_flag_called_context.schema.json new file mode 100644 index 000000000..1829dde76 --- /dev/null +++ b/posthog/test/fixtures/feature_flag_rules_v2/feature_flag_called_context.schema.json @@ -0,0 +1,692 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://posthog.com/contracts/feature_flag_rules_v2/feature_flag_called_context.schema.json", + "title": "$feature_flag_called context properties", + "type": "object", + "additionalProperties": false, + "required": [ + "$feature_flag", + "$feature_flag_response" + ], + "properties": { + "$feature_flag": { + "type": "string", + "minLength": 1 + }, + "$feature_flag_response": { + "oneOf": [ + { + "type": "boolean" + }, + { + "type": "string", + "minLength": 1 + } + ] + }, + "$feature_flag_config_version": { + "enum": [ + 1, + 2 + ] + }, + "$feature_flag_variant_key": { + "type": "string" + }, + "$feature_flag_rule_id": { + "type": "string", + "format": "uuid" + }, + "$feature_flag_rule_type": { + "enum": [ + "targeted_release", + "percentage_rollout", + "experiment" + ] + }, + "$feature_flag_experiment_id": { + "type": "integer" + }, + "$feature_flag_reason_code": { + "type": "string", + "minLength": 1 + }, + "$feature_flag_reason": {}, + "$feature_flag_forced_variant": { + "const": true + } + }, + "description": "Closed projection of the flag-access context properties. Other existing capture fields are outside this projection. Complete v2 producer context follows the terminal outcome. A tolerant reader drops the whole attribution of a record that breaks its reason's rows (a rule ID without its rule type or the reverse, a rule type the reason does not allow, or a malformed split): for every reason that names a rule, a call may keep the config version and reason code and omit all rule, variant and experiment attribution, while a partly populated or mismatched tuple is rejected. A config version 2 call whose reason code is missing or not registered keeps the raw code, if any, and carries no attribution; a registered code only matches its own reason's branches. A config version 2 call never carries $feature_flag_reason, the reason description that only version 1 calls carry.", + "allOf": [ + { + "if": { + "required": [ + "$feature_flag_config_version" + ], + "properties": { + "$feature_flag_config_version": { + "const": 2 + } + } + }, + "then": { + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_forced_variant" + ] + }, + { + "required": [ + "$feature_flag_reason" + ] + } + ] + }, + "oneOf": [ + { + "properties": { + "$feature_flag_reason_code": { + "const": "targeting_match" + }, + "$feature_flag_rule_type": { + "const": "targeted_release" + } + }, + "required": [ + "$feature_flag_reason_code", + "$feature_flag_rule_type", + "$feature_flag_rule_id" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant_key" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "targeting_match" + }, + "$feature_flag_rule_type": { + "const": "percentage_rollout" + } + }, + "required": [ + "$feature_flag_reason_code", + "$feature_flag_rule_type", + "$feature_flag_rule_id" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant_key" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "experiment_split" + }, + "$feature_flag_rule_type": { + "const": "experiment" + } + }, + "required": [ + "$feature_flag_reason_code", + "$feature_flag_rule_type", + "$feature_flag_rule_id", + "$feature_flag_variant_key" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_experiment_id" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "rollout_miss" + }, + "$feature_flag_rule_type": { + "const": "percentage_rollout" + } + }, + "required": [ + "$feature_flag_reason_code", + "$feature_flag_rule_type", + "$feature_flag_rule_id" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant_key" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "rollout_miss" + }, + "$feature_flag_rule_type": { + "const": "experiment" + } + }, + "required": [ + "$feature_flag_reason_code", + "$feature_flag_rule_type", + "$feature_flag_rule_id" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant_key" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "experiment_paused" + }, + "$feature_flag_rule_type": { + "const": "experiment" + } + }, + "required": [ + "$feature_flag_reason_code", + "$feature_flag_rule_type", + "$feature_flag_rule_id" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant_key" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "holdout" + }, + "$feature_flag_rule_type": { + "const": "experiment" + } + }, + "required": [ + "$feature_flag_reason_code", + "$feature_flag_rule_type", + "$feature_flag_rule_id" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant_key" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "no_rule_match" + } + }, + "required": [ + "$feature_flag_reason_code" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_rule_type" + ] + }, + { + "required": [ + "$feature_flag_rule_id" + ] + }, + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant_key" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "missing_group_key" + } + }, + "required": [ + "$feature_flag_reason_code" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_rule_type" + ] + }, + { + "required": [ + "$feature_flag_rule_id" + ] + }, + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant_key" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "dependency_error" + } + }, + "required": [ + "$feature_flag_reason_code" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_rule_type" + ] + }, + { + "required": [ + "$feature_flag_rule_id" + ] + }, + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant_key" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "error" + } + }, + "required": [ + "$feature_flag_reason_code" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_rule_type" + ] + }, + { + "required": [ + "$feature_flag_rule_id" + ] + }, + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant_key" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "targeting_match" + } + }, + "required": [ + "$feature_flag_reason_code" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_rule_type" + ] + }, + { + "required": [ + "$feature_flag_rule_id" + ] + }, + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant_key" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "experiment_split" + } + }, + "required": [ + "$feature_flag_reason_code" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_rule_type" + ] + }, + { + "required": [ + "$feature_flag_rule_id" + ] + }, + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant_key" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "rollout_miss" + } + }, + "required": [ + "$feature_flag_reason_code" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_rule_type" + ] + }, + { + "required": [ + "$feature_flag_rule_id" + ] + }, + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant_key" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "experiment_paused" + } + }, + "required": [ + "$feature_flag_reason_code" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_rule_type" + ] + }, + { + "required": [ + "$feature_flag_rule_id" + ] + }, + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant_key" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "const": "holdout" + } + }, + "required": [ + "$feature_flag_reason_code" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_rule_type" + ] + }, + { + "required": [ + "$feature_flag_rule_id" + ] + }, + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant_key" + ] + } + ] + } + }, + { + "properties": { + "$feature_flag_reason_code": { + "not": { + "enum": [ + "targeting_match", + "experiment_split", + "rollout_miss", + "experiment_paused", + "holdout", + "no_rule_match", + "missing_group_key", + "dependency_error", + "error", + "flag_disabled" + ] + } + } + }, + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_rule_type" + ] + }, + { + "required": [ + "$feature_flag_rule_id" + ] + }, + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant_key" + ] + } + ] + } + }, + { + "description": "A local-only outcome with no matrix row: the server never returns flag_disabled, and the call carries no rule or experiment attribution.", + "properties": { + "$feature_flag_reason_code": { + "const": "flag_disabled" + } + }, + "required": [ + "$feature_flag_reason_code" + ], + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_rule_type" + ] + }, + { + "required": [ + "$feature_flag_rule_id" + ] + }, + { + "required": [ + "$feature_flag_experiment_id" + ] + }, + { + "required": [ + "$feature_flag_variant_key" + ] + } + ] + } + } + ] + }, + "else": { + "not": { + "anyOf": [ + { + "required": [ + "$feature_flag_rule_id" + ] + }, + { + "required": [ + "$feature_flag_rule_type" + ] + } + ] + } + } + } + ] +} diff --git a/posthog/test/fixtures/feature_flag_rules_v2/readers.json b/posthog/test/fixtures/feature_flag_rules_v2/readers.json index 7ed8c252b..add97e172 100644 --- a/posthog/test/fixtures/feature_flag_rules_v2/readers.json +++ b/posthog/test/fixtures/feature_flag_rules_v2/readers.json @@ -992,7 +992,10 @@ "value": "compact", "config_version": 1, "experiment_exposure": false, - "error_code": null + "error_code": null, + "legacy_payload": { + "columns": 2 + } }, "producer_expected": "valid" }, @@ -1018,7 +1021,8 @@ "value": true, "config_version": 1, "experiment_exposure": false, - "error_code": "GENERAL" + "error_code": "GENERAL", + "legacy_payload": null }, "producer_expected": "valid" }, diff --git a/posthog/test/released_flags_scenarios.py b/posthog/test/released_flags_scenarios.py index a1fccb10c..2e8246191 100644 --- a/posthog/test/released_flags_scenarios.py +++ b/posthog/test/released_flags_scenarios.py @@ -31,6 +31,7 @@ ) API_KEY = "phc_released_flags_scenarios" DISTINCT_ID = "user-1" +ACCEPTED_UUID = "70000000-0000-4000-8000-000000000000" REQUEST_ID = "60000000-0000-4000-8000-000000000000" EVALUATED_AT = 1800000000000 MATCHED = { @@ -171,7 +172,8 @@ def flags(*args: Any, **kwargs: Any) -> Any: def enqueue(message: Dict[str, Any], *args: Any, **kwargs: Any) -> Optional[str]: events.append([message["event"], _flag_event_properties(message)]) - return None + # Accept the event, as the real queue does, by returning an event UUID. + return ACCEPTED_UUID try: with ( diff --git a/posthog/test/test_experiment_exposure.py b/posthog/test/test_experiment_exposure.py new file mode 100644 index 000000000..c12152833 --- /dev/null +++ b/posthog/test/test_experiment_exposure.py @@ -0,0 +1,1841 @@ +"""``$experiment_exposure`` and the attribution of ``$feature_flag_called``. + +A remotely evaluated config version 2 record that assigns an experiment variant sends +``$experiment_exposure`` next to its ``$feature_flag_called``, and every config version +2 record adds its rule attribution to ``$feature_flag_called``. The reader fixtures and +event schemas are copied unchanged from a ``PostHog/posthog-sdk-test-harness`` contract; +``fixtures/feature_flag_rules_v2/SOURCE.json`` records which. +""" + +from __future__ import annotations + +import copy +import json +import threading +import warnings +from dataclasses import dataclass +from typing import Any, Callable, Dict, Iterator, List, Mapping, Optional +from unittest import mock + +import jsonschema +import pytest + +from posthog import AsyncPosthog +from posthog import client as posthog_client +from posthog.client import ( + _MINIMAL_FLAG_CALLED_EVENT_PROPERTIES, + _MINIMAL_V2_FLAG_CALLED_EVENT_PROPERTIES, + MAX_DICT_SIZE, + Client, +) +from posthog.test.test_flags_v4_response import FIXTURES, READERS, _build +from posthog.test.test_utils import FAKE_TEST_API_KEY +from posthog.types import FeatureFlagResult +from posthog.utils import SizeLimitedDict + +FLAG = "pricing-page" +DISTINCT_ID = "visitor-42" +RULE = "7f3c2a10-8d4e-4b6a-9c21-5e0f1a2b3c4d" +OTHER_RULE = "2b9d4e61-0c7a-4f35-8e12-6a4b3c2d1e0f" +REQUEST_ID = "5d1e2f30-4a5b-4c6d-8e7f-90a1b2c3d4e5" +OTHER_REQUEST_ID = "6e2f3a41-5b6c-4d7e-9f80-a1b2c3d4e5f6" +EVALUATED_AT = 1_800_000_000_000 + +EXPOSURE = "$experiment_exposure" +FLAG_CALLED = "$feature_flag_called" +ATTRIBUTION_PROPERTIES = ( + "$feature_flag_config_version", + "$feature_flag_reason_code", + "$feature_flag_rule_id", + "$feature_flag_rule_type", + "$feature_flag_variant_key", +) + + +# $experiment_exposure leaves these out. The config version, rule type and reason code +# stay on the $feature_flag_called of the same read. The schema projection drops +# undeclared properties, so their absence is asserted separately. +EXPOSURE_LEAVES_OUT = ( + "$feature_flag_config_version", + "$feature_flag_rule_type", + "$feature_flag_reason_code", + "$experiment_exposure_source", +) + + +def _schema(name: str) -> jsonschema.Draft202012Validator: + schema = json.loads((FIXTURES / name).read_text(encoding="utf-8")) + return jsonschema.Draft202012Validator(schema) + + +FLAG_CALLED_SCHEMA = _schema("feature_flag_called_context.schema.json") +EXPOSURE_SCHEMA = _schema("experiment_exposure_properties.schema.json") + + +def _projection( + validator: jsonschema.Draft202012Validator, properties: Mapping[str, Any] +) -> Dict[str, Any]: + """The properties a closed contract projection covers.""" + names = validator.schema["properties"] + return {key: value for key, value in properties.items() if key in names} + + +# --- Responses ---------------------------------------------------------------- + + +def _record( + code: str, + value: Any = True, + *, + key: str = FLAG, + rule_type: Optional[str] = None, + rule_id: Optional[str] = None, + variant_key: Optional[str] = None, + config_version: int = 2, + has_experiment: bool = False, + description: Optional[str] = None, + failed: bool = False, +) -> Dict[str, Any]: + metadata: Dict[str, Any] = { + "id": 77, + "version": 3, + "config_version": config_version, + "description": None, + "payload": None, + "has_experiment": has_experiment, + } + for field, field_value in ( + ("rule_type", rule_type), + ("rule_id", rule_id), + ("variant_key", variant_key), + ): + if field_value is not None: + metadata[field] = field_value + reason: Dict[str, Any] = { + "code": code, + "condition_index": 0 if rule_id else None, + } + if description is not None: + reason["description"] = description + record = {"key": key, "value": value, "reason": reason, "metadata": metadata} + if failed: + record["failed"] = True + return record + + +def _split( + value: Any = "annual", + variant_key: str = "annual-first", + rule_id: str = RULE, + **overrides: Any, +) -> Dict[str, Any]: + fields: Dict[str, Any] = { + "rule_type": "experiment", + "rule_id": rule_id, + "variant_key": variant_key, + "has_experiment": True, + **overrides, + } + return _record("experiment_split", value, **fields) + + +def _response( + *records: Dict[str, Any], + request_id: str = REQUEST_ID, + evaluated_at: int = EVALUATED_AT, + **envelope: Any, +) -> Dict[str, Any]: + return { + "flags": {record["key"]: record for record in records}, + "errorsWhileComputingFlags": False, + "requestId": request_id, + "evaluatedAt": evaluated_at, + **envelope, + } + + +# --- Recording events ----------------------------------------------------------- + + +@dataclass +class _Event: + name: str + distinct_id: Any + properties: Dict[str, Any] + groups: Any + property_allowlist: Any + uuid: Optional[str] + + +def _record_events(client: Any) -> tuple[Any, List[_Event]]: + """Patch ``client.capture`` to record each call, then capture as usual.""" + events: List[_Event] = [] + capture = client.capture + + def record(event: str, **kwargs: Any) -> Optional[str]: + uuid = capture(event, **kwargs) + events.append( + _Event( + name=event, + distinct_id=kwargs.get("distinct_id"), + properties=copy.deepcopy(kwargs.get("properties") or {}), + groups=kwargs.get("groups"), + property_allowlist=kwargs.get("_property_allowlist"), + uuid=uuid, + ) + ) + return uuid + + return mock.patch.object(client, "capture", side_effect=record), events + + +def _named(events: List[_Event], name: str, key: str = FLAG) -> List[_Event]: + return [ + event + for event in events + if event.name == name and event.properties.get("$feature_flag") == key + ] + + +def _attribution(properties: Mapping[str, Any]) -> Dict[str, Any]: + return { + name: properties[name] for name in ATTRIBUTION_PROPERTIES if name in properties + } + + +# --- Reading through each path ------------------------------------------------------ + +# Every sync read that sends flag events. Each takes the client, the key, the +# distinct ID and the groups. +SYNC_READS: Dict[str, Callable[..., Any]] = { + "get_feature_flag": lambda client, key, distinct_id, groups: ( + client.get_feature_flag(key, distinct_id, groups=groups) + ), + "feature_enabled": lambda client, key, distinct_id, groups: ( + client.feature_enabled(key, distinct_id, groups=groups) + ), + "get_feature_flag_result": lambda client, key, distinct_id, groups: ( + client.get_feature_flag_result(key, distinct_id, groups=groups) + ), + "snapshot.get_flag": lambda client, key, distinct_id, groups: ( + client.evaluate_flags(distinct_id, groups=groups).get_flag(key) + ), + "snapshot.is_enabled": lambda client, key, distinct_id, groups: ( + client.evaluate_flags(distinct_id, groups=groups).is_enabled(key) + ), +} +SINGLE_FLAG_READS = ["get_feature_flag", "feature_enabled", "get_feature_flag_result"] +SNAPSHOT_READS = ["snapshot.get_flag", "snapshot.is_enabled"] + + +@pytest.fixture +def client() -> Iterator[Client]: + client = Client(FAKE_TEST_API_KEY, send=False) + yield client + client.shutdown() + + +def _serving(*responses: Dict[str, Any]) -> Any: + """Patch ``/flags`` to answer with each response in turn, the last one after that.""" + remaining = list(responses) + + def answer(*args: Any, **kwargs: Any) -> Dict[str, Any]: + response = remaining.pop(0) if len(remaining) > 1 else remaining[0] + return copy.deepcopy(response) + + return mock.patch("posthog.client.flags", side_effect=answer) + + +def _read( + client: Client, + path: str, + key: str = FLAG, + distinct_id: Any = DISTINCT_ID, + groups: Optional[Dict[str, Any]] = None, +) -> Any: + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + return SYNC_READS[path](client, key, distinct_id, groups or {}) + + +def _read_events( + path: str, + *responses: Dict[str, Any], + reads: int = 1, + client: Optional[Client] = None, + **read_kwargs: Any, +) -> List[_Event]: + """Read ``FLAG`` ``reads`` times through ``path``, one response per read.""" + own_client = client is None + client = client or Client(FAKE_TEST_API_KEY, send=False) + recording, events = _record_events(client) + try: + with _serving(*responses), recording: + for _ in range(reads): + _read(client, path, **read_kwargs) + finally: + if own_client: + client.shutdown() + return events + + +async def _read_async_events( + responses: List[Dict[str, Any]], + reads: List[Callable[[Any], Any]], +) -> List[_Event]: + """Run the reads on one async client, answering each evaluation with the next + response, and the last one after that.""" + remaining = list(responses) + + def answer(*args: Any, **kwargs: Any) -> Dict[str, Any]: + response = remaining.pop(0) if len(remaining) > 1 else remaining[0] + return copy.deepcopy(response) + + with mock.patch( + "posthog.async_client._async_flags", new=mock.AsyncMock(side_effect=answer) + ): + client = AsyncPosthog("project-key", send=False) + recording, events = _record_events(client) + with recording: + for read in reads: + await read(client) + await client.shutdown() + return events + + +# --- The contract's reader cases -------------------------------------------------------- + +READER_CASES = [case for case in READERS["cases"] if case["expected"] == "reader"] + + +def _check_reader_events(case: Dict[str, Any], events: List[_Event]) -> None: + reader = case["reader"] + expectations = {reader["requested_key"]: reader} + expectations.update(case.get("sibling_expectations", {})) + for key, expected in expectations.items(): + exposures = _named(events, EXPOSURE, key) + assert len(exposures) == (1 if expected["experiment_exposure"] else 0), key + [called] = _named(events, FLAG_CALLED, key) + config_version = expected.get("config_version") + if expected.get("error_code") == "PARSE_ERROR": + config_version = None + assert called.properties.get("$feature_flag_config_version") == ( + 2 if config_version == 2 else None + ), key + if called.properties["$feature_flag_response"] is not None: + FLAG_CALLED_SCHEMA.validate( + _projection(FLAG_CALLED_SCHEMA, called.properties) + ) + for exposure in exposures: + EXPOSURE_SCHEMA.validate(_projection(EXPOSURE_SCHEMA, exposure.properties)) + assert not set(EXPOSURE_LEAVES_OUT) & set(exposure.properties), key + assert ( + exposure.properties["$feature_flag_response"] + == (called.properties["$feature_flag_response"]) + ) + assert exposure.uuid is not None and exposure.uuid != called.uuid + for event in [called, *exposures]: + # A flag may be named anything, so `$feature/` is left out. + for name in event.properties: + if not name.startswith("$feature/"): + for part in ("experiment_id", "holdout", "seed"): + assert part not in name, name + + +@pytest.mark.parametrize("path", list(SYNC_READS)) +@pytest.mark.parametrize("case", READER_CASES, ids=lambda case: case["id"]) +def test_reader_cases_send_an_exposure_exactly_when_the_contract_expects(case, path): + response = _build(READERS, case) + client = Client(FAKE_TEST_API_KEY, send=False) + recording, events = _record_events(client) + try: + with _serving(response), recording: + _read(client, path, case["reader"]["requested_key"]) + for sibling in case.get("sibling_expectations", {}): + _read(client, path, sibling) + finally: + client.shutdown() + + _check_reader_events(case, events) + + +@pytest.mark.parametrize("method", ["get_flag", "is_enabled"]) +@pytest.mark.parametrize("case", READER_CASES, ids=lambda case: case["id"]) +async def test_async_reader_cases_send_an_exposure_exactly_when_the_contract_expects( + case, method +): + keys = [case["reader"]["requested_key"], *case.get("sibling_expectations", {})] + + async def read(client: Any) -> None: + snapshot = await client.evaluate_flags(DISTINCT_ID) + for key in keys: + getattr(snapshot, method)(key) + + events = await _read_async_events([_build(READERS, case)], [read]) + + _check_reader_events(case, events) + + +# --- Attribution on $feature_flag_called -------------------------------------------------- + +_RULE_ID = "10000000-0000-4000-8000-000000000001" +ATTRIBUTION_CASES = [ + # (reader template or case id, attribution on $feature_flag_called) + ( + "experiment_split", + { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "experiment_split", + "$feature_flag_rule_id": _RULE_ID, + "$feature_flag_rule_type": "experiment", + "$feature_flag_variant_key": "arm_a", + }, + ), + ( + "targeting_match_targeted_release", + { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "targeting_match", + "$feature_flag_rule_id": _RULE_ID, + "$feature_flag_rule_type": "targeted_release", + }, + ), + ( + "targeting_match_percentage_rollout", + { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "targeting_match", + "$feature_flag_rule_id": _RULE_ID, + "$feature_flag_rule_type": "percentage_rollout", + }, + ), + ( + "rollout_miss_percentage_rollout", + { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "rollout_miss", + "$feature_flag_rule_id": _RULE_ID, + "$feature_flag_rule_type": "percentage_rollout", + }, + ), + ( + "rollout_miss_experiment", + { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "rollout_miss", + "$feature_flag_rule_id": _RULE_ID, + "$feature_flag_rule_type": "experiment", + }, + ), + ( + "experiment_paused", + { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "experiment_paused", + "$feature_flag_rule_id": _RULE_ID, + "$feature_flag_rule_type": "experiment", + }, + ), + ( + "holdout", + { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "holdout", + "$feature_flag_rule_id": _RULE_ID, + "$feature_flag_rule_type": "experiment", + }, + ), + # A holdout assigns no variant, even when its record names one. + ( + "readers.holdout_variant_key", + { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "holdout", + "$feature_flag_rule_id": _RULE_ID, + "$feature_flag_rule_type": "experiment", + }, + ), + ( + "no_rule_match", + { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "no_rule_match", + }, + ), + ( + "missing_group_key", + { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "missing_group_key", + }, + ), + ( + "dependency_error", + { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "dependency_error", + }, + ), + ( + "error", + {"$feature_flag_config_version": 2, "$feature_flag_reason_code": "error"}, + ), + # A malformed split keeps its config version and reason, and loses the whole rule + # tuple. + *( + ( + case_id, + { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "experiment_split", + }, + ) + for case_id in ( + "readers.partial_split_missing_rule_type", + "readers.partial_split_missing_rule_id", + "readers.partial_split_missing_variant_key", + "readers.wrong_rule_type_literal", + "readers.partial_split_string_value", + ) + ), + # Config version 1, from either kind of server, is unchanged. + ("v1", {}), + ("old_server", {}), + ("readers.partial_split_missing_config_version", {}), +] + + +def _attribution_response(source: str) -> Dict[str, Any]: + if source.startswith("readers."): + [case] = [case for case in READERS["cases"] if case["id"] == source] + return _build(READERS, case) + return _build(READERS, {"template": source}) + + +@pytest.mark.parametrize("path", ["get_feature_flag_result", "snapshot.get_flag"]) +@pytest.mark.parametrize( + "source, expected", ATTRIBUTION_CASES, ids=[c[0] for c in ATTRIBUTION_CASES] +) +def test_flag_called_attribution_follows_the_record(source, expected, path): + events = _read_events(path, _attribution_response(source), key="garden-layout") + + [called] = _named(events, FLAG_CALLED, "garden-layout") + assert _attribution(called.properties) == expected + FLAG_CALLED_SCHEMA.validate(_projection(FLAG_CALLED_SCHEMA, called.properties)) + exposures = _named(events, EXPOSURE, "garden-layout") + assert len(exposures) == (1 if "$feature_flag_variant_key" in expected else 0) + + +# A record whose reason names no rule, or that failed, reports no rule even when it +# carries one. Neither sends an exposure. The last element says whether the record is +# a valid server outcome, which the contract's event schema then accepts. +STRAY_RULE_CASES = [ + ("no_rule_match", {}, True), + ("missing_group_key", {}, True), + ("flag_disabled", {}, True), + ("dependency_error", {"failed": True, "value": None}, True), + ("error", {"failed": True, "value": None}, True), + # A failed record reports no rule, whatever its reason code. + ( + "experiment_split", + {"failed": True, "value": None, "variant_key": "annual-first"}, + True, + ), + ("holdout", {"failed": True, "value": None}, False), + # A reason code this SDK does not know. + ("future_reason", {}, False), +] + + +@pytest.mark.parametrize("path", ["get_feature_flag_result", "snapshot.get_flag"]) +@pytest.mark.parametrize( + "code, fields, valid_outcome", + STRAY_RULE_CASES, + ids=[f"{c}-{sorted(f)}" for c, f, _ in STRAY_RULE_CASES], +) +def test_rule_fields_are_sent_only_for_reasons_that_name_a_rule( + code, fields, valid_outcome, path +): + fields = {"value": False, **fields} + record = _record( + code, rule_type="experiment", rule_id=RULE, has_experiment=True, **fields + ) + + events = _read_events(path, _response(record)) + + [called] = _named(events, FLAG_CALLED) + assert _attribution(called.properties) == { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": code, + } + assert _named(events, EXPOSURE) == [] + if valid_outcome: + FLAG_CALLED_SCHEMA.validate(_projection(FLAG_CALLED_SCHEMA, called.properties)) + + +@pytest.mark.parametrize("method", ["get_flag", "is_enabled"]) +async def test_async_a_failed_split_sends_no_exposure(method): + # The record keeps its complete split metadata, but it failed. + record = _split(value=None, failed=True) + + async def read(client: Any) -> None: + snapshot = await client.evaluate_flags(DISTINCT_ID) + getattr(snapshot, method)(FLAG) + + events = await _read_async_events([_response(record)], [read]) + + [called] = _named(events, FLAG_CALLED) + assert _attribution(called.properties) == { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "experiment_split", + } + assert _named(events, EXPOSURE) == [] + + +@pytest.mark.parametrize("path", ["get_feature_flag_result", "snapshot.get_flag"]) +@pytest.mark.parametrize( + "code, rule_type", + [ + ("targeting_match", "targeted_release"), + ("targeting_match", "percentage_rollout"), + ("rollout_miss", "percentage_rollout"), + ("rollout_miss", "experiment"), + ("experiment_paused", "experiment"), + ("holdout", "experiment"), + ], +) +def test_rule_matches_misses_pauses_and_holdouts_name_their_rule(code, rule_type, path): + record = _record(code, False, rule_type=rule_type, rule_id=RULE) + + events = _read_events(path, _response(record)) + + [called] = _named(events, FLAG_CALLED) + assert _attribution(called.properties) == { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": code, + "$feature_flag_rule_id": RULE, + "$feature_flag_rule_type": rule_type, + } + assert _named(events, EXPOSURE) == [] + FLAG_CALLED_SCHEMA.validate(_projection(FLAG_CALLED_SCHEMA, called.properties)) + + +# The rule ID and rule type are sent together, with a rule type the reason allows, or +# not at all. A call that drops a broken pair keeps its config version and reason code. +BROKEN_RULE_PAIR_CASES = [ + # Half pairs. + ("targeting_match", {"rule_id": RULE}), + ("rollout_miss", {"rule_id": RULE}), + ("experiment_paused", {"rule_id": RULE}), + ("holdout", {"rule_id": RULE}), + ("targeting_match", {"rule_type": "targeted_release"}), + ("rollout_miss", {"rule_type": "percentage_rollout"}), + ("experiment_paused", {"rule_type": "experiment"}), + ("holdout", {"rule_type": "experiment"}), + # Rule types the reason code does not allow. + ("targeting_match", {"rule_id": RULE, "rule_type": "experiment"}), + ("rollout_miss", {"rule_id": RULE, "rule_type": "targeted_release"}), + ("experiment_paused", {"rule_id": RULE, "rule_type": "percentage_rollout"}), + ("holdout", {"rule_id": RULE, "rule_type": "targeted_release"}), + # A rule type this SDK does not know. + ("targeting_match", {"rule_id": RULE, "rule_type": "future_rule_type"}), +] + + +@pytest.mark.parametrize("path", ["get_feature_flag_result", "snapshot.get_flag"]) +@pytest.mark.parametrize( + "code, fields", + BROKEN_RULE_PAIR_CASES, + ids=[f"{c}-{sorted(f.items())}" for c, f in BROKEN_RULE_PAIR_CASES], +) +def test_a_half_or_disallowed_rule_pair_is_not_sent(code, fields, path): + record = _record(code, False, **fields) + + events = _read_events(path, _response(record)) + + [called] = _named(events, FLAG_CALLED) + assert _attribution(called.properties) == { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": code, + } + assert _named(events, EXPOSURE) == [] + FLAG_CALLED_SCHEMA.validate(_projection(FLAG_CALLED_SCHEMA, called.properties)) + + +@pytest.mark.parametrize("path", list(SYNC_READS)) +def test_a_targeting_match_that_names_an_experiment_rule_sends_no_exposure(path): + # Even with a variant key and a linked experiment, a targeting match assigns no + # arm, so it reports no rule and sends no exposure. + record = _record( + "targeting_match", + "annual", + rule_type="experiment", + rule_id=RULE, + variant_key="annual-first", + has_experiment=True, + ) + + events = _read_events(path, _response(record)) + + [called] = _named(events, FLAG_CALLED) + assert _attribution(called.properties) == { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "targeting_match", + } + assert _named(events, EXPOSURE) == [] + FLAG_CALLED_SCHEMA.validate(_projection(FLAG_CALLED_SCHEMA, called.properties)) + + +def test_an_older_server_record_reports_no_attribution(): + # A record without a typed value comes from a server before /flags?v=4. Its + # metadata is not read as config version 2, whatever it says. + response = _response( + { + "key": FLAG, + "enabled": True, + "variant": "annual", + "reason": {"code": "condition_match", "condition_index": 0}, + "metadata": { + "id": 77, + "version": 3, + "config_version": 2, + "rule_type": "experiment", + "rule_id": RULE, + "variant_key": "annual-first", + }, + } + ) + for path in SYNC_READS: + events = _read_events(path, response) + [called] = _named(events, FLAG_CALLED) + assert _attribution(called.properties) == {} + assert _named(events, EXPOSURE) == [] + + +# --- $feature_flag_reason ------------------------------------------------------------------ + +REASON_DESCRIPTION = "Matched rule 1" + + +@pytest.mark.parametrize("path", list(SYNC_READS)) +@pytest.mark.parametrize( + "record", + [ + _record( + "targeting_match", + rule_type="targeted_release", + rule_id=RULE, + description=REASON_DESCRIPTION, + ), + _record("no_rule_match", False, description=REASON_DESCRIPTION), + _record("error", None, failed=True, description=REASON_DESCRIPTION), + _split(description=REASON_DESCRIPTION), + ], + ids=["targeting_match", "no_rule_match", "failed", "experiment_split"], +) +def test_a_config_version_2_call_reports_its_reason_by_code_only(record, path): + events = _read_events(path, _response(record)) + + [called] = _named(events, FLAG_CALLED) + assert called.properties["$feature_flag_reason_code"] == record["reason"]["code"] + assert "$feature_flag_reason" not in called.properties + + +@pytest.mark.parametrize("path", list(SYNC_READS)) +@pytest.mark.parametrize( + "record", + [ + _record( + "condition_match", + "annual", + config_version=1, + description=REASON_DESCRIPTION, + ), + # A record from a server before /flags?v=4, which has no config version. + { + "key": FLAG, + "enabled": True, + "variant": "annual", + "reason": { + "code": "condition_match", + "condition_index": 0, + "description": REASON_DESCRIPTION, + }, + "metadata": {"id": 77, "version": 3}, + }, + ], + ids=["config_version_1", "older_server"], +) +def test_a_call_without_config_version_2_keeps_its_reason_description(record, path): + events = _read_events(path, _response(record)) + + [called] = _named(events, FLAG_CALLED) + assert called.properties["$feature_flag_reason"] == REASON_DESCRIPTION + assert _attribution(called.properties) == {} + + +@pytest.mark.parametrize( + "config_version, code, reason", + [(1, "condition_match", REASON_DESCRIPTION), (2, "targeting_match", None)], +) +def test_minimal_calls_keep_the_reason_description_only_without_config_version_2( + config_version, code, reason +): + messages: List[Dict[str, Any]] = [] + + def before_send(message: Dict[str, Any]) -> Dict[str, Any]: + messages.append(message) + return message + + client = Client(FAKE_TEST_API_KEY, send=False, before_send=before_send) + record = _record( + code, config_version=config_version, description=REASON_DESCRIPTION + ) + try: + with _serving(_response(record, minimalFlagCalledEvents=True)): + client.evaluate_flags(DISTINCT_ID).get_flag(FLAG) + client.get_feature_flag_result(FLAG, "visitor-43") + finally: + client.shutdown() + + allowlist = ( + _MINIMAL_FLAG_CALLED_EVENT_PROPERTIES + if config_version == 1 + else _MINIMAL_V2_FLAG_CALLED_EVENT_PROPERTIES + ) + assert [message["event"] for message in messages] == [FLAG_CALLED, FLAG_CALLED] + for message in messages: + assert set(message["properties"]) <= allowlist + assert message["properties"].get("$feature_flag_reason") == reason + + +def test_minimal_config_version_1_calls_drop_super_properties_named_like_attribution(): + attribution_names = ( + _MINIMAL_V2_FLAG_CALLED_EVENT_PROPERTIES - _MINIMAL_FLAG_CALLED_EVENT_PROPERTIES + ) + messages: List[Dict[str, Any]] = [] + + def before_send(message: Dict[str, Any]) -> Dict[str, Any]: + messages.append(message) + return message + + client = Client( + FAKE_TEST_API_KEY, + send=False, + before_send=before_send, + super_properties={name: "set-by-the-app" for name in attribution_names}, + ) + record = _record( + "condition_match", config_version=1, description=REASON_DESCRIPTION + ) + try: + with _serving(_response(record, minimalFlagCalledEvents=True)): + client.get_feature_flag_result(FLAG, DISTINCT_ID) + finally: + client.shutdown() + + [message] = messages + assert not attribution_names & set(message["properties"]) + + +# --- The exact events of a split --------------------------------------------------------- + + +# The attribution of a split's $feature_flag_called. +EXPECTED_ATTRIBUTION = { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "experiment_split", + "$feature_flag_rule_id": RULE, + "$feature_flag_rule_type": "experiment", + "$feature_flag_variant_key": "annual-first", +} +EXPECTED_EXPOSURE = { + "$feature_flag": FLAG, + "$feature_flag_response": "annual", + "$feature_flag_variant_key": "annual-first", + "$feature_flag_rule_id": RULE, + "locally_evaluated": False, + "$feature_flag_id": 77, + "$feature_flag_version": 3, + "$feature_flag_request_id": REQUEST_ID, + "$feature_flag_evaluated_at": EVALUATED_AT, +} + + +@pytest.mark.parametrize("path", list(SYNC_READS)) +def test_a_split_sends_both_events_with_exact_properties(path): + response = _response(_split(description="Assigned variant 'annual-first'")) + + events = _read_events(path, response, groups={"company": "company-7"}) + + assert [event.name for event in events] == [FLAG_CALLED, EXPOSURE] + called, exposure = events + assert called.properties == { + "$feature_flag": FLAG, + "$feature_flag_response": "annual", + "locally_evaluated": False, + f"$feature/{FLAG}": "annual", + "$feature_flag_request_id": REQUEST_ID, + "$feature_flag_evaluated_at": EVALUATED_AT, + # The reason code replaces the record's reason description. + "$feature_flag_version": 3, + "$feature_flag_id": 77, + "$feature_flag_has_experiment": True, + **EXPECTED_ATTRIBUTION, + } + assert exposure.properties == EXPECTED_EXPOSURE + assert exposure.distinct_id == called.distinct_id == DISTINCT_ID + assert exposure.groups == called.groups == {"company": "company-7"} + assert exposure.property_allowlist is None + + +async def test_async_split_sends_both_events_with_exact_properties(): + async def read(client: Any) -> None: + snapshot = await client.evaluate_flags( + DISTINCT_ID, groups={"company": 7}, disable_geoip=True + ) + snapshot.get_flag(FLAG) + + response = _response(_split(description="Assigned variant 'annual-first'")) + events = await _read_async_events([response], [read]) + + assert [event.name for event in events] == [FLAG_CALLED, EXPOSURE] + called, exposure = events + assert exposure.properties == EXPECTED_EXPOSURE + assert _attribution(called.properties) == EXPECTED_ATTRIBUTION + assert "$feature_flag_reason" not in called.properties + # The async client sends group keys as strings. + assert exposure.groups == called.groups == {"company": "7"} + assert exposure.distinct_id == called.distinct_id == DISTINCT_ID + + +def test_the_pair_shares_capture_identity_but_not_its_uuid(): + messages: List[Dict[str, Any]] = [] + + def before_send(message: Dict[str, Any]) -> Dict[str, Any]: + messages.append(message) + return message + + client = Client(FAKE_TEST_API_KEY, send=False, before_send=before_send) + try: + with _serving(_response(_split())): + snapshot = client.evaluate_flags( + DISTINCT_ID, groups={"company": "company-7"}, disable_geoip=True + ) + assert snapshot.get_flag(FLAG) == "annual" + finally: + client.shutdown() + + called, exposure = messages + assert (called["event"], exposure["event"]) == (FLAG_CALLED, EXPOSURE) + assert called["uuid"] and exposure["uuid"] and called["uuid"] != exposure["uuid"] + assert exposure["distinct_id"] == called["distinct_id"] == DISTINCT_ID + for name in ("$groups", "$lib", "$lib_version", "$geoip_disable"): + assert exposure["properties"][name] == called["properties"][name], name + assert exposure["properties"]["$groups"] == {"company": "company-7"} + + +@pytest.mark.parametrize("path", SNAPSHOT_READS) +def test_an_assignment_is_reported_whatever_the_read_returns(path): + # An experiment arm whose value is false still assigned the visitor to that arm. + response = _response(_split(value=False, variant_key="no-banner")) + + events = _read_events(path, response) + + [exposure] = _named(events, EXPOSURE) + assert exposure.properties["$feature_flag_response"] is False + assert exposure.properties["$feature_flag_variant_key"] == "no-banner" + + +# --- What sends no exposure ------------------------------------------------------------ + + +@pytest.mark.parametrize("path", SINGLE_FLAG_READS) +def test_opting_out_of_flag_events_suppresses_both(client, path): + recording, events = _record_events(client) + with _serving(_response(_split())), recording, warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + getattr(client, path)(FLAG, DISTINCT_ID, send_feature_flag_events=False) + + assert events == [] + + +def test_a_payload_read_sends_no_flag_events(client): + recording, events = _record_events(client) + with _serving(_response(_split())), recording, warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + client.get_feature_flag_payload(FLAG, DISTINCT_ID) + snapshot = client.evaluate_flags(DISTINCT_ID) + snapshot.get_flag_payload(FLAG) + + assert events == [] + + +def test_a_payload_read_that_sends_flag_events_also_sends_the_exposure(client): + recording, events = _record_events(client) + with _serving(_response(_split())), recording, warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + client.get_feature_flag_payload( + FLAG, DISTINCT_ID, send_feature_flag_events=True + ) + + assert [event.name for event in events] == [FLAG_CALLED, EXPOSURE] + + +MATCH_VALUE_CASES = [ + ("another value", _split(value="annual"), "monthly"), + ("the record's own value", _split(value="annual"), "annual"), + # Object and number arms both read as True, so equal values prove nothing about + # where a value came from. + ("an object arm", _split(value={"layout": "grid"}), True), + ("a number arm", _split(value=12.5), True), +] + + +@pytest.mark.parametrize("later_read", list(SYNC_READS)) +@pytest.mark.parametrize( + "record, match_value", + [case[1:] for case in MATCH_VALUE_CASES], + ids=[case[0] for case in MATCH_VALUE_CASES], +) +def test_a_payload_lookup_for_a_match_value_sends_no_exposure( + client, record, match_value, later_read +): + # The lookup reports the caller's match_value, not the record's value, so it + # sends no exposure, whatever the value. A later read of the record does. + recording, events = _record_events(client) + with _serving(_response(record)), recording, warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + client.get_feature_flag_payload( + FLAG, DISTINCT_ID, match_value=match_value, send_feature_flag_events=True + ) + [called] = events + assert called.name == FLAG_CALLED + assert called.properties["$feature_flag_response"] == match_value + # The record did not supply that value, so the call does not carry the + # record's attribution. + assert _attribution(called.properties) == {} + + _read(client, later_read) + + [exposure] = _named(events, EXPOSURE) + assert exposure.properties["$feature_flag_variant_key"] == "annual-first" + + +def test_a_stale_cached_value_sends_no_exposure(client): + # The record arrived, but caching it failed, so the read reports the stale cached + # value instead. That value did not come from the record, even though it is equal. + cache = mock.Mock() + cache._set_cached_remote_flag.side_effect = RuntimeError("cache unavailable") + cache.get_stale_cached_flag.return_value = FeatureFlagResult( + key=FLAG, enabled=True, variant="annual", payload=None, reason=None + ) + client.flag_cache = cache + recording, events = _record_events(client) + with _serving(_response(_split())), recording: + result = client.get_feature_flag_result(FLAG, DISTINCT_ID) + + assert result is not None and result.get_value() == "annual" + cache.get_stale_cached_flag.assert_called_once() + [called] = events + assert called.name == FLAG_CALLED + # The stale value is not attributed to the record's rule or variant. + assert _attribution(called.properties) == {} + + +@pytest.mark.parametrize("path", list(SYNC_READS)) +def test_a_locally_evaluated_flag_is_unchanged(path): + client = Client(FAKE_TEST_API_KEY, send=False, secret_key="secret-key") + client.feature_flags = [ + { + "id": 9, + "name": "Pricing page", + "key": FLAG, + "active": True, + "filters": { + "groups": [{"properties": [], "rollout_percentage": 100}], + "multivariate": { + "variants": [{"key": "annual", "rollout_percentage": 100}] + }, + }, + } + ] + recording, events = _record_events(client) + try: + with _serving(_response(_split())) as remote, recording: + _read(client, path) + finally: + client.shutdown() + + assert remote.call_count == 0 + [called] = events + assert called.name == FLAG_CALLED + assert called.properties["locally_evaluated"] is True + assert _attribution(called.properties) == {} + + +def test_an_empty_snapshot_sends_nothing(client): + recording, events = _record_events(client) + with _serving(_response(_split())), recording: + snapshot = client.evaluate_flags() + snapshot.get_flag(FLAG) + snapshot.is_enabled(FLAG) + + assert events == [] + + +def test_a_failed_exposure_capture_does_not_change_the_read(): + def before_send(message: Dict[str, Any]) -> Dict[str, Any]: + if message["event"] == EXPOSURE: + raise ValueError("rejected") + return message + + client = Client(FAKE_TEST_API_KEY, send=False, before_send=before_send) + try: + with _serving(_response(_split())), warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + assert client.get_feature_flag(FLAG, DISTINCT_ID) == "annual" + assert client.evaluate_flags(DISTINCT_ID).get_flag(FLAG) == "annual" + finally: + client.shutdown() + + +# --- Dedupe ------------------------------------------------------------------------ + + +@pytest.mark.parametrize( + "first, second", + [ + ("get_feature_flag", "get_feature_flag"), + ("snapshot.get_flag", "snapshot.is_enabled"), + ("feature_enabled", "snapshot.get_flag"), + ("snapshot.get_flag", "get_feature_flag_result"), + ], +) +def test_a_repeated_read_sends_nothing_new(client, first, second): + # The second evaluation has a new request ID and time, which neither event's + # dedupe reads. + later = _response( + _split(), request_id=OTHER_REQUEST_ID, evaluated_at=EVALUATED_AT + 1 + ) + recording, events = _record_events(client) + with _serving(_response(_split()), later), recording: + _read(client, first) + _read(client, second) + + assert [event.name for event in events] == [FLAG_CALLED, EXPOSURE] + + +@pytest.mark.parametrize("path", list(SYNC_READS)) +def test_equal_valued_arms_are_each_reported(path): + # An A/A experiment: both arms return the same value. + events = _read_events( + path, + _response(_split(value=True, variant_key="control")), + _response(_split(value=True, variant_key="test")), + reads=2, + ) + + exposures = _named(events, EXPOSURE) + assert [e.properties["$feature_flag_variant_key"] for e in exposures] == [ + "control", + "test", + ] + assert len(_named(events, FLAG_CALLED)) == 2 + + +@pytest.mark.parametrize("path", list(SYNC_READS)) +def test_a_later_experiment_on_the_flag_is_reported_again(path): + events = _read_events( + path, + _response(_split(rule_id=RULE)), + _response(_split(rule_id=OTHER_RULE)), + reads=2, + ) + + exposures = _named(events, EXPOSURE) + assert [e.properties["$feature_flag_rule_id"] for e in exposures] == [ + RULE, + OTHER_RULE, + ] + assert len(_named(events, FLAG_CALLED)) == 2 + + +@pytest.mark.parametrize("path", list(SYNC_READS)) +def test_the_exposure_dedupe_leaves_out_the_value(path): + # The arm's value changed, but the visitor is still in the same arm. + events = _read_events( + path, + _response(_split(value="annual")), + _response(_split(value="annual-v2")), + reads=2, + ) + + assert len(_named(events, FLAG_CALLED)) == 2 + assert len(_named(events, EXPOSURE)) == 1 + + +@pytest.mark.parametrize("path", list(SYNC_READS)) +def test_a_holdout_and_a_split_of_one_rule_are_both_reported(path): + holdout = _record( + "holdout", False, rule_type="experiment", rule_id=RULE, has_experiment=True + ) + events = _read_events( + path, _response(holdout), _response(_split(value=False)), reads=2 + ) + + called = _named(events, FLAG_CALLED) + assert [e.properties["$feature_flag_reason_code"] for e in called] == [ + "holdout", + "experiment_split", + ] + assert len(_named(events, EXPOSURE)) == 1 + + +def test_groups_are_part_of_the_exposure_identity(client): + recording, events = _record_events(client) + with _serving(_response(_split())), recording: + _read(client, "snapshot.get_flag", groups={"company": "a", "team": "b"}) + _read(client, "get_feature_flag", groups={"team": "b", "company": "a"}) + _read(client, "snapshot.get_flag", groups={"company": "c"}) + + exposures = _named(events, EXPOSURE) + assert [e.groups for e in exposures] == [ + {"company": "a", "team": "b"}, + {"company": "c"}, + ] + + +def test_the_exposure_dedupe_normalizes_the_distinct_id(client): + recording, events = _record_events(client) + with _serving(_response(_split())), recording: + _read(client, "get_feature_flag_result", distinct_id=42) + _read(client, "snapshot.get_flag", distinct_id="42") + + assert len(_named(events, EXPOSURE)) == 1 + + +def test_a_reported_call_never_suppresses_its_exposure(client): + # The two events dedupe apart. When only the exposure key is gone, as after an + # eviction, the next read sends the exposure again, although its call is a + # duplicate. + recording, events = _record_events(client) + with _serving(_response(_split())), recording: + _read(client, "get_feature_flag") + client._experiment_exposures_reported.clear() + _read(client, "get_feature_flag") + + assert [event.name for event in events] == [FLAG_CALLED, EXPOSURE, EXPOSURE] + + +async def test_async_a_reported_call_never_suppresses_its_exposure(): + async def read(client: Any) -> None: + snapshot = await client.evaluate_flags(DISTINCT_ID) + snapshot.get_flag(FLAG) + client._experiment_exposures_reported.clear() + snapshot.get_flag(FLAG) + + events = await _read_async_events([_response(_split())], [read]) + + assert [event.name for event in events] == [FLAG_CALLED, EXPOSURE, EXPOSURE] + + +def test_a_read_whose_value_the_record_did_not_supply_is_reported_as_before(client): + # Such a read sends the config version 1 call and no exposure, and takes neither + # config version 2 dedupe key: a later read of the record sends both its events. + with _serving(_response(_split(description="Assigned variant annual-first"))): + details = client.evaluate_flags(DISTINCT_ID)._flags[FLAG].details + recording, events = _record_events(client) + with recording: + for value_from_details in (False, True): + client._capture_feature_flag_called_if_needed( + distinct_id=DISTINCT_ID, + key=FLAG, + response="annual", + properties={ + "$feature_flag": FLAG, + "$feature_flag_response": "annual", + "$feature_flag_reason": "Assigned variant annual-first", + }, + details=details, + value_from_details=value_from_details, + ) + + assert [event.name for event in events] == [FLAG_CALLED, FLAG_CALLED, EXPOSURE] + as_before, attributed = _named(events, FLAG_CALLED) + assert _attribution(as_before.properties) == {} + assert as_before.properties["$feature_flag_reason"] == ( + "Assigned variant annual-first" + ) + assert _attribution(attributed.properties) == EXPECTED_ATTRIBUTION + assert "$feature_flag_reason" not in attributed.properties + + +async def test_async_snapshots_dedupe_both_events(): + async def read(client: Any) -> None: + snapshot = await client.evaluate_flags(DISTINCT_ID) + snapshot.get_flag(FLAG) + snapshot.is_enabled(FLAG) + snapshot.only([FLAG]).get_flag(FLAG) + + events = await _read_async_events( + [ + _response(_split(variant_key="control", value=True)), + _response(_split(variant_key="control", value=True)), + _response(_split(variant_key="test", value=True)), + ], + [read, read, read], + ) + + assert [e.properties["$feature_flag_variant_key"] for e in events] == [ + "control", + "control", + "test", + "test", + ] + assert [e.name for e in events] == [FLAG_CALLED, EXPOSURE] * 2 + + +def test_the_v1_dedupe_key_is_unchanged(client): + with _serving(_build(READERS, {"template": "v1"})): + _read(client, "get_feature_flag", "garden-layout") + + assert client.distinct_ids_feature_flags_reported[DISTINCT_ID] == { + ("garden-layout", "compact", ()) + } + assert len(client._experiment_exposures_reported) == 0 + + +def test_shutdown_clears_the_exposure_dedupe(): + client = Client(FAKE_TEST_API_KEY, send=False) + with _serving(_response(_split())): + _read(client, "snapshot.get_flag") + assert len(client._experiment_exposures_reported) == 1 + + client.shutdown() + + assert len(client._experiment_exposures_reported) == 0 + + +async def test_async_shutdown_clears_the_exposure_dedupe(): + with mock.patch( + "posthog.async_client._async_flags", + new=mock.AsyncMock(return_value=_response(_split())), + ): + client = AsyncPosthog("project-key", send=False) + (await client.evaluate_flags(DISTINCT_ID)).get_flag(FLAG) + assert len(client._experiment_exposures_reported) == 1 + await client.shutdown() + + assert len(client._experiment_exposures_reported) == 0 + + +# --- Minimal events ------------------------------------------------------------------- + + +def test_minimal_flag_called_events_keep_the_attribution(): + messages: List[Dict[str, Any]] = [] + + def before_send(message: Dict[str, Any]) -> Dict[str, Any]: + messages.append(message) + return message + + client = Client( + FAKE_TEST_API_KEY, + send=False, + before_send=before_send, + super_properties={"plan": "team"}, + ) + # A split of an experiment rule no experiment links to: the flag reports no + # experiment, so its $feature_flag_called is minimal. + response = _response(_split(has_experiment=False), minimalFlagCalledEvents=True) + try: + with _serving(response): + client.evaluate_flags(DISTINCT_ID).get_flag(FLAG) + finally: + client.shutdown() + + called, exposure = messages + assert set(called["properties"]) <= _MINIMAL_V2_FLAG_CALLED_EVENT_PROPERTIES + assert "plan" not in called["properties"] + assert _attribution(called["properties"]) == EXPECTED_ATTRIBUTION + # The exposure is a full event. + assert exposure["event"] == EXPOSURE + assert exposure["properties"]["plan"] == "team" + assert _attribution(exposure["properties"]) == _attribution(EXPECTED_EXPOSURE) + + +# --- Reason codes, rule IDs and variant keys --------------------------------------------- + + +@pytest.mark.parametrize("path", ["get_feature_flag_result", "snapshot.get_flag"]) +@pytest.mark.parametrize("code", ["future_reason", None], ids=["unknown", "missing"]) +def test_an_unknown_or_missing_reason_code_still_reports_config_version_2(code, path): + record = _split() + if code is None: + del record["reason"] + else: + record["reason"]["code"] = code + + events = _read_events(path, _response(record)) + + [called] = _named(events, FLAG_CALLED) + expected: Dict[str, Any] = {"$feature_flag_config_version": 2} + if code is not None: + expected["$feature_flag_reason_code"] = code + assert _attribution(called.properties) == expected + assert "$feature_flag_reason" not in called.properties + assert _named(events, EXPOSURE) == [] + FLAG_CALLED_SCHEMA.validate(_projection(FLAG_CALLED_SCHEMA, called.properties)) + + +NOT_A_UUID = ["rule-1", "", "7F3C2A108D4E4B6A9C215E0F1A2B3C4D", "{" + RULE + "}"] + + +@pytest.mark.parametrize("path", ["get_feature_flag_result", "snapshot.get_flag"]) +@pytest.mark.parametrize("rule_id", NOT_A_UUID) +def test_a_rule_id_that_is_not_a_uuid_counts_as_missing(rule_id, path): + for record in ( + _split(rule_id=rule_id), + _record("targeting_match", rule_type="targeted_release", rule_id=rule_id), + ): + events = _read_events(path, _response(record)) + + [called] = _named(events, FLAG_CALLED) + assert _attribution(called.properties) == { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": record["reason"]["code"], + } + assert _named(events, EXPOSURE) == [] + FLAG_CALLED_SCHEMA.validate(_projection(FLAG_CALLED_SCHEMA, called.properties)) + + +def test_an_upper_case_uuid_rule_id_is_kept(): + events = _read_events( + "get_feature_flag_result", _response(_split(rule_id=RULE.upper())) + ) + + [exposure] = _named(events, EXPOSURE) + assert exposure.properties["$feature_flag_rule_id"] == RULE.upper() + + +@pytest.mark.parametrize("path", list(SYNC_READS)) +def test_an_empty_variant_key_is_a_complete_split(path): + events = _read_events(path, _response(_split(variant_key=""))) + + [called] = _named(events, FLAG_CALLED) + assert called.properties["$feature_flag_variant_key"] == "" + [exposure] = _named(events, EXPOSURE) + assert exposure.properties["$feature_flag_variant_key"] == "" + EXPOSURE_SCHEMA.validate(_projection(EXPOSURE_SCHEMA, exposure.properties)) + + +# --- Capture failures ---------------------------------------------------------------------- + + +def _dropping(event: str) -> Callable[[Dict[str, Any]], Optional[Dict[str, Any]]]: + """A before_send hook that drops the first ``event`` and keeps everything else.""" + dropped: List[str] = [] + + def before_send(message: Dict[str, Any]) -> Optional[Dict[str, Any]]: + if message["event"] == event and not dropped: + dropped.append(event) + return None + return message + + return before_send + + +def _failing_once(client: Client, event: str, failure: str) -> Any: + """Patch ``client.capture`` to fail for the first ``event``, then capture as usual. + + ``failure`` is ``"raise"`` for a raising capture, or ``"unqueued"`` for a capture + that returns ``None`` without ``before_send`` dropping the event, as a full or + closed queue, opt-out or a disabled client does. + """ + capture = client.capture + failed: List[str] = [] + + def maybe_fail(name: str, **kwargs: Any) -> Optional[str]: + if name == event and not failed: + failed.append(name) + if failure == "raise": + raise RuntimeError("capture failed") + return None + return capture(name, **kwargs) + + return mock.patch.object(client, "capture", side_effect=maybe_fail) + + +def _sent(client: Client, path: str, reads: int = 2) -> List[str]: + """The events that reached the queue over ``reads`` reads of a complete split.""" + sent: List[str] = [] + enqueue = client._enqueue + + def record(msg: Dict[str, Any], *args: Any, **kwargs: Any) -> Optional[str]: + uuid = enqueue(msg, *args, **kwargs) + if uuid is not None: + sent.append(msg["event"]) + return uuid + + with ( + _serving(_response(_split())), + mock.patch.object(client, "_enqueue", side_effect=record), + ): + for _ in range(reads): + assert _read(client, path) == "annual" + return sent + + +@pytest.mark.parametrize("path", ["get_feature_flag", "snapshot.get_flag"]) +@pytest.mark.parametrize("event", [FLAG_CALLED, EXPOSURE]) +def test_an_event_dropped_by_before_send_keeps_its_dedupe_key(event, path): + client = Client(FAKE_TEST_API_KEY, send=False, before_send=_dropping(event)) + try: + sent = _sent(client, path) + finally: + client.shutdown() + + # The application dropped the event, so the second read does not send it again. + # The other event of the first read is still sent, once. + assert sent == [EXPOSURE if event == FLAG_CALLED else FLAG_CALLED] + + +async def test_async_an_event_dropped_by_before_send_keeps_its_dedupe_key(): + seen: List[str] = [] + batches: List[Any] = [] + + def before_send(message: Dict[str, Any]) -> Optional[Dict[str, Any]]: + seen.append(message["event"]) + return None if message["event"] == EXPOSURE else message + + async def batch_post(*args: Any, **kwargs: Any) -> None: + batches.append(kwargs["batch"]) + + with ( + mock.patch( + "posthog.async_client._async_flags", + new=mock.AsyncMock(return_value=_response(_split())), + ), + mock.patch("posthog._async_consumer.async_batch_post", side_effect=batch_post), + ): + client = AsyncPosthog("project-key", before_send=before_send, flush_interval=30) + for _ in range(2): + snapshot = await client.evaluate_flags(DISTINCT_ID) + assert snapshot.get_flag(FLAG) == "annual" + await client.flush(timeout_seconds=1) + await client.shutdown() + + # before_send saw each event once: the dropped exposure kept its key, so the + # second read sent neither event again. + assert seen == [FLAG_CALLED, EXPOSURE] + sent = [event["event"] for batch in batches for event in batch] + assert sent == [FLAG_CALLED] + + +@pytest.mark.parametrize("path", ["get_feature_flag", "snapshot.get_flag"]) +@pytest.mark.parametrize("event", [FLAG_CALLED, EXPOSURE]) +@pytest.mark.parametrize("failure", ["raise", "unqueued"]) +def test_a_failed_capture_keeps_no_dedupe_key(failure, event, path, client): + with _failing_once(client, event, failure): + sent = _sent(client, path) + + # The other event of the first read is still sent, and the failed one is sent + # by the second read, once. + assert sorted(sent) == sorted([FLAG_CALLED, EXPOSURE]) + + +def test_an_unqueued_config_version_1_call_is_deduped_as_before(client): + record = _record("condition_match", "annual", config_version=1) + calls: List[str] = [] + + def unqueued(name: str, **kwargs: Any) -> None: + calls.append(name) + return None + + with ( + _serving(_response(record)), + mock.patch.object(client, "capture", side_effect=unqueued), + ): + _read(client, "get_feature_flag") + _read(client, "get_feature_flag") + + # Config version 1 dedupe is unchanged: the attempt takes the key. + assert calls == [FLAG_CALLED] + + +class _Rendezvous: + """Holds each of the first two callers until the other one arrives. + + When the dedupe check and the reservation are one step, the second read waits + for the first one to finish both, so the wait times out instead. + """ + + def __init__(self) -> None: + self._barrier = threading.Barrier(2, timeout=0.25) + self._lock = threading.Lock() + self.arrivals = 0 + + def __call__(self) -> None: + with self._lock: + self.arrivals += 1 + arrival = self.arrivals + if arrival <= 2: + try: + self._barrier.wait() + except threading.BrokenBarrierError: + pass + + +class _MeetingSet(set): + """A dedupe set: each membership check reads the set, then meets the other read.""" + + def __init__(self, meet: _Rendezvous) -> None: + super().__init__() + self._meet = meet + + def __contains__(self, item: Any) -> bool: + found = super().__contains__(item) + self._meet() + return found + + +class _MeetingDict(SizeLimitedDict): + """A dedupe map: each lookup reads the map, then meets the other read.""" + + def __init__(self, meet: _Rendezvous) -> None: + super().__init__(MAX_DICT_SIZE, set) + self._meet = meet + + def get(self, key: Any, default: Any = None) -> Any: + found = super().get(key, default) + self._meet() + return found + + +@pytest.mark.parametrize("event", [FLAG_CALLED, EXPOSURE]) +@pytest.mark.parametrize("distinct_id", ["seen", "new"]) +def test_concurrent_reads_of_one_outcome_send_each_event_once( + client, event, distinct_id +): + # Two reads of one outcome both look up the dedupe key before either goes on. + # Unless the check and the reservation are one step, both find no key and both + # send the event. "new" also races on the distinct ID's first entry. + meet = _Rendezvous() + attribute = ( + "distinct_ids_feature_flags_reported" + if event == FLAG_CALLED + else "_experiment_exposures_reported" + ) + if distinct_id == "seen": + getattr(client, attribute)[DISTINCT_ID] = _MeetingSet(meet) + else: + setattr(client, attribute, _MeetingDict(meet)) + recording, events = _record_events(client) + with _serving(_response(_split())), recording: + reads = [ + threading.Thread(target=_read, args=(client, "get_feature_flag")) + for _ in range(2) + ] + for read in reads: + read.start() + for read in reads: + read.join(5) + + assert meet.arrivals == 2 + sent = [e.name for e in events if e.uuid is not None] + assert sorted(sent) == sorted([FLAG_CALLED, EXPOSURE]) + + +@pytest.mark.parametrize("delivered", [True, False], ids=["delivered", "failed"]) +def test_a_read_during_a_sync_mode_send_does_not_send_it_again(delivered): + # The second read runs while the first read's exposure is on the wire. It sends + # neither event. A send that fails releases the key, so a third read sends the + # exposure. + client = Client(FAKE_TEST_API_KEY, sync_mode=True, capture_mode="v0") + sending = threading.Event() + finish = threading.Event() + posted: List[str] = [] + + def batch_post(*args: Any, batch: List[Dict[str, Any]], **kwargs: Any) -> None: + [message] = batch + posted.append(message["event"]) + if message["event"] == EXPOSURE and posted.count(EXPOSURE) == 1: + sending.set() + finish.wait(5) + if not delivered: + raise RuntimeError("connection reset") + + try: + with ( + _serving(_response(_split())), + mock.patch("posthog.client.batch_post", side_effect=batch_post), + ): + first = threading.Thread(target=_read, args=(client, "get_feature_flag")) + first.start() + assert sending.wait(5) + assert _read(client, "get_feature_flag") == "annual" + finish.set() + first.join(5) + assert _read(client, "get_feature_flag") == "annual" + finally: + finish.set() + client.shutdown() + + assert posted == [FLAG_CALLED, EXPOSURE] + ([] if delivered else [EXPOSURE]) + + +def test_a_forked_child_replaces_the_dedupe_lock(client): + # A parent thread can hold the lock when the process forks. The child replaces + # it, so its reads do not wait for a thread the child does not have. + with posthog_client._flag_event_keys_lock: + posthog_client._reinit_flag_event_keys_lock_after_fork() + events = _read_events("get_feature_flag", _response(_split()), client=client) + + assert [event.name for event in events] == [FLAG_CALLED, EXPOSURE] + + +def test_a_flags_reload_does_not_repeat_an_exposure(client): + recording, events = _record_events(client) + with _serving(_response(_split())), recording: + _read(client, "get_feature_flag") + client._update_flag_state( + {"flags": [], "group_type_mapping": {}, "cohorts": {}} + ) + _read(client, "get_feature_flag") + client.evaluate_flags(DISTINCT_ID).get_flag(FLAG) + + assert len(_named(events, EXPOSURE)) == 1 + + +# --- Super properties ------------------------------------------------------------------------ + +SPOOFED_SUPER_PROPERTIES = { + "$feature_flag_config_version": 7, + "$feature_flag_reason_code": "spoofed", + "$feature_flag_rule_type": "spoofed", + "$feature_flag_rule_id": OTHER_RULE, + "$feature_flag_variant_key": "spoofed", + "$feature_flag_reason": "spoofed", + "$feature_flag_forced_variant": True, + "$feature_flag_experiment_id": 301, + "$experiment_exposure_source": "spoofed", + # The flag identity of the read. + "$feature_flag": "spoofed", + "$feature_flag_response": "spoofed", + "locally_evaluated": True, +} + +# What both events of a split read report about the read itself. +EXPECTED_FLAG_IDENTITY = { + "$feature_flag": FLAG, + "$feature_flag_response": "annual", + "locally_evaluated": False, +} + + +def _flag_identity(properties: Mapping[str, Any]) -> Dict[str, Any]: + return { + name: properties[name] for name in EXPECTED_FLAG_IDENTITY if name in properties + } + + +def _messages_with_super_properties(record: Dict[str, Any]) -> List[Dict[str, Any]]: + messages: List[Dict[str, Any]] = [] + + def before_send(message: Dict[str, Any]) -> Dict[str, Any]: + messages.append(copy.deepcopy(message)) + return message + + client = Client( + FAKE_TEST_API_KEY, + send=False, + before_send=before_send, + super_properties=dict(SPOOFED_SUPER_PROPERTIES), + ) + try: + with _serving(_response(record)): + _read(client, "get_feature_flag") + finally: + client.shutdown() + return messages + + +def test_super_properties_cannot_change_config_version_2_attribution(): + messages = _messages_with_super_properties(_split()) + + [called] = [m for m in messages if m["event"] == FLAG_CALLED] + assert _attribution(called["properties"]) == EXPECTED_ATTRIBUTION + for name in ( + "$feature_flag_reason", + "$feature_flag_forced_variant", + "$feature_flag_experiment_id", + ): + assert name not in called["properties"], name + [exposure] = [m for m in messages if m["event"] == EXPOSURE] + assert _attribution(exposure["properties"]) == _attribution(EXPECTED_EXPOSURE) + assert not set(EXPOSURE_LEAVES_OUT) & set(exposure["properties"]) + for name in ("$feature_flag_experiment_id", "$feature_flag_forced_variant"): + assert name not in exposure["properties"], name + EXPOSURE_SCHEMA.validate(_projection(EXPOSURE_SCHEMA, exposure["properties"])) + + +def test_super_properties_cannot_change_the_flag_identity_of_config_version_2_events(): + messages = _messages_with_super_properties(_split()) + + # Both events report the same flag, response and evaluation source. + [called] = [m for m in messages if m["event"] == FLAG_CALLED] + assert _flag_identity(called["properties"]) == EXPECTED_FLAG_IDENTITY + [exposure] = [m for m in messages if m["event"] == EXPOSURE] + assert _flag_identity(exposure["properties"]) == EXPECTED_FLAG_IDENTITY + + +def test_super_properties_cannot_add_rule_fields_a_config_version_2_call_omits(): + messages = _messages_with_super_properties(_record("no_rule_match", False)) + + [called] = messages + assert _attribution(called["properties"]) == { + "$feature_flag_config_version": 2, + "$feature_flag_reason_code": "no_rule_match", + } + + +def test_super_properties_still_apply_to_config_version_1_calls(): + record = _record("condition_match", "annual", config_version=1) + [called] = _messages_with_super_properties(record) + + for name, value in SPOOFED_SUPER_PROPERTIES.items(): + assert called["properties"][name] == value, name + + +async def test_async_super_properties_cannot_change_config_version_2_attribution(): + messages: List[Dict[str, Any]] = [] + with mock.patch( + "posthog.async_client._async_flags", + new=mock.AsyncMock(return_value=_response(_split())), + ): + client = AsyncPosthog( + "project-key", + send=False, + super_properties=dict(SPOOFED_SUPER_PROPERTIES), + ) + prepare = client._prepare_event + + def record(*args: Any, **kwargs: Any) -> Any: + prepared, uuid = prepare(*args, **kwargs) + messages.append(copy.deepcopy(prepared)) + return prepared, uuid + + with mock.patch.object(client, "_prepare_event", side_effect=record): + snapshot = await client.evaluate_flags(DISTINCT_ID) + snapshot.get_flag(FLAG) + await client.shutdown() + + [called] = [m for m in messages if m["event"] == FLAG_CALLED] + assert _attribution(called["properties"]) == EXPECTED_ATTRIBUTION + assert _flag_identity(called["properties"]) == EXPECTED_FLAG_IDENTITY + assert "$feature_flag_reason" not in called["properties"] + [exposure] = [m for m in messages if m["event"] == EXPOSURE] + assert _attribution(exposure["properties"]) == _attribution(EXPECTED_EXPOSURE) + assert _flag_identity(exposure["properties"]) == EXPECTED_FLAG_IDENTITY + assert not set(EXPOSURE_LEAVES_OUT) & set(exposure["properties"]) diff --git a/posthog/test/test_flags_released_behaviour.py b/posthog/test/test_flags_released_behaviour.py index 3141dd891..7f5dd12b9 100644 --- a/posthog/test/test_flags_released_behaviour.py +++ b/posthog/test/test_flags_released_behaviour.py @@ -30,7 +30,7 @@ def results(request: Any) -> Dict[str, Any]: return scenarios.run_all(request.param) -def test_every_legacy_getter_and_event_matches_7_82_1(results): +def test_every_legacy_getter_and_event_matches_7_82_1(request, results): expected = copy.deepcopy(RELEASED["getters"]) # A record without a flag ID or version reads them as absent (None), where 7.82.1 # read 0. Events already left out both, so only the parsed record differs. @@ -58,6 +58,21 @@ def test_every_legacy_getter_and_event_matches_7_82_1(results): for properties in failed_events: assert properties.pop("$feature_flag_payload") == {"columns": 2} + if request.node.callspec.params["results"] == "v4": + # A $feature_flag_called for a config version 2 record carries its config + # version and reason code in place of the reason text. Only the v4 shape says + # a record is config version 2; config version 1 events are unchanged. + for name, properties in expected["events"]: + if name == "$feature_flag_called" and properties[ + "$feature_flag" + ].startswith("v2-"): + assert ( + properties.pop("$feature_flag_reason") + == scenarios.MATCHED["description"] + ) + properties["$feature_flag_config_version"] = 2 + properties["$feature_flag_reason_code"] = scenarios.MATCHED["code"] + assert results["getters"] == expected diff --git a/posthog/test/test_flags_v4_response.py b/posthog/test/test_flags_v4_response.py index 30e22a7fb..352c793aa 100644 --- a/posthog/test/test_flags_v4_response.py +++ b/posthog/test/test_flags_v4_response.py @@ -1,8 +1,8 @@ -"""Reading the ``/flags?v=4`` response, checked against the released wire contract. +"""Reading the ``/flags?v=4`` response, checked against the harness wire contract. -The files in ``fixtures/feature_flag_rules_v2`` are copied unchanged from a released -``PostHog/posthog-sdk-test-harness`` contract. ``SOURCE.json`` records the release and -the digest of each file. +The files in ``fixtures/feature_flag_rules_v2`` are copied unchanged from a +``PostHog/posthog-sdk-test-harness`` contract. ``SOURCE.json`` records the revision, +its release and the digest of each file. """ from __future__ import annotations @@ -83,8 +83,9 @@ def _evaluate(client: Client, response: Dict[str, Any]) -> FeatureFlagEvaluation def test_vendored_fixtures_match_the_recorded_release_digests(): source = _load("SOURCE.json") - assert source["release"] == "1.22.0" - assert source["contract_version"] == "7.0.0" + assert source["release"] == "1.30.0" + assert source["revision"] == "6414114501396d96aadf2412fdd8054a03076782" + assert source["contract_version"] == "10.0.0" for name, entry in source["files"].items(): digest = hashlib.sha256((FIXTURES / name).read_bytes()).hexdigest() assert digest == entry["sha256"], name @@ -1112,11 +1113,27 @@ def _parity_response(shape: str) -> Dict[str, Any]: } +# A config version 2 record adds these to its flag events, and leaves out +# $feature_flag_reason. A v2-shaped response has no config version 2 records, so they +# are left out of the comparison of the v2-* flags and checked in +# test_experiment_exposure.py. +_CONFIG_VERSION_2_EVENT_PROPERTIES = { + "$feature_flag_config_version", + "$feature_flag_reason_code", + "$feature_flag_rule_id", + "$feature_flag_rule_type", + "$feature_flag_variant_key", +} + + def _flag_event_properties(message: Dict[str, Any]) -> Dict[str, Any]: + left_out = set(_CONFIG_VERSION_2_EVENT_PROPERTIES) + if str(message["properties"].get("$feature_flag", "")).startswith("v2-"): + left_out.add("$feature_flag_reason") return { key: value for key, value in message["properties"].items() - if key.startswith("$feature") + if (key.startswith("$feature") and key not in left_out) or key in ("$active_feature_flags", "locally_evaluated") } @@ -1132,7 +1149,8 @@ def response(*args: Any, **kwargs: Any) -> Dict[str, Any]: def enqueue(message: Dict[str, Any], *args: Any, **kwargs: Any) -> Optional[str]: events.append((message["event"], _flag_event_properties(message))) - return None + # Accept the event, as the real queue does, by returning an event UUID. + return "70000000-0000-4000-8000-000000000000" try: with (