Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 19 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -377,11 +377,22 @@ Script generates:
- MAQL transformation log to `metrics_maql.log`
- Failed Cloud metrics to `cloud_failed_metrics.json`
- Skipped objects IDs to `cloud_skipped_metrics.json`
- Computed attribute mappings to `computed_attribute_mappings.csv`
- Failed / skipped computed attributes to `cloud_failed_computed_attributes.json` / `cloud_skipped_computed_attributes.json`
- Computed attribute dumps to `legacy_computed_attributes.json` / `cloud_computed_attributes.json` (with `--dump-legacy` / `--dump-cloud`)

Script reads:

- LDM mappings from `ldm_mappings.csv`

Computed attributes:

- Legacy computed attributes (`attr.comp.X` + `label.comp.X`) are migrated together with metrics as Cloud computed attributes with ID `label.comp.X`. The Legacy `CASE WHEN` definition is translated to `SELECT CASE WHEN SUM({metric/...}) <= n THEN "bucket" ... END BY {label/...} ALL OTHER`.
- Metrics, insights and dashboards reference them as `{computed_attribute/label.comp.X}` / `"type": "computedAttribute"`.
- Deployment order is: metrics → computed attributes → metrics using computed attributes.
- Requires the `enableComputedAttributes` feature flag in Cloud, on the parent workspace and on every child workspace that uses them (it is not inherited). Cloud computed attributes are in Beta: exports, date dimension and period-over-period are not supported with them.
- Only one-metric, numeric-threshold definitions are supported; other definitions are logged as errors and skipped.

Notes:

- If any element values are missing within the Legacy metric, they are replaced with `--MISSING VALUE--`. Such metrics can be found in `metrics_maql.log` file and these metrics are also marked with [WARN] in their title (unless `--suppress-migration-warnings` is used). Warnings are always printed to console.
Expand Down Expand Up @@ -561,7 +572,7 @@ gooddata-legacy2cloud dashboard-permissions
**What this script does:**

1. Fetches selected Legacy dashboards based on filter parameters
2. Identifies the creator (account that created the dashboard) from Legacy
2. Identifies the creator (`author`) and last modifier (`contributor`) from Legacy
3. Collects sharing information (grantees) from Legacy dashboard permissions
4. Maps Legacy users to Cloud users based on either Legacy `email` field or Legacy `login` field
5. Maps Legacy user groups to Cloud user groups based on group names
Expand All @@ -588,6 +599,10 @@ gooddata-legacy2cloud dashboard-permissions

**Dashboard Permissions-specific options:**

**--include-metrics** - Also migrate creator/modifier of metrics (Legacy `author` → Cloud `createdBy`, Legacy `contributor` → Cloud `modifiedBy`). Metrics have no permissions, only these two fields are updated. Reads `metric_mappings.csv` (`--metric-mapping-file`).

**--include-insights** - Also migrate creator/modifier of insights, same as above. Reads `insight_mappings.csv` (`--insight-mapping-file`).

**--dump-layout** - Store the layout JSON locally before and after modifications for debugging purposes

**--use-email** - Use Legacy email field instead of login field when matching Legacy users to Cloud users. By default, the script uses the login field from Legacy users to match against Cloud user emails. With this parameter it uses email field from Legacy users to match against Cloud user emails.
Expand Down Expand Up @@ -625,6 +640,9 @@ gooddata-legacy2cloud dashboard-permissions --permission VIEW

# Update with client prefix
gooddata-legacy2cloud dashboard-permissions --client-prefix client1_ --legacy-ws client1_legacy_ws --cloud-ws client1_cloud_ws

# Also set creator/modifier of metrics and insights
gooddata-legacy2cloud dashboard-permissions --include-metrics --include-insights
```

**Important notes:**
Expand Down
12 changes: 12 additions & 0 deletions src/gooddata_legacy2cloud/arg_parsing/arg_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -648,6 +648,18 @@ def parse_dashboard_permission_cli_args() -> argparse.Namespace:
add_object_filter_arguments(parser)
add_dashboard_mapping_arguments(parser)
add_pp_dashboard_mapping_arguments(parser)
add_metric_mapping_arguments(parser)
add_insight_mapping_arguments(parser)
parser.add_argument(
"--include-metrics",
action="store_true",
help="Also migrate creator/modifier (author/contributor) of metrics",
)
parser.add_argument(
"--include-insights",
action="store_true",
help="Also migrate creator/modifier (author/contributor) of insights",
)
parser.add_argument(
"--dump-layout",
action="store_true",
Expand Down
22 changes: 22 additions & 0 deletions src/gooddata_legacy2cloud/backends/cloud/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,28 @@ def update_metric(self, metric_id, metric):
url = f"api/v1/entities/workspaces/{self.ws}/metrics/{metric_id}"
return self._put(url, metric)

def get_computed_attributes(self, page=0, origin: str = "ALL"):
"""Sends a GET request to the server to get all computed attributes."""
url = f"api/v1/entities/workspaces/{self.ws}/computedAttributes?origin={origin}&size={CLOUD_REQUEST_PAGE_SIZE}&page={page}"
try:
result = self._get(url)
data = result["data"]
if len(data) == CLOUD_REQUEST_PAGE_SIZE:
data.extend(self.get_computed_attributes(page + 1))
return data
except KeyError:
return []

def create_computed_attribute(self, computed_attribute):
"""Post a computed attribute to the server."""
url = f"api/v1/entities/workspaces/{self.ws}/computedAttributes"
return self._post(url, computed_attribute)

def update_computed_attribute(self, computed_attribute_id, computed_attribute):
"""Update a computed attribute on the server."""
url = f"api/v1/entities/workspaces/{self.ws}/computedAttributes/{computed_attribute_id}"
return self._put(url, computed_attribute)

def remove_metric(self, metric_id):
"""Sends a DELETE request to the server."""
url = f"api/v1/entities/workspaces/{self.ws}/metrics/{metric_id}"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@
CLOUD_FAILED_REPORTS_FILE = "cloud_failed_reports.json"
CLOUD_FAILED_PP_DASHBOARDS_FILE = "cloud_failed_pp_dashboards.json"
CLOUD_SKIPPED_METRICS_FILE = "cloud_skipped_metrics.json"
CLOUD_FAILED_COMPUTED_ATTRIBUTES_FILE = "cloud_failed_computed_attributes.json"
CLOUD_SKIPPED_COMPUTED_ATTRIBUTES_FILE = "cloud_skipped_computed_attributes.json"
CLOUD_SKIPPED_INSIGHTS_FILE = "cloud_skipped_insights.json"
CLOUD_SKIPPED_KPI_COMPARISON_INSIGHTS_FILE = (
"cloud_skipped_kpi_comparison_insights.json"
Expand Down Expand Up @@ -147,6 +149,28 @@ def get_object_type(self) -> str:
return "metric"


class ComputedAttributeCreationStrategy(CreationStrategy):
"""Strategy for creating computed attributes"""

def get_existing_objects(self) -> List[Dict]:
return self.cloud_client.get_computed_attributes()

def create_object(self, obj: T) -> Any:
return self.cloud_client.create_computed_attribute(obj)

def update_object(self, obj: T) -> Any:
return self.cloud_client.update_computed_attribute(obj["data"]["id"], obj)

def disable_object(self, obj: T, response: Response) -> Any:
return obj # no placeholder for computed attributes

def get_object_link(self, domain: str, workspace_id: str, obj_id: str) -> str:
return f"{domain}/api/v1/entities/workspaces/{workspace_id}/computedAttributes/{obj_id}"

def get_object_type(self) -> str:
return "computed_attribute"


class InsightCreationStrategy(CreationStrategy):
"""Strategy for creating insights"""

Expand Down Expand Up @@ -463,6 +487,12 @@ class ObjectConfig(TypedDict):
"skipped_file": CLOUD_SKIPPED_METRICS_FILE,
"log_file_mode": "w",
},
"computed_attribute": {
"strategy_class": ComputedAttributeCreationStrategy,
"failed_file": CLOUD_FAILED_COMPUTED_ATTRIBUTES_FILE,
"skipped_file": CLOUD_SKIPPED_COMPUTED_ATTRIBUTES_FILE,
"log_file_mode": "w",
},
"insight": {
"strategy_class": InsightCreationStrategy,
"failed_file": CLOUD_FAILED_INSIGHTS_FILE,
Expand Down
17 changes: 17 additions & 0 deletions src/gooddata_legacy2cloud/config/configuration_objects.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,25 @@ class DashboardPermissionsConfig(BaseConfig):
+ "is treated as default and also used for output, all files are used "
+ f"for input. Default: [{PIXEL_PERFECT_DASHBOARD_MAPPING_FILE}]",
)
metric_mapping_file: list[str] = Field(
default_factory=lambda: [METRIC_MAPPING_FILE],
description=f"Metric mappings file name(s). Default: [{METRIC_MAPPING_FILE}]",
)
insight_mapping_file: list[str] = Field(
default_factory=lambda: [INSIGHT_MAPPING_FILE],
description="Insight mappings file name(s). "
+ f"Default: [{INSIGHT_MAPPING_FILE}]",
)

# Custom migration arguments
include_metrics: bool = Field(
default=False,
description="Also migrate creator/modifier of metrics",
)
include_insights: bool = Field(
default=False,
description="Also migrate creator/modifier of insights",
)
dump_layout: bool = Field(
default=False,
description="Save the layout JSON before and after modifications",
Expand Down
1 change: 1 addition & 0 deletions src/gooddata_legacy2cloud/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

LDM_MAPPING_FILE = "ldm_mappings.csv"
METRIC_MAPPING_FILE = "metric_mappings.csv"
COMPUTED_ATTRIBUTE_MAPPING_FILE = "computed_attribute_mappings.csv"
INSIGHT_MAPPING_FILE = "insight_mappings.csv"
DASHBOARD_MAPPING_FILE = "dashboard_mappings.csv"
REPORT_MAPPING_FILE = "report_mappings.csv"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,15 @@ def format_change_line(change: ActualChange) -> str:
Formatted string containing dashboard, action, and target context.
"""

prefix = f" Dashboard: '{change.dashboard_title}' (ID: {change.dashboard_id}) - "
prefix = (
f" {change.object_type.capitalize()}: '{change.dashboard_title}' "
f"(ID: {change.dashboard_id}) - "
)

if change.change_type == "creator_updated":
return (
f"{prefix}Changed creator from '{change.old_value}' to '{change.new_value}'"
f"{prefix}Changed creator / modifier from '{change.old_value}' "
f"to '{change.new_value}'"
)

if change.change_type == "permission_removed":
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ class ActualChange:
new_value: str = field(default="")
old_permission: str = field(default="")
new_permission: str = field(default="")
object_type: str = field(default="dashboard")


@dataclass(frozen=True)
Expand Down Expand Up @@ -67,3 +68,5 @@ class DashboardPermissionContext:
keep_existing_permissions: bool = field(default=False)
print_user_mappings: bool = field(default=False)
client_prefix: str | None = field(default=None)
metric_mappings: IdMappings | None = field(default=None)
insight_mappings: IdMappings | None = field(default=None)
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from gooddata_legacy2cloud.dashboard_permissions.data_classes import (
DashboardPermissionContext,
)
from gooddata_legacy2cloud.id_mappings import IdMappings
from gooddata_legacy2cloud.layout.layout_manager import (
update_object_creators_in_layout,
)
Expand Down Expand Up @@ -47,6 +48,8 @@ def __init__(self, context: DashboardPermissionContext):
"""
self.ctx = context
self.legacy_dashboards: list[dict] = []
self.legacy_metrics: list[dict] = []
self.legacy_insights: list[dict] = []
self.legacy_user_map: dict[str, str] = {}
self.cloud_user_map: dict[str, str] = {}
self.user_mappings: dict[str, UserMapping] = {}
Expand All @@ -59,18 +62,26 @@ def __init__(self, context: DashboardPermissionContext):
self.identifier_plural = f"{self.identifier_type}s"

def process_dashboards(
self, legacy_dashboards: list[dict]
self,
legacy_dashboards: list[dict],
legacy_metrics: list[dict] | None = None,
legacy_insights: list[dict] | None = None,
) -> tuple[list[ObjectUpdate], dict[str, int]]:
"""
Process Legacy dashboards and prepare permission updates.
Process Legacy dashboards (permissions + creators) and optionally
metrics and insights (creators only) and prepare layout updates.

Args:
legacy_dashboards: List of Legacy dashboard objects
legacy_metrics: List of Legacy metric objects
legacy_insights: List of Legacy insight objects

Returns:
tuple: (list of ObjectUpdate instances, skip statistics dictionary)
"""
self.legacy_dashboards = legacy_dashboards
self.legacy_metrics = legacy_metrics or []
self.legacy_insights = legacy_insights or []

# Fetch and map users
self._fetch_and_map_users()
Expand Down Expand Up @@ -118,10 +129,12 @@ def _fetch_and_map_users(self) -> None:
legacy_users, self.identifier_type
)

# Extract creator IDs
user_ids = extract_creator_modifier_ids(self.legacy_dashboards)
# Extract creator and modifier IDs
user_ids = extract_creator_modifier_ids(
self.legacy_dashboards + self.legacy_metrics + self.legacy_insights
)
logger.info(
"Found %s unique creator %s in dashboards",
"Found %s unique creator/modifier %s in objects",
len(user_ids),
self.identifier_plural,
)
Expand Down Expand Up @@ -317,8 +330,63 @@ def _build_object_updates(self) -> tuple[list[ObjectUpdate], dict[str, int]]:
if update:
object_updates.append(update)

for objects, object_type, mappings in (
(self.legacy_metrics, "metric", self.ctx.metric_mappings),
(self.legacy_insights, "insight", self.ctx.insight_mappings),
):
if objects:
logger.info("----Processing %ss----", object_type)
for obj in objects:
update = self._process_creator_only(
obj, object_type, mappings, skip_stats
)
if update:
object_updates.append(update)

return object_updates, skip_stats

def _process_creator_only(
self,
obj: dict,
object_type: str,
mappings: IdMappings | None,
skip_stats: dict[str, int],
) -> ObjectUpdate | None:
"""Build a creator/modifier-only update for a metric or an insight."""
if not obj or len(obj) != 1 or mappings is None:
return None
root_obj = next(iter(obj.values()))
if not isinstance(root_obj, dict) or "meta" not in root_obj:
return None
meta = root_obj["meta"]
cloud_object_id = mappings.get_value_by_key(meta.get("identifier"))
if not cloud_object_id:
skip_stats["no_cloud_mapping"] += 1
return None
legacy_creator_id = self._extract_profile_id(meta.get("author"))
cloud_creator_id, creator_email = self._get_mapped_user(legacy_creator_id)
if not cloud_creator_id:
skip_stats["no_user_mapping"] += 1
logger.warning(
"Creator '%s' not mapped for %s '%s' - skipped",
creator_email or legacy_creator_id,
object_type,
meta.get("title", ""),
)
return None
legacy_modifier_id = self._extract_profile_id(meta.get("contributor"))
cloud_modifier_id, _ = self._get_mapped_user(legacy_modifier_id)
return ObjectUpdate(
legacy_object_id=meta.get("identifier"),
legacy_object_title=meta.get("title", ""),
cloud_object_id=cloud_object_id,
object_type=object_type,
legacy_creator_id=legacy_creator_id,
legacy_modifier_id=legacy_modifier_id,
cloud_creator_id=cloud_creator_id,
cloud_modifier_id=cloud_modifier_id,
)

def _process_single_dashboard(
self, obj: dict, idx: int, skip_stats: dict[str, int]
) -> ObjectUpdate | None:
Expand All @@ -344,6 +412,8 @@ def _process_single_dashboard(
creator_uri = dashboard_meta.get("author")
legacy_creator_id = self._extract_profile_id(creator_uri)
cloud_creator_id, creator_email = self._get_mapped_user(legacy_creator_id)
legacy_modifier_id = self._extract_profile_id(dashboard_meta.get("contributor"))
cloud_modifier_id, _ = self._get_mapped_user(legacy_modifier_id)

# Process grantees if sharing is not skipped
grantee_user_ids, grantee_mappings, grantee_group_ids, group_mappings = (
Expand Down Expand Up @@ -376,9 +446,9 @@ def _process_single_dashboard(
cloud_object_id=cloud_object_id,
object_type="dashboard",
legacy_creator_id=legacy_creator_id,
legacy_modifier_id=None,
legacy_modifier_id=legacy_modifier_id,
cloud_creator_id=cloud_creator_id,
cloud_modifier_id=None,
cloud_modifier_id=cloud_modifier_id,
grantee_user_ids=grantee_user_ids if grantee_user_ids else None,
grantee_group_ids=grantee_group_ids if grantee_group_ids else None,
)
Expand Down
3 changes: 2 additions & 1 deletion src/gooddata_legacy2cloud/dashboards/drill_converter.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from typing import Optional

from gooddata_legacy2cloud.dashboards.data_classes import DashboardContext
from gooddata_legacy2cloud.helpers import cloud_display_form_type

logger = logging.getLogger("migration")

Expand Down Expand Up @@ -61,7 +62,7 @@ def _get_attribute_identifier(self, legacy_attribute_id: str) -> dict:
return {
"identifier": {
"id": cloud_id,
"type": "label", # Most Legacy attributes map to Cloud labels
"type": cloud_display_form_type(legacy_attribute_id),
}
}

Expand Down
Loading
Loading