From a6ff5c77713677ccf99b65e185aa45faa26780ae Mon Sep 17 00:00:00 2001 From: Karthik Satheesh Kumar Date: Wed, 15 Jul 2026 02:19:25 +0400 Subject: [PATCH 1/3] Enhance device metrics export workflow to support JSON output and include optional location data - Updated README to reflect new CSV and JSON output formats and added optional AP floorplan location data. - Refactored account_credentials.yaml to unify credential structure. - Modified device_metrics_export.py to fetch and process device location data, including new command line options for output formats. - Updated requirements.txt to use pycentral version 2.0a22. - Enhanced utils.py with functions to derive site IDs and fetch device locations. --- device-metrics-export/README.md | 124 ++++++---- .../account_credentials.yaml | 8 +- .../device_metrics_export.py | 111 +++++++-- device-metrics-export/requirements.txt | 2 +- device-metrics-export/utils.py | 222 ++++++++++++++++-- 5 files changed, 376 insertions(+), 91 deletions(-) diff --git a/device-metrics-export/README.md b/device-metrics-export/README.md index 19edc5e..1a3cb70 100644 --- a/device-metrics-export/README.md +++ b/device-metrics-export/README.md @@ -1,11 +1,11 @@ # Device Metrics Export -This workflow retrieves comprehensive device information from both HPE Aruba Networking Central and GreenLake Platform (GLP) APIs. It consolidates device attributes, monitoring data, connectivity status, and subscription details into a single CSV file for easy analysis and reporting. +This workflow retrieves comprehensive device information from both HPE Aruba Networking Central and GreenLake Platform (GLP) APIs. It consolidates device attributes, monitoring data, connectivity status, subscription details, and optional AP floorplan location data into CSV and JSON files for analysis, automation, and reporting. ## Prerequisites - Python 3.8 or higher -- API credentials for HPE Aruba Networking Central & GLP (JSON or YAML format) +- API credentials for HPE Aruba Networking Central and GLP (JSON or YAML format) ## Installation @@ -25,93 +25,115 @@ source venv/bin/activate # On Windows use: venv\Scripts\activate pip install -r requirements.txt ``` -This workflow is tested on the `pycentral` SDK (version: `v2.0a10`). Please check compatibility before executing on older/newer versions as there may be changes. +This workflow is tested on the `pycentral` SDK (version: `v2.0a22`). Please check compatibility before executing on older/newer versions as there may be changes. ## Configuration -#### New HPE Aruba Networking Central & GLP Credentials (account_credentials.yaml) +#### Credentials (account_credentials.yaml) -For API operations in new HPE Aruba Networking Central & GLP: +This workflow requires API credentials generated for GLP platform as the script makes API calls to GLP & Central. ```yaml -new_central: - cluster_name: # or base_url: - client_id: - client_secret: -glp: - client_id: - client_secret: +unified: + cluster_name: # e.g. EU-1, or use base_url: + workspace_id: + client_id: + client_secret: ``` + **Sample Input:** See [`account_credentials.yaml`](./account_credentials.yaml) in this repository for an example credential file. > [!TIP] > **Where to find these:** -> - [New Central API Gateway Base URLs](https://developer.arubanetworks.com/new-hpe-anw-central/docs/getting-started-with-rest-apis#api-gateway-base-urls) -> - [How to get API Credentials for new Central](https://developer.arubanetworks.com/new-hpe-anw-central/docs/generating-and-managing-access-tokens) -> - [How to get API Credentials for GLP](https://developer.greenlake.hpe.com/docs/greenlake/guides/public/authentication/authentication#creating-a-personal-api-client) +> - [Finding your Central API Gateway Base URL](https://developer.arubanetworks.com/new-central/docs/getting-started-with-rest-apis#finding-your-base-url) +> - [Generating & Managing Access Token](https://developer.arubanetworks.com/new-central/docs/generating-and-managing-access-tokens) ## Execution This workflow is executed by the script device_metrics_export.py and covers the following: - Loads and validates the credentials file (JSON/YAML) & establishes a connection to Central & GLP - Runs API calls - - - New Central - [Get a list of devices](https://developer.arubanetworks.com/new-central/reference/getdevices) - - New Central - [Get a device inventory list](https://developer.arubanetworks.com/new-central/reference/getdeviceinventory) - - GLP - [Get Devices managed in a workspace](https://developer.greenlake.hpe.com/docs/greenlake/services/device-management/public/openapi/nbapi-inventory-latest/openapi/devices-v1/getdevicesv1) - - GLP - [Get subscriptions of a workspace](https://developer.greenlake.hpe.com/docs/greenlake/services/subscription-management/public/openapi/nbapi-subscription/subscriptions-v1/getsubscriptionsv1) -- Processes results from the above mentioned APIs and outputs the result in a CSV format for usage. + - GLP + - [Get Devices managed in a workspace](https://developer.greenlake.hpe.com/docs/greenlake/services/device-management/public/openapi/nbapi-inventory-latest/openapi/devices-v1/getdevicesv1) + - [Get subscriptions of a workspace](https://developer.greenlake.hpe.com/docs/greenlake/services/subscription-management/public/openapi/nbapi-subscription/subscriptions-v1/getsubscriptionsv1) + - New Central + - [Get a list of devices](https://developer.arubanetworks.com/new-central/reference/getdevices) + - [Get a device inventory list](https://developer.arubanetworks.com/new-central/reference/getdeviceinventory) + - Optionally, [List devices with location information](https://developer.arubanetworks.com/new-central/reference/getdevicelocationsv1) +Processes results from the above mentioned APIs and outputs the result in CSV and structured JSON formats. Example usage: ```bash -python device_metrics_export.py -c --output +python device_metrics_export.py -c --output [--output-json ] [--include-floorplan] [--include-raw-location] ``` ### Command Line Options -The script accepts the following command line argument +The script accepts the following command line arguments -| Name | Type | Description | Required | -|-------------|--------|----------------------|----------| -| credentials_file | string | Path to file with New Central & GLP credentials (JSON or YAML). Must end in .json, .yaml, or .yml. | Yes | -| output | string | Path to output CSV file. Defaults to device_data.csv. | No | +| Name | Type | Description | Required | +|------------------|--------|-----------------------------------------------------------------------------|----------| +| credentials_file | string | Path to credentials (JSON or YAML). Must end in .json, .yaml, or .yml. | Yes | +| output | string | Path to output CSV file. Defaults to device_data.csv. | No | +| --output-json | string | Optional path to write structured JSON output (single array). Defaults to same name as CSV with .json extension. | No | +| --include-floorplan | flag | If set, fetch per-site device locations from the Central and include location fields in output. | No | +| --include-raw-location | flag | If set, include the Raw Location column in the CSV and JSON output, and also fetch per-site device locations (implies `--include-floorplan`). | No | ## Output -A CSV file (default: device_data.csv) containing merged device details from Central and GLP. You can find a sample output file [here](device_data.csv) - -CSV column titles (ordered): -- Serial Number -- Mac Address -- Device Name -- Device Type -- Device Model -- Deployment -- IPv4 -- Firmware Version -- Site -- Device Group -- Status -- Uptime -- Last Seen At -- Config Status -- Config Last Modified At -- Subscription Key -- Subscription Tier -- Subscription Type -- Subscription End Time +This workflow emits both a CSV (traditional tabular output) and a structured JSON file (single JSON array of device objects). + +- CSV (default: device_data.csv): tabular rows where missing values are empty strings. The CSV includes a Raw Location column which contains a JSON string (serialized) when location details are present. +- JSON (default: device_data.json alongside CSV unless overridden): a single JSON array of device objects. Fields with no data are serialized as null in JSON. Location data (Raw Location) is kept as a nested object when available. + +CSV / JSON fields (ordered for CSV; JSON objects contain the same keys): + +| Field | Description | +|-------|-------------| +| Serial Number | Device serial number from Central inventory. | +| Mac Address | Device MAC address. | +| Device Name | Device name configured in Central. | +| Device Type | Device category, such as access point, switch, or gateway. | +| Device Model | Device hardware model. | +| Deployment | Deployment mode reported by Central inventory. | +| IPv4 | Device IPv4 address. | +| Firmware Version | Software or firmware version reported for the device. | +| Site | Central site name associated with the device. | +| Device Group | Central compatible device group name. | +| Status | Device status from Central inventory. | +| Uptime | Device uptime converted to a human-readable duration. | +| Last Seen At | Last time the device was seen, converted to UTC. | +| Config Status | Device configuration status from monitoring data. | +| Config Last Modified At | Last configuration update time, converted to UTC. | +| Subscription Key | GLP subscription key assigned to the device. | +| Subscription Tier | GLP subscription tier. | +| Subscription Type | Derived subscription type, such as Foundation or Advanced. | +| Subscription End Time | GLP subscription end time, converted to UTC. | +| Floorplan ID | Floorplan identifier from Central device-location data. | +| Building ID | Building identifier from Central device-location data. | +| X Coordinate | Device's X coordinate on the floorplan. | +| Y Coordinate | Device's Y coordinate on the floorplan. | +| Floor Coordinate Unit | Unit used for floorplan's X/Y coordinates. | +| Latitude | Device's latitude from location data. | +| Longitude | Device's longitude from location data. | +| Raw Location | Full normalized location payload. In CSV this is serialized as a JSON string; in JSON this remains a nested object. | > [!TIP] -> **Device Group**: The Device Group field will only be populated for groups that are New Central compatible device groups. Classic Central groups will not appear in this export. +> The Device Group field will only be populated for groups that are Central compatible device groups. Classic Central groups will not appear in this export. + +> Location fields from Floorplan ID through Longitude are populated only when `--include-floorplan` (or `--include-raw-location`) is provided. The Raw Location column is additionally gated behind `--include-raw-location`; passing this flag also implies `--include-floorplan`, so no separate flag is needed. All other location columns remain present with blank/null values when location data is unavailable. + If no data is returned from the APIs or post-processing yields an empty list, the script prints "No data to save." ## Troubleshooting -- Authentication / tokens: Ensure your credentials file is complete and has valid credentials for GLP & New Central. +- Authentication / tokens: Ensure your credentials file contains a top-level `unified` section with `workspace_id`, `client_id`, `client_secret`, and `cluster_name` or `base_url`. - Empty output: Confirm that devices exist in Central inventory and that GLP subscriptions/devices exist for the account. -- SDK compatibility: If API calls fail unexpectedly, confirm the installed pycentral version matches tested versions (v2.0a10) or update helpers accordingly. +- Missing location fields: Confirm APs have `siteId` values and floorplan/device-location data exists for the site. Devices remain in the output with blank/null location fields when location data is unavailable. +- SDK compatibility: If API calls fail unexpectedly, confirm the installed pycentral version matches tested versions (v2.0a22) or update helpers accordingly. ## Support diff --git a/device-metrics-export/account_credentials.yaml b/device-metrics-export/account_credentials.yaml index 4f3a23b..246dcab 100644 --- a/device-metrics-export/account_credentials.yaml +++ b/device-metrics-export/account_credentials.yaml @@ -1,7 +1,5 @@ -new_central: - cluster_name: #e.g. EU-1 - client_id: - client_secret: -glp: +unified: + cluster_name: #e.g. EU-1 client_id: client_secret: + workspace_id: diff --git a/device-metrics-export/device_metrics_export.py b/device-metrics-export/device_metrics_export.py index 38cec96..e85bd1d 100644 --- a/device-metrics-export/device_metrics_export.py +++ b/device-metrics-export/device_metrics_export.py @@ -1,3 +1,5 @@ +import json +import os from pycentral.new_monitoring import MonitoringDevices from pycentral import NewCentralBase from pycentral.glp import Devices, Subscriptions @@ -11,6 +13,8 @@ processed_data, ensure_tokens_available, get_all_device_inventory, + derive_site_ids_with_aps, + fetch_device_locations, ) CSV_COLUMNS = [ @@ -36,8 +40,19 @@ "Subscription Tier", "Subscription Type", "Subscription End Time", + # Location fields (may be empty) + "Floorplan ID", + "Building ID", + "X Coordinate", + "Y Coordinate", + "Floor Coordinate Unit", + "Latitude", + "Longitude", + "Raw Location", ] +LOCATION_FETCH_WORKERS = 3 + def main(): args = parse_args() @@ -66,23 +81,70 @@ def main(): conn=new_central_conn, select="id,key,endTime,tier" ), } - # Fetch and process data raw_data = run_concurrent_tasks(tasks) - processed = process_all_data(raw_data) - # Save results to CSV + processed_inventory = process_list(_safe("device_inventory", raw_data), "serialNumber") + processed_devices = process_monitoring_data(_safe("monitoring_devices", raw_data)) + processed_locations = {} + if args.include_floorplan or args.include_raw_location: + site_ids = derive_site_ids_with_aps(processed_inventory, processed_devices) + if site_ids: + processed_locations = fetch_device_locations( + new_central_conn, site_ids, max_workers=LOCATION_FETCH_WORKERS + ) + else: + print("No site IDs with APs found; skipping floorplan/device-location fetch.") + + processed = process_all_data(raw_data, processed_locations) + + # Save results to CSV and structured JSON output = processed_data(**processed) + effective_columns = ( + CSV_COLUMNS + if args.include_raw_location + else [col for col in CSV_COLUMNS if col != "Raw Location"] + ) if output: - df = pd.DataFrame(output) - df = df.reindex(columns=CSV_COLUMNS, fill_value="") + # Prepare CSV rows: ensure Raw Location is serialized to a JSON string + csv_rows = [] + for row in output: + r = row.copy() + raw = r.get("Raw Location", "") + if isinstance(raw, (dict, list)): + r["Raw Location"] = json.dumps(raw, ensure_ascii=False) + elif raw is None: + r["Raw Location"] = "" + csv_rows.append(r) + + df = pd.DataFrame(csv_rows) + df = df.reindex(columns=effective_columns, fill_value="") df = df.sort_values( by="Device Type", ascending=True, na_position="last", ) df.to_csv(args.output, index=False) + + # Determine JSON output path + json_path = getattr(args, "output_json", None) + if not json_path: + base, ext = os.path.splitext(args.output) + json_path = f"{base}.json" if ext else f"{args.output}.json" + + # Prepare JSON rows: convert empty-string placeholders to null (None) in JSON + json_rows = [] + for row in output: + jr = {} + for column in effective_columns: + value = row.get(column, "") + jr[column] = None if value == "" else value + json_rows.append(jr) + + with open(json_path, "w", encoding="utf-8") as jf: + json.dump(json_rows, jf, indent=2, ensure_ascii=False) + print( - f"{len(output)} Central devices processed. Device data is saved to {args.output}" + f"{len(output)} Central devices processed. Device data is saved to {args.output} and {json_path}" ) else: print("No data to save.") @@ -90,11 +152,11 @@ def main(): def parse_args(): """Parse command line arguments""" - parser = argparse.ArgumentParser(description="Device Onboarding") + parser = argparse.ArgumentParser(description="Device Metrics Export") parser.add_argument( "-c", "--credentials", - help="Credentials file for New Central API (JSON or YAML)", + help="Credentials file for New Central API (JSON or YAML). Unified PyCentral credential files are supported.", required=True, type=validate_file_format, ) @@ -105,6 +167,24 @@ def parse_args(): required=False, default="device_data.csv", ) + parser.add_argument( + "--include-floorplan", + help="Include per-site AP floorplan/location data (optional)", + action="store_true", + default=False, + ) + parser.add_argument( + "--include-raw-location", + help="Include the Raw Location column in the output. Also implies --include-floorplan (fetches per-site device locations).", + action="store_true", + default=False, + ) + parser.add_argument( + "--output-json", + help="Optional path to write structured JSON output (single array). Defaults to replacing CSV extension with .json", + required=False, + default=None, + ) return parser.parse_args() @@ -119,19 +199,22 @@ def validate_file_format(file_path): return file_path -def process_all_data(raw_data): - """Process raw API data into structured format""" +def process_all_data(raw_data, processed_locations=None): + """Process raw API data into structured format. Accept optional processed_locations map.""" + processed_inventory = process_list( + _safe("device_inventory", raw_data), "serialNumber" + ) return { "processed_devices": process_monitoring_data( _safe("monitoring_devices", raw_data) ), - "processed_inventory": process_list( - _safe("device_inventory", raw_data), "serialNumber" - ), + "processed_inventory": processed_inventory, "processed_glp_devices": process_glp_device( - process_list(_safe("glp_devices", raw_data), "serialNumber") + process_list(_safe("glp_devices", raw_data), "serialNumber"), + known_serials=processed_inventory, ), "processed_subs": process_list(_safe("glp_subs", raw_data), "id"), + "processed_locations": processed_locations or {}, } diff --git a/device-metrics-export/requirements.txt b/device-metrics-export/requirements.txt index 6dd4cee..de151ff 100644 --- a/device-metrics-export/requirements.txt +++ b/device-metrics-export/requirements.txt @@ -1,2 +1,2 @@ pandas==2.3.3 -pycentral==2.0a10 \ No newline at end of file +pycentral==2.0a22 \ No newline at end of file diff --git a/device-metrics-export/utils.py b/device-metrics-export/utils.py index 4074ebe..cbcd539 100644 --- a/device-metrics-export/utils.py +++ b/device-metrics-export/utils.py @@ -1,3 +1,4 @@ +import json from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime, timezone @@ -18,23 +19,30 @@ def run_concurrent_tasks(tasks, max_workers=4): def process_list(list_of_dicts, key): - """Turn list of dicts into dict keyed by given key (consumes key from item).""" + """Turn list of dicts into dict keyed by given key.""" result = {} for item in list_of_dicts: if key in item: - result[item.pop(key)] = item + result[item[key]] = {k: v for k, v in item.items() if k != key} return result -def process_glp_device(device_dict): - """Replace device entry with subscription id if present.""" - for serial in list(device_dict): - if ( - "subscription" in device_dict[serial] - and device_dict[serial]["subscription"] - ): - device_dict[serial] = device_dict[serial]["subscription"][0]["id"] - return device_dict +def process_glp_device(device_dict, known_serials=None): + """Return serial-to-subscription-id mapping for subscribed GLP devices.""" + known_serials = set(known_serials or []) + result = {} + for serial, device in device_dict.items(): + if known_serials and serial not in known_serials: + continue + if not isinstance(device, dict): + continue + subscription = device.get("subscription") or [] + if not isinstance(subscription, list) or not subscription: + continue + subscription_id = subscription[0].get("id") if isinstance(subscription[0], dict) else None + if subscription_id: + result[serial] = subscription_id + return result def millis_to_human(ms): @@ -56,13 +64,20 @@ def processed_data( processed_inventory, processed_glp_devices=None, processed_subs=None, + processed_locations=None, ): - """Build list of device dicts for CSV output.""" + """Build list of device dicts for CSV output. + + processed_locations is an optional mapping keyed by serialNumber with normalized + location fields. Missing location data should not omit device rows. + """ processed_glp_devices = processed_glp_devices or {} processed_subs = processed_subs or {} + processed_locations = processed_locations or {} result = [] for serial_number in processed_inventory: device = processed_inventory[serial_number] + # base entry entry = { "Serial Number": serial_number, "Mac Address": device.get("macAddress", ""), @@ -76,6 +91,7 @@ def processed_data( "Device Group": device.get("deviceGroupName", ""), "Status": device.get("status", ""), } + # connectivity/config if serial_number in processed_devices: raw_uptime = processed_devices[serial_number].get("uptimeInMillis", "") entry.update( @@ -92,11 +108,10 @@ def processed_data( ), } ) + # subscription if serial_number in processed_glp_devices: sub_id = processed_glp_devices[serial_number] sub = processed_subs.get(sub_id, {}) or {} - if sub == {}: - continue # skip if subscription details not found tier = sub.get("tier", "") or "" tier_lower = tier.lower() entry.update( @@ -111,19 +126,40 @@ def processed_data( "Subscription End Time": iso_to_human(sub.get("endTime", "")), } ) + # attach location fields if present (do not omit rows if missing) + loc = processed_locations.get(serial_number, {}) or {} + entry.update( + { + "Floorplan ID": loc.get("floorplan_id", ""), + "Building ID": loc.get("building_id", ""), + "X Coordinate": loc.get("floor_x", ""), + "Y Coordinate": loc.get("floor_y", ""), + "Floor Coordinate Unit": loc.get("floor_coordinate_unit", ""), + "Latitude": loc.get("device_latitude", ""), + "Longitude": loc.get("device_longitude", ""), + "Raw Location": loc.get("raw_location", ""), + } + ) result.append(entry) return result def ensure_tokens_available(new_central_conn): - """Simple token presence check; raise with actionable message if missing.""" - tokens = getattr(new_central_conn, "token_info", None) or {} - if not tokens.get("new_central") or not tokens.get("glp"): + """Ensure unified token information can reach Central and GLP APIs.""" + tokens = getattr(new_central_conn, "token_info", None) + if not tokens or "unified" not in tokens: + raise Exception( + "Unified token information is missing. Provide a valid PyCentral " + "unified credentials file with client_id, client_secret, workspace_id, " + "and cluster_name or base_url." + ) + routes = getattr(new_central_conn, "_app_routes", {}) + missing_routes = {"new_central", "glp"} - set(routes) + if missing_routes: raise Exception( - "Required tokens (new_central or glp) are missing. " - "Please ensure the credentials file contains valid credentials." + "Unified credentials must enable both Central and GLP API routes. " + "Provide workspace_id and cluster_name or base_url under unified." ) - # caller can print confirmation if desired return True @@ -191,3 +227,149 @@ def _parse_iso(ts): except Exception: # fallback: return minimal so it won't be chosen over real timestamps return datetime.min.replace(tzinfo=timezone.utc) + + +def derive_site_ids_with_aps(*device_maps) -> list: + """Return sorted site IDs that have APs based on known device data. + + Device maps are dictionaries keyed by serial number with device dictionaries as + values. The device-location API requires siteId, so records without siteId are + ignored. + """ + site_ids = set() + for device_map in device_maps: + for dev in (device_map or {}).values(): + if not isinstance(dev, dict) or not _is_ap_device(dev): + continue + site_identifier = dev.get("siteId") + if site_identifier: + site_ids.add(str(site_identifier)) + return sorted(site_ids) + + +def _is_ap_device(device): + device_type = ( + device.get("deviceType") + or device.get("type") + or device.get("device_type") + or "" + ).lower() + model = (device.get("model") or "").lower() + return ( + "access point" in device_type + or "access_point" in device_type + or device_type in {"ap", "iap"} + or model.startswith("ap-") + ) + + +def _central_get(central_conn, path: str, params: dict): + """GET a New Central API path through PyCentral's command interface.""" + resp = central_conn.command("GET", path, app_name="new_central", api_params=params) + if isinstance(resp, dict) and "msg" in resp: + return resp["msg"] + return resp + + +def fetch_device_locations(central_conn, site_ids: set, max_workers: int = 4) -> dict: + """Fetch device-location items for given site ids and return map keyed by serialNumber. + + - Handles pagination with 'next' and 'total' when present. + - Returns mapping: serialNumber -> normalized location dict (see below) + + Normalized fields per device: + floorplan_id, building_id, floor_x, floor_y, floor_coordinate_unit, + device_latitude, device_longitude, raw_location + + On any per-site fetch failure a warning is printed and that site's devices are skipped. + """ + if not site_ids: + return {} + + results = [] + + def _fetch_site(site_identifier): + path = f"network-services/v1/sites/{site_identifier}/device-locations" + items = [] + next_page = None + try: + while True: + params = {"with-location": "true"} + if next_page: + params["next"] = next_page + resp = _central_get(central_conn, path, params) + if not resp: + break + # expect items list + page_items = resp.get("items") if isinstance(resp, dict) else None + if page_items is None and isinstance(resp, list): + page_items = resp + if page_items: + items.extend(page_items) + total = resp.get("total") if isinstance(resp, dict) else None + # advance + nxt = resp.get("next") if isinstance(resp, dict) else None + if not nxt or (total and len(items) >= total): + break + next_page = nxt + return items + except Exception as e: + print(f"Warning: failed to fetch device locations for site {site_identifier}: {e}") + return [] + + # parallelize site fetches + with ThreadPoolExecutor(max_workers=min(max_workers, max(1, len(site_ids)))) as ex: + futures = {ex.submit(_fetch_site, sid): sid for sid in site_ids} + for fut in as_completed(futures): + sid = futures[fut] + try: + items = fut.result() + if items: + results.extend(items) + except Exception as e: + print(f"Warning: site {sid} fetch raised: {e}") + + # build serial-number keyed map + serial_map = {} + for item in results: + if not item: + continue + serial = item.get("serialNumber") + if not serial: + # we merge by serial only + continue + loc = item.get("consolidatedLocation") or item.get("consolidatedLocationJson") or {} + # consolidatedLocation may be a JSON string + if isinstance(loc, str): + try: + loc = json.loads(loc) + except json.JSONDecodeError: + pass + # extract cartesian coordinates + cart = loc.get("cartesianCoordinates") if isinstance(loc, dict) else None + x = y = unit = "" + if isinstance(cart, dict): + x = cart.get("x_position") or cart.get("x") or cart.get("xPos") or "" + y = cart.get("y_position") or cart.get("y") or cart.get("yPos") or "" + unit = cart.get("unit") or "" + # center -> lat/long + center = loc.get("center") if isinstance(loc, dict) else None + lat = lon = "" + if isinstance(center, dict): + lat = center.get("latitude") or center.get("lat") or "" + lon = center.get("longitude") or center.get("lon") or center.get("lng") or "" + normalized = { + "floorplan_id": item.get("floorId") + or (loc.get("floorId") if isinstance(loc, dict) else ""), + "building_id": item.get("buildingId") + or (loc.get("buildingId") if isinstance(loc, dict) else ""), + "floor_x": x, + "floor_y": y, + "floor_coordinate_unit": unit, + "device_latitude": lat, + "device_longitude": lon, + "raw_location": loc, + } + serial_map[serial] = normalized + + return serial_map From b985161f02c190cb7ce47c895a174f77193d936a Mon Sep 17 00:00:00 2001 From: Karthik Satheesh Kumar Date: Wed, 15 Jul 2026 02:33:49 +0400 Subject: [PATCH 2/3] Refactor device metrics export to conditionally include location data in outputs based on command line arguments --- device-metrics-export/README.md | 4 +- .../device_metrics_export.py | 77 ++++++++++++------- device-metrics-export/utils.py | 50 +++++------- 3 files changed, 69 insertions(+), 62 deletions(-) diff --git a/device-metrics-export/README.md b/device-metrics-export/README.md index 1a3cb70..7409f30 100644 --- a/device-metrics-export/README.md +++ b/device-metrics-export/README.md @@ -123,7 +123,7 @@ CSV / JSON fields (ordered for CSV; JSON objects contain the same keys): > [!TIP] > The Device Group field will only be populated for groups that are Central compatible device groups. Classic Central groups will not appear in this export. -> Location fields from Floorplan ID through Longitude are populated only when `--include-floorplan` (or `--include-raw-location`) is provided. The Raw Location column is additionally gated behind `--include-raw-location`; passing this flag also implies `--include-floorplan`, so no separate flag is needed. All other location columns remain present with blank/null values when location data is unavailable. +> Location fields from Floorplan ID through Longitude are only added to the CSV/JSON when `--include-floorplan` (or `--include-raw-location`) is provided. The Raw Location column is additionally gated behind `--include-raw-location`; passing this flag also implies `--include-floorplan`. When neither flag is provided, location columns are omitted entirely from outputs. If no data is returned from the APIs or post-processing yields an empty list, the script prints "No data to save." @@ -132,7 +132,7 @@ If no data is returned from the APIs or post-processing yields an empty list, th - Authentication / tokens: Ensure your credentials file contains a top-level `unified` section with `workspace_id`, `client_id`, `client_secret`, and `cluster_name` or `base_url`. - Empty output: Confirm that devices exist in Central inventory and that GLP subscriptions/devices exist for the account. -- Missing location fields: Confirm APs have `siteId` values and floorplan/device-location data exists for the site. Devices remain in the output with blank/null location fields when location data is unavailable. +-- Missing location fields: Confirm APs have `siteId` values and floorplan/device-location data exists for the site. If you want location columns in the output, run with `--include-floorplan` (or `--include-raw-location`). - SDK compatibility: If API calls fail unexpectedly, confirm the installed pycentral version matches tested versions (v2.0a22) or update helpers accordingly. ## Support diff --git a/device-metrics-export/device_metrics_export.py b/device-metrics-export/device_metrics_export.py index e85bd1d..add1278 100644 --- a/device-metrics-export/device_metrics_export.py +++ b/device-metrics-export/device_metrics_export.py @@ -11,13 +11,12 @@ process_list, process_glp_device, processed_data, - ensure_tokens_available, get_all_device_inventory, derive_site_ids_with_aps, fetch_device_locations, ) -CSV_COLUMNS = [ +BASE_COLUMNS = [ # Device information columns "Serial Number", "Mac Address", @@ -40,7 +39,9 @@ "Subscription Tier", "Subscription Type", "Subscription End Time", - # Location fields (may be empty) +] + +LOCATION_COLUMNS = [ "Floorplan ID", "Building ID", "X Coordinate", @@ -48,21 +49,17 @@ "Floor Coordinate Unit", "Latitude", "Longitude", - "Raw Location", ] +RAW_LOCATION_COLUMN = "Raw Location" + LOCATION_FETCH_WORKERS = 3 def main(): args = parse_args() new_central_conn = NewCentralBase(token_info=args.credentials, log_level="ERROR") - try: - ensure_tokens_available(new_central_conn) - except Exception as e: - print(f"Error: {e}") - return - + devices_api = Devices() subscriptions_api = Subscriptions() @@ -95,25 +92,38 @@ def main(): else: print("No site IDs with APs found; skipping floorplan/device-location fetch.") - processed = process_all_data(raw_data, processed_locations) + processed = process_all_data( + raw_data, + processed_locations, + processed_inventory, + processed_devices, + ) # Save results to CSV and structured JSON - output = processed_data(**processed) - effective_columns = ( - CSV_COLUMNS - if args.include_raw_location - else [col for col in CSV_COLUMNS if col != "Raw Location"] + output = processed_data( + **processed, + include_floorplan=(args.include_floorplan or args.include_raw_location), + include_raw_location=args.include_raw_location, ) + + # Build columns based on flags: omit location columns unless requested + effective_columns = list(BASE_COLUMNS) + if args.include_floorplan or args.include_raw_location: + effective_columns += LOCATION_COLUMNS + if args.include_raw_location: + effective_columns.append(RAW_LOCATION_COLUMN) if output: # Prepare CSV rows: ensure Raw Location is serialized to a JSON string csv_rows = [] for row in output: r = row.copy() - raw = r.get("Raw Location", "") - if isinstance(raw, (dict, list)): - r["Raw Location"] = json.dumps(raw, ensure_ascii=False) - elif raw is None: - r["Raw Location"] = "" + # only serialize Raw Location when it's part of effective columns + if RAW_LOCATION_COLUMN in effective_columns: + raw = r.get(RAW_LOCATION_COLUMN, "") + if isinstance(raw, (dict, list)): + r[RAW_LOCATION_COLUMN] = json.dumps(raw, ensure_ascii=False) + elif raw is None: + r[RAW_LOCATION_COLUMN] = "" csv_rows.append(r) df = pd.DataFrame(csv_rows) @@ -199,15 +209,24 @@ def validate_file_format(file_path): return file_path -def process_all_data(raw_data, processed_locations=None): - """Process raw API data into structured format. Accept optional processed_locations map.""" - processed_inventory = process_list( - _safe("device_inventory", raw_data), "serialNumber" - ) +def process_all_data( + raw_data, processed_locations=None, processed_inventory=None, processed_devices=None +): + """Process raw API data into structured format. + + Accept optional precomputed `processed_inventory` and `processed_devices` to + avoid recomputing large mappings when callers already have them. + """ + # compute only if not provided to preserve backward compatibility + if processed_inventory is None: + processed_inventory = process_list( + _safe("device_inventory", raw_data), "serialNumber" + ) + if processed_devices is None: + processed_devices = process_monitoring_data(_safe("monitoring_devices", raw_data)) + return { - "processed_devices": process_monitoring_data( - _safe("monitoring_devices", raw_data) - ), + "processed_devices": processed_devices, "processed_inventory": processed_inventory, "processed_glp_devices": process_glp_device( process_list(_safe("glp_devices", raw_data), "serialNumber"), diff --git a/device-metrics-export/utils.py b/device-metrics-export/utils.py index cbcd539..7aa9a16 100644 --- a/device-metrics-export/utils.py +++ b/device-metrics-export/utils.py @@ -65,6 +65,8 @@ def processed_data( processed_glp_devices=None, processed_subs=None, processed_locations=None, + include_floorplan: bool = False, + include_raw_location: bool = False, ): """Build list of device dicts for CSV output. @@ -126,41 +128,27 @@ def processed_data( "Subscription End Time": iso_to_human(sub.get("endTime", "")), } ) - # attach location fields if present (do not omit rows if missing) - loc = processed_locations.get(serial_number, {}) or {} - entry.update( - { - "Floorplan ID": loc.get("floorplan_id", ""), - "Building ID": loc.get("building_id", ""), - "X Coordinate": loc.get("floor_x", ""), - "Y Coordinate": loc.get("floor_y", ""), - "Floor Coordinate Unit": loc.get("floor_coordinate_unit", ""), - "Latitude": loc.get("device_latitude", ""), - "Longitude": loc.get("device_longitude", ""), - "Raw Location": loc.get("raw_location", ""), - } - ) + # attach location fields only when requested (do not omit rows if missing) + if include_floorplan: + loc = processed_locations.get(serial_number, {}) or {} + entry.update( + { + "Floorplan ID": loc.get("floorplan_id", ""), + "Building ID": loc.get("building_id", ""), + "X Coordinate": loc.get("floor_x", ""), + "Y Coordinate": loc.get("floor_y", ""), + "Floor Coordinate Unit": loc.get("floor_coordinate_unit", ""), + "Latitude": loc.get("device_latitude", ""), + "Longitude": loc.get("device_longitude", ""), + } + ) + if include_raw_location: + # include raw_location (may be dict) only when explicitly requested + entry["Raw Location"] = loc.get("raw_location", "") result.append(entry) return result -def ensure_tokens_available(new_central_conn): - """Ensure unified token information can reach Central and GLP APIs.""" - tokens = getattr(new_central_conn, "token_info", None) - if not tokens or "unified" not in tokens: - raise Exception( - "Unified token information is missing. Provide a valid PyCentral " - "unified credentials file with client_id, client_secret, workspace_id, " - "and cluster_name or base_url." - ) - routes = getattr(new_central_conn, "_app_routes", {}) - missing_routes = {"new_central", "glp"} - set(routes) - if missing_routes: - raise Exception( - "Unified credentials must enable both Central and GLP API routes. " - "Provide workspace_id and cluster_name or base_url under unified." - ) - return True def get_all_device_inventory(MonitoringDevices, central_conn): From 244552a00117c22b12a15df781afd05838fd08d7 Mon Sep 17 00:00:00 2001 From: Karthik Satheesh Kumar <49326615+KarthikSKumar98@users.noreply.github.com> Date: Wed, 15 Jul 2026 10:35:35 +0400 Subject: [PATCH 3/3] Potential fix for pull request finding 'Empty except' Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> --- device-metrics-export/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/device-metrics-export/utils.py b/device-metrics-export/utils.py index 7aa9a16..08e15d5 100644 --- a/device-metrics-export/utils.py +++ b/device-metrics-export/utils.py @@ -331,8 +331,8 @@ def _fetch_site(site_identifier): if isinstance(loc, str): try: loc = json.loads(loc) - except json.JSONDecodeError: - pass + except json.JSONDecodeError as e: + print(f"Warning: failed to parse consolidated location JSON for serial {serial}: {e}") # extract cartesian coordinates cart = loc.get("cartesianCoordinates") if isinstance(loc, dict) else None x = y = unit = ""