monitoring.ts

import * as Alchemy from "alchemy";
import * as Axiom from "@alchemy.run/axiom";
import * as Output from "alchemy/Output";
import * as Effect from "effect/Effect";
import * as Option from "effect/Option";

import { relayResourceNameForStage } from "../deploymentConfig.ts";
import { notifierConfig } from "../monitoring.ts";
import { isPreviewStage } from "../previewStages.ts";

const RUNBOOK_URL =
  "https://github.com/pingdotgg/t3code/blob/main/docs/operations/relay-observability.md#mobile-crashes-and-errors";

// A new dataset has none of these fields until a report carries them, and an
// unknown field fails the whole query, so every one has a default.
const field = (name: string, fallback: string) => `column_ifexists('${name}', ${fallback})`;
const reports = (dataset: string) =>
  [
    `['${dataset}']`,
    `| extend source = tostring(${field("attributes.expo.error.source", "''")}),`,
    `    fatal = tobool(${field("attributes.expo.error.is_fatal", "false")}),`,
    `    js_fatal = tobool(${field("attributes.relay.crash.js_fatal", "false")}),`,
    `    platform = tostring(${field("resource.os.name", "''")}),`,
    `    install = tostring(${field("resource.expo.eas_client.id", "''")}),`,
    `    error_type = tostring(${field("attributes.exception.type", "''")}),`,
    `    message = substring(tostring(${field("attributes.exception.message", "''")}), 0, 140),`,
    `    version = strcat(tostring(${field("service.version", "''")}), ' (', tostring(${field("resource.expo.app.build_number", "''")}), ')')`,
    `| where isnotempty(source) and isnotempty(platform)`,
    // A fatal JS error also ends in a native abort; counting both would double every JS crash.
    `| extend kind = case(source == 'nativeCrash' and js_fatal, 'abort after JS crash', source == 'nativeCrash', 'native crash', fatal, 'JS crash', 'handled error')`,
  ].join("\n");
const crashes = (dataset: string) =>
  `${reports(dataset)}\n| where kind in ('JS crash', 'native crash')`;
const handledErrors = (dataset: string) => `${reports(dataset)}\n| where kind == 'handled error'`;
const ingest = (traces: string) =>
  `['${traces}']\n| where name == 'relay.mobile_telemetry.ingest'\n| extend signal = tostring(['attributes.custom']['relay.mobile_telemetry.signal']), outcome = tostring(['attributes.custom']['relay.mobile_telemetry.outcome'])`;

type Severity = "page" | "warn";

interface Datasets {
  readonly logs: string;
  readonly traces: string;
}

interface MobileMonitor {
  readonly id: string;
  readonly severity: Severity;
  readonly name: string;
  /** Why it fires. The runbook link is appended. */
  readonly description: string;
  readonly query: (datasets: Datasets) => string;
  readonly props: Pick<
    Axiom.MonitorProps,
    | "type"
    | "operator"
    | "threshold"
    | "rangeMinutes"
    | "intervalMinutes"
    | "compareDays"
    | "tolerance"
    | "alertOnNoData"
    | "triggerFromNRuns"
  >;
}

// No production baseline exists yet: the spike monitor learns its own, and the
// thresholds below are starting points to tune after two weeks of data.
const mobileMonitors: ReadonlyArray<MobileMonitor> = [
  {
    id: "MobileCrashSpikeMonitor",
    severity: "page",
    name: "T3 Code mobile: crash spike",
    description:
      "Crashes (JS and native, each counted once) per 15 min well above the same time over the last 7 days. Usually a bad OTA update or store build: compare app versions on the dashboard.",
    query: ({ logs }) => `${crashes(logs)}\n| summarize crashes = count() by bin_auto(_time)`,
    props: {
      type: "AnomalyDetection",
      operator: "Above",
      compareDays: 7,
      tolerance: 50,
      rangeMinutes: 15,
      intervalMinutes: 15,
    },
  },
  {
    id: "MobileCrashSpreadMonitor",
    severity: "page",
    name: "T3 Code mobile: one crash hitting many installs",
    description:
      "The most widespread crash reached this many installs in an hour. Its message and versions are at the top of the dashboard's Top crashes table.",
    query: ({ logs }) =>
      `${crashes(logs)}\n| summarize installs = dcount(install) by platform, error_type, message\n| summarize worst = max(installs)`,
    props: {
      type: "Threshold",
      operator: "AboveOrEqual",
      threshold: 25,
      rangeMinutes: 60,
      intervalMinutes: 15,
    },
  },
  {
    id: "MobileTelemetryStoppedMonitor",
    severity: "warn",
    name: "T3 Code mobile: reports stopped",
    description:
      "No startup metrics forwarded in an hour. Every launch sends some, so the relay route, the Axiom token, or the app's endpoint is broken, and crashes go unseen too.",
    query: ({ traces }) =>
      `${ingest(traces)}\n| where signal == 'metrics' and outcome == 'forwarded'\n| summarize forwarded = count()`,
    props: {
      type: "Threshold",
      operator: "Below",
      threshold: 1,
      rangeMinutes: 60,
      intervalMinutes: 15,
      alertOnNoData: true,
      triggerFromNRuns: 2,
    },
  },
  {
    id: "MobileSymbolicationFailingMonitor",
    severity: "warn",
    name: "T3 Code mobile: crash stacks not symbolicated",
    description:
      "Share of release JS exceptions over 6 h whose stack stayed minified. Map uploads are failing or the relay cannot read them; relay.symbolication.status says which.",
    query: ({ logs }) =>
      [
        `['${logs}']`,
        `| extend status = tostring(${field("attributes.relay.symbolication.status", "''")})`,
        `| where isnotempty(status) and status !in ('no_bundle_id', 'no_bundle_frames')`,
        `| summarize total = count(), minified = countif(status != 'symbolicated')`,
        `| project minified_pct = iff(total >= 20, 100.0 * minified / total, 0.0)`,
      ].join("\n"),
    props: {
      type: "Threshold",
      operator: "Above",
      threshold: 50,
      rangeMinutes: 360,
      intervalMinutes: 60,
    },
  },
];

interface QueryChart {
  readonly id: string;
  readonly name: string;
  readonly type: "TimeSeries" | "Table" | "Statistic";
  readonly query: { readonly apl: string };
  /** Grid columns out of 12. */
  readonly width: 4 | 6 | 12;
}

const chart = (
  type: QueryChart["type"],
  width: QueryChart["width"],
  id: string,
  name: string,
  apl: string,
): QueryChart => ({ id, name, type, width, query: { apl } });

const mobileDashboardCharts = ({ logs, traces }: Datasets): ReadonlyArray<QueryChart> => [
  chart("Statistic", 4, "crashes", "Crashes", `${crashes(logs)}\n| summarize count()`),
  chart(
    "Statistic",
    4,
    "crashed-installs",
    "Installs that crashed",
    `${crashes(logs)}\n| summarize dcount(install)`,
  ),
  chart(
    "Statistic",
    4,
    "handled-errors",
    "Handled errors",
    `${handledErrors(logs)}\n| summarize count()`,
  ),
  chart(
    "TimeSeries",
    6,
    "crashes-over-time",
    "Crashes by platform",
    `${crashes(logs)}\n| summarize count() by bin_auto(_time), series = strcat(platform, ' ', kind)`,
  ),
  chart(
    "TimeSeries",
    6,
    "handled-over-time",
    "Handled errors by platform",
    `${handledErrors(logs)}\n| summarize count() by bin_auto(_time), platform`,
  ),
  chart(
    "Table",
    12,
    "top-crashes",
    "Top crashes",
    `${crashes(logs)}\n| summarize installs = dcount(install), crashes = count(), first_seen = min(_time), last_seen = max(_time), versions = make_set(version, 3) by platform, kind, error_type, message\n| order by installs desc, crashes desc\n| take 15`,
  ),
  chart(
    "Table",
    12,
    "top-handled",
    "Top handled errors",
    `${handledErrors(logs)}\n| summarize installs = dcount(install), errors = count(), last_seen = max(_time), versions = make_set(version, 3) by platform, error_type, message\n| order by installs desc, errors desc\n| take 15`,
  ),
  chart(
    "Table",
    6,
    "by-version",
    "Crashes by app version",
    `${crashes(logs)}\n| summarize crashes = count(), installs = dcount(install) by platform, version\n| order by crashes desc\n| take 10`,
  ),
  chart(
    "TimeSeries",
    6,
    "symbolication",
    "JS stacks by symbolication outcome",
    `['${logs}']\n| extend status = tostring(${field("attributes.relay.symbolication.status", "''")})\n| where isnotempty(status)\n| summarize count() by bin_auto(_time), status`,
  ),
  chart(
    "TimeSeries",
    12,
    "ingest",
    "Reports reaching the relay",
    `${ingest(traces)}\n| summarize count() by bin_auto(_time), series = strcat(signal, ' ', outcome)`,
  ),
];

/** Lays charts left to right, wrapping at Axiom's 12-column grid. */
function layout(charts: ReadonlyArray<QueryChart>) {
  let x = 0;
  let y = 0;
  let rowHeight = 0;
  return charts.map((chart) => {
    const h = chart.type === "Statistic" ? 3 : 6;
    if (x + chart.width > 12) {
      x = 0;
      y += rowHeight;
      rowHeight = 0;
    }
    const cell = { i: chart.id, x, y, w: chart.width, h };
    x += chart.width;
    rowHeight = Math.max(rowHeight, h);
    return cell;
  });
}

/**
 * Monitors follow the relay's: a deployment opts in by naming its paging notifier. The
 * dashboard also comes up on PR previews, so a change to mobile telemetry can be checked
 * there; the preview's teardown removes it.
 */
export const MobileMonitoring = Effect.fn("MobileMonitoring")(function* (
  logs: Axiom.Dataset,
  traces: Axiom.Dataset,
) {
  const { stage } = yield* Alchemy.Stack;
  const page = yield* notifierConfig("RELAY_ALERT_NOTIFIER_ID");
  if (Option.isNone(page) && !isPreviewStage(stage)) return;
  const datasets = Output.map(Output.all(logs.name, traces.name), ([logs, traces]) => ({
    logs,
    traces,
  }));

  if (Option.isSome(page)) {
    const notifiers: Record<Severity, Option.Option<string>> = {
      page,
      warn: yield* notifierConfig("RELAY_WARNING_NOTIFIER_ID"),
    };
    for (const monitor of mobileMonitors) {
      yield* Axiom.Monitor(monitor.id, {
        ...monitor.props,
        name: stage === "prod" ? monitor.name : `[${stage}] ${monitor.name}`,
        description: `${monitor.description} Runbook: ${RUNBOOK_URL}`,
        aplQuery: Output.map(datasets, monitor.query),
        triggerFromNRuns: monitor.props.triggerFromNRuns ?? 1,
        alertOnNoData: monitor.props.alertOnNoData ?? false,
        notifierIds: Option.toArray(notifiers[monitor.severity]),
      });
    }
  }

  yield* Axiom.Dashboard("MobileHealthDashboard", {
    dashboard: Output.map(datasets, (names) => {
      const charts = mobileDashboardCharts(names);
      return {
        name: relayResourceNameForStage("t3-code-mobile-health", stage),
        owner: "X-AXIOM-EVERYONE",
        description: `T3 Code mobile app crashes and errors. Crashes count JS and native once each. Runbook: ${RUNBOOK_URL}`,
        refreshTime: 300,
        schemaVersion: 2,
        timeWindowStart: "qr-now-7d",
        timeWindowEnd: "qr-now",
        charts: charts.map(({ width: _width, ...chart }) => chart),
        layout: layout(charts),
      };
    }),
  });
});