diff --git a/under-the-hood/LIVE_RESTRICTION_STATUS.md b/under-the-hood/LIVE_RESTRICTION_STATUS.md new file mode 100644 index 00000000..b84f4a1f --- /dev/null +++ b/under-the-hood/LIVE_RESTRICTION_STATUS.md @@ -0,0 +1,38 @@ +# Live restriction status + +`UnderTheHoodReport.liveRestrictionStatus` is an additive, typed GraphQL view of +recommendation restrictions that are visible to the authenticated creator. It complements the +existing monthly `reportJson`; it does not replace that historical aggregate. + +## Coverage and freshness + +- Current account labels come from the Gizmoduck composite user record. +- Post labels come from `visibility/baseTweetSafetyLabelMap` for at most 100 unique, authored + posts returned by the creator timeline, restricted to the last 30 days. +- Retweets are excluded. A retweeted original's label is not attributed to the creator. +- Only labels in `underTheHoodLabels`' public account and post allowlists are returned. +- A label is current when its applied timestamp is not in the future and its optional expiry is + still in the future. Missing expiry means the published source supplied no expiry. +- `postScanTruncated` and `postScanStatus` say when the timeline limit prevented a complete scan. + An empty result therefore means no allowlisted active restriction was found within the stated + coverage, not that every X enforcement system was queried. +- `accountLabelStatus` is `UNAVAILABLE` if the Gizmoduck user record or its requested label field + is absent; an empty account-label list is not silently presented as complete in that case. + +The outer `underTheHoodReport` column resolves only the authenticated viewer's user id. Its +internal keyed implementation retains the existing limited administrative access policy. + +## Privacy and unavailable metadata + +Raw account-label `source` / `byUser` values and post-label source variants are never returned. +Those values can contain internal actor names, LDAP identifiers, or rule ids. `provenance` +therefore remains absent until a stable, explicitly public provenance taxonomy exists. + +The published label stores do not provide a reliable end-to-end model version or per-label appeal +availability or a machine-readable public surface taxonomy. `modelVersion`, +`appealAvailability`, and `affectedSurfaces` remain absent and are listed in `unavailableFields`. +Timestamps are returned only when the source record actually supplies them; no applied, expiry, +review, model, appeal, or surface value is inferred. + +The human-readable `effect` remains the authoritative public description of affected product +surfaces. It does not expose internal rule or pipeline names. diff --git a/under-the-hood/strato/columns/underTheHoodReport.User.strato b/under-the-hood/strato/columns/underTheHoodReport.User.strato index 6be87004..1302c9de 100644 --- a/under-the-hood/strato/columns/underTheHoodReport.User.strato +++ b/under-the-hood/strato/columns/underTheHoodReport.User.strato @@ -1,5 +1,6 @@ import import +import type Key = Long type View = Unit @@ -296,14 +297,15 @@ def monthDataReady( val fetchOp = Op.fetch[Key, View, Value] { ctx => val now = Time.now - val gizUserOpt = userWithQueryFields(ctx.key, Set(Account, Safety)) + val gizUserOpt = userWithQueryFields(ctx.key, Set(Account, Safety, Labels)) if (gizUserOpt.map { user => isSuspended(user) }.getOrElse(false)) { Ok.missing } else { - val ageOk = - gizUserOpt.map({ user => - meetsAccountAgeMinimum(user, now) - }).getOrElse(false) + val liveStatus = liveRestrictionStatus.build(ctx.key, gizUserOpt, now) + val ageOk = + gizUserOpt.map({ user => + meetsAccountAgeMinimum(user, now) + }).getOrElse(false) val primaryMonth = reportMonthBucket(now) val primaryUserKey: UthUserMonthKey = { @@ -340,10 +342,18 @@ val fetchOp = Op.fetch[Key, View, Value] { ctx => case None => monthMetaOpt match { case None => - Ok.missing + Ok.value(Some({ + reportJson = None, + reportInfo = None, + liveRestrictionStatus = Some(liveStatus), + })) case Some(meta) => if (!monthObservationComplete(monthBucket, meta.daysIncluded)) { - Ok.missing + Ok.value(Some({ + reportJson = None, + reportInfo = None, + liveRestrictionStatus = Some(liveStatus), + })) } else { val reportInfo = { reportPeriod = Some(reportPeriodLabel(monthBucket)), @@ -355,13 +365,18 @@ val fetchOp = Op.fetch[Key, View, Value] { ctx => Ok.value(Some({ reportJson = None, reportInfo = Some(reportInfo), + liveRestrictionStatus = Some(liveStatus), })) } } case Some(userMonth) => val daysIncludedForTs = userMonth.daysIncluded if (!monthObservationComplete(monthBucket, daysIncludedForTs)) { - Ok.missing + Ok.value(Some({ + reportJson = None, + reportInfo = None, + liveRestrictionStatus = Some(liveStatus), + })) } else { val postCount = userMonth.eligiblePostAgg .map { days => sumDayCounts(Some(days)) } @@ -386,6 +401,7 @@ val fetchOp = Op.fetch[Key, View, Value] { ctx => Ok.value(Some({ reportJson = reportJson, reportInfo = Some(reportInfo), + liveRestrictionStatus = Some(liveStatus), })) } } @@ -408,7 +424,9 @@ Managed({ "Picks the latest prior UTC month past the 10-day calendar floor (keep at 10+); if that " + "month lacks complete observation (finalThroughDay), falls back one month. " + "Uses the per-user row when present; if missing, userId=-1 month-meta can still return " + - "reportInfo without reportJson. Missing only when both user and meta are absent/incomplete. " + - "eligibilityChecks = posts/age only. Called by underTheHoodReport for the authenticated viewer." + "reportInfo without reportJson. The additive liveRestrictionStatus is returned even when " + + "monthly data is absent or incomplete; it performs a bounded scan of the authenticated " + + "creator's current account labels and recent authored posts. eligibilityChecks = posts/age " + + "only. Called by underTheHoodReport for the authenticated viewer." ), }) diff --git a/under-the-hood/strato/graphql/under_the_hood.thrift b/under-the-hood/strato/graphql/under_the_hood.thrift index f32a202d..3d6e0a97 100644 --- a/under-the-hood/strato/graphql/under_the_hood.thrift +++ b/under-the-hood/strato/graphql/under_the_hood.thrift @@ -6,8 +6,42 @@ struct UnderTheHoodReportInfo { ) } (strato.graphql.typename = "UnderTheHoodReportInfo") +// A creator-visible, best-effort view of a currently active recommendation restriction. +// This contract intentionally does not expose enforcement actors, internal rule identifiers, +// or raw label-source payloads. +struct UnderTheHoodLiveRestriction { + 1: optional string scope + 2: optional i64 postId ( + GraphQlEncoding = "Int53", + personalDataType = "TweetId" + ) + 3: optional string label (personalDataType = "TweetSafetyLabels, UserSafetyLabels") + 4: optional string about + 5: optional string effect + 6: optional list affectedSurfaces + 7: optional i64 appliedAt (GraphQlEncoding = "Int53") + 8: optional i64 expiresAt (GraphQlEncoding = "Int53") + 9: optional string provenance + 10: optional string modelVersion + 11: optional string appealAvailability +} (strato.graphql.typename = "UnderTheHoodLiveRestriction") + +struct UnderTheHoodLiveRestrictionStatus { + 1: optional i64 generatedAt (GraphQlEncoding = "Int53") + 2: optional string freshness + 3: optional list restrictions + 4: optional i32 postScanLimit + 5: optional i32 postLookbackDays + 6: optional i32 postsScanned + 7: optional bool postScanTruncated + 8: optional string postScanStatus + 9: optional list unavailableFields + 10: optional string accountLabelStatus +} (strato.graphql.typename = "UnderTheHoodLiveRestrictionStatus") + struct UnderTheHoodReport { 1: optional i64 generatedAt (GraphQlEncoding = "Int53") 2: optional string reportJson 3: optional UnderTheHoodReportInfo reportInfo + 4: optional UnderTheHoodLiveRestrictionStatus liveRestrictionStatus } (strato.graphql.typename = "UnderTheHoodReport") diff --git a/under-the-hood/strato/lib/liveRestrictionStatus.strato b/under-the-hood/strato/lib/liveRestrictionStatus.strato new file mode 100644 index 00000000..36c50e71 --- /dev/null +++ b/under-the-hood/strato/lib/liveRestrictionStatus.strato @@ -0,0 +1,154 @@ +import +import +import + +type User = com.twitter.gizmoduck.User +type LiveRestriction = com.twitter.strato.graphql.UnderTheHoodLiveRestriction +type LiveRestrictionStatus = com.twitter.strato.graphql.UnderTheHoodLiveRestrictionStatus + +// Bound both fanout and history. This is a creator-facing status check, not an +// unrestricted label-history API. +val postScanLimit = 100 +val postLookbackDays = 30 + +def accountLabelIsActive(createdAtMsec: Long, expiresAtMsec: Option[Long], nowMs: Long): Boolean = + createdAtMsec <= nowMs && expiresAtMsec.map { expiry => expiry > nowMs }.getOrElse(true) + +def postLabelIsActive( + createdAtMsec: Option[Long], + expiresAtMsec: Option[Long], + nowMs: Long +): Boolean = + createdAtMsec.map { applied => applied <= nowMs }.getOrElse(true) && + expiresAtMsec.map { expiry => expiry > nowMs }.getOrElse(true) + +def build(userId: Long, userOpt: Option[User], now: Time.T): LiveRestrictionStatus = { + val nowMs = now.inMillis + val accountRestrictions: Seq[LiveRestriction] = userOpt + .flatMap { user => user.labels } + .map { labels => labels.labels } + .getOrElse(Seq.empty) + .flatMap { label => + val raw = Thrift.labelOfEnum(label.labelValue) + if ( + underTheHoodLabels.isAccountLabel(raw) && + accountLabelIsActive(label.createdAtMsec, label.expiresAtMsec, nowMs) + ) { + Some({ + scope = Some("ACCOUNT"), + postId = None, + label = Some(underTheHoodLabels.accountLabelName(raw)), + about = Some(underTheHoodLabels.accountLabelAbout(raw)), + effect = Some(underTheHoodLabels.accountLabelEffect(raw)), + // The source label does not carry a machine-readable public surface taxonomy. + // The human-readable effect above is the only authoritative surface description. + affectedSurfaces = None, + appliedAt = Some(label.createdAtMsec), + expiresAt = label.expiresAtMsec, + // Raw Gizmoduck source/byUser values are deliberately not surfaced: they may + // contain internal actor identifiers and do not form a stable public taxonomy. + provenance = None, + modelVersion = None, + appealAvailability = None, + }) + } else { + None + } + } + + val timelineEdges = # + .paginate(Begin({ + source = #User(userId), + limit = postScanLimit, + startAt = None, + direction = Descending, + })) + .data + val timelinePostIds = timelineEdges + .map { edge => edge.to } + .distinct + val recentAuthoredPostIds = timelinePostIds.filter { postId => + val originalPostId = tweetUtil.originalTweetForRetweet(postId) + postId == originalPostId && + Snowflake.getTime(postId).map { created => + created.inMillis >= now.minus(postLookbackDays.days).inMillis + }.getOrElse(false) + } + + val postRestrictions: Seq[LiveRestriction] = recentAuthoredPostIds.flatMap { postId => + # + .fetch(postId, ()) + .v + .map { value => value.labels.getOrElse(Map.empty).toSeq } + .getOrElse(Seq.empty) + .flatMap { case (labelType, label) => + val raw = Thrift.labelOfEnum(labelType) + if ( + underTheHoodLabels.isPostLabel(raw) && + postLabelIsActive(label.createdAtMsec, label.expiresAtMsec, nowMs) + ) { + Some({ + scope = Some("POST"), + postId = Some(postId), + label = Some(underTheHoodLabels.postLabelName(raw)), + about = Some(underTheHoodLabels.postLabelAbout(raw)), + effect = Some(underTheHoodLabels.postLabelEffect(raw)), + // Do not infer a surface list from prose; the label store does not publish one. + affectedSurfaces = None, + appliedAt = label.createdAtMsec, + expiresAt = label.expiresAtMsec, + // SafetyLabel.source can encode actors or rule ids, so do not surface it. + provenance = None, + modelVersion = None, + appealAvailability = None, + }) + } else { + None + } + } + } + + // Pagination does not publish a total. A full page is conservatively reported as partial, + // even if it happens to be the user's complete timeline. + val truncated = timelineEdges.size >= postScanLimit + { + generatedAt = Some(nowMs), + freshness = Some("LIVE_BEST_EFFORT"), + restrictions = Some(accountRestrictions ++ postRestrictions), + postScanLimit = Some(postScanLimit), + postLookbackDays = Some(postLookbackDays), + postsScanned = Some(recentAuthoredPostIds.size), + postScanTruncated = Some(truncated), + postScanStatus = Some( + if (truncated) { "PARTIAL_LIMIT_REACHED" } else { "COMPLETE_WITHIN_LOOKBACK" } + ), + accountLabelStatus = Some( + userOpt.flatMap { user => user.labels } match { + case Some(_) => "AVAILABLE" + case None => "UNAVAILABLE" + } + ), + unavailableFields = Some(Seq( + "affectedSurfaces", + "provenance", + "modelVersion", + "appealAvailability", + )), + } +} + +Library({ + export = { + postScanLimit = postScanLimit, + postLookbackDays = postLookbackDays, + accountLabelIsActive = accountLabelIsActive, + postLabelIsActive = postLabelIsActive, + build = build, + }, + description = PlainText( + "Builds the bounded, creator-visible live restriction status. Only public allowlisted " + + "labels are returned; raw source, actor, and rule provenance is never exposed." + ), + contactInfo = sharedConfig.contactInfo, + lifecycle = Production, +})