Merge pull request #3698 from code-yeongyu/fix/posthog-cost-reduction

fix(telemetry): cut PostHog cost via HAU removal + plugin_loaded daily dedupe
2026-04-28 15:08:55 +09:00
parent 90c2c9c31a 669e0667be
commit 4eb597b10f
7 changed files with 365 additions and 55 deletions
@@ -104,6 +104,12 @@ function installModuleMocks(): void {
    createPluginPostHog: mockCreatePluginPostHog,
    getPostHogDistinctId: mockGetPostHogDistinctId,
  }))
+  mock.module("./shared/posthog-activity-state", () => ({
+    getPluginLoadedCaptureState: () => ({
+      dayUTC: "2026-04-18",
+      capturePluginLoaded: true,
+    }),
+  }))
 }

 describe("oh-my-openagent telemetry isolation", () => {
@@ -18,6 +18,7 @@ import { installAgentSortShim } from "./shared/agent-sort-shim"
 import { detectExternalSkillPlugin, getSkillPluginConflictWarning } from "./shared/external-plugin-detector"
 import { startBackgroundCheck as startTmuxCheck } from "./tools/interactive-bash"
 import { createPluginPostHog, getPostHogDistinctId } from "./shared/posthog"
+import { getPluginLoadedCaptureState } from "./shared/posthog-activity-state"

 const serverPlugin: Plugin = async (input, _options): Promise<Hooks> => {
  installAgentSortShim()
@@ -43,19 +44,27 @@ const serverPlugin: Plugin = async (input, _options): Promise<Hooks> => {
  } catch {
    // telemetry failure is non-fatal, silently ignore
  }
+  let pluginLoadedCaptureState: ReturnType<typeof getPluginLoadedCaptureState> | null = null
  try {
-    posthog.capture({
-      distinctId,
-      event: "plugin_loaded",
-      properties: {
-        entry_point: "plugin",
-        has_openclaw: !!pluginConfig.openclaw,
-        tmux_enabled: isTmuxIntegrationEnabled(pluginConfig),
-      },
-    })
+    pluginLoadedCaptureState = getPluginLoadedCaptureState()
  } catch {
    // telemetry failure is non-fatal, silently ignore
  }
+  if (pluginLoadedCaptureState?.capturePluginLoaded) {
+    try {
+      posthog.capture({
+        distinctId,
+        event: "plugin_loaded",
+        properties: {
+          entry_point: "plugin",
+          has_openclaw: !!pluginConfig.openclaw,
+          tmux_enabled: isTmuxIntegrationEnabled(pluginConfig),
+        },
+      })
+    } catch {
+      // telemetry failure is non-fatal, silently ignore
+    }
+  }
  if (pluginConfig.openclaw) {
    await initializeOpenClaw(pluginConfig.openclaw)
  }
@@ -1,5 +1,5 @@
 import { afterEach, describe, expect, it } from "bun:test"
-import { mkdirSync, rmSync, writeFileSync } from "node:fs"
+import { mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"
 import { join } from "node:path"
 import { tmpdir } from "node:os"

@@ -37,9 +37,7 @@ describe("getPostHogActivityCaptureState", () => {
    // then
    expect(result).toEqual({
      dayUTC: "2026-04-11",
-      hourUTC: "2026-04-11T10",
      captureDaily: true,
-      captureHourly: true,
    })

    rmSync(dataHomePath, { recursive: true, force: true })
@@ -60,9 +58,7 @@ describe("getPostHogActivityCaptureState", () => {
    // then
    expect(result).toEqual({
      dayUTC: "2026-04-11",
-      hourUTC: "2026-04-11T10",
      captureDaily: true,
-      captureHourly: true,
    })

    rmSync(dataHomePath, { recursive: true, force: true })
@@ -83,9 +79,7 @@ describe("getPostHogActivityCaptureState", () => {
    // then
    expect(result).toEqual({
      dayUTC: "2026-04-11",
-      hourUTC: "2026-04-11T10",
      captureDaily: true,
-      captureHourly: true,
    })

    rmSync(dataHomePath, { recursive: true, force: true })
@@ -112,11 +106,190 @@ describe("getPostHogActivityCaptureState", () => {
    // then
    expect(result).toEqual({
      dayUTC: "2026-04-11",
-      hourUTC: "2026-04-11T10",
      captureDaily: false,
-      captureHourly: false,
+    })
+
+    rmSync(dataHomePath, { recursive: true, force: true })
+  })
+
+  it("reads legacy hourly state without crashing", async () => {
+    // given
+    const dataHomePath = createDataHomePath()
+    const cachePath = join(dataHomePath, "oh-my-opencode")
+    mkdirSync(cachePath, { recursive: true })
+    writeFileSync(
+      join(cachePath, "posthog-activity.json"),
+      `${JSON.stringify({
+        lastActiveHourUTC: "2026-04-11T10",
+      })}\n`,
+    )
+    process.env.XDG_DATA_HOME = dataHomePath
+    const { getPostHogActivityCaptureState } = await importPostHogActivityStateModule()
+
+    // when
+    const result = getPostHogActivityCaptureState(new Date("2026-04-11T10:15:00.000Z"))
+
+    // then
+    expect(result).toEqual({
+      dayUTC: "2026-04-11",
+      captureDaily: true,
+    })
+
+    rmSync(dataHomePath, { recursive: true, force: true })
+  })
+
+  it("preserves lastPluginLoadedDayUTC when writing lastActiveDayUTC", async () => {
+    // given
+    const dataHomePath = createDataHomePath()
+    const cachePath = join(dataHomePath, "oh-my-opencode")
+    mkdirSync(cachePath, { recursive: true })
+    writeFileSync(
+      join(cachePath, "posthog-activity.json"),
+      `${JSON.stringify({
+        lastActiveDayUTC: "2026-04-10",
+        lastPluginLoadedDayUTC: "2026-04-11",
+      })}\n`,
+    )
+    process.env.XDG_DATA_HOME = dataHomePath
+    const { getPostHogActivityCaptureState } = await importPostHogActivityStateModule()
+
+    // when
+    getPostHogActivityCaptureState(new Date("2026-04-11T10:15:00.000Z"))
+
+    // then
+    const persistedState = JSON.parse(
+      readFileSync(join(cachePath, "posthog-activity.json"), "utf-8"),
+    )
+    expect(persistedState).toEqual({
+      lastActiveDayUTC: "2026-04-11",
+      lastPluginLoadedDayUTC: "2026-04-11",
    })

    rmSync(dataHomePath, { recursive: true, force: true })
  })
 })
+
+describe("getPluginLoadedCaptureState", () => {
+  it("returns capturePluginLoaded=true when activity file does not exist", async () => {
+    // given
+    const dataHomePath = createDataHomePath()
+    process.env.XDG_DATA_HOME = dataHomePath
+    const { getPluginLoadedCaptureState } = await importPostHogActivityStateModule()
+
+    // when
+    const result = getPluginLoadedCaptureState(new Date("2026-04-11T10:15:00.000Z"))
+
+    // then
+    expect(result).toEqual({
+      dayUTC: "2026-04-11",
+      capturePluginLoaded: true,
+    })
+
+    rmSync(dataHomePath, { recursive: true, force: true })
+  })
+
+  it("returns capturePluginLoaded=false when lastPluginLoadedDayUTC matches today", async () => {
+    // given
+    const dataHomePath = createDataHomePath()
+    const cachePath = join(dataHomePath, "oh-my-opencode")
+    mkdirSync(cachePath, { recursive: true })
+    writeFileSync(
+      join(cachePath, "posthog-activity.json"),
+      `${JSON.stringify({
+        lastPluginLoadedDayUTC: "2026-04-11",
+      })}\n`,
+    )
+    process.env.XDG_DATA_HOME = dataHomePath
+    const { getPluginLoadedCaptureState } = await importPostHogActivityStateModule()
+
+    // when
+    const result = getPluginLoadedCaptureState(new Date("2026-04-11T10:15:00.000Z"))
+
+    // then
+    expect(result).toEqual({
+      dayUTC: "2026-04-11",
+      capturePluginLoaded: false,
+    })
+
+    rmSync(dataHomePath, { recursive: true, force: true })
+  })
+
+  it("returns capturePluginLoaded=true when lastPluginLoadedDayUTC is from a previous day", async () => {
+    // given
+    const dataHomePath = createDataHomePath()
+    const cachePath = join(dataHomePath, "oh-my-opencode")
+    mkdirSync(cachePath, { recursive: true })
+    writeFileSync(
+      join(cachePath, "posthog-activity.json"),
+      `${JSON.stringify({
+        lastPluginLoadedDayUTC: "2026-04-10",
+      })}\n`,
+    )
+    process.env.XDG_DATA_HOME = dataHomePath
+    const { getPluginLoadedCaptureState } = await importPostHogActivityStateModule()
+
+    // when
+    const result = getPluginLoadedCaptureState(new Date("2026-04-11T10:15:00.000Z"))
+
+    // then
+    expect(result).toEqual({
+      dayUTC: "2026-04-11",
+      capturePluginLoaded: true,
+    })
+
+    rmSync(dataHomePath, { recursive: true, force: true })
+  })
+
+  it("preserves lastActiveDayUTC when writing lastPluginLoadedDayUTC", async () => {
+    // given
+    const dataHomePath = createDataHomePath()
+    const cachePath = join(dataHomePath, "oh-my-opencode")
+    mkdirSync(cachePath, { recursive: true })
+    writeFileSync(
+      join(cachePath, "posthog-activity.json"),
+      `${JSON.stringify({
+        lastActiveDayUTC: "2026-04-11",
+        lastPluginLoadedDayUTC: "2026-04-10",
+      })}\n`,
+    )
+    process.env.XDG_DATA_HOME = dataHomePath
+    const { getPluginLoadedCaptureState } = await importPostHogActivityStateModule()
+
+    // when
+    getPluginLoadedCaptureState(new Date("2026-04-11T10:15:00.000Z"))
+
+    // then
+    const persistedState = JSON.parse(
+      readFileSync(join(cachePath, "posthog-activity.json"), "utf-8"),
+    )
+    expect(persistedState).toEqual({
+      lastActiveDayUTC: "2026-04-11",
+      lastPluginLoadedDayUTC: "2026-04-11",
+    })
+
+    rmSync(dataHomePath, { recursive: true, force: true })
+  })
+
+  it("does not rewrite state when lastPluginLoadedDayUTC matches today", async () => {
+    // given
+    const dataHomePath = createDataHomePath()
+    const cachePath = join(dataHomePath, "oh-my-opencode")
+    mkdirSync(cachePath, { recursive: true })
+    const initialPayload = `${JSON.stringify({
+      lastActiveDayUTC: "2026-04-10",
+      lastPluginLoadedDayUTC: "2026-04-11",
+    })}\n`
+    writeFileSync(join(cachePath, "posthog-activity.json"), initialPayload)
+    process.env.XDG_DATA_HOME = dataHomePath
+    const { getPluginLoadedCaptureState } = await importPostHogActivityStateModule()
+
+    // when
+    getPluginLoadedCaptureState(new Date("2026-04-11T10:15:00.000Z"))
+
+    // then
+    const persistedPayload = readFileSync(join(cachePath, "posthog-activity.json"), "utf-8")
+    expect(persistedPayload).toBe(initialPayload)
+
+    rmSync(dataHomePath, { recursive: true, force: true })
+  })
+})
@@ -8,14 +8,17 @@ import { writeFileAtomically } from "./write-file-atomically"

 type PostHogActivityState = {
  lastActiveDayUTC?: string
-  lastActiveHourUTC?: string
+  lastPluginLoadedDayUTC?: string
 }

 type PostHogActivityCaptureState = {
  dayUTC: string
-  hourUTC: string
  captureDaily: boolean
-  captureHourly: boolean
+}
+
+type PluginLoadedCaptureState = {
+  dayUTC: string
+  capturePluginLoaded: boolean
 }

 const POSTHOG_ACTIVITY_STATE_FILE = "posthog-activity.json"
@@ -28,10 +31,6 @@ function getUtcDayString(date: Date): string {
  return date.toISOString().slice(0, 10)
 }

-function getUtcHourString(date: Date): string {
-  return date.toISOString().slice(0, 13)
-}
-
 function isPostHogActivityState(value: unknown): value is PostHogActivityState {
  return value !== null && typeof value === "object" && !Array.isArray(value)
 }
@@ -75,22 +74,37 @@ function writePostHogActivityState(nextState: PostHogActivityState): void {
 export function getPostHogActivityCaptureState(now: Date = new Date()): PostHogActivityCaptureState {
  const state = readPostHogActivityState()
  const dayUTC = getUtcDayString(now)
-  const hourUTC = getUtcHourString(now)

  const captureDaily = state.lastActiveDayUTC !== dayUTC
-  const captureHourly = state.lastActiveHourUTC !== hourUTC

-  if (captureDaily || captureHourly) {
+  if (captureDaily) {
    writePostHogActivityState({
-      lastActiveDayUTC: captureDaily ? dayUTC : state.lastActiveDayUTC,
-      lastActiveHourUTC: captureHourly ? hourUTC : state.lastActiveHourUTC,
+      ...state,
+      lastActiveDayUTC: dayUTC,
    })
  }

  return {
    dayUTC,
-    hourUTC,
    captureDaily,
-    captureHourly,
+  }
+}
+
+export function getPluginLoadedCaptureState(now: Date = new Date()): PluginLoadedCaptureState {
+  const state = readPostHogActivityState()
+  const dayUTC = getUtcDayString(now)
+
+  const capturePluginLoaded = state.lastPluginLoadedDayUTC !== dayUTC
+
+  if (capturePluginLoaded) {
+    writePostHogActivityState({
+      ...state,
+      lastPluginLoadedDayUTC: dayUTC,
+    })
+  }
+
+  return {
+    dayUTC,
+    capturePluginLoaded,
  }
 }
@@ -1,23 +1,54 @@
-import { afterEach, describe, expect, it, mock } from "bun:test"
+import { afterEach, beforeEach, describe, expect, it, mock } from "bun:test"
+
+type CapturedPostHogMessage = {
+  distinctId: string
+  event: string
+  properties?: Record<string, unknown>
+}

 async function importPostHogModule(): Promise<typeof import("./posthog")> {
  return import(`./posthog?test=${Date.now()}-${Math.random()}`)
 }

+function enableTelemetryEnv(): void {
+  process.env.OMO_DISABLE_POSTHOG = "0"
+  process.env.OMO_SEND_ANONYMOUS_TELEMETRY = "1"
+  process.env.POSTHOG_API_KEY = "test-api-key"
+}
+
+function clearTelemetryEnv(): void {
+  delete process.env.OMO_DISABLE_POSTHOG
+  delete process.env.OMO_SEND_ANONYMOUS_TELEMETRY
+  delete process.env.POSTHOG_API_KEY
+  delete process.env.POSTHOG_HOST
+}
+
+function mockPostHogNode(capturedMessages: CapturedPostHogMessage[]): void {
+  mock.module("posthog-node", () => ({
+    PostHog: class {
+      capture(message: CapturedPostHogMessage): void {
+        capturedMessages.push(message)
+      }
+      captureException(): void {}
+      async shutdown(): Promise<void> {}
+    },
+  }))
+}
+
 describe("posthog client creation", () => {
+  beforeEach(() => {
+    mock.restore()
+    clearTelemetryEnv()
+  })
+
  afterEach(() => {
    mock.restore()
-    delete process.env.OMO_DISABLE_POSTHOG
-    delete process.env.OMO_SEND_ANONYMOUS_TELEMETRY
-    delete process.env.POSTHOG_API_KEY
-    delete process.env.POSTHOG_HOST
+    clearTelemetryEnv()
  })

  it("returns a no-op client when PostHog construction throws", async () => {
    // given
-    process.env.OMO_DISABLE_POSTHOG = "0"
-    process.env.OMO_SEND_ANONYMOUS_TELEMETRY = "1"
-    process.env.POSTHOG_API_KEY = "test-api-key"
+    enableTelemetryEnv()

    mock.module("posthog-node", () => ({
      PostHog: class {
@@ -100,3 +131,73 @@ describe("posthog client creation", () => {
    await expect(pluginPostHog.shutdown()).resolves.toBeUndefined()
  })
 })
+
+describe("posthog trackActive emission contract", () => {
+  let resetActivityStateProvider: (() => void) | null = null
+
+  beforeEach(() => {
+    mock.restore()
+    clearTelemetryEnv()
+  })
+
+  afterEach(() => {
+    resetActivityStateProvider?.()
+    resetActivityStateProvider = null
+    mock.restore()
+    clearTelemetryEnv()
+  })
+
+  it("emits exactly one omo_daily_active and never omo_hourly_active when captureDaily is true", async () => {
+    // given
+    enableTelemetryEnv()
+    const captured: CapturedPostHogMessage[] = []
+    mockPostHogNode(captured)
+    const posthogModule = await importPostHogModule()
+    posthogModule.__setActivityStateProviderForTesting(() => ({
+      dayUTC: "2026-04-18",
+      captureDaily: true,
+    }))
+    resetActivityStateProvider = posthogModule.__resetActivityStateProviderForTesting
+    const client = posthogModule.createCliPostHog()
+
+    // when
+    client.trackActive("distinct-cli", "run_started")
+
+    // then
+    expect(captured).toHaveLength(1)
+    const emittedEvents = captured.map((message) => message.event)
+    expect(emittedEvents).not.toContain("omo_hourly_active")
+    const [dailyEvent] = captured
+    expect(dailyEvent?.event).toBe("omo_daily_active")
+    expect(dailyEvent?.distinctId).toBe("distinct-cli")
+    expect(dailyEvent?.properties).toMatchObject({
+      day_utc: "2026-04-18",
+      reason: "run_started",
+      source: "cli",
+    })
+    expect(dailyEvent?.properties).not.toHaveProperty("hour_utc")
+  })
+
+  it("emits nothing and never omo_hourly_active when captureDaily is false", async () => {
+    // given
+    enableTelemetryEnv()
+    const captured: CapturedPostHogMessage[] = []
+    mockPostHogNode(captured)
+    const posthogModule = await importPostHogModule()
+    posthogModule.__setActivityStateProviderForTesting(() => ({
+      dayUTC: "2026-04-18",
+      captureDaily: false,
+    }))
+    resetActivityStateProvider = posthogModule.__resetActivityStateProviderForTesting
+    const client = posthogModule.createPluginPostHog()
+
+    // when
+    client.trackActive("distinct-plugin", "plugin_loaded")
+
+    // then
+    expect(captured).toHaveLength(0)
+    const emittedEvents = captured.map((message) => message.event)
+    expect(emittedEvents).not.toContain("omo_daily_active")
+    expect(emittedEvents).not.toContain("omo_hourly_active")
+  })
+})
@@ -5,6 +5,25 @@ import packageJson from "../../package.json" with { type: "json" }
 import { PLUGIN_NAME, PUBLISHED_PACKAGE_NAME } from "./plugin-identity"
 import { getPostHogActivityCaptureState } from "./posthog-activity-state"

+/** @internal test-only seam: keep null in production to use the real implementation. */
+let activityStateProviderOverride: typeof getPostHogActivityCaptureState | null = null
+
+function resolveActivityState(): ReturnType<typeof getPostHogActivityCaptureState> {
+  return (activityStateProviderOverride ?? getPostHogActivityCaptureState)()
+}
+
+/** @internal test-only */
+export function __setActivityStateProviderForTesting(
+  provider: typeof getPostHogActivityCaptureState,
+): void {
+  activityStateProviderOverride = provider
+}
+
+/** @internal test-only */
+export function __resetActivityStateProviderForTesting(): void {
+  activityStateProviderOverride = null
+}
+
 const DEFAULT_POSTHOG_HOST = "https://us.i.posthog.com"
 const DEFAULT_POSTHOG_API_KEY = "phc_CFJhj5HyvA62QPhvyaUCtaq23aUfznnijg5VaaGkNk74"

@@ -128,7 +147,7 @@ function createPostHogClient(
      })
    },
    trackActive: (distinctId, reason) => {
-      const activityState = getPostHogActivityCaptureState()
+      const activityState = resolveActivityState()

      if (activityState.captureDaily) {
        configuredClient.capture({
@@ -141,18 +160,6 @@ function createPostHogClient(
          },
        })
      }
-
-      if (activityState.captureHourly) {
-        configuredClient.capture({
-          distinctId,
-          event: "omo_hourly_active",
-          properties: {
-            ...sharedProperties,
-            hour_utc: activityState.hourUTC,
-            reason,
-          },
-        })
-      }
    },
    shutdown: async () => configuredClient.shutdown(),
  }