diff --git a/README.md b/README.md index a94cd78..5be43bd 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ For Node bots, use a durable Chat SDK state adapter and a durable Matrix store. ### Raw Matrix core (no Chat SDK) ```ts -import { createMatrixClient } from "better-matrix-js/node"; +import { createMatrixClient, onMessage } from "better-matrix-js/node"; import { createFileMatrixStore } from "@better-matrix-js/state-file"; const client = createMatrixClient({ @@ -60,7 +60,7 @@ const client = createMatrixClient({ recoveryKey: process.env.MATRIX_RECOVERY_KEY, }); -client.events.onMessage(async (event) => { +await onMessage(client, undefined, async (event) => { if (event.sender.isMe) return; await client.messages.send({ roomId: event.roomId, @@ -68,9 +68,6 @@ client.events.onMessage(async (event) => { replyTo: event.eventId, }); }); - -await client.connect(); -await client.sync.start(); ``` ### Cloudflare Worker @@ -83,6 +80,16 @@ Use `MatrixSyncDurableObject` for live sync and feed each webhook body to `clien Use `@better-matrix-js/state-memory` for tests, `@better-matrix-js/state-file` or `@better-matrix-js/state-sqlite` in Node, `@better-matrix-js/state-indexeddb` in browsers, and `@better-matrix-js/cloudflare` for Durable Object or KV storage. For anything custom, wrap a simple getter/setter with `@better-matrix-js/state-simple`. +Docker-backed storage smoke tests are available for service-style stores: + +```sh +pnpm test:docker +pnpm test:docker:down +``` + +The Redis smoke uses `@better-matrix-js/state-simple` against a real Redis container +to prove the minimal Matrix store contract works with external server-side storage. + Browser apps should load `matrix-core.wasm` with `wasmUrl`, `wasmBytes`, or a bundler-provided `wasmModule`, and should persist Matrix state in IndexedDB. ## Feature support matrix @@ -92,7 +99,7 @@ Browser apps should load `matrix-core.wasm` with `wasmUrl`, `wasmBytes`, or a bu | Node bots | Supported via `better-matrix-js/node` and file, SQLite, or custom stores. | | Browser apps | Supported with explicit WASM loading and IndexedDB-backed state. | | Cloudflare Workers | Supported with Durable Object state and `MatrixSyncDurableObject`. | -| Live `/sync` loop | Supported with `client.sync.start()` in long-lived runtimes. | +| Live `/sync` loop | Supported with `client.subscribe(filter, handler)` in long-lived runtimes. | | Serverless sync | Supported by applying webhooked responses with `applyResponse`. | | E2EE | Supported when the crypto store, `pickleKey`, and optional `recoveryKey` are durable. | | Beeper ephemeral events | Supported only on Beeper homeservers. | diff --git a/TODO.md b/TODO.md index 1b75ab6..e293db8 100644 --- a/TODO.md +++ b/TODO.md @@ -1,127 +1,305 @@ -# Matrix SDK Alignment TODO - -## API Alignment - -- [x] Remove `recoveryCode`; keep only top-level `recoveryKey` in JS and Go contracts. -- [x] Keep `recoveryKey` top-level in `MatrixClientOptions` and adapter config. -- [x] Keep one sync ingestion primitive: `client.sync.applyResponse({ response, since })`. -- [x] Do not add a Matrix-client `applyEnvelope`; encrypted webhook envelopes belong to app/transport helpers. -- [x] Make `MatrixAdapter` explicitly satisfy Chat SDK adapter requirements. -- [x] Delete duplicated `MatrixRawMessage` in the Chat adapter; use the core `MatrixMessageEvent` as raw data. -- [x] Collapse duplicated public/runtime/generated Matrix event types where possible; keep generated runtime and public client event types separate because they model different wire/client shapes. -- [x] Keep package public entrypoints, but stop internal imports from convenience barrels. -- [x] Normalize option naming to one public spelling for each concern. -- [x] Make Beeper-specific features explicit under `client.beeper`. - -## Main Library Capabilities - -- [x] Expose `client.beeper.ephemeral.send(...)` for Beeper homeservers. -- [x] Implement Chat SDK `postEphemeral` through Beeper ephemeral events when supported. -- [x] Make non-Beeper `postEphemeral` fail clearly instead of pretending to be portable Matrix. -- [x] Move generic streaming orchestration from `@better-matrix-js/chat-adapter` into `better-matrix-js`. -- [x] Expose `client.streams.send(...)` with automatic Beeper-native or edit-fallback mode. -- [x] Keep Beeper native streaming available under `client.beeper.streams`. -- [x] Move AI-specific stream conversion into `@better-matrix-js/ai-sdk`. -- [x] Expose `client.crypto.status()` or equivalent queryable E2EE status. -- [x] Expose pending decryption count/status. -- [x] Expose recovery/backup status after startup. -- [x] Add profile APIs: get/set own display name and avatar. -- [x] Add room creation API beyond DM creation. -- [x] Add room permission/power-level inspection. -- [x] Add generic room state read/send APIs for advanced users. -- [x] Add member listing and member event/profile APIs. -- [x] Add room alias resolution and optional directory lookup. -- [x] Add optional media thumbnail support. -- [x] Decide whether URL previews belong in scope; document unsupported if not. - -## Go/Core Ownership - -- [x] Move Matrix attachment extraction fully into Go/core event normalization. -- [x] Move mention detection fully into Go/core event normalization. -- [x] Move relation parsing fully into Go/core event normalization. -- [x] Normalize replies, threads, edits, annotations/reactions, and references in core. -- [x] Normalize inbound redactions in core. -- [x] Normalize inbound edits in core without Chat adapter raw-content inference. -- [x] Move reaction target/thread lookup into core state. -- [x] Make reaction removal work across cold starts. -- [x] Make `openDM(userId)` reuse existing `m.direct` rooms by default. -- [x] Add an option to force creating a new DM when needed. -- [x] Use `m.direct` account data for DM detection before member-count fallback. -- [x] Make Beeper sync options conditional instead of setting `BeeperStreaming: true` for every homeserver. -- [x] Add Beeper capability detection beyond hostname fallback. -- [x] Keep encrypted media behavior in Go; remove duplicate TS parsing paths. -- [x] Ensure fetch-message pagination always returns chronological page order. -- [x] Ensure sync response replay is idempotent at the emitted event level. +# Matrix SDK v1 Completion Plan -## Serverless +## Product Intention -- [x] Document two modes: live sync and serverless apply-response. -- [x] Make cursor ownership explicit for Cloudflare DO syncer versus core `nextBatch`. -- [x] Ensure serverless `applyResponse` works with E2EE cold starts and durable crypto state. -- [x] Add a single-writer story for E2EE stores. -- [x] Recommend Durable Objects or other serialized storage for encrypted bots. -- [x] Warn against concurrent KV writes for active E2EE devices. -- [x] Add optional Cloudflare encrypted webhook helper outside the Matrix client sync API. -- [x] Add webhook replay/idempotency guidance. +Build `better-matrix-js` as a Node-first Matrix client SDK that is agents/bots-first but capable of becoming a full client SDK. The public API should feel like a Vercel SDK: synchronous factory, lazy async methods, plain serializable account objects, one live subscription primitive, pure helper functions, and no sync ceremony during construction. -## Chat SDK Adapter +The Chat SDK adapter must be a very thin translator over `better-matrix-js`. If Chat SDK needs Matrix behavior, the behavior belongs in core or a focused helper exported by core. The adapter should map Chat SDK concepts to Matrix calls, not maintain a parallel Matrix client model. + +Non-standard Beeper features are first-class but must remain explicit under `client.beeper` or Beeper-namespaced event/content handling. Standard Matrix behavior should stay standard and portable. + +No backward compatibility is required. Prefer deleting old API shapes and duplicated layers over preserving aliases. + +## API Contract Target + +- [x] `createMatrixClient(options)` is synchronous and inert. +- [x] First awaited Matrix method lazily boots runtime/core/store/crypto. +- [x] `client.boot()` exists for apps that want startup failure early. +- [x] Remove public `connect()`. +- [x] Use `MatrixAccount` as the serializable account/session object. +- [x] Make `deviceId` immutable identity from login/whoami, not a user-editable client option. +- [x] Keep `client.sync` minimal with only `applyResponse({ response, since })`. +- [x] Make `client.subscribe(filter, handler)` the only core live event primitive. +- [x] `subscribe` returns `{ stop, catchUp, done }`. +- [x] Default subscription delivery is future-only. +- [x] `catchUp()` explicitly replays missed events from stored cursor. +- [x] Move ergonomic event helpers to pure exports: `onMessage`, `onReaction`, `onInvite`, `onRawEvent`. +- [x] Add short namespaces that were still missing: + - [x] `client.accountData.*` + - [x] `client.toDevice.*` + - [x] `client.receipts.*` + - [x] `client.raw.request(...)` +- [x] Confirm every public namespace has one canonical method name per operation and no aliases. +- [x] Audit package exports so there are no convenience barrel imports inside package source. + +## Boot, Account, And Login + +- [x] `boot()` initializes runtime/account/store/crypto only. +- [x] Request methods work without sync for CLI usage. +- [x] `whoami()` boots if needed and confirms account identity. +- [x] Generic password/token login returns `MatrixAccount`. +- [x] Remove public login option for caller-selected `deviceId`. +- [x] Add logout helper. +- [x] Add token/JWT login coverage for returned `MatrixAccount`. +- [x] Persist/use account metadata without making it required for runtime identity. +- [x] Add Beeper signup/login flow object under `client.beeper` or a Beeper login helper package. +- [x] Ensure no QA-specific secrets, OTP assumptions, or fixed Beeper test behavior ship in public code. + +## Sync And Subscription Semantics + +- [x] Sync starts only through `client.subscribe(...)`, pure handler helpers, or `sync.applyResponse(...)`. +- [x] Reused accounts do not replay backlog unless `catchUp()` is called. +- [x] Explicit pagination reads history independently of subscription delivery. +- [x] Stopping the last subscription stops the internal sync runner. +- [x] Make `sub.done` reject on unrecoverable sync loop errors from Go, not only subscription handler errors. +- [x] Add subscription options for runtime sync tuning if needed without exposing `sync.start`. +- [x] Add tests for multiple subscribers sharing one sync runner. +- [x] Add tests for stopping one subscriber while another remains active. +- [x] Add tests for handler failures and `done` rejection behavior. +- [x] Add tests that `boot()` does not emit app events. +- [x] Add tests that default subscription is future-only with a reused stored cursor. +- [x] Add tests that `catchUp()` emits missed events and only through that subscription. -- [x] Make the Chat adapter a thin translator over `better-matrix-js`. -- [x] Delete old Chat adapter streaming drivers; keep streaming delegated to core plus the public `MatrixStream` type/helper exports. -- [x] Remove Chat adapter Matrix attachment parser after core emits normalized attachments. -- [x] Remove Chat adapter Matrix mention parser after core emits normalized mentions. -- [x] Remove Chat adapter Matrix relation parser after core emits normalized relations. -- [x] Remove Chat adapter in-memory reaction/thread authority. -- [x] Keep only Chat message construction, formatting conversion, slash dispatch, and Chat SDK method mapping. -- [x] Wire Chat SDK streaming to `client.streams.send(...)`. -- [x] Wire Chat SDK ephemeral messages to `client.beeper.ephemeral.send(...)`. - -## Code Organization - -- [x] Split `packages/core/src/client.ts` public interfaces into `client-types.ts`. -- [x] Split `packages/core/src/client.ts` event normalization into `events.ts`. -- [x] Split `packages/core/src/client.ts` streaming orchestration into `streams.ts`. -- [x] Continue shrinking `packages/core/src/client.ts` by moving media byte helpers if it keeps growing. -- [x] Keep card/action support fallback-only unless Beeper interactive product scope is explicitly added. -- [x] Document unsupported Chat SDK features: native modals and native scheduled messages. +## Raw Event Access + +- [x] `onRawEvent(...)` helper exists. +- [x] Raw helper currently exposes mapped events plus available raw payload. +- [x] Implement true granular raw Matrix sync events from Go for: + - [x] joined room timeline events + - [x] invited room state + - [x] left room timeline/state + - [x] room state events + - [x] ephemeral room events + - [x] account data events + - [x] to-device events + - [x] device list changes + - [x] presence events if supported +- [x] Raw events must include unmodified Matrix payload and source metadata: + - [x] sync cursor `since` + - [x] next batch when available + - [x] room id if applicable + - [x] event class/source section + - [x] event type + - [x] encrypted/decrypted status where applicable +- [x] Ensure raw event delivery shares the same subscription runner and filter path. +- [x] Add unit tests for raw event filtering and metadata. +- [ ] Add e2e coverage for raw encrypted timeline events. + +## Normalized Event Model + +- [x] Message events. +- [x] Reaction events. +- [x] Invite events. +- [x] Sync status events. +- [x] Crypto status events. +- [x] Decryption error events. +- [x] Redaction events as first-class normalized events. +- [x] Membership events as first-class normalized events. +- [x] Room state events as first-class normalized events. +- [x] Account data events as first-class normalized events. +- [x] To-device events as first-class normalized events. +- [x] Ephemeral events as first-class normalized events. +- [x] Receipt events as first-class normalized events. +- [x] Typing events as first-class normalized events. +- [ ] Room summary/update events if needed for client UIs. +- [ ] Decryption lifecycle events for pending, retried, failed, and recovered decryptions. +- [x] Make event filter matching work consistently across `kind`, `roomId`, `type`, sender, relation, and thread root where available. +- [x] Add unit tests for every normalized event mapper. +- [x] Add Go tests for every event emitted from `/sync`. + +## Core Matrix Capabilities + +- [x] Messages: send, edit, redact, get, list, mark read. +- [x] Reactions: send, redact, cold-start removal state. +- [x] Media: upload/download encrypted and unencrypted media. +- [x] Rooms: create, join, leave, invite, ban, kick, unban, open DM. +- [x] Room state read/send. +- [x] Room power-level inspection. +- [x] Thread listing. +- [x] Profile get/set own display name/avatar. +- [x] Account data get/set helpers. +- [x] Room account data get/set helpers. +- [x] To-device send helper. +- [x] Receipt send helper. +- [x] Generic raw Matrix request helper with typed method/path/body/query. +- [ ] Account data delete helper if a homeserver-supported delete shape is needed. +- [ ] Receipt fetch helpers if a product need appears beyond sync receipt events. +- [ ] Pagination helpers that cleanly support old encrypted history. +- [ ] Room membership timeline/history helpers for full-client usage. +- [ ] Room summary cache exposed without becoming a gomuks-style timeline DB. +- [ ] Relation summary cache for reactions/threads/edit summaries. +- [ ] Bounded recent cache configuration and tests. ## E2EE -- [x] Require or strongly recommend explicit `pickleKey` for durable E2EE bot deployments. -- [x] Reconsider access-token fallback as pickle key before release. -- [x] Provide a clear bot onboarding flow: login, device ID, store persistence, recovery key restore. -- [ ] Test fresh-device historical decryption via recovery key. Requires live Matrix credentials that can create a fresh device and access a real key backup. -- [x] Test missing backup/recovery status behavior. -- [x] Test encrypted media upload/download roundtrip. -- [x] Test decryption retry and pending queue persistence. - -## Tests And Verification - -- [x] Add compile-time Chat SDK adapter conformance test. -- [x] Add Go relation parsing tests. -- [x] Add Go redaction/edit normalization tests. -- [x] Add reaction removal after cold start test. -- [x] Add `openDM` reuse test. -- [ ] Add serverless encrypted-room `applyResponse` test. Requires a captured encrypted `/sync` fixture plus matching durable crypto store, or live Matrix credentials. -- [x] Add serverless replay/idempotency test. -- [x] Add core streaming tests for Beeper-native and edit-fallback modes. -- [x] Add Cloudflare Worker smoke with Durable Object store and WASM. -- [ ] Add browser smoke with IndexedDB and WASM. Requires a browser harness that can load the packaged WASM asset and IndexedDB store. -- [x] Add Node smoke with file/sqlite store and E2EE. -- [x] Run `pnpm typecheck`. -- [x] Run `pnpm test`. -- [x] Run `pnpm test:go`. -- [x] Run `pnpm build`. -- [x] Run package consumer and Cloudflare smoke tests. - -## Documentation - -- [x] Document browser setup. -- [x] Document Node setup. -- [x] Document Cloudflare Worker setup. -- [x] Document serverless sync apply-response flow. -- [x] Document E2EE bot storage requirements. -- [x] Document recovery key usage. -- [x] Document Beeper-only ephemeral support. -- [x] Document Beeper-native streaming and edit-fallback streaming. -- [x] Document feature support matrix. +- [x] E2EE is initialized through mautrix/go crypto helper. +- [x] Durable crypto store support. +- [x] Recovery key support. +- [x] Crypto status query includes pending decryption count and backup status. +- [x] Pending decryptions are persisted/retried. +- [x] Encrypted media roundtrip test coverage exists. +- [ ] Fresh-device historical decryption via recovery key e2e. +- [ ] Existing-device reused account decrypts old encrypted history e2e. +- [ ] Existing accounts with old devices and old rooms e2e. +- [ ] Multi-device same account behavior tests. +- [ ] Multi-client same-process isolation tests. +- [ ] Missing room key behavior tests. +- [ ] Key backup unavailable/unverified behavior tests in JS unit coverage. +- [ ] Browser E2EE smoke once browser harness exists. +- [ ] Cloudflare E2EE smoke after API migration. + +## Storage + +- [x] Keep separate storage packages: memory, file, sqlite, indexeddb, cloudflare. +- [x] Store fast-boot state: crypto, cursor, pending decryptions, reaction summaries. +- [x] Audit all stores against the new lazy lifecycle. +- [x] Add shared conformance tests for all storage adapters. +- [x] Confirm no store tries to model a full gomuks timeline DB. +- [ ] Persist room summaries needed for bot/client startup. +- [ ] Persist relation summaries needed for reactions/threads. +- [ ] Add bounded recent event cache and eviction tests. +- [x] Document single-writer requirements per store. +- [x] Cloudflare Durable Object store smoke after subscription API change. +- [x] IndexedDB smoke after subscription API change. + +## Beeper + +- [x] Existing Beeper stream primitives under `client.beeper.streams`. +- [x] Existing Beeper ephemeral send under `client.beeper.ephemeral`. +- [x] Beeper stream auto mode for Beeper homeservers. +- [ ] Move all remaining non-standard Beeper APIs under `client.beeper`. +- [ ] Add Beeper capability discovery beyond hostname where possible. +- [x] Add Beeper login/signup flow as stateless request functions. +- [x] Add tests ensuring non-standard event/content keys remain namespaced. +- [x] Document Beeper-first behavior and standard Matrix fallback behavior. + +## Chat SDK Adapter + +- [x] Adapter initializes with `whoami()` and no public `connect()`. +- [x] `sync.enabled: false` disables live subscription. +- [x] Live mode uses `client.subscribe(...)`. +- [x] Webhook/serverless mode uses `client.sync.applyResponse(...)`. +- [x] Streaming delegates to core `client.streams.send(...)`. +- [x] Ephemeral delegates to `client.beeper.ephemeral.send(...)`. +- [ ] Move remaining generic Matrix parsing/rendering from adapter to core: + - [ ] content parsing + - [ ] mentions + - [ ] media normalization + - [ ] relation/thread mapping + - [ ] Beeper content primitives +- [x] Audit cards/actions behavior: + - [x] text fallback only when no unsupported interactivity is implied + - [x] throw clearly for unsupported interactive cards/actions + - [x] tests for unsupported behavior +- [x] Add tests for live subscription mode. +- [x] Add tests for sync-disabled CLI/request mode. +- [x] Add tests for webhook/apply mode with raw JSON payloads. +- [x] Confirm adapter does not keep parallel Matrix event systems or stores. + +## Serverless + +- [x] `client.sync.applyResponse({ response, since })` accepts externally supplied raw `/sync`. +- [x] Serverless encrypted sync payload decryption is intentionally outside core sync API. +- [x] Add optional stateless helper in adapter or companion package for encrypted webhook payloads. +- [x] Add replay/idempotency tests for serverless apply. +- [ ] Add encrypted-room `applyResponse` fixture test with matching crypto store. +- [x] Document cursor ownership in live, webhook, and Durable Object modes. +- [x] Document how to avoid concurrent writers for encrypted devices. +- [x] Cloudflare Worker smoke using current subscription/apply APIs. + +## Node, Browser, Cloudflare Compatibility + +- [x] Node entrypoint lazily loads packaged WASM. +- [x] Generic entrypoint still accepts `wasmBytes`, `wasmModule`, or `wasmUrl`. +- [x] Node smoke against packaged build after API migration. +- [ ] Browser smoke with IndexedDB and WASM asset. +- [x] Cloudflare Worker smoke with Durable Object store and WASM. +- [x] Confirm no Node-only imports leak into browser/core entrypoint. +- [x] Confirm Node helper exports are available from `better-matrix-js/node`. +- [x] Confirm package exports support direct helper imports if needed. + +## AI Streaming + +- [x] Core accepts generic async iterable text/delta input. +- [x] AI-specific helper package exists separately. +- [x] Confirm optional AI helper has no required runtime dependency on AI SDK. +- [x] Add type-only/dev import audit. +- [x] Add streaming tests for generic string chunks, text deltas, markdown chunks, and empty streams. + +## Public Documentation + +- [x] README updated away from old `connect/events/sync.start` API. +- [x] Core README updated away from old API. +- [x] Add a dedicated API overview with: + - [x] inert factory and lazy boot + - [x] `MatrixAccount` + - [x] CLI usage without sync + - [x] live subscription usage + - [x] `catchUp()` + - [x] serverless `applyResponse` + - [x] raw event helper + - [x] E2EE store/recovery guidance +- [x] Add migration note stating no backward compatibility is intended pre-release. +- [x] Add Chat SDK adapter usage docs for live, disabled-sync, and webhook modes. +- [x] Add Beeper-specific docs. +- [x] Add unsupported features docs for cards/actions/modals/scheduled messages. +- [x] Add e2e README explaining external Beeper setup and cached account reuse. + +## E2E Test Plan + +- [ ] Move public e2e tests into this repo with a README. +- [x] Keep e2e out of default CI. +- [x] Reuse cached Beeper accounts by default. +- [ ] Scenario: lazy client can send/fetch without sync. +- [ ] Scenario: `boot()` initializes but does not emit app events. +- [ ] Scenario: `whoami()` confirms account/device identity. +- [ ] Scenario: `client.subscribe(...)` returns `{ stop, catchUp, done }`. +- [ ] Scenario: default subscription receives future events only. +- [ ] Scenario: `catchUp()` replays missed events. +- [ ] Scenario: `onRawEvent(...)` receives raw granular Matrix payloads. +- [ ] Scenario: encrypted messages. +- [ ] Scenario: edits. +- [ ] Scenario: reactions and reaction removals. +- [ ] Scenario: media upload/download. +- [ ] Scenario: threads. +- [ ] Scenario: invites and auto-join. +- [ ] Scenario: room state. +- [ ] Scenario: account data. +- [ ] Scenario: to-device. +- [ ] Scenario: receipts. +- [ ] Scenario: reused accounts paginate and decrypt old encrypted history. +- [ ] Scenario: fresh and existing devices behave correctly. +- [ ] Scenario: multi-client same-process isolation. +- [ ] Scenario: Chat SDK live subscription mode. +- [ ] Scenario: Chat SDK sync-disabled mode. +- [ ] Scenario: Chat SDK webhook/apply mode. + +## Unit And Type Test Plan + +- [x] Core lazy boot behavior. +- [x] Subscription controller lifecycle basics. +- [x] Chat adapter type conformance. +- [x] Subscription multi-subscriber lifecycle. +- [x] Subscription handler error behavior. +- [x] Pure helper behavior for `onMessage`, `onReaction`, `onInvite`, `onRawEvent`. +- [x] Normalized event mapping for every event kind. +- [x] Raw event path. +- [x] Storage adapter conformance. +- [x] Unsupported Chat SDK card/action behavior. +- [x] `raw.request` request construction and error handling. +- [x] Account data/to-device/receipt helper tests. +- [x] Raw request helper tests. + +## Release Readiness + +- [x] `pnpm typecheck` +- [x] `pnpm test` +- [x] `pnpm test:go` +- [x] `pnpm build` +- [x] Package consumer smoke. +- [x] Cloudflare smoke. +- [ ] Browser smoke. +- [ ] Node live e2e with cached accounts. +- [x] Review public exports for duplicate or fake layers. +- [x] Review code for duplicate types and adapters owning core logic. +- [x] Review docs for stale old API references. diff --git a/docker-compose.e2e.yml b/docker-compose.e2e.yml new file mode 100644 index 0000000..8caa85b --- /dev/null +++ b/docker-compose.e2e.yml @@ -0,0 +1,11 @@ +services: + redis: + image: ${BMJS_E2E_REDIS_IMAGE:-public.ecr.aws/docker/library/redis:7.4-alpine} + command: ["redis-server", "--save", "", "--appendonly", "no"] + ports: + - "${BMJS_E2E_REDIS_PORT:-6380}:6379" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 1s + timeout: 1s + retries: 30 diff --git a/docs/API.md b/docs/API.md new file mode 100644 index 0000000..13ff5c4 --- /dev/null +++ b/docs/API.md @@ -0,0 +1,197 @@ +# better-matrix-js API Overview + +`better-matrix-js` is a Matrix client SDK built around one lifecycle model: + +- `createMatrixClient(options)` is synchronous and inert. +- First awaited Matrix method lazily boots WASM, store, account identity, and crypto. +- `client.boot()` exists when an app wants startup failures early. +- Live events only flow through `client.subscribe(filter, handler)`. +- Serverless sync payloads enter through `client.sync.applyResponse({ response, since })`. + +There is no public `connect()`, `events`, `sync.start()`, `sync.once()`, or `sync.stop()`. + +## Migration Stance + +This SDK has not had a stable public release. The v1 API intentionally deletes old generated shapes instead of preserving aliases. Treat stale examples that mention `connect()`, root `events`, or public `sync.start()` as obsolete. + +## Account Objects + +Use `MatrixAccount` as the serializable account/session shape: + +```ts +type MatrixAccount = { + homeserver: string; + userId: string; + deviceId: string; + accessToken: string; + metadata?: Record; +}; +``` + +`deviceId` is immutable identity returned by Matrix login/whoami. Do not generate or edit it as a runtime option for an existing access token. + +```ts +const login = createMatrixLogin({ homeserver: "https://matrix.example.com" }); +const account = await login.token({ token: process.env.MATRIX_LOGIN_TOKEN! }); + +const client = createMatrixClient({ + account, + pickleKey: process.env.MATRIX_PICKLE_KEY!, + store, +}); + +await client.whoami(); +``` + +`metadata` is carried through login results if provided, but it is never used as Matrix identity. Runtime identity is always `userId`, `deviceId`, `homeserver`, and `accessToken`. + +## CLI Usage Without Sync + +Request-style programs can send/fetch and exit without subscribing: + +```ts +const client = createMatrixClient({ account, store, pickleKey }); + +await client.messages.send({ + roomId: "!room:example.com", + text: "done", +}); + +await client.close(); +``` + +No `/sync` loop is started by construction, `boot()`, `whoami()`, send, fetch, or pagination. + +## Live Subscriptions + +Use one root live primitive: + +```ts +const sub = await client.subscribe({ kind: "message", roomId }, async (event) => { + if (event.kind !== "message" || event.sender.isMe) return; + await client.messages.send({ + roomId: event.roomId, + text: "ack", + replyTo: event.eventId, + }); +}); + +await sub.stop(); +await sub.done; +``` + +The first subscriber starts the internal sync runner. Stopping the last subscriber stops it. Multiple subscribers share one runner. + +Optional sync tuning lives on the subscription call, not under `client.sync`: + +```ts +await client.subscribe(filter, handler, { + timeoutMs: 30_000, + retryDelayMs: 1_000, +}); +``` + +## Catch-Up + +Subscriptions are future-only by default. A reused account does not replay stored-cursor backlog unless the caller asks: + +```ts +const sub = await client.subscribe({ kind: "message" }, onMessage); +await sub.catchUp(); +``` + +`catchUp()` replays missed events from the stored cursor through that subscription. + +## Helper Functions + +Pure helpers are thin wrappers over `client.subscribe`: + +```ts +import { onInvite, onMessage, onRawEvent, onReaction } from "better-matrix-js"; + +await onMessage(client, { roomId }, handler); +await onReaction(client, { relationEventId: "$event" }, handler); +await onInvite(client, undefined, handler); +await onRawEvent(client, { roomId }, handler); +``` + +They are not separate event systems. + +## Serverless Apply + +When `/sync` is owned by another process, pass raw Matrix sync JSON to the client: + +```ts +await client.sync.applyResponse({ response, since }); +``` + +Only one writer should advance an encrypted Matrix device cursor and crypto store at a time. In serverless deployments, serialize work through a Durable Object, a lock, or another single-writer mechanism. + +Live mode owns the cursor inside `client.subscribe(...)`. Webhook mode owns the cursor in the external sync producer and applies the payload to the account client. Cloudflare mode should use one sync Durable Object to poll Matrix and one account Durable Object to apply responses and run bot code. + +## Raw Requests + +Use `client.raw.request` for advanced Matrix endpoints without adding throwaway wrappers: + +```ts +const result = await client.raw.request({ + method: "POST", + path: "/_matrix/client/v3/rooms/!room:example/send/m.room.message/txn", + body: { msgtype: "m.text", body: "hello" }, +}); +``` + +The path must be relative to the homeserver. + +## E2EE Storage + +Encrypted bots should always use durable storage and a stable `pickleKey`: + +```ts +const client = createMatrixClient({ + account, + store, + pickleKey: process.env.MATRIX_PICKLE_KEY!, + recoveryKey: process.env.MATRIX_RECOVERY_KEY, +}); +``` + +`recoveryKey` unlocks Matrix key backup for historical encrypted messages. `pickleKey` protects local crypto state and must remain stable for the device/store pair. + +## Store Ownership + +Each Matrix account/device store is single-writer. Do not run two live clients, webhook consumers, or Durable Objects against the same store prefix at the same time. Multiple logical bots can share a process by creating separate clients with separate account/device stores. + +The storage adapters persist fast-boot state only: account/session material supplied by the app, crypto state, sync cursors, pending decryptions, and small summaries/caches. They intentionally do not model a full gomuks-style timeline database. + +## Unsupported Chat SDK Features + +Matrix has no native portable equivalent for Chat SDK modals, scheduled messages, or interactive cards/actions. The adapter may render plain text only when that does not imply unsupported interactivity; otherwise it should throw clearly. + +## Beeper + +Beeper is first-class, but non-standard behavior stays explicit. Native stream events and ephemeral sends live under `client.beeper.*`, and the Chat SDK adapter only uses them when the homeserver is Beeper or `beeper: true` is configured. Standard Matrix homeservers use Matrix edit-based streaming and reject Beeper-only ephemeral sends. + +Beeper login helpers are stateless request functions: + +```ts +import { createBeeperLogin } from "better-matrix-js/beeper-login"; + +const beeper = createBeeperLogin(); +const token = await beeper.requestEmailToken({ + clientSecret, + email, + sendAttempt: 1, +}); + +const registered = await beeper.register({ + auth: { + type: "m.login.email.identity", + threepid_creds: { sid: token.sid, client_secret: clientSecret }, + }, + password, + username, +}); +``` + +The helper does not embed QA secrets, fixed OTP values, or environment-specific assumptions. diff --git a/e2e-scripts/.gitignore b/e2e-scripts/.gitignore new file mode 100644 index 0000000..4efeb1b --- /dev/null +++ b/e2e-scripts/.gitignore @@ -0,0 +1 @@ +.out/ diff --git a/e2e-scripts/README.md b/e2e-scripts/README.md new file mode 100644 index 0000000..65daf96 --- /dev/null +++ b/e2e-scripts/README.md @@ -0,0 +1,52 @@ +# E2E Scripts + +These scripts were moved from the private E2E harness so we can keep the test +coverage close to the SDK. They intentionally do not include Beeper QA account +creation, fixed OTP flows, private tokens, or any account provisioning logic. + +They are not CI tests by default. They require reusable Matrix/Beeper accounts +with access tokens and, for encrypted-history coverage, recovery keys. + +## Account File + +Create `e2e-scripts/.out/accounts.json` yourself: + +```json +{ + "accounts": [ + { + "homeserverUrl": "https://example.com", + "userId": "@user:example.com", + "deviceId": "DEVICEID", + "accessToken": "ACCESS_TOKEN", + "recoveryKey": "OPTIONAL_RECOVERY_KEY", + "loginToken": "OPTIONAL_JWT_LOGIN_TOKEN_FOR_FRESH_DEVICE_TESTS", + "username": "stable-label" + } + ] +} +``` + +The test suite reuses accounts and stores by default so old history, old devices, +and recovery behavior stay exercised. Use `MATRIX_E2E_RESET_STORES=1` when you +want to force a clean local store. Use `MATRIX_E2E_FRESH_DEVICE=1` only when the +accounts include reusable Matrix JWT login tokens. + +## Running + +Build the SDK first: + +```sh +pnpm build +``` + +Then run from this directory or from the repo root: + +```sh +cd e2e-scripts +MATRIX_E2E_SDK_ROOT=.. npm run test:surface +MATRIX_E2E_SDK_ROOT=.. npm test +``` + +The Chat SDK adapter test requires the upstream `chat` package to be resolvable +in Node, for example by installing/linking it in your local environment. diff --git a/e2e-scripts/package.json b/e2e-scripts/package.json new file mode 100644 index 0000000..86567d9 --- /dev/null +++ b/e2e-scripts/package.json @@ -0,0 +1,13 @@ +{ + "name": "better-matrix-js-e2e-scripts", + "private": true, + "type": "module", + "scripts": { + "test": "node --test --test-concurrency=1 --test-timeout=420000 test/*.test.mjs", + "test:browser:serve": "node src/browser-smoke-server.mjs", + "test:core": "node --test --test-timeout=420000 test/core-e2e.test.mjs", + "test:adapter": "node --test --test-timeout=420000 test/chat-adapter-e2e.test.mjs", + "test:cloudflare": "node src/cloudflare-runtime-smoke.mjs", + "test:surface": "node --test --test-timeout=420000 test/sdk-surface-happy-path-e2e.test.mjs" + } +} diff --git a/e2e-scripts/src/accounts.mjs b/e2e-scripts/src/accounts.mjs new file mode 100644 index 0000000..df2206f --- /dev/null +++ b/e2e-scripts/src/accounts.mjs @@ -0,0 +1,22 @@ +import { readFile } from "node:fs/promises"; +import { ACCOUNTS_PATH, HOMESERVER_URL } from "./config.mjs"; + +export async function loadAccounts(count) { + const data = JSON.parse(await readFile(ACCOUNTS_PATH, "utf8")); + const accounts = Array.isArray(data) ? data : data.accounts; + if (!Array.isArray(accounts)) { + throw new Error(`Expected ${ACCOUNTS_PATH} to contain an accounts array`); + } + if (accounts.length < count) { + throw new Error(`Expected at least ${count} E2E accounts in ${ACCOUNTS_PATH}, found ${accounts.length}`); + } + return accounts.slice(0, count).map(normalizeAccount); +} + +function normalizeAccount(account) { + const homeserverUrl = account.homeserverUrl ?? account.homeserver ?? HOMESERVER_URL; + return { + ...account, + homeserverUrl, + }; +} diff --git a/e2e-scripts/src/browser-smoke-server.mjs b/e2e-scripts/src/browser-smoke-server.mjs new file mode 100644 index 0000000..d51cb2e --- /dev/null +++ b/e2e-scripts/src/browser-smoke-server.mjs @@ -0,0 +1,208 @@ +import { createReadStream } from "node:fs"; +import { readFile } from "node:fs/promises"; +import { createServer } from "node:http"; +import { extname, join, normalize } from "node:path"; +import { ACCOUNTS_PATH, SDK_ROOT } from "./config.mjs"; + +const accountIndex = Number(process.env.MATRIX_E2E_BROWSER_ACCOUNT_INDEX ?? 0); +const port = Number(process.env.MATRIX_E2E_BROWSER_PORT ?? 8765); +const account = JSON.parse(await readFile(ACCOUNTS_PATH, "utf8")).accounts[accountIndex]; + +if (!account) { + throw new Error(`No browser smoke account at index ${accountIndex}; add reusable sessions to ${ACCOUNTS_PATH}.`); +} + +const mime = { + ".html": "text/html", + ".js": "text/javascript", + ".wasm": "application/wasm", +}; + +const html = ` + + + + better-matrix-js browser smoke + + + +
+

better-matrix-js browser smoke

+
starting
+
+ + + +`; + +const server = createServer((request, response) => { + const url = new URL(request.url ?? "/", `http://127.0.0.1:${port}`); + if (url.pathname === "/") { + response.writeHead(200, { "content-type": "text/html" }); + response.end(html); + return; + } + if (url.pathname.startsWith("/sdk/")) { + const relative = normalize(url.pathname.slice("/sdk/".length)); + if (relative.startsWith("..")) { + response.writeHead(403); + response.end("forbidden"); + return; + } + const file = join(SDK_ROOT, relative); + response.writeHead(200, { "content-type": mime[extname(file)] ?? "application/octet-stream" }); + createReadStream(file).on("error", () => { + response.writeHead(404); + response.end("not found"); + }).pipe(response); + return; + } + response.writeHead(404); + response.end("not found"); +}); + +server.listen(port, "127.0.0.1", () => { + console.log(`browser smoke server http://127.0.0.1:${port}`); +}); diff --git a/e2e-scripts/src/cloudflare-runtime-smoke.mjs b/e2e-scripts/src/cloudflare-runtime-smoke.mjs new file mode 100644 index 0000000..a8415b9 --- /dev/null +++ b/e2e-scripts/src/cloudflare-runtime-smoke.mjs @@ -0,0 +1,105 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { spawnSync } from "node:child_process"; +import assert from "node:assert/strict"; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), "..", ".."); +const outDir = resolve(root, "e2e-scripts/.out/cloudflare-runtime-smoke"); +const accountId = process.env.CLOUDFLARE_ACCOUNT_ID ?? "2d6696feb60377216e949e7a39f904a1"; +const name = process.env.BMJS_CLOUDFLARE_SMOKE_NAME ?? + `better-matrix-js-cf-smoke-${new Date().toISOString().replaceAll(/\D/g, "").slice(0, 14)}`; + +await mkdir(outDir, { recursive: true }); +const configPath = resolve(outDir, "wrangler.jsonc"); +await writeFile(configPath, JSON.stringify({ + account_id: accountId, + compatibility_date: "2026-04-24", + durable_objects: { + bindings: [ + { class_name: "MatrixStoreSmokeObject", name: "MATRIX_STORE_SMOKE" }, + { class_name: "MatrixSyncSmokeObject", name: "MATRIX_SYNC" }, + ], + }, + main: "../../src/cloudflare-runtime-worker.mjs", + migrations: [ + { + new_sqlite_classes: ["MatrixStoreSmokeObject", "MatrixSyncSmokeObject"], + tag: "v1", + }, + ], + name, + vars: { + MATRIX_SYNC_WEBHOOK_URL: "https://example.invalid/matrix/webhook", + SMOKE_WEBHOOK_SECRET: "better-matrix-js-cloudflare-runtime-smoke", + }, + workers_dev: true, +}, null, 2)); + +const deploy = run("pnpm", [ + "--dir", + "examples/cloudflare-worker", + "exec", + "wrangler", + "deploy", + "--config", + configPath, +]); +const workerUrl = deploy.stdout.match(/https:\/\/[^\s]+\.workers\.dev/)?.[0]; +assert.ok(workerUrl, `Could not find workers.dev URL in Wrangler output:\n${deploy.stdout}`); + +const rootResponse = await getJson(workerUrl); +assert.deepEqual(rootResponse, { ok: true }); + +const cryptoResponse = await getJson(`${workerUrl}/crypto`); +assert.equal(cryptoResponse.payload.since, "before"); +assert.equal(cryptoResponse.payload.response.next_batch, "cloudflare-smoke"); +assert.equal(cryptoResponse.envelope.alg, "AES-GCM-256"); + +const storeResponse = await getJson(`${workerUrl}/store`); +assert.deepEqual(storeResponse.stored, [1, 3, 3, 7]); +assert.deepEqual(storeResponse.keys, ["runtime"]); +assert.equal(storeResponse.deleted, true); + +const syncStatus = await getJson(`${workerUrl}/sync/status`); +assert.equal(syncStatus.enabled, false); + +const syncStart = await postJson(`${workerUrl}/sync/start`); +assert.equal(syncStart.ok, true); +assert.equal(syncStart.status.enabled, true); +assert.match(syncStart.status.lastError, /MATRIX_SYNC_ACCESS_TOKEN|MATRIX_SYNC_HOMESERVER_URL/); + +console.log(JSON.stringify({ + name, + ok: true, + workerUrl, +}, null, 2)); + +async function getJson(url) { + const response = await fetch(url); + const text = await response.text(); + assert.equal(response.ok, true, `${url} failed: HTTP ${response.status} ${text}`); + return JSON.parse(text); +} + +async function postJson(url) { + const response = await fetch(url, { method: "POST" }); + const text = await response.text(); + assert.equal(response.ok, true, `${url} failed: HTTP ${response.status} ${text}`); + return JSON.parse(text); +} + +function run(command, args) { + const result = spawnSync(command, args, { + cwd: root, + encoding: "utf8", + env: { + ...process.env, + CLOUDFLARE_ACCOUNT_ID: accountId, + }, + }); + if (result.status !== 0) { + throw new Error(`${command} ${args.join(" ")} failed\n${result.stdout}\n${result.stderr}`); + } + return result; +} diff --git a/e2e-scripts/src/cloudflare-runtime-worker.mjs b/e2e-scripts/src/cloudflare-runtime-worker.mjs new file mode 100644 index 0000000..478de2a --- /dev/null +++ b/e2e-scripts/src/cloudflare-runtime-worker.mjs @@ -0,0 +1,59 @@ +import { + MatrixSyncDurableObject, + createDurableObjectMatrixStore, + decryptMatrixSyncWebhookEnvelope, + encryptMatrixSyncWebhookPayload, +} from "../../packages/cloudflare/dist/index.js"; + +export class MatrixStoreSmokeObject { + constructor(state) { + this.state = state; + } + + async fetch() { + const store = createDurableObjectMatrixStore(this.state.storage, { + prefix: "smoke/", + }); + const key = "runtime"; + const value = new Uint8Array([1, 3, 3, 7]); + await store.set(key, value); + const stored = await store.get(key); + const keys = await store.list(""); + await store.delete(key); + const deleted = await store.get(key); + return Response.json({ + deleted: deleted === null, + keys, + stored: stored ? [...stored] : null, + }); + } +} + +export class MatrixSyncSmokeObject extends MatrixSyncDurableObject {} + +export default { + async fetch(request, env) { + const url = new URL(request.url); + + if (url.pathname === "/crypto") { + const payload = { response: { next_batch: "cloudflare-smoke" }, since: "before" }; + const envelope = await encryptMatrixSyncWebhookPayload(payload, env.SMOKE_WEBHOOK_SECRET); + return Response.json({ + envelope, + payload: await decryptMatrixSyncWebhookEnvelope(envelope, env.SMOKE_WEBHOOK_SECRET), + }); + } + + if (url.pathname === "/store") { + const id = env.MATRIX_STORE_SMOKE.idFromName("default"); + return env.MATRIX_STORE_SMOKE.get(id).fetch(request); + } + + if (url.pathname.startsWith("/sync")) { + const id = env.MATRIX_SYNC.idFromName("default"); + return env.MATRIX_SYNC.get(id).fetch(request); + } + + return Response.json({ ok: true }); + }, +}; diff --git a/e2e-scripts/src/config.mjs b/e2e-scripts/src/config.mjs new file mode 100644 index 0000000..e122329 --- /dev/null +++ b/e2e-scripts/src/config.mjs @@ -0,0 +1,32 @@ +import { mkdir } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +export const ROOT = dirname(dirname(fileURLToPath(import.meta.url))); +export const SDK_ROOT = process.env.MATRIX_E2E_SDK_ROOT + ? resolve(process.env.MATRIX_E2E_SDK_ROOT) + : resolve(ROOT, "../matrix-chat-sdk"); +export const OUT_DIR = process.env.MATRIX_E2E_OUT_DIR + ? resolve(process.env.MATRIX_E2E_OUT_DIR) + : resolve(ROOT, ".out"); +export const STORE_DIR = resolve(OUT_DIR, "stores"); +export const ACCOUNTS_PATH = resolve(OUT_DIR, "accounts.json"); + +export const HOMESERVER_URL = + process.env.MATRIX_E2E_HOMESERVER_URL ?? "https://matrix-client.matrix.org"; +export const BEEPER_DOMAIN = + process.env.MATRIX_E2E_BEEPER_DOMAIN ?? new URL(HOMESERVER_URL).hostname; + +export const RUN_ID = + process.env.MATRIX_E2E_RUN_ID ?? + `${Date.now()}${Math.floor(Math.random() * 1000) + .toString() + .padStart(3, "0")}`; + +export async function ensureOutDirs() { + await mkdir(STORE_DIR, { recursive: true }); +} + +export function sdkDist(path) { + return resolve(SDK_ROOT, path); +} diff --git a/e2e-scripts/src/harness.mjs b/e2e-scripts/src/harness.mjs new file mode 100644 index 0000000..882acb7 --- /dev/null +++ b/e2e-scripts/src/harness.mjs @@ -0,0 +1,359 @@ +import { readFile, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import assert from "node:assert/strict"; +import { OUT_DIR, STORE_DIR, ensureOutDirs, sdkDist } from "./config.mjs"; + +const { createMatrixClient } = await import(sdkDist("packages/core/dist/node.js")); +const { createMatrixLogin } = await import(sdkDist("packages/core/dist/login.js")); +const { createFileMatrixStore } = await import(sdkDist("packages/state-file/dist/index.js")); + +export async function makeCore(account, label) { + await ensureOutDirs(); + const storeDir = join(STORE_DIR, label); + if (process.env.MATRIX_E2E_RESET_STORES === "1") { + await rm(storeDir, { force: true, recursive: true }); + } + const sdkAccount = await accountForLabel(account, label); + const events = []; + const core = createCompatCore(sdkAccount, storeDir); + core.events.on((event) => events.push(toRuntimeEvent(event))); + const initOptions = {}; + if (account.recoveryKey) { + initOptions.recoveryKey = account.recoveryKey; + } + const whoami = await core.connect(initOptions); + assert.equal(whoami.userId, account.userId); + return { account: sdkAccount, core, events, label, storeDir, userId: whoami.userId }; +} + +async function accountForLabel(account, label) { + const cached = process.env.MATRIX_E2E_RESET_STORES === "1" ? null : await loadCachedSession(label, account); + if (cached) { + return cached; + } + if (process.env.MATRIX_E2E_FRESH_DEVICE === "1" && account.loginToken) { + const fresh = await loginFreshDevice(account, label); + await saveCachedSession(label, fresh); + return fresh; + } + return account; +} + +const SESSIONS_PATH = join(OUT_DIR, "sessions.json"); + +async function loadCachedSession(label, account) { + try { + const data = JSON.parse(await readFile(SESSIONS_PATH, "utf8")); + const session = data[label]; + if (session?.username === account.username && session?.accessToken && session?.deviceId) { + return { ...account, ...session }; + } + } catch { + // No reusable session yet. + } + return null; +} + +async function saveCachedSession(label, account) { + let data = {}; + try { + data = JSON.parse(await readFile(SESSIONS_PATH, "utf8")); + } catch { + // Create below. + } + data[label] = { + accessToken: account.accessToken, + deviceId: account.deviceId, + homeserverUrl: account.homeserverUrl, + recoveryKey: account.recoveryKey, + userId: account.userId, + username: account.username, + }; + await writeFile(SESSIONS_PATH, JSON.stringify(data, null, 2)); +} + +async function loginFreshDevice(account, label) { + return retry(`fresh Matrix login ${label}`, async () => { + const session = await createMatrixLogin({ + homeserver: account.homeserverUrl, + initialDeviceDisplayName: `better-matrix-js private e2e ${label}`, + }).token({ + token: account.loginToken, + type: "org.matrix.login.jwt", + }); + return { + ...account, + accessToken: session.accessToken, + deviceId: session.deviceId, + userId: session.userId, + }; + }, 5, 5000); +} + +export async function sync(account, count = 1, timeoutMs = 1000) { + for (let index = 0; index < count; index += 1) { + await retry("sync once", () => account.core.sync.once({ timeoutMs }), 3, 1000); + } +} + +export async function closeAll(...accounts) { + await Promise.allSettled(accounts.map((account) => account?.core?.close())); +} + +export async function eventually(label, fn, timeoutMs = 90000, intervalMs = 1000) { + logProgress(`wait ${label}`); + const started = Date.now(); + let lastError; + while (Date.now() - started < timeoutMs) { + try { + const value = await fn(); + if (value) { + logProgress(`ok ${label}`); + return value; + } + } catch (error) { + lastError = error; + } + await new Promise((resolve) => setTimeout(resolve, intervalMs)); + } + throw new Error(`${label} timed out${lastError ? `: ${lastError.message}` : ""}`); +} + +export async function retry(label, fn, attempts = 4, delayMs = 1500) { + logProgress(`try ${label}`); + let lastError; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + try { + const value = await fn(); + logProgress(`ok ${label}`); + return value; + } catch (error) { + lastError = error; + if (attempt === attempts || !isTransient(error)) { + throw error; + } + await new Promise((resolve) => setTimeout(resolve, delayMs * attempt)); + } + } + throw lastError; +} + +function logProgress(message) { + if (process.env.MATRIX_E2E_VERBOSE === "1") { + console.error(`[e2e ${new Date().toISOString()}] ${message}`); + } +} + +export async function syncUntil(label, account, predicate, timeoutMs = 90000) { + return eventually( + label, + async () => { + await sync(account, 1, 1000); + return predicate(account.events); + }, + timeoutMs + ); +} + +export function messageEvent(events, roomId, predicate) { + return events.find( + (event) => event.type === "message" && event.event.roomId === roomId && predicate(event.event) + )?.event; +} + +export function reactionEvent(events, roomId, messageId, key, added = true) { + return events.find( + (event) => + event.type === "reaction" && + event.event.roomId === roomId && + event.event.relatesToEventId === messageId && + event.event.key === key && + (event.event.added ?? true) === added + )?.event; +} + +export function cryptoStatus(events, status) { + if (status === "enabled") { + return events.find((event) => + event.type === "crypto_status" && + ["enabled", "recoveryKeyCached", "recoveryKeyLoaded", "recoveryRestored"].includes(event.status) + ); + } + return events.find((event) => event.type === "crypto_status" && event.status === status); +} + +function isTransient(error) { + const message = error?.message ?? String(error); + return /\b(408|425|429|500|502|503|504)\b/.test(message) || + /timeout|temporar|ECONNRESET|ETIMEDOUT|failed to query keys/i.test(message); +} + +export function createCompatCore(account, storeDir) { + const client = createMatrixClient({ + homeserver: account.homeserverUrl, + recoveryKey: account.recoveryKey, + store: createFileMatrixStore(storeDir), + token: account.accessToken, + }); + const listeners = new Set(); + let subscription; + const compat = Object.create(client); + const overrides = { + accountData: client.accountData, + beeper: client.beeper, + boot: (...args) => client.boot(...args), + crypto: client.crypto, + logout: (...args) => client.logout(...args), + media: client.media, + messages: client.messages, + raw: client.raw, + reactions: client.reactions, + receipts: client.receipts, + rooms: client.rooms, + streams: client.streams, + subscribe: (...args) => client.subscribe(...args), + toDevice: client.toDevice, + typing: client.typing, + users: client.users, + whoami: (...args) => client.whoami(...args), + close: async () => { + await subscription?.stop(); + await subscription?.done.catch(() => {}); + subscription = undefined; + }, + connect: async (options = {}) => { + await client.boot(options); + const whoami = await client.whoami(); + const cryptoStatus = await client.crypto.status(); + if (cryptoStatus.state && cryptoStatus.state !== "disabled") { + for (const listener of listeners) { + listener({ kind: "crypto", state: cryptoStatus.state }); + } + } + subscription ??= await client.subscribe({}, (event) => { + for (const listener of listeners) { + listener(event); + } + }); + return whoami; + }, + events: { + on: (listener) => { + listeners.add(listener); + return () => listeners.delete(listener); + }, + }, + sync: { + applyResponse: (...args) => client.sync.applyResponse(...args), + once: async ({ timeoutMs = 1000 } = {}) => { + await new Promise((resolve) => setTimeout(resolve, timeoutMs)); + }, + }, + addReaction: ({ emoji, messageId, roomId }) => + client.reactions.send({ eventId: messageId, key: emoji, roomId }), + applySyncResponse: (options) => client.sync.applyResponse(options), + deleteMessage: ({ messageId, reason: _reason, roomId }) => + client.messages.redact({ eventId: messageId, roomId }), + downloadEncryptedMedia: async ({ file }) => toBytesBase64Result(await client.media.downloadEncrypted({ file })), + downloadMedia: async ({ contentUri }) => toBytesBase64Result(await client.media.download({ contentUri })), + editMessage: ({ body, formattedBody, messageId, roomId }) => + client.messages.edit({ eventId: messageId, html: formattedBody, roomId, text: body }), + fetchJoinedRooms: () => client.rooms.listJoined(), + fetchMessage: async ({ messageId, roomId }) => { + const result = await client.messages.get({ eventId: messageId, roomId }); + return { message: result.message ? toRuntimeMessage(result.message) : null }; + }, + fetchMessages: async ({ cursor, limit, roomId, threadRootEventId }) => { + const result = await client.messages.list({ cursor, limit, roomId, threadRoot: threadRootEventId }); + return { + messages: result.messages.map(toRuntimeMessage), + nextCursor: result.nextCursor, + }; + }, + fetchRoom: ({ roomId }) => client.rooms.get({ roomId }), + createRoom: ({ initialState, invite, isDirect, name, preset, topic, visibility }) => + client.rooms.create({ initialState, invite, isDirect, name, preset, topic, visibility }), + getUser: ({ userId }) => client.users.get({ userId }), + leaveRoom: ({ reason, roomId }) => client.rooms.leave({ reason, roomId }), + joinRoom: ({ roomIdOrAlias }) => client.rooms.join({ roomIdOrAlias }), + listRoomThreads: ({ limit, roomId }) => client.rooms.threads.list({ limit, roomId }), + markRead: ({ eventId, roomId }) => client.messages.markRead({ eventId, roomId }), + openDM: ({ userId }) => client.rooms.openDM({ userId }), + postMediaMessage: ({ bytesBase64, contentType, filename, height, msgtype, roomId, width }) => + client.messages.sendMedia({ + bytes: Buffer.from(bytesBase64, "base64"), + contentType, + filename, + height, + kind: msgtype?.startsWith("m.") ? msgtype.slice(2) : "file", + roomId, + width, + }), + postMessage: ({ body, formattedBody, mentions, roomId, threadRootEventId }) => + client.messages.send({ html: formattedBody, mentions, roomId, text: body, threadRoot: threadRootEventId }), + removeReaction: ({ emoji, messageId, roomId }) => + client.reactions.redact({ eventId: messageId, key: emoji, roomId }), + setTyping: ({ roomId, timeoutMs, typing }) => client.typing.set({ roomId, timeoutMs, typing }), + }; + return Object.defineProperties( + compat, + Object.fromEntries(Object.entries(overrides).map(([key, value]) => [ + key, + { configurable: true, enumerable: true, value, writable: true }, + ])) + ); +} + +function toBytesBase64Result(result) { + return { + ...result, + bytesBase64: Buffer.from(result.bytes).toString("base64"), + }; +} + +export function toRuntimeEvent(event) { + if (event.kind === "message") { + return { event: toRuntimeMessage(event), type: "message" }; + } + if (event.kind === "reaction") { + return { + event: { + ...event, + relatesToEventId: event.relatesTo, + }, + type: "reaction", + }; + } + if (event.kind === "crypto") { + return { status: event.state, type: "crypto_status" }; + } + if (event.kind === "sync") { + return { status: event.state, type: "sync_status" }; + } + return event; +} + +function toRuntimeMessage(message) { + return { + ...message, + attachments: message.attachments?.map((attachment) => ({ + contentUri: attachment.contentUri, + encryptedFile: attachment.encryptedFile, + filename: attachment.filename, + info: { + contentType: attachment.contentType, + duration: attachment.duration, + height: attachment.height, + size: attachment.size, + width: attachment.width, + }, + msgtype: `m.${attachment.kind}`, + })) ?? [], + body: message.text, + formattedBody: message.html, + isEdited: message.edited, + isEncrypted: message.encrypted, + msgtype: message.messageType, + threadRootEventId: message.threadRoot, + }; +} diff --git a/e2e-scripts/src/matrix-rest.mjs b/e2e-scripts/src/matrix-rest.mjs new file mode 100644 index 0000000..7217140 --- /dev/null +++ b/e2e-scripts/src/matrix-rest.mjs @@ -0,0 +1,97 @@ +export class MatrixREST { + constructor(account) { + this.account = account; + } + + async request(method, path, body) { + let lastError; + const homeserverUrl = this.account.homeserverUrl ?? this.account.homeserver; + if (!homeserverUrl) { + throw new Error("MatrixREST account is missing homeserverUrl"); + } + for (let attempt = 1; attempt <= 5; attempt += 1) { + const response = await fetch(new URL(path, homeserverUrl), { + body: body === undefined ? undefined : JSON.stringify(body), + headers: { + authorization: `Bearer ${this.account.accessToken}`, + "content-type": "application/json", + }, + method, + }); + const text = await response.text(); + const data = text ? JSON.parse(text) : {}; + if (response.ok) { + return data; + } + lastError = new Error(`${method} ${path} failed: HTTP ${response.status} ${JSON.stringify(data)}`); + if (![408, 425, 429, 500, 502, 503, 504].includes(response.status) || attempt === 5) { + throw lastError; + } + const retryAfterMs = Number(data.retry_after_ms) || attempt * 1500; + await new Promise((resolve) => setTimeout(resolve, retryAfterMs)); + } + throw lastError; + } + + createRoom({ historyVisibility, initialState = [], invite = [], name, topic } = {}) { + const state = [...initialState]; + if (historyVisibility) { + state.push({ + content: { history_visibility: historyVisibility }, + state_key: "", + type: "m.room.history_visibility", + }); + } + return this.request("POST", "/_matrix/client/v3/createRoom", { + invite, + initial_state: state, + is_direct: false, + name, + preset: "private_chat", + topic, + }); + } + + createEncryptedRoom({ historyVisibility, invite = [], name, topic } = {}) { + const initialState = [ + { + content: { + algorithm: "m.megolm.v1.aes-sha2", + rotation_period_ms: 604800000, + rotation_period_msgs: 100, + }, + state_key: "", + type: "m.room.encryption", + }, + ]; + return this.createRoom({ + historyVisibility, + invite, + initialState, + name, + topic, + }); + } + + invite(roomId, userId) { + return this.request("POST", `/_matrix/client/v3/rooms/${encodeURIComponent(roomId)}/invite`, { + user_id: userId, + }); + } + + join(roomIdOrAlias) { + return this.request( + "POST", + `/_matrix/client/v3/join/${encodeURIComponent(roomIdOrAlias)}`, + {} + ); + } + + sync({ since, timeout = 0 } = {}) { + const params = new URLSearchParams({ timeout: String(timeout), set_presence: "offline" }); + if (since) { + params.set("since", since); + } + return this.request("GET", `/_matrix/client/v3/sync?${params.toString()}`); + } +} diff --git a/e2e-scripts/test/chat-adapter-e2e.test.mjs b/e2e-scripts/test/chat-adapter-e2e.test.mjs new file mode 100644 index 0000000..abe5d23 --- /dev/null +++ b/e2e-scripts/test/chat-adapter-e2e.test.mjs @@ -0,0 +1,207 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { sdkDist } from "../src/config.mjs"; +import { loadAccounts } from "../src/accounts.mjs"; +import { MatrixREST } from "../src/matrix-rest.mjs"; +import { closeAll, eventually, makeCore, messageEvent, sync, syncUntil } from "../src/harness.mjs"; + +const { Chat } = await import(sdkDist("packages/chat-adapter/node_modules/chat/dist/index.js")); +const { createMatrixAdapter } = await import(sdkDist("packages/chat-adapter/dist/index.js")); + +function memoryState() { + const values = new Map(); + const locks = new Map(); + const subscriptions = new Set(); + return { + async acquireLock(threadId, ttlMs) { + const existing = locks.get(threadId); + if (existing?.expiresAt > Date.now()) { + return null; + } + const lock = { expiresAt: Date.now() + ttlMs, threadId, token: String(Math.random()) }; + locks.set(threadId, lock); + return lock; + }, + async connect() {}, + async delete() {}, + async disconnect() {}, + async get() { + return values.get(arguments[0]); + }, + async isSubscribed(threadId) { + return subscriptions.has(threadId); + }, + async list() { + return []; + }, + async releaseLock(lock) { + if (locks.get(lock.threadId)?.token === lock.token) { + locks.delete(lock.threadId); + } + }, + async subscribe(threadId) { + subscriptions.add(threadId); + }, + async set(key, value) { + values.set(key, value); + }, + async setIfNotExists(key, value) { + if (values.has(key)) { + return false; + } + values.set(key, value); + return true; + }, + async unsubscribe(threadId) { + subscriptions.delete(threadId); + }, + }; +} + +async function* textStream(chunks) { + for (const chunk of chunks) { + await new Promise((resolve) => setTimeout(resolve, 25)); + yield chunk; + } +} + +async function deliverSync(adapter, account) { + return adapter.handleWebhook( + new Request("https://example.invalid/matrix", { + body: JSON.stringify({ response: await new MatrixREST(account).sync({ timeout: 0 }) }), + method: "POST", + }) + ); +} + +test("chat adapter: Chat SDK posting, streaming fallback, webhook sync, slash dispatch, metadata", async () => { + const [botAccount, peerAccount, thirdAccount] = await loadAccounts(3); + const bot = await makeCore(botAccount, "adapter-bot"); + const peer = await makeCore(peerAccount, "adapter-peer"); + const third = await makeCore(thirdAccount, "adapter-third"); + const peerRest = new MatrixREST(peerAccount); + const thirdRest = new MatrixREST(thirdAccount); + let slash; + + const adapter = createMatrixAdapter({ + commandPrefix: "/", + client: bot.core, + homeserver: bot.account.homeserverUrl, + inviteAutoJoin: { inviterAllowlist: [peer.userId] }, + recoveryKey: bot.account.recoveryKey, + sync: { enabled: false }, + token: bot.account.accessToken, + userName: "matrix-e2e-bot", + }); + const chat = new Chat({ + adapters: { matrix: adapter }, + fallbackStreamingPlaceholderText: "...", + state: memoryState(), + streamingUpdateIntervalMs: 50, + userName: "matrix-e2e-bot", + }); + chat.onSlashCommand("/status", async (event) => { + slash = event; + }); + + try { + await chat.initialize(); + + const allowedInvite = await peerRest.createRoom({ + invite: [bot.userId], + name: `adapter allowed invite ${Date.now()}`, + }); + await deliverSync(adapter, bot.account); + await eventually("adapter auto-joins allowed invite", async () => { + await deliverSync(adapter, bot.account); + const joined = await bot.core.fetchJoinedRooms(); + return joined.roomIds.includes(allowedInvite.room_id); + }); + + const deniedInvite = await thirdRest.createRoom({ + invite: [bot.userId], + name: `adapter denied invite ${Date.now()}`, + }); + await deliverSync(adapter, bot.account); + const joinedAfterDenied = await bot.core.fetchJoinedRooms(); + assert.equal(joinedAfterDenied.roomIds.includes(deniedInvite.room_id), false); + + const threadId = await adapter.openDM(peer.userId); + const roomId = adapter.decodeThreadId(threadId).roomId; + await peer.core.joinRoom({ roomIdOrAlias: roomId }); + await Promise.all([sync(bot, 8), sync(peer, 8)]); + + const channelInfo = await adapter.fetchChannelInfo(adapter.channelIdFromThreadId(threadId)); + assert.equal(channelInfo.metadata.encrypted, true); + assert.equal(channelInfo.isDM, true); + + const user = await adapter.getUser(peer.userId); + assert.equal(user.userId, peer.userId); + + const posted = await adapter.postMessage(threadId, { + attachments: [ + { + data: Buffer.from("adapter attachment", "utf8"), + mimeType: "text/plain", + name: "adapter.txt", + type: "file", + }, + ], + markdown: `hello <@(${peer.userId})>`, + }); + assert.ok(posted.id); + await syncUntil("peer receives adapter message", peer, (events) => + messageEvent(events, roomId, (event) => event.eventId === posted.id) + ); + + const thread = chat.createThread(adapter, threadId, undefined, false); + const streamed = await thread.post(textStream(["stream ", "**from** ", "chat sdk"])); + assert.ok(streamed.id); + const streamedSeen = await syncUntil("peer receives streamed final edit", peer, (events) => + messageEvent( + events, + roomId, + (event) => event.eventId === streamed.id && event.body.replaceAll("**", "").includes("stream from chat sdk") + ) + ); + assert.equal(streamedSeen.isEdited, true); + + const fetched = await adapter.fetchMessage(threadId, streamed.id); + assert.equal(fetched?.text.replaceAll("**", ""), "stream from chat sdk"); + assert.equal(fetched?.metadata.edited, true); + const history = await adapter.fetchMessages(threadId, { limit: 20 }); + assert.ok(history.messages.some((message) => message.id === posted.id)); + + await adapter.startTyping(threadId, "thinking"); + await adapter.addReaction(threadId, posted.id, "✅"); + await adapter.removeReaction(threadId, posted.id, "✅"); + const edited = await adapter.editMessage(threadId, posted.id, "edited through adapter"); + assert.equal(edited.id, posted.id); + + const syncPayload = await new MatrixREST(bot.account).sync({ timeout: 0 }); + const webhookResponse = await adapter.handleWebhook( + new Request("https://example.invalid/matrix", { + body: JSON.stringify({ response: syncPayload }), + method: "POST", + }) + ); + assert.equal(webhookResponse.status, 200); + + const command = await peer.core.postMessage({ body: "/status verbose", roomId }); + await syncUntil("bot receives slash command", bot, (events) => + messageEvent(events, roomId, (event) => event.eventId === command.eventId) + ); + await adapter.handleWebhook( + new Request("https://example.invalid/matrix", { + body: JSON.stringify({ response: await new MatrixREST(bot.account).sync({ timeout: 0 }) }), + method: "POST", + }) + ); + assert.equal(slash?.command, "/status"); + assert.equal(slash?.text, "verbose"); + } finally { + await chat.shutdown(); + await adapter.disconnect(); + await closeAll(bot, peer, third); + } +}); diff --git a/e2e-scripts/test/core-e2e.test.mjs b/e2e-scripts/test/core-e2e.test.mjs new file mode 100644 index 0000000..4165b44 --- /dev/null +++ b/e2e-scripts/test/core-e2e.test.mjs @@ -0,0 +1,306 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { loadAccounts } from "../src/accounts.mjs"; +import { MatrixREST } from "../src/matrix-rest.mjs"; +import { + closeAll, + cryptoStatus, + makeCore, + messageEvent, + reactionEvent, + retry, + sync, + syncUntil, +} from "../src/harness.mjs"; + +test("core: encrypted rooms, messages, edits, reactions, media, history, webhook sync ingestion", async () => { + const [botAccount, peerAccount, thirdAccount, lateAccount] = await loadAccounts(4); + const bot = await makeCore(botAccount, "core-bot"); + const peer = await makeCore(peerAccount, "core-peer"); + const third = await makeCore(thirdAccount, "core-third"); + const late = await makeCore(lateAccount, "core-late"); + const botRest = new MatrixREST(botAccount); + const peerRest = new MatrixREST(peerAccount); + const thirdRest = new MatrixREST(thirdAccount); + const lateRest = new MatrixREST(lateAccount); + + try { + assert.ok(cryptoStatus(bot.events, "enabled"), "bot crypto should initialize"); + assert.ok(cryptoStatus(peer.events, "enabled"), "peer crypto should initialize"); + assert.ok(cryptoStatus(late.events, "enabled"), "late member crypto should initialize"); + + await Promise.all([sync(bot, 2), sync(peer, 2), sync(third, 2), sync(late, 2)]); + + const profile = await bot.core.getUser({ userId: peer.userId }); + assert.equal(profile.userId, peer.userId); + + const dm = await bot.core.openDM({ userId: peer.userId }); + await peer.core.joinRoom({ roomIdOrAlias: dm.roomId }); + await Promise.all([sync(bot, 8), sync(peer, 8)]); + + const dmInfo = await bot.core.fetchRoom({ roomId: dm.roomId }); + assert.equal(dmInfo.encrypted, true); + assert.equal(dmInfo.isDM, true); + + const group = await bot.core.createRoom({ + initialState: [{ + content: { algorithm: "m.megolm.v1.aes-sha2" }, + stateKey: "", + type: "m.room.encryption", + }], + invite: [peer.userId, third.userId], + name: `better-matrix-js e2e ${Date.now()}`, + preset: "private_chat", + topic: "private automated SDK coverage", + }); + await Promise.all([ + peer.core.joinRoom({ roomIdOrAlias: group.roomId }), + third.core.joinRoom({ roomIdOrAlias: group.roomId }), + ]); + await Promise.all([sync(bot, 8), sync(peer, 8), sync(third, 8)]); + + const groupInfo = await bot.core.fetchRoom({ roomId: group.roomId }); + assert.equal(groupInfo.encrypted, true); + assert.notEqual(groupInfo.isDM, true); + assert.ok(groupInfo.memberCount >= 3); + + const plainRoom = await botRest.createRoom({ + invite: [peer.userId], + name: `better-matrix-js plain ${Date.now()}`, + topic: "unencrypted SDK coverage", + }); + await peer.core.joinRoom({ roomIdOrAlias: plainRoom.room_id }); + await Promise.all([sync(bot, 4), sync(peer, 4)]); + const plainInfo = await bot.core.fetchRoom({ roomId: plainRoom.room_id }); + assert.equal(plainInfo.encrypted, false); + const plainBody = `unencrypted ${Date.now()}`; + const plainSent = await retry("send unencrypted message", () => + bot.core.postMessage({ body: plainBody, roomId: plainRoom.room_id }) + ); + const plainSeen = await syncUntil("peer receives unencrypted message", peer, (events) => + messageEvent(events, plainRoom.room_id, (event) => event.eventId === plainSent.eventId) + ); + assert.equal(plainSeen.body, plainBody); + assert.notEqual(plainSeen.isEncrypted, true); + const imagePayload = Buffer.from("fake png payload", "utf8"); + const plainImage = await retry("send unencrypted image media", () => bot.core.postMediaMessage({ + bytesBase64: imagePayload.toString("base64"), + contentType: "image/png", + filename: "matrix-e2e.png", + height: 12, + msgtype: "m.image", + roomId: plainRoom.room_id, + width: 16, + })); + const plainImageSeen = await syncUntil("peer receives unencrypted media", peer, (events) => + messageEvent(events, plainRoom.room_id, (event) => event.eventId === plainImage.eventId) + ); + const plainAttachment = plainImageSeen.attachments?.[0]; + assert.equal(plainAttachment?.contentUri?.startsWith("mxc://"), true); + assert.equal(plainAttachment?.encryptedFile, undefined); + assert.equal(plainAttachment?.info?.contentType, "image/png"); + assert.equal(plainAttachment?.info?.height, 12); + assert.equal(plainAttachment?.info?.width, 16); + const plainDownloaded = await peer.core.downloadMedia({ contentUri: plainAttachment.contentUri }); + assert.equal(Buffer.from(plainDownloaded.bytesBase64, "base64").toString("utf8"), imagePayload.toString("utf8")); + + const historyPrefix = `history ${Date.now()}`; + const historySent = []; + for (let index = 0; index < 18; index += 1) { + historySent.push(await retry(`send group history ${index}`, () => + bot.core.postMessage({ body: `${historyPrefix} ${String(index).padStart(2, "0")}`, roomId: group.roomId }) + )); + } + await syncUntil("peer receives latest group history message", peer, (events) => + messageEvent(events, group.roomId, (event) => event.eventId === historySent.at(-1).eventId) + ); + const historyPage1 = await peer.core.fetchMessages({ limit: 8, roomId: group.roomId }); + assert.equal(historyPage1.messages.length, 8); + const historyPage2 = await peer.core.fetchMessages({ + cursor: historyPage1.nextCursor, + limit: 8, + roomId: group.roomId, + }); + assert.equal(historyPage2.messages.length, 8); + const pagedHistoryIds = new Set([...historyPage1.messages, ...historyPage2.messages].map((message) => message.eventId)); + assert.equal(pagedHistoryIds.size, 16); + assert.ok([...historyPage1.messages, ...historyPage2.messages].some((message) => message.body.startsWith(historyPrefix))); + + const lateRoom = await bot.core.createRoom({ + initialState: [ + { + content: { algorithm: "m.megolm.v1.aes-sha2" }, + stateKey: "", + type: "m.room.encryption", + }, + { + content: { history_visibility: "shared" }, + stateKey: "", + type: "m.room.history_visibility", + }, + ], + invite: [peer.userId], + name: `better-matrix-js late join ${Date.now()}`, + preset: "private_chat", + topic: "late membership coverage", + }); + await peer.core.joinRoom({ roomIdOrAlias: lateRoom.roomId }); + await Promise.all([sync(bot, 8), sync(peer, 8)]); + const beforeLateJoin = []; + for (let index = 0; index < 8; index += 1) { + beforeLateJoin.push(await retry(`send pre-join encrypted history ${index}`, () => + bot.core.postMessage({ body: `pre-late ${Date.now()} ${index}`, roomId: lateRoom.roomId }) + )); + } + await syncUntil("peer receives pre-join encrypted history", peer, (events) => + messageEvent(events, lateRoom.roomId, (event) => event.eventId === beforeLateJoin.at(-1).eventId) + ); + await botRest.invite(lateRoom.roomId, late.userId); + await late.core.joinRoom({ roomIdOrAlias: lateRoom.roomId }); + await Promise.all([sync(bot, 8), sync(late, 8)]); + const latePreJoinHistory = await late.core.fetchMessages({ limit: 20, roomId: lateRoom.roomId }); + const preJoinBodies = new Set(beforeLateJoin.map((message) => message.eventId)); + assert.equal( + latePreJoinHistory.messages.some((message) => preJoinBodies.has(message.eventId)), + false, + "late encrypted member should not get decryptable pre-join messages" + ); + const afterLateJoinBody = `post-late ${Date.now()}`; + const afterLateJoin = await retry("send post-join encrypted message", () => + bot.core.postMessage({ body: afterLateJoinBody, roomId: lateRoom.roomId }) + ); + const lateSeen = await syncUntil("late member decrypts post-join message", late, (events) => + messageEvent(events, lateRoom.roomId, (event) => event.eventId === afterLateJoin.eventId) + ); + assert.equal(lateSeen.body, afterLateJoinBody); + assert.equal(lateSeen.isEncrypted, true); + + const lateDevice = await makeCore(peerAccount, "core-peer-late-device"); + try { + await Promise.all([sync(bot, 3), sync(lateDevice, 3)]); + const lateDeviceBody = `late-device ${Date.now()}`; + const lateDeviceMessage = await retry("send message after fresh device discovery", () => + bot.core.postMessage({ body: lateDeviceBody, roomId: dm.roomId }) + ); + const lateDeviceSeen = await syncUntil("fresh recovered device decrypts existing room message", lateDevice, (events) => + messageEvent(events, dm.roomId, (event) => event.eventId === lateDeviceMessage.eventId) + ); + assert.equal(lateDeviceSeen.body, lateDeviceBody); + assert.equal(lateDeviceSeen.isEncrypted, true); + } finally { + await closeAll(lateDevice); + } + + const body = `plain ${Date.now()}`; + const sent = await retry("send plain message", () => + bot.core.postMessage({ body, roomId: dm.roomId }) + ); + const received = await syncUntil("peer receives encrypted bot message", peer, (events) => + messageEvent(events, dm.roomId, (event) => event.body === body) + ); + assert.equal(received.isEncrypted, true); + + const formatted = `formatted ${Date.now()}`; + const formattedSent = await retry("send formatted message", () => peer.core.postMessage({ + body: formatted, + formattedBody: `${formatted}`, + mentions: { userIds: [bot.userId] }, + roomId: dm.roomId, + })); + const formattedSeen = await syncUntil("bot receives formatted mention", bot, (events) => + messageEvent(events, dm.roomId, (event) => event.eventId === formattedSent.eventId) + ); + assert.equal(formattedSeen.formattedBody.includes(""), true); + assert.equal(formattedSeen.isEncrypted, true); + + const editedBody = `${body} edited`; + await retry("edit message", () => bot.core.editMessage({ + body: editedBody, + messageId: sent.eventId, + roomId: dm.roomId, + })); + const editedSeen = await syncUntil("peer receives edit", peer, (events) => + messageEvent(events, dm.roomId, (event) => event.eventId === sent.eventId && event.body === editedBody) + ); + assert.equal(editedSeen.isEdited, true); + const fetchedEdited = await peer.core.fetchMessage({ messageId: sent.eventId, roomId: dm.roomId }); + assert.equal(fetchedEdited.message.body, editedBody); + assert.equal(fetchedEdited.message.isEdited, true); + + await retry("add reaction", () => + peer.core.addReaction({ emoji: "✅", messageId: sent.eventId, roomId: dm.roomId }) + ); + await syncUntil("bot receives reaction", bot, (events) => + reactionEvent(events, dm.roomId, sent.eventId, "✅", true) + ); + await peer.core.removeReaction({ emoji: "✅", messageId: sent.eventId, roomId: dm.roomId }); + await syncUntil("bot receives reaction removal", bot, (events) => + reactionEvent(events, dm.roomId, sent.eventId, "✅", false) + ); + + await bot.core.setTyping({ roomId: dm.roomId, timeoutMs: 5000, typing: true }); + await bot.core.setTyping({ roomId: dm.roomId, timeoutMs: 0, typing: false }); + + const mediaPayload = Buffer.from(`media ${Date.now()}`, "utf8"); + const media = await retry("send media", () => bot.core.postMediaMessage({ + bytesBase64: mediaPayload.toString("base64"), + contentType: "text/plain", + filename: "better-matrix-js-e2e.txt", + roomId: dm.roomId, + })); + const mediaSeen = await syncUntil("peer receives encrypted media", peer, (events) => + messageEvent(events, dm.roomId, (event) => event.eventId === media.eventId) + ); + const attachment = mediaSeen.attachments?.[0]; + assert.ok(attachment); + assert.ok(attachment.encryptedFile, "media in encrypted room should use encrypted file metadata"); + const downloaded = await peer.core.downloadEncryptedMedia({ file: attachment.encryptedFile }); + assert.equal(Buffer.from(downloaded.bytesBase64, "base64").toString("utf8"), mediaPayload.toString("utf8")); + + const threadRoot = await retry("send thread root", () => bot.core.postMessage({ + body: `thread root ${Date.now()}`, + roomId: group.roomId, + })); + await syncUntil("peer receives thread root", peer, (events) => + messageEvent(events, group.roomId, (event) => event.eventId === threadRoot.eventId) + ); + const threadReply = await retry("send thread reply", () => peer.core.postMessage({ + body: `thread reply ${Date.now()}`, + roomId: group.roomId, + threadRootEventId: threadRoot.eventId, + })); + await syncUntil("bot receives thread reply", bot, (events) => + messageEvent(events, group.roomId, (event) => event.eventId === threadReply.eventId) + ); + const threadMessages = await bot.core.fetchMessages({ + limit: 10, + roomId: group.roomId, + threadRootEventId: threadRoot.eventId, + }); + assert.ok(threadMessages.messages.some((message) => message.eventId === threadReply.eventId)); + const threads = await bot.core.listRoomThreads({ limit: 10, roomId: group.roomId }); + assert.ok(threads.threads.some((thread) => thread.root.eventId === threadRoot.eventId)); + + const history = await bot.core.fetchMessages({ limit: 20, roomId: dm.roomId }); + assert.ok(history.messages.some((message) => + message.eventId === sent.eventId && message.body === editedBody && message.isEdited === true + )); + + const rawSync = await new MatrixREST(botAccount).sync({ timeout: 0 }); + await bot.core.applySyncResponse({ response: rawSync }); + + const joined = await bot.core.fetchJoinedRooms(); + assert.ok(joined.roomIds.includes(dm.roomId)); + assert.ok(joined.roomIds.includes(group.roomId)); + + await bot.core.deleteMessage({ messageId: sent.eventId, reason: "e2e cleanup", roomId: dm.roomId }); + await sync(peer, 2); + const deleted = await peer.core.fetchMessage({ messageId: sent.eventId, roomId: dm.roomId }); + assert.equal(deleted.message, null); + await bot.core.markRead({ eventId: formattedSent.eventId, roomId: dm.roomId }); + await third.core.leaveRoom({ reason: "e2e complete", roomId: group.roomId }); + await late.core.leaveRoom({ reason: "e2e complete", roomId: lateRoom.roomId }); + } finally { + await closeAll(bot, peer, third, late); + } +}); diff --git a/e2e-scripts/test/restart-persistence-e2e.test.mjs b/e2e-scripts/test/restart-persistence-e2e.test.mjs new file mode 100644 index 0000000..572fd0c --- /dev/null +++ b/e2e-scripts/test/restart-persistence-e2e.test.mjs @@ -0,0 +1,85 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { loadAccounts } from "../src/accounts.mjs"; +import { + closeAll, + createCompatCore, + cryptoStatus, + makeCore, + messageEvent, + retry, + sync, + syncUntil, + toRuntimeEvent, +} from "../src/harness.mjs"; + +async function openCoreFromStore(account, storeDir) { + const events = []; + const core = createCompatCore(account, storeDir); + core.events.on((event) => events.push(toRuntimeEvent(event))); + const initOptions = {}; + if (account.recoveryKey) { + initOptions.recoveryKey = account.recoveryKey; + } + const whoami = await core.connect(initOptions); + assert.equal(whoami.userId, account.userId); + return { account, core, events, storeDir, userId: whoami.userId }; +} + +test("core: encrypted DM survives FileMatrixStore restart", async () => { + const [botAccount, peerAccount] = await loadAccounts(2); + let bot = await makeCore(botAccount, "restart-bot"); + const peer = await makeCore(peerAccount, "restart-peer"); + + try { + assert.ok(cryptoStatus(bot.events, "enabled"), "bot crypto should initialize"); + assert.ok(cryptoStatus(peer.events, "enabled"), "peer crypto should initialize"); + + await Promise.all([sync(bot, 2), sync(peer, 2)]); + + const dm = await bot.core.openDM({ userId: peer.userId }); + await peer.core.joinRoom({ roomIdOrAlias: dm.roomId }); + await Promise.all([sync(bot, 8), sync(peer, 8)]); + + const firstBody = `before restart ${Date.now()}`; + const firstSent = await retry("send encrypted message before restart", () => + peer.core.postMessage({ body: firstBody, roomId: dm.roomId }) + ); + const firstSeen = await syncUntil("bot decrypts message before restart", bot, (events) => + messageEvent(events, dm.roomId, (event) => event.eventId === firstSent.eventId) + ); + assert.equal(firstSeen.body, firstBody); + assert.equal(firstSeen.isEncrypted, true); + + const persistedStoreDir = bot.storeDir; + const persistedAccount = bot.account; + await closeAll(bot); + bot = await openCoreFromStore(persistedAccount, persistedStoreDir); + assert.ok(cryptoStatus(bot.events, "enabled"), "bot crypto should reinitialize after restart"); + + await Promise.all([sync(bot, 4), sync(peer, 2)]); + + const fetchedFirst = await bot.core.fetchMessage({ + messageId: firstSent.eventId, + roomId: dm.roomId, + }); + assert.equal(fetchedFirst.message.body, firstBody); + assert.equal(fetchedFirst.message.isEncrypted, true); + + const secondBody = `after restart ${Date.now()}`; + const secondSent = await retry("send encrypted message after restart", () => + bot.core.postMessage({ body: secondBody, roomId: dm.roomId }) + ); + const secondSeen = await syncUntil("peer decrypts message after restart", peer, (events) => + messageEvent(events, dm.roomId, (event) => event.eventId === secondSent.eventId) + ); + assert.equal(secondSeen.body, secondBody); + assert.equal(secondSeen.isEncrypted, true); + + const fetchedHistory = await bot.core.fetchMessages({ limit: 10, roomId: dm.roomId }); + assert.ok(fetchedHistory.messages.some((message) => message.eventId === firstSent.eventId)); + assert.ok(fetchedHistory.messages.some((message) => message.eventId === secondSent.eventId)); + } finally { + await closeAll(bot, peer); + } +}); diff --git a/e2e-scripts/test/sdk-surface-happy-path-e2e.test.mjs b/e2e-scripts/test/sdk-surface-happy-path-e2e.test.mjs new file mode 100644 index 0000000..a9cbde7 --- /dev/null +++ b/e2e-scripts/test/sdk-surface-happy-path-e2e.test.mjs @@ -0,0 +1,319 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { BEEPER_DOMAIN } from "../src/config.mjs"; +import { loadAccounts } from "../src/accounts.mjs"; +import { + closeAll, + eventually, + makeCore, + messageEvent, + retry, + sync, + syncUntil, +} from "../src/harness.mjs"; + +const tinyPng = Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=", + "base64" +); + +test("sdk surface: rooms, account data, to-device, receipts, redactions, relations, media, raw, isolation", async () => { + const [adminAccount, peerAccount, inviteeAccount, kickedAccount, bannedAccount, otherAccount] = + await loadAccounts(6); + const admin = await makeCore(adminAccount, "surface-admin"); + const peer = await makeCore(peerAccount, "surface-peer"); + const invitee = await makeCore(inviteeAccount, "surface-invitee"); + const kicked = await makeCore(kickedAccount, "surface-kicked"); + const banned = await makeCore(bannedAccount, "surface-banned"); + const other = await makeCore(otherAccount, "surface-other"); + + try { + await Promise.all([sync(admin, 2), sync(peer, 2), sync(invitee, 2), sync(kicked, 2), sync(banned, 2), sync(other, 2)]); + + const aliasLocalpart = `bmjs-surface-${Date.now()}`; + const room = await admin.core.rooms.create({ + invite: [peer.userId], + name: "surface room initial", + preset: "private_chat", + roomAliasName: aliasLocalpart, + topic: "surface topic initial", + visibility: "public", + }); + await peer.core.rooms.join({ roomIdOrAlias: room.roomId }); + await Promise.all([sync(admin, 4), sync(peer, 4)]); + + const alias = `#${aliasLocalpart}:${BEEPER_DOMAIN}`; + const resolved = await admin.core.rooms.resolveAlias({ alias }); + assert.equal(resolved.roomId, room.roomId); + assert.ok(Array.isArray(resolved.servers)); + + const avatar = await admin.core.media.upload({ + bytes: tinyPng, + contentType: "image/png", + filename: "avatar.png", + }); + await admin.core.rooms.sendStateEvent({ + content: { name: "surface room renamed" }, + eventType: "m.room.name", + roomId: room.roomId, + stateKey: "", + }); + const topicEvent = await admin.core.rooms.sendStateEvent({ + content: { topic: "surface topic renamed" }, + eventType: "m.room.topic", + roomId: room.roomId, + stateKey: "", + }); + await admin.core.rooms.sendStateEvent({ + content: { url: avatar.contentUri }, + eventType: "m.room.avatar", + roomId: room.roomId, + stateKey: "", + }); + + const power = await admin.core.rooms.getPowerLevels({ roomId: room.roomId }); + await admin.core.rooms.sendStateEvent({ + content: { + ...power.raw, + users: { ...(power.users ?? {}), [peer.userId]: 50 }, + }, + eventType: "m.room.power_levels", + roomId: room.roomId, + stateKey: "", + }); + const updatedPower = await admin.core.rooms.getPowerLevels({ roomId: room.roomId }); + assert.equal(updatedPower.users?.[peer.userId], 50); + + await eventually("room state reflects name/topic/avatar", async () => { + const info = await admin.core.rooms.get({ roomId: room.roomId }); + assert.equal(info.name, "surface room renamed"); + assert.equal(info.topic, "surface topic renamed"); + const avatarState = await admin.core.rooms.getStateEvent({ + eventType: "m.room.avatar", + roomId: room.roomId, + stateKey: "", + }); + assert.equal(avatarState.content.url, avatar.contentUri); + const allState = await admin.core.rooms.getState({ roomId: room.roomId }); + assert.ok(allState.events.some((event) => event.type === "m.room.power_levels")); + return true; + }); + + await admin.core.rooms.invite({ reason: "surface invite", roomId: room.roomId, userId: invitee.userId }); + await syncUntil("invitee sees invite", invitee, (events) => + events.some((event) => event.kind === "invite" && event.roomId === room.roomId) + ); + await invitee.core.rooms.join({ roomIdOrAlias: room.roomId }); + await sync(admin, 3); + await invitee.core.rooms.leave({ reason: "surface leave", roomId: room.roomId }); + + await admin.core.rooms.invite({ roomId: room.roomId, userId: kicked.userId }); + await kicked.core.rooms.join({ roomIdOrAlias: room.roomId }); + await sync(admin, 3); + await admin.core.rooms.kick({ reason: "surface kick", roomId: room.roomId, userId: kicked.userId }); + await eventually("kick is visible in members", async () => { + const members = await admin.core.rooms.listMembers({ membership: "leave", roomId: room.roomId }); + assert.ok(members.members.some((member) => member.userId === kicked.userId)); + return true; + }); + + await admin.core.rooms.ban({ reason: "surface ban", redactEvents: false, roomId: room.roomId, userId: banned.userId }); + await eventually("ban is visible in members", async () => { + const members = await admin.core.rooms.listMembers({ membership: "ban", roomId: room.roomId }); + assert.ok(members.members.some((member) => member.userId === banned.userId)); + return true; + }); + await admin.core.rooms.unban({ reason: "surface unban", roomId: room.roomId, userId: banned.userId }); + + const globalType = `com.beeper.bmjs.surface.global.${Date.now()}`; + const roomType = `com.beeper.bmjs.surface.room.${Date.now()}`; + await admin.core.accountData.set({ eventType: globalType, content: { ok: true, scope: "global" } }); + await admin.core.accountData.setRoom({ eventType: roomType, roomId: room.roomId, content: { ok: true, scope: "room" } }); + assert.deepEqual((await admin.core.accountData.get({ eventType: globalType })).content, { ok: true, scope: "global" }); + assert.deepEqual((await admin.core.accountData.getRoom({ eventType: roomType, roomId: room.roomId })).content, { ok: true, scope: "room" }); + await syncUntil("account data sync emits both scopes", admin, (events) => + events.some((event) => event.kind === "accountData" && event.type === globalType) && + events.some((event) => event.kind === "accountData" && event.type === roomType && event.roomId === room.roomId) + ); + + const toDeviceType = `com.beeper.bmjs.surface.to_device.${Date.now()}`; + await admin.core.toDevice.send({ + eventType: toDeviceType, + transactionId: `txn-${Date.now()}`, + userId: peer.userId, + deviceId: peer.account.deviceId, + content: { hello: "device" }, + }); + await admin.core.toDevice.send({ + eventType: `${toDeviceType}.bulk`, + messages: { [peer.userId]: { [peer.account.deviceId]: { hello: "bulk" } } }, + }); + await syncUntil("peer receives custom to-device events", peer, (events) => + events.some((event) => event.kind === "toDevice" && event.type === toDeviceType && event.content.hello === "device") && + events.some((event) => event.kind === "toDevice" && event.type === `${toDeviceType}.bulk` && event.content.hello === "bulk") + ); + + const receiptMessage = await retry("send receipt target", () => + admin.core.messages.send({ roomId: room.roomId, text: `receipt target ${Date.now()}` }) + ); + await syncUntil("peer sees receipt target", peer, (events) => + messageEvent(events, room.roomId, (event) => event.eventId === receiptMessage.eventId) + ); + await peer.core.receipts.send({ eventId: receiptMessage.eventId, receiptType: "m.read", roomId: room.roomId }); + await peer.core.receipts.send({ + content: { extra: true }, + eventId: receiptMessage.eventId, + receiptType: "m.read.private", + roomId: room.roomId, + threadId: "main", + }); + await syncUntil("admin receives read receipt", admin, (events) => + events.some((event) => event.kind === "receipt" && event.roomId === room.roomId) + ); + + const customTimeline = await admin.core.raw.request({ + method: "PUT", + path: `/_matrix/client/v3/rooms/${encodeURIComponent(room.roomId)}/send/com.beeper.bmjs.surface.custom/surface-${Date.now()}`, + body: { body: "surface custom event" }, + }); + assert.equal(customTimeline.status, 200); + assert.ok(customTimeline.body.event_id); + await sync(peer, 2); + const redaction = await admin.core.raw.request({ + method: "PUT", + path: `/_matrix/client/v3/rooms/${encodeURIComponent(room.roomId)}/redact/${encodeURIComponent(customTimeline.body.event_id)}/surface-${Date.now()}`, + body: { reason: "surface custom redaction" }, + }); + assert.equal(redaction.status, 200); + await eventually("custom event is redacted", async () => { + const fetched = await peer.core.raw.request({ + method: "GET", + path: `/_matrix/client/v3/rooms/${encodeURIComponent(room.roomId)}/event/${encodeURIComponent(customTimeline.body.event_id)}`, + }); + assert.deepEqual(fetched.body.content, {}); + assert.ok(fetched.body.unsigned?.redacted_because); + return true; + }); + + const roots = []; + for (let rootIndex = 0; rootIndex < 3; rootIndex += 1) { + roots.push(await retry(`send thread root ${rootIndex}`, () => + admin.core.messages.send({ roomId: room.roomId, text: `surface thread root ${rootIndex} ${Date.now()}` }) + )); + } + for (let index = 0; index < 12; index += 1) { + await retry(`send thread reply ${index}`, () => + peer.core.messages.send({ + roomId: room.roomId, + text: `surface thread reply ${index} ${Date.now()}`, + threadRoot: roots[0].eventId, + }) + ); + } + await syncUntil("admin receives last thread reply", admin, (events) => + messageEvent(events, room.roomId, (event) => event.threadRootEventId === roots[0].eventId || event.threadRoot === roots[0].eventId) + ); + const firstThreadPage = await admin.core.messages.list({ limit: 5, roomId: room.roomId, threadRoot: roots[0].eventId }); + assert.ok(firstThreadPage.messages.length >= 1); + assert.ok(firstThreadPage.nextCursor); + const secondThreadPage = await admin.core.messages.list({ + cursor: firstThreadPage.nextCursor, + limit: 5, + roomId: room.roomId, + threadRoot: roots[0].eventId, + }); + assert.ok(secondThreadPage.messages.length >= 1); + const threadList = await eventually("thread list contains created roots", async () => { + const result = await admin.core.rooms.threads.list({ limit: 10, roomId: room.roomId }); + assert.ok(result.threads.some((thread) => thread.root.eventId === roots[0].eventId)); + return result; + }); + assert.ok(threadList.threads.length >= 1); + + const uploadedPlain = await admin.core.media.upload({ + bytes: tinyPng, + contentType: "image/png", + filename: "plain.png", + height: 1, + width: 1, + }); + const downloadedPlain = await admin.core.media.download({ contentUri: uploadedPlain.contentUri }); + assert.deepEqual(Buffer.from(downloadedPlain.bytes), tinyPng); + const thumbnail = await admin.core.media.downloadThumbnail({ + contentUri: uploadedPlain.contentUri, + height: 32, + method: "scale", + width: 32, + }); + assert.ok(thumbnail.bytes.byteLength > 0); + const uploadedEncrypted = await admin.core.media.uploadEncrypted({ + bytes: tinyPng, + contentType: "image/png", + filename: "encrypted.png", + }); + const downloadedEncrypted = await admin.core.media.downloadEncrypted({ file: uploadedEncrypted.file }); + assert.deepEqual(Buffer.from(downloadedEncrypted.bytes), tinyPng); + + const rawJoined = await admin.core.raw.request({ + method: "GET", + path: "/_matrix/client/v3/joined_rooms", + query: { surface: "1" }, + headers: { "x-bmjs-surface": "true" }, + }); + assert.equal(rawJoined.status, 200); + assert.ok(Array.isArray(rawJoined.body.joined_rooms)); + const rawState = await admin.core.raw.request({ + method: "PUT", + path: `/_matrix/client/v3/rooms/${encodeURIComponent(room.roomId)}/state/com.beeper.bmjs.surface.raw/${encodeURIComponent("state")}`, + body: { ok: true }, + }); + assert.equal(rawState.status, 200); + const rawStateRead = await admin.core.rooms.getStateEvent({ + eventType: "com.beeper.bmjs.surface.raw", + roomId: room.roomId, + stateKey: "state", + }); + assert.equal(rawStateRead.content.ok, true); + + const isolationRoomA = await admin.core.rooms.create({ + invite: [peer.userId], + name: `surface isolation A ${Date.now()}`, + preset: "private_chat", + }); + const isolationRoomB = await other.core.rooms.create({ + invite: [invitee.userId], + name: `surface isolation B ${Date.now()}`, + preset: "private_chat", + }); + await Promise.all([ + peer.core.rooms.join({ roomIdOrAlias: isolationRoomA.roomId }), + invitee.core.rooms.join({ roomIdOrAlias: isolationRoomB.roomId }), + ]); + await Promise.all([sync(admin, 3), sync(peer, 3), sync(other, 3), sync(invitee, 3)]); + const isolationMessages = []; + for (let index = 0; index < 8; index += 1) { + isolationMessages.push(await admin.core.messages.send({ + roomId: isolationRoomA.roomId, + text: `surface isolation A ${index} ${Date.now()}`, + })); + isolationMessages.push(await other.core.messages.send({ + roomId: isolationRoomB.roomId, + text: `surface isolation B ${index} ${Date.now()}`, + })); + await Promise.all([sync(admin, 1, 250), sync(peer, 1, 250), sync(other, 1, 250), sync(invitee, 1, 250)]); + } + await syncUntil("peer receives only isolation room A messages", peer, (events) => { + const seenA = isolationMessages.filter((message) => + message.roomId === isolationRoomA.roomId && + messageEvent(events, isolationRoomA.roomId, (event) => event.eventId === message.eventId) + ); + const leakedB = isolationMessages.some((message) => + message.roomId === isolationRoomB.roomId && + messageEvent(events, isolationRoomB.roomId, (event) => event.eventId === message.eventId) + ); + return seenA.length >= 8 && !leakedB; + }); + } finally { + await closeAll(admin, peer, invitee, kicked, banned, other); + } +}); diff --git a/e2e-scripts/test/unencrypted-room-e2e.test.mjs b/e2e-scripts/test/unencrypted-room-e2e.test.mjs new file mode 100644 index 0000000..8482533 --- /dev/null +++ b/e2e-scripts/test/unencrypted-room-e2e.test.mjs @@ -0,0 +1,107 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { loadAccounts } from "../src/accounts.mjs"; +import { MatrixREST } from "../src/matrix-rest.mjs"; +import { closeAll, makeCore, messageEvent, retry, sync, syncUntil } from "../src/harness.mjs"; + +const PNG_1X1_BASE64 = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAFgwJ/luzO7wAAAABJRU5ErkJggg=="; + +test("core: unencrypted room metadata, messages, and media stay plain", async () => { + const [botAccount, peerAccount] = await loadAccounts(2); + const bot = await makeCore(botAccount, "unencrypted-bot"); + const peer = await makeCore(peerAccount, "unencrypted-peer"); + const botRest = new MatrixREST(botAccount); + const peerRest = new MatrixREST(peerAccount); + + try { + await Promise.all([sync(bot, 2), sync(peer, 2)]); + + const name = `better-matrix-js unencrypted ${Date.now()}`; + const topic = "unencrypted live e2e coverage"; + const room = await botRest.createRoom({ + invite: [peer.userId], + name, + topic, + }); + await peerRest.join(room.room_id); + await Promise.all([sync(bot, 6), sync(peer, 6)]); + + const roomInfo = await bot.core.fetchRoom({ roomId: room.room_id }); + assert.equal(roomInfo.encrypted, false); + assert.equal(roomInfo.name, name); + assert.equal(roomInfo.topic, topic); + assert.ok(roomInfo.memberCount >= 2); + + const body = `unencrypted plain ${Date.now()}`; + const sent = await retry("send unencrypted plain message", () => + bot.core.postMessage({ body, roomId: room.room_id }) + ); + const seen = await syncUntil("peer receives unencrypted plain message", peer, (events) => + messageEvent(events, room.room_id, (event) => event.eventId === sent.eventId) + ); + assert.equal(seen.body, body); + assert.notEqual(seen.isEncrypted, true); + assert.equal(seen.content?.msgtype, "m.text"); + assert.equal(seen.content?.body, body); + assert.equal(seen.content?.ciphertext, undefined); + + const textPayload = Buffer.from(`unencrypted file ${Date.now()}`, "utf8"); + const textMedia = await retry("send unencrypted file media", () => + bot.core.postMediaMessage({ + bytesBase64: textPayload.toString("base64"), + contentType: "text/plain", + filename: "unencrypted-e2e.txt", + msgtype: "m.file", + roomId: room.room_id, + size: textPayload.byteLength, + }) + ); + const textMediaSeen = await syncUntil("peer receives unencrypted file media", peer, (events) => + messageEvent(events, room.room_id, (event) => event.eventId === textMedia.eventId) + ); + const textAttachment = textMediaSeen.attachments?.[0]; + assert.ok(textAttachment); + assert.equal(textMediaSeen.msgtype, "m.file"); + assert.equal(textAttachment.msgtype, "m.file"); + assert.equal(textAttachment.filename, "unencrypted-e2e.txt"); + assert.equal(textAttachment.info?.contentType, "text/plain"); + assert.equal(textAttachment.info?.size, textPayload.byteLength); + assert.ok(textAttachment.contentUri?.startsWith("mxc://")); + assert.equal(textAttachment.encryptedFile, undefined); + const textDownloaded = await peer.core.downloadMedia({ contentUri: textAttachment.contentUri }); + assert.equal(Buffer.from(textDownloaded.bytesBase64, "base64").toString("utf8"), textPayload.toString("utf8")); + + const imagePayload = Buffer.from(PNG_1X1_BASE64, "base64"); + const imageMedia = await retry("send unencrypted image media", () => + bot.core.postMediaMessage({ + bytesBase64: PNG_1X1_BASE64, + contentType: "image/png", + filename: "unencrypted-pixel.png", + height: 1, + msgtype: "m.image", + roomId: room.room_id, + size: imagePayload.byteLength, + width: 1, + }) + ); + const imageMediaSeen = await syncUntil("peer receives unencrypted image media", peer, (events) => + messageEvent(events, room.room_id, (event) => event.eventId === imageMedia.eventId) + ); + const imageAttachment = imageMediaSeen.attachments?.[0]; + assert.ok(imageAttachment); + assert.equal(imageMediaSeen.msgtype, "m.image"); + assert.equal(imageAttachment.msgtype, "m.image"); + assert.equal(imageAttachment.filename, "unencrypted-pixel.png"); + assert.equal(imageAttachment.info?.contentType, "image/png"); + assert.equal(imageAttachment.info?.height, 1); + assert.equal(imageAttachment.info?.size, imagePayload.byteLength); + assert.equal(imageAttachment.info?.width, 1); + assert.ok(imageAttachment.contentUri?.startsWith("mxc://")); + assert.equal(imageAttachment.encryptedFile, undefined); + const imageDownloaded = await peer.core.downloadMedia({ contentUri: imageAttachment.contentUri }); + assert.deepEqual(Buffer.from(imageDownloaded.bytesBase64, "base64"), imagePayload); + } finally { + await closeAll(bot, peer); + } +}); diff --git a/e2e/README.md b/e2e/README.md new file mode 100644 index 0000000..79ca7a8 --- /dev/null +++ b/e2e/README.md @@ -0,0 +1,68 @@ +# Matrix Live E2E + +Live E2E tests are intentionally not part of default CI. They require real Matrix/Beeper accounts, durable stores, and recovery material for encrypted-history coverage. + +## Account Strategy + +Prefer cached Beeper accounts and stable device stores by default. Reusing accounts is required to catch the cases bots usually break: + +- old encrypted rooms +- old Megolm sessions +- existing devices +- recovery after process and store reload +- history pagination across encrypted events +- multi-client same-process isolation + +Fresh-device tests should be explicit opt-in scenarios because they create new Matrix devices. + +## Required Environment + +Use separate accounts for bot and peer: + +```sh +export MATRIX_HOMESERVER_URL=https://matrix.beeper.com +export MATRIX_BOT_ACCESS_TOKEN=... +export MATRIX_PEER_ACCESS_TOKEN=... +export MATRIX_BOT_RECOVERY_KEY=... +export MATRIX_PEER_RECOVERY_KEY=... +export MATRIX_LIVE_E2E_STORE_DIR=.matrix-e2e-store +``` + +`MATRIX_LIVE_E2E_STORE_DIR` should be reused between runs unless a test explicitly validates fresh-device behavior. + +## Required Scenarios + +- lazy client can send/fetch without sync +- `boot()` initializes but does not emit app events +- `whoami()` confirms immutable account/device identity +- `client.subscribe(...)` returns `{ stop, catchUp, done }` +- default subscription receives future events only +- `catchUp()` replays missed events +- `onRawEvent(...)` receives granular Matrix payloads +- encrypted messages +- edits +- reactions and reaction removals +- media upload/download +- threads +- invites and auto-join +- room state +- account data +- to-device +- receipts +- reused accounts paginate and decrypt old encrypted history +- fresh and existing devices behave correctly +- multi-client same-process isolation +- Chat SDK live subscription mode +- Chat SDK sync-disabled mode +- Chat SDK webhook/apply mode + +## Running + +The current live smoke entrypoint is: + +```sh +pnpm build +pnpm test:live -- --keep-store +``` + +Do not add this to default CI until the account provisioning and secret handling are automated. diff --git a/examples/beeper-streaming-smoke/src/index.mjs b/examples/beeper-streaming-smoke/src/index.mjs index f5c9b5c..e376046 100644 --- a/examples/beeper-streaming-smoke/src/index.mjs +++ b/examples/beeper-streaming-smoke/src/index.mjs @@ -5,7 +5,7 @@ import { Chat } from "chat"; import { createMatrixLogin } from "better-matrix-js"; import { createMatrixClient } from "better-matrix-js/node"; import { createMatrixAdapter } from "@better-matrix-js/chat-adapter"; -import { FileState, MatrixState } from "./file-state.mjs"; +import { FileState, MatrixState } from "../../shared/file-state.mjs"; const loremSentenceCorpus = [ "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", diff --git a/examples/cloudflare-worker/src/index.js b/examples/cloudflare-worker/src/index.js index 7907b4e..5e08a1f 100644 --- a/examples/cloudflare-worker/src/index.js +++ b/examples/cloudflare-worker/src/index.js @@ -49,7 +49,7 @@ export class MatrixClientObject { })); const core = await this.corePromise; if (this.env.MATRIX_ACCESS_TOKEN && this.env.MATRIX_HOMESERVER_URL) { - this.initPromise ??= core.connect(); + this.initPromise ??= core.boot(); await this.initPromise; } return core; diff --git a/examples/cloudflare-worker/wrangler.jsonc b/examples/cloudflare-worker/wrangler.jsonc index ca93acb..e372216 100644 --- a/examples/cloudflare-worker/wrangler.jsonc +++ b/examples/cloudflare-worker/wrangler.jsonc @@ -17,7 +17,7 @@ "migrations": [ { "tag": "v1", - "new_classes": ["MatrixClientObject", "MatrixSyncObject"] + "new_sqlite_classes": ["MatrixClientObject", "MatrixSyncObject"] } ], "vars": { diff --git a/examples/dummybridge-bot/README.md b/examples/dummybridge-bot/README.md new file mode 100644 index 0000000..99c9bce --- /dev/null +++ b/examples/dummybridge-bot/README.md @@ -0,0 +1,35 @@ +# DummyBridge Bot Example + +A Node-first Matrix bot example that mirrors the useful DummyBridge demo surface: +it accepts every invite, replies to all incoming messages, streams synthetic +agent output, demonstrates tools/approvals/artifacts in text, reacts to handled +messages, and keeps a small local state directory. + +```sh +pnpm build +cd examples/dummybridge-bot +MATRIX_HOMESERVER_URL=https://matrix.example.org \ +MATRIX_ACCESS_TOKEN=... \ +MATRIX_RECOVERY_KEY='optional recovery key' \ +pnpm start +``` + +Password login is also supported: + +```sh +MATRIX_HOMESERVER_URL=https://matrix.example.org \ +MATRIX_USERNAME=@bot:example.org \ +MATRIX_PASSWORD=... \ +pnpm start +``` + +Try these messages in any room the bot is invited to: + +- `help` +- `stream-lorem 4096 --reasoning=1200 --steps=3 --sources=4 --documents=3 --files=2 --meta --chunk-chars=48:160` +- `stream-tools 2500 search#delta#prelim approval#deny weather#provider --reasoning=800 --steps=3` +- `stream-random --actions=24 --profile=artifacts` +- `stream-chaos 4 --max-actions=16` +- `error` + +The example uses the core SDK directly, not the Chat SDK adapter. diff --git a/examples/dummybridge-bot/package.json b/examples/dummybridge-bot/package.json new file mode 100644 index 0000000..c176256 --- /dev/null +++ b/examples/dummybridge-bot/package.json @@ -0,0 +1,15 @@ +{ + "name": "@better-matrix-js/example-dummybridge-bot", + "private": true, + "type": "module", + "scripts": { + "start": "node src/index.mjs" + }, + "dependencies": { + "@better-matrix-js/state-file": "workspace:*", + "better-matrix-js": "workspace:*" + }, + "engines": { + "node": ">=20" + } +} diff --git a/examples/dummybridge-bot/src/dummy-runtime.mjs b/examples/dummybridge-bot/src/dummy-runtime.mjs new file mode 100644 index 0000000..dbb3a10 --- /dev/null +++ b/examples/dummybridge-bot/src/dummy-runtime.mjs @@ -0,0 +1,355 @@ +const lorem = [ + "Lorem ipsum dolor sit amet, consectetur adipiscing elit.", + "Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.", + "Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.", + "Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.", + "Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.", + "Integer nec odio praesent libero sed cursus ante dapibus diam.", + "Nulla quis sem at nibh elementum imperdiet duis sagittis ipsum.", + "Praesent mauris fusce nec tellus sed augue semper porta.", + "Mauris massa vestibulum lacinia arcu eget nulla.", + "Curabitur ullamcorper ultricies nisi nam eget dui etiam rhoncus.", +]; + +const max = { + actions: 64, + chars: 20000, + collections: 16, + delayMs: 30000, + steps: 32, + tools: 16, + turns: 16, +}; + +export function helpText() { + return [ + "**DummyBridge bot commands**", + "", + "`stream-lorem [chars]` streams long markdown with reasoning, sources, documents, files, metadata, data parts, abort/error/finish states.", + "`stream-tools [chars] tool#tag...` demonstrates tool input, deltas, preliminary output, approvals, denial, provider execution, and failures.", + "`stream-random [seconds]` emits a deterministic mix of text, tools, artifacts, and approvals.", + "`stream-chaos [turns]` runs multiple demo turns back to back.", + "", + "Useful options: `--reasoning=1200`, `--steps=3`, `--sources=4`, `--documents=3`, `--files=2`, `--meta`, `--data=name`, `--data-transient=name`, `--chunk-chars=48:160`, `--delay-ms=0:250`, `--finish=length`, `--abort`, `--error`, `--seed=42`, `--profile=artifacts`.", + "", + "Tool tags: `delta`, `prelim`, `approval`, `deny`, `fail`, `inputerror`, `provider`.", + ].join("\n"); +} + +export async function* dummybridgeTextStream(input, options = {}) { + const command = parseCommand(input); + if (command.name === "help") { + yield helpText(); + return; + } + if (command.name === "chaos") { + for (let turn = 0; turn < command.turns; turn += 1) { + yield `\n\n## Chaos turn ${turn + 1}/${command.turns}\n\n`; + yield* runOne({ ...command, name: "random", seed: command.seed + turn, actions: command.maxActions }); + await sleep(sampleInt(makeRng(command.seed + turn), command.staggerMinMs, command.staggerMaxMs), options.signal); + } + return; + } + yield* runOne(command, options); +} + +export function parseCommand(input) { + const tokens = String(input || "").trim().split(/\s+/).filter(Boolean); + const first = normalizeCommand(tokens[0] || "stream-lorem"); + if (first === "help") return { name: "help" }; + if (first === "stream-chaos") { + const turns = clamp(readPositionalInt(tokens, 1, 4), 1, max.turns); + return { + ...common(tokens), + maxActions: clamp(readOptionInt(tokens, "max-actions", 16), 1, max.actions), + name: "chaos", + staggerMaxMs: readRange(tokens, "stagger-ms", 250, 1500).max, + staggerMinMs: readRange(tokens, "stagger-ms", 250, 1500).min, + turns, + }; + } + if (first === "stream-random") { + return { + ...common(tokens), + actions: clamp(readOptionInt(tokens, "actions", 24), 1, max.actions), + chars: clamp(readOptionInt(tokens, "chars", 6000), 1, max.chars), + name: "random", + profile: readOption(tokens, "profile", "balanced"), + }; + } + if (first === "stream-tools") { + const rawTools = tokens.slice(2).filter((token) => !token.startsWith("--")); + return { + ...common(tokens), + chars: clamp(readPositionalInt(tokens, 1, 3200), 1, max.chars), + name: "tools", + tools: rawTools.slice(0, max.tools).map(parseTool).concat(rawTools.length ? [] : defaultTools()), + }; + } + return { + ...common(tokens), + chars: clamp(readPositionalInt(tokens, 1, first === "error" ? 1200 : 4096), 1, max.chars), + error: first === "error" || tokens.includes("--error"), + name: "lorem", + tools: first === "tools" ? defaultTools() : [], + }; +} + +async function* runOne(command, options = {}) { + const rng = makeRng(command.seed); + const steps = clamp(command.steps, 1, max.steps); + const tools = command.name === "random" ? randomTools(rng, command) : command.tools ?? []; + const visible = markdownCorpus(command.chars, rng); + const reasoning = prose(command.reasoning, rng); + + yield `\n`; + if (reasoning) { + yield `\n
Reasoning\n\n`; + for (const chunk of chunks(reasoning, rng, command.chunkMin, command.chunkMax)) { + yield chunk; + await sleep(sampleInt(rng, command.delayMinMs, command.delayMaxMs), options.signal); + } + yield `\n\n
\n\n`; + } + yield `I heard: \`${escapeBackticks(command.raw || command.name)}\`\n\n`; + + for (let step = 0; step < steps; step += 1) { + yield `\n### Step ${step + 1}/${steps}\n\n`; + yield decorations(command, step, steps); + for (const chunk of chunks(slice(visible, steps, step), rng, command.chunkMin, command.chunkMax)) { + yield chunk; + await sleep(sampleInt(rng, command.delayMinMs, command.delayMaxMs), options.signal); + } + if (tools[step]) { + yield* renderTool(tools[step], step + 1, rng, command, options); + } + } + for (let index = steps; index < tools.length; index += 1) { + yield* renderTool(tools[index], index + 1, rng, command, options); + } + + if (command.abort) { + yield "\n\n**Aborted:** DummyBridge synthetic abort.\n"; + return; + } + if (command.error) { + yield "\n\n**Error:** DummyBridge synthetic error was emitted.\n"; + return; + } + yield `\n\n**Finished:** ${command.finishReason || "stop"}\n`; +} + +async function* renderTool(tool, sequence, rng, command, options) { + const id = `dummy-tool-${sequence}-${slug(tool.name)}`; + yield `\n\n#### Tool: ${title(tool.name)}\n\n`; + yield `- id: \`${id}\`\n- provider executed: \`${Boolean(tool.provider)}\`\n- tags: ${(tool.tags ?? []).map((tag) => `\`${tag}\``).join(", ") || "_none_"}\n`; + if (tool.delta) { + yield "- input delta:\n\n```json\n"; + for (const chunk of chunks(JSON.stringify({ sequence, tool: tool.name, tags: tool.tags ?? [] }, null, 2), rng, command.chunkMin, command.chunkMax)) { + yield chunk; + await sleep(sampleInt(rng, command.delayMinMs, command.delayMaxMs), options.signal); + } + yield "\n```\n"; + } + if (tool.inputError) { + yield "\nInput error: DummyBridge synthetic input error.\n"; + } + if (tool.prelim) { + yield "\nPreliminary output: `{ \"status\": \"streaming\" }`\n"; + } + if (tool.approval || tool.deny) { + yield `\nApproval requested for \`${id}\`: ${tool.deny ? "denied" : "approved"} automatically by the demo.\n`; + } + if (tool.fail || tool.inputError) { + yield "\nTool output error: DummyBridge synthetic tool failure.\n"; + return; + } + yield `\nTool output: \`{"status":"ok","sequence":${sequence}}\`\n`; +} + +function common(tokens) { + const delay = readRange(tokens, "delay-ms", 0, 0); + const chunk = readRange(tokens, "chunk-chars", 48, 160); + return { + abort: tokens.includes("--abort"), + chunkMax: clamp(chunk.max, 1, 512), + chunkMin: clamp(chunk.min, 1, 512), + data: readOption(tokens, "data", "demo"), + dataTransient: readOption(tokens, "data-transient", "demo-transient"), + delayMaxMs: clamp(delay.max, 0, max.delayMs), + delayMinMs: clamp(delay.min, 0, max.delayMs), + documents: clamp(readOptionInt(tokens, "documents", 2), 0, max.collections), + error: tokens.includes("--error"), + files: clamp(readOptionInt(tokens, "files", 1), 0, max.collections), + finishReason: readOption(tokens, "finish", "stop"), + meta: tokens.includes("--meta"), + raw: tokens.join(" "), + reasoning: clamp(readOptionInt(tokens, "reasoning", 1200), 0, max.chars), + seed: readOptionInt(tokens, "seed", 42), + sources: clamp(readOptionInt(tokens, "sources", 3), 0, max.collections), + steps: clamp(readOptionInt(tokens, "steps", 2), 1, max.steps), + }; +} + +function decorations(command, step, steps) { + const lines = []; + if (command.meta) lines.push(`Metadata: \`${JSON.stringify({ step: step + 1, seed: command.seed })}\``); + for (let i = 0; i < split(command.sources, steps, step); i += 1) { + lines.push(`Source: [Demo Source ${step + 1}.${i + 1}](https://dummybridge.local/source/${step + 1}-${i + 1})`); + } + for (let i = 0; i < split(command.documents, steps, step); i += 1) { + lines.push(`Document: \`demo-doc-${step + 1}-${i + 1}.txt\``); + } + for (let i = 0; i < split(command.files, steps, step); i += 1) { + lines.push(`File: \`mxc://dummybridge/demo-file-${step + 1}-${i + 1}\``); + } + if (step === 0 && command.data) lines.push(`Data part: \`${command.data}\``); + if (step === 0 && command.dataTransient) lines.push(`Transient data part: \`${command.dataTransient}\``); + return lines.length ? `${lines.map((line) => `- ${line}`).join("\n")}\n\n` : ""; +} + +function markdownCorpus(chars, rng) { + const blocks = []; + while (blocks.join("\n\n").length < chars + 120) { + const kind = sampleInt(rng, 0, 5); + if (kind === 0) blocks.push(prose(sampleInt(rng, 160, 360), rng)); + else if (kind === 1) blocks.push(`${prose(sampleInt(rng, 90, 180), rng)} Review the [release notes](https://dummybridge.local/docs/streaming) for **incremental** output.`); + else if (kind === 2) blocks.push("- Confirm seeded output.\n- Exercise markdown rendering.\n- Preserve readable deltas."); + else if (kind === 3) blocks.push(`> Streaming output should feel alive.\n>\n> ${prose(sampleInt(rng, 80, 160), rng)}`); + else if (kind === 4) blocks.push("```js\nconst preview = chunks.filter(Boolean).join(\"\");\n```"); + else blocks.push("| Metric | Value |\n| --- | --- |\n| stream | active |\n| renderer | markdown |"); + } + return trim(blocks.join("\n\n"), chars); +} + +function prose(chars, rng) { + let out = ""; + while (out.length < chars + 80) out += `${out ? " " : ""}${lorem[sampleInt(rng, 0, lorem.length - 1)]}`; + return trim(out, chars); +} + +function chunks(text, rng, min, maxValue) { + const out = []; + let offset = 0; + while (offset < text.length) { + const size = sampleInt(rng, min, maxValue); + out.push(text.slice(offset, offset + size)); + offset += size; + } + return out; +} + +function randomTools(rng, command) { + const pool = [ + "search#delta#prelim#provider", + "fetch#fail#provider", + "approval#deny", + "render_artifact#provider", + "shell#approval", + ]; + return Array.from({ length: Math.min(command.actions, 12) }, () => parseTool(pool[sampleInt(rng, 0, pool.length - 1)])); +} + +function defaultTools() { + return ["lookup_room_context#delta#prelim#provider", "approval_gate#deny", "dummy_weather#provider"].map(parseTool); +} + +function parseTool(token) { + const [name = "tool", ...tags] = token.split("#"); + return { + approval: tags.includes("approval"), + delta: tags.includes("delta"), + deny: tags.includes("deny"), + fail: tags.includes("fail"), + inputError: tags.includes("inputerror"), + name, + prelim: tags.includes("prelim"), + provider: tags.includes("provider"), + tags, + }; +} + +function makeRng(seed) { + let state = BigInt.asUintN(32, BigInt(seed || 1)); + return () => { + state = BigInt.asUintN(32, state * 1664525n + 1013904223n); + return Number(state) / 0x100000000; + }; +} + +function sampleInt(rng, min, maxValue) { + return Math.floor(min + rng() * (maxValue - min + 1)); +} + +function readOption(tokens, name, fallback) { + const prefix = `--${name}=`; + return tokens.find((token) => token.startsWith(prefix))?.slice(prefix.length) ?? fallback; +} + +function readOptionInt(tokens, name, fallback) { + const value = Number.parseInt(readOption(tokens, name, String(fallback)), 10); + return Number.isFinite(value) ? value : fallback; +} + +function readPositionalInt(tokens, index, fallback) { + const value = Number.parseInt(tokens[index] ?? "", 10); + return Number.isFinite(value) ? value : fallback; +} + +function readRange(tokens, name, fallbackMin, fallbackMax) { + const raw = readOption(tokens, name, ""); + if (!raw) return { min: fallbackMin, max: fallbackMax }; + const [left, right = left] = raw.split(":"); + const min = Number.parseInt(left, 10); + const maxValue = Number.parseInt(right, 10); + if (!Number.isFinite(min) || !Number.isFinite(maxValue)) return { min: fallbackMin, max: fallbackMax }; + return { min: Math.min(min, maxValue), max: Math.max(min, maxValue) }; +} + +function normalizeCommand(value) { + return String(value || "").trim().toLowerCase().replace(/^!dummybridge\s+/, ""); +} + +function split(total, steps, step) { + const start = Math.floor((total * step) / steps); + const end = Math.floor((total * (step + 1)) / steps); + return end - start; +} + +function slice(text, steps, step) { + return text.slice(Math.floor((text.length * step) / steps), Math.floor((text.length * (step + 1)) / steps)); +} + +function trim(text, chars) { + if (text.length <= chars) return text; + const sliced = text.slice(0, chars); + const lastSpace = sliced.lastIndexOf(" "); + return `${sliced.slice(0, lastSpace > chars * 0.75 ? lastSpace : chars).trimEnd()}.`; +} + +function clamp(value, min, maxValue) { + return Math.max(min, Math.min(maxValue, value)); +} + +function slug(value) { + return String(value || "tool").toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "") || "tool"; +} + +function title(value) { + return slug(value).split("_").map((part) => part.charAt(0).toUpperCase() + part.slice(1)).join(" "); +} + +function escapeBackticks(value) { + return String(value).replaceAll("`", "\\`"); +} + +function sleep(ms, signal) { + if (!ms) return Promise.resolve(); + return new Promise((resolve, reject) => { + const timer = setTimeout(resolve, ms); + signal?.addEventListener("abort", () => { + clearTimeout(timer); + reject(signal.reason ?? new Error("aborted")); + }, { once: true }); + }); +} diff --git a/examples/dummybridge-bot/src/index.mjs b/examples/dummybridge-bot/src/index.mjs new file mode 100644 index 0000000..f79c23a --- /dev/null +++ b/examples/dummybridge-bot/src/index.mjs @@ -0,0 +1,196 @@ +import { readFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { createMatrixLogin } from "better-matrix-js/login"; +import { createMatrixClient, onInvite, onMessage, onReaction } from "better-matrix-js/node"; +import { createFileMatrixStore } from "@better-matrix-js/state-file"; +import { FileState } from "../../shared/file-state.mjs"; +import { dummybridgeTextStream, helpText } from "./dummy-runtime.mjs"; + +const root = dirname(dirname(fileURLToPath(import.meta.url))); +await loadEnvFile(process.env.MATRIX_ENV_FILE || join(root, ".env")); + +const homeserver = env("MATRIX_HOMESERVER_URL", env("MATRIX_HOMESERVER")); +if (!homeserver) throw new Error("Missing MATRIX_HOMESERVER_URL"); + +const stateDir = env("MATRIX_STATE_DIR", join(root, ".matrix-state")); +const state = new FileState(join(stateDir, "state.json")); +await state.connect(); +const session = await resolveSession(homeserver, state); + +const client = createMatrixClient({ + account: { + accessToken: session.accessToken, + deviceId: session.deviceId, + homeserver, + userId: session.userId, + }, + beeper: env("MATRIX_BEEPER_STREAMS", "auto") === "1" ? true : undefined, + recoveryKey: env("MATRIX_RECOVERY_KEY"), + store: createFileMatrixStore(join(stateDir, "matrix")), + verifyRecoveryOnStart: env("MATRIX_VERIFY_RECOVERY_ON_START") === "1", +}); + +const whoami = await client.boot(); +const crypto = await client.crypto.status(); +console.log(`bot_user_id=${whoami.userId}`); +console.log(`bot_device_id=${whoami.deviceId}`); +console.log(`crypto_state=${crypto.state}`); + +const inFlight = new Set(); +const inviteSub = await onInvite(client, undefined, async (invite) => { + console.log(`invite room=${invite.roomId} inviter=${invite.inviter ?? ""}`); + await client.rooms.join({ roomIdOrAlias: invite.roomId }).catch((error) => { + console.error(`join_failed room=${invite.roomId}`, error); + }); + await client.messages.send({ + roomId: invite.roomId, + text: [ + "DummyBridge bot joined.", + "", + helpText(), + ].join("\n"), + }).catch((error) => console.error(`welcome_failed room=${invite.roomId}`, error)); +}); + +const messageSub = await onMessage(client, undefined, async (message) => { + if (message.sender.isMe || !message.text.trim()) return; + if (inFlight.has(message.eventId)) return; + inFlight.add(message.eventId); + try { + console.log(`message room=${message.roomId} sender=${message.sender.userId} id=${message.eventId}`); + await client.typing.set({ roomId: message.roomId, timeoutMs: 15000, typing: true }); + await client.reactions.send({ eventId: message.eventId, key: "👀", roomId: message.roomId }).catch(() => {}); + const sent = await client.streams.send({ + mode: env("MATRIX_STREAM_MODE", "auto"), + roomId: message.roomId, + stream: dummybridgeTextStream(message.text), + text: "DummyBridge is thinking...", + threadRoot: env("MATRIX_REPLY_IN_THREADS", "1") === "1" ? message.eventId : undefined, + updateIntervalMs: Number(env("MATRIX_STREAM_UPDATE_MS", "500")), + }); + await client.reactions.send({ eventId: message.eventId, key: "✅", roomId: message.roomId }).catch(() => {}); + await state.appendToList("dummybridge-bot:handled", { + at: new Date().toISOString(), + inputEventId: message.eventId, + outputEventId: sent.eventId, + roomId: message.roomId, + sender: message.sender.userId, + }, { maxLength: 200 }); + } catch (error) { + console.error(`message_failed id=${message.eventId}`, error); + await client.messages.send({ + roomId: message.roomId, + text: `DummyBridge error: ${error?.message ?? String(error)}`, + threadRoot: env("MATRIX_REPLY_IN_THREADS", "1") === "1" ? message.eventId : undefined, + }).catch(() => {}); + } finally { + await client.typing.set({ roomId: message.roomId, timeoutMs: 0, typing: false }).catch(() => {}); + inFlight.delete(message.eventId); + } +}); + +const reactionSub = await onReaction(client, undefined, async (reaction) => { + if (reaction.sender.isMe) return; + console.log(`reaction room=${reaction.roomId} sender=${reaction.sender.userId} key=${reaction.key} target=${reaction.relatesTo}`); +}); + +if (env("MATRIX_CATCH_UP_ON_START") === "1") { + await inviteSub.catchUp?.(); + await messageSub.catchUp?.(); +} + +console.log("dummybridge_bot=ready; invite this account to any room"); + +process.on("SIGINT", () => { + void shutdown(); +}); +process.on("SIGTERM", () => { + void shutdown(); +}); + +async function shutdown() { + console.log("dummybridge_bot=stopping"); + await Promise.allSettled([inviteSub.stop(), messageSub.stop(), reactionSub.stop()]); + await Promise.allSettled([inviteSub.done, messageSub.done, reactionSub.done]); + await client.close(); + await state.disconnect(); + process.exit(0); +} + +async function resolveSession(homeserverUrl, fileState) { + if (process.env.MATRIX_ACCESS_TOKEN) { + const userId = process.env.MATRIX_USER_ID; + const deviceId = process.env.MATRIX_DEVICE_ID; + if (userId && deviceId) { + return { + accessToken: process.env.MATRIX_ACCESS_TOKEN, + deviceId, + userId, + }; + } + const accountClient = createMatrixClient({ + homeserver: homeserverUrl, + store: createFileMatrixStore(join(stateDir, "whoami")), + token: process.env.MATRIX_ACCESS_TOKEN, + }); + const whoami = await accountClient.whoami(); + await accountClient.close(); + return { + accessToken: process.env.MATRIX_ACCESS_TOKEN, + deviceId: whoami.deviceId, + userId: whoami.userId, + }; + } + if (!process.env.MATRIX_USERNAME || !process.env.MATRIX_PASSWORD) { + throw new Error("Missing MATRIX_ACCESS_TOKEN or MATRIX_USERNAME/MATRIX_PASSWORD"); + } + const cacheKey = "dummybridge-bot:login-session"; + const cached = await fileState.get(cacheKey); + if (cached?.homeserver === homeserverUrl && cached.username === process.env.MATRIX_USERNAME) { + return cached; + } + const login = await createMatrixLogin({ + homeserver: homeserverUrl, + initialDeviceDisplayName: "better-matrix-js dummybridge bot", + }).password({ + password: process.env.MATRIX_PASSWORD, + username: process.env.MATRIX_USERNAME, + }); + const session = { ...login, username: process.env.MATRIX_USERNAME }; + await fileState.set(cacheKey, session); + return session; +} + +function env(name, fallback) { + return process.env[name] || fallback; +} + +async function loadEnvFile(path) { + if (!path) return; + let text; + try { + text = await readFile(path, "utf8"); + } catch (error) { + if (error?.code === "ENOENT") return; + throw error; + } + for (const line of text.split(/\r?\n/)) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith("#")) continue; + const match = trimmed.match(/^([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/); + if (!match) continue; + const [, key, raw] = match; + if (process.env[key]) continue; + process.env[key] = parseEnv(raw); + } +} + +function parseEnv(value) { + const trimmed = value.trim(); + if ((trimmed.startsWith('"') && trimmed.endsWith('"')) || (trimmed.startsWith("'") && trimmed.endsWith("'"))) { + return trimmed.slice(1, -1); + } + const comment = trimmed.search(/\s#/); + return comment === -1 ? trimmed : trimmed.slice(0, comment).trim(); +} diff --git a/examples/beeper-streaming-smoke/src/file-state.mjs b/examples/shared/file-state.mjs similarity index 100% rename from examples/beeper-streaming-smoke/src/file-state.mjs rename to examples/shared/file-state.mjs diff --git a/package.json b/package.json index 688920b..11f8d86 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,8 @@ "scripts": { "build": "pnpm -r build", "build:wasm": "pnpm --filter better-matrix-js build:wasm", - "check": "pnpm typecheck && pnpm test && pnpm test:go && pnpm build && pnpm pack:packages && pnpm smoke:consumer && pnpm smoke:cloudflare", + "audit:surface": "node scripts/audit-package-surface.mjs", + "check": "pnpm audit:surface && pnpm typecheck && pnpm test && pnpm test:go && pnpm build && pnpm pack:packages && pnpm smoke:consumer && pnpm smoke:cloudflare", "clean": "pnpm -r clean", "pack:packages": "mkdir -p .packs && pnpm -r --filter './packages/*' pack --pack-destination ./.packs", "publish:packages": "pnpm --filter better-matrix-js publish --access public --no-git-checks && pnpm --filter @better-matrix-js/state-memory publish --access public --no-git-checks && pnpm --filter @better-matrix-js/state-simple publish --access public --no-git-checks && pnpm --filter @better-matrix-js/state-file publish --access public --no-git-checks && pnpm --filter @better-matrix-js/state-sqlite publish --access public --no-git-checks && pnpm --filter @better-matrix-js/state-indexeddb publish --access public --no-git-checks && pnpm --filter @better-matrix-js/cloudflare publish --access public --no-git-checks && pnpm --filter @better-matrix-js/chat-adapter publish --access public --no-git-checks && pnpm --filter @better-matrix-js/ai-sdk publish --access public --no-git-checks", @@ -15,6 +16,8 @@ "smoke:package-consumer": "node scripts/package-consumer-smoke.mjs", "test:go": "cd packages/core/native && go test -tags goolm ./...", "test:live": "node scripts/live-e2e.mjs", + "test:docker": "pnpm build && BMJS_E2E_REDIS_PORT=${BMJS_E2E_REDIS_PORT:-6380} docker compose -f docker-compose.e2e.yml up -d --wait redis && BMJS_E2E_REDIS_PORT=${BMJS_E2E_REDIS_PORT:-6380} node scripts/docker-redis-store-smoke.mjs", + "test:docker:down": "docker compose -f docker-compose.e2e.yml down -v", "test": "pnpm -r test", "typecheck": "pnpm -r typecheck" }, diff --git a/packages/chat-adapter/README.md b/packages/chat-adapter/README.md index 0c8674d..89f55f1 100644 --- a/packages/chat-adapter/README.md +++ b/packages/chat-adapter/README.md @@ -33,7 +33,7 @@ bot.onNewMention(async (thread, message) => { await bot.initialize(); ``` -That's it. The adapter starts long-sync `/sync` automatically and forwards Matrix events into Chat SDK threads. +That's it. The adapter subscribes automatically and forwards future Matrix events into Chat SDK threads. For E2EE bots, keep both Chat SDK state and Matrix client state durable. `recoveryKey` lets the bot restore backed-up room keys, while `pickleKey` protects the local crypto pickles; keep `pickleKey` stable for the lifetime of the Matrix device. @@ -91,6 +91,8 @@ await matrix.handleSyncResponse({ response, since }); In this mode the external sync runner owns the cursor. Do not also let the adapter run live sync for the same Matrix account. +For encrypted rooms, keep webhook application single-writer for each Matrix device. The external sync runner should deliver raw Matrix JSON to `handleSyncResponse`; the adapter does not decrypt or unwrap custom webhook envelopes itself. + ## Thread IDs Chat SDK thread IDs encode `{ roomId, eventId? }`. Use the helpers when you need to cross between Matrix room IDs and Chat SDK thread IDs: @@ -111,7 +113,7 @@ createMatrixAdapter({ recoveryKey | pickleKey, // optional, for E2EE inviteAutoJoin: { inviterAllowlist }, // optional roomAllowlist, // optional - sync: { enabled, retryDelayMs, timeoutMs }, + sync: { enabled }, typingTimeoutMs, commandPrefix, }); @@ -124,7 +126,7 @@ createMatrixAdapter({ | Messages, replies, reactions, threads | Supported. | | Streaming responses | Beeper native streaming on Beeper homeservers; Matrix edit fallback elsewhere. | | Ephemeral messages | Beeper-only. Non-Beeper homeservers reject this operation. | -| Cards and actions | Fallback text only; interactive Matrix/Beeper UI is not exposed. | +| Cards and actions | Non-interactive cards can be rendered as text; interactive cards/actions throw clearly. | | Native modals | Unsupported because Matrix has no equivalent native surface. | | Scheduled messages | Unsupported; schedule work in your app and send later. | | URL previews | Unsupported by design; send explicit text or rendered content instead. | diff --git a/packages/chat-adapter/src/adapter.test.ts b/packages/chat-adapter/src/adapter.test.ts index da9b382..cb0a768 100644 --- a/packages/chat-adapter/src/adapter.test.ts +++ b/packages/chat-adapter/src/adapter.test.ts @@ -156,6 +156,7 @@ function makeCore(overrides: Partial = {}) { }) ); const listeners = new Set<(event: MatrixClientEvent) => void>(); + const subscribeOptions: unknown[] = []; const client: MatrixClient = { beeper: { ephemeral: { @@ -167,28 +168,12 @@ function makeCore(overrides: Partial = {}) { register: core.registerBeeperStream, }, }, - close: core.close, - connect: () => + boot: () => core.init({ accessToken: "token", homeserverUrl: "https://matrix.beeper.com", }), - events: { - on: (next) => { - listeners.add(next); - return () => { - listeners.delete(next); - }; - }, - onMessage: (next) => { - listeners.add(next as (event: MatrixClientEvent) => void); - return () => undefined; - }, - onReaction: (next) => { - listeners.add(next as (event: MatrixClientEvent) => void); - return () => undefined; - }, - }, + close: core.close, media: { download: async (options) => { const result = await core.downloadMedia(options); @@ -278,11 +263,19 @@ function makeCore(overrides: Partial = {}) { streams: { send: sendStream, }, + subscribe: async (_filter, next, options) => { + subscribeOptions.push(options); + listeners.add(next as (event: MatrixClientEvent) => void); + return { + catchUp: core.syncOnce, + done: Promise.resolve(), + stop: async () => { + listeners.delete(next as (event: MatrixClientEvent) => void); + }, + }; + }, sync: { applyResponse: core.applySyncResponse, - once: core.syncOnce, - start: async () => undefined, - stop: async () => undefined, }, typing: { set: core.setTyping }, users: { get: core.getUser }, @@ -299,6 +292,7 @@ function makeCore(overrides: Partial = {}) { } }, sendStream, + subscribeOptions, }; } @@ -373,10 +367,7 @@ describe("MatrixAdapter", () => { await adapter.initialize(makeChat()); - expect(core.init).toHaveBeenCalledWith({ - accessToken: "token", - homeserverUrl: "https://matrix.beeper.com", - }); + expect(core.whoami).toHaveBeenCalledOnce(); }); it("connects the injected Matrix client", async () => { @@ -384,16 +375,12 @@ describe("MatrixAdapter", () => { const adapter = new MatrixAdapter({ token: "token", client, - deviceId: "DEVICE", homeserver: "https://matrix.example.com", - initialSync: "persisted", - since: "s123", - sync: { enabled: false }, - userId: "@bot:example.com", + sync: { enabled: true }, }); await adapter.initialize(makeChat()); - expect(core.init).toHaveBeenCalledOnce(); + expect(core.whoami).toHaveBeenCalledOnce(); }); it("parses Matrix formatted HTML and m.mentions into Chat SDK messages", async () => { @@ -402,7 +389,7 @@ describe("MatrixAdapter", () => { token: "token", client, homeserver: "https://matrix.example.com", - sync: { enabled: false }, + sync: { enabled: true }, }); await adapter.initialize(makeChat()); @@ -437,7 +424,7 @@ describe("MatrixAdapter", () => { token: "token", client, homeserver: "https://matrix.example.com", - sync: { enabled: false }, + sync: { enabled: true }, }); await adapter.initialize(makeChat()); @@ -586,7 +573,7 @@ describe("MatrixAdapter", () => { token: "token", client, homeserver: "https://matrix.example.com", - sync: { enabled: false }, + sync: { enabled: true }, }); await adapter.initialize(chat); @@ -622,7 +609,7 @@ describe("MatrixAdapter", () => { client, homeserver: "https://matrix.example.com", inviteAutoJoin: { inviterAllowlist: ["@alice:example.com"] }, - sync: { enabled: false }, + sync: { enabled: true }, }); await adapter.initialize(makeChat()); @@ -651,7 +638,7 @@ describe("MatrixAdapter", () => { client, homeserver: "https://matrix.example.com", inviteAutoJoin: { inviterAllowlist: ["@alice:example.com"] }, - sync: { enabled: false }, + sync: { enabled: true }, }); await adapter.initialize(makeChat()); @@ -680,7 +667,7 @@ describe("MatrixAdapter", () => { commandPrefix: "/", client, homeserver: "https://matrix.example.com", - sync: { enabled: false }, + sync: { enabled: true }, }); await adapter.initialize(chat); @@ -711,6 +698,36 @@ describe("MatrixAdapter", () => { ); }); + it("uses a non-live subscription when sync is disabled", async () => { + const { client, emit, subscribeOptions } = makeCore(); + const chat = makeChat(); + const adapter = new MatrixAdapter({ + token: "token", + client, + homeserver: "https://matrix.example.com", + sync: { enabled: false }, + }); + await adapter.initialize(chat); + expect(subscribeOptions).toEqual([{ live: false }]); + + emit({ + event: { + body: "live", + content: { body: "live", msgtype: "m.text" }, + eventId: "$live", + isMe: false, + msgtype: "m.text", + raw: {}, + roomId: "!room:example.com", + sender: "@alice:example.com", + type: "m.room.message", + }, + type: "message", + }); + + expect(chat.processMessage).toHaveBeenCalledOnce(); + }); + it("applies sync responses directly through the Matrix core", async () => { const { client, core } = makeCore(); const adapter = new MatrixAdapter({ @@ -738,6 +755,55 @@ describe("MatrixAdapter", () => { }); }); + it("posts non-interactive cards as text fallback only", async () => { + const { client, core } = makeCore(); + const adapter = new MatrixAdapter({ + token: "token", + client, + homeserver: "https://matrix.example.com", + sync: { enabled: false }, + }); + await adapter.initialize(makeChat()); + + await adapter.postMessage(encodeMatrixChatThreadRef({ roomId: "!room:example.com" }), { + card: { + children: [{ content: "Body", type: "text" }], + title: "Status", + type: "card", + }, + } as never); + + expect(core.postMessage).toHaveBeenCalledWith(expect.objectContaining({ + body: "Status\nBody", + roomId: "!room:example.com", + })); + }); + + it("throws clearly for unsupported interactive cards/actions", async () => { + const { client } = makeCore(); + const adapter = new MatrixAdapter({ + token: "token", + client, + homeserver: "https://matrix.example.com", + sync: { enabled: false }, + }); + await adapter.initialize(makeChat()); + + await expect(adapter.postMessage(encodeMatrixChatThreadRef({ roomId: "!room:example.com" }), { + card: { + children: [ + { + children: [{ id: "approve", label: "Approve", type: "button" }], + type: "actions", + }, + ], + title: "Approval", + type: "card", + }, + fallbackText: "Approval requested", + } as never)).rejects.toThrow("interactive Chat SDK cards/actions"); + }); + it("delegates Beeper homeserver streams to the core stream API", async () => { const { client, sendStream } = makeCore(); const adapter = new MatrixAdapter({ diff --git a/packages/chat-adapter/src/adapter.ts b/packages/chat-adapter/src/adapter.ts index 8414f93..2538889 100644 --- a/packages/chat-adapter/src/adapter.ts +++ b/packages/chat-adapter/src/adapter.ts @@ -3,8 +3,10 @@ import { type MatrixAttachment as MatrixClientAttachment, type MatrixClient, type MatrixClientEvent, + type MatrixClientOptions, type MatrixEncryptedFile, type MatrixMessageEvent, + type MatrixSubscription, type RoomInfo, type MatrixStore, } from "better-matrix-js"; @@ -117,7 +119,7 @@ export class MatrixAdapter implements Adapter>(); #roomCache = new Map(); #roomAllowlist: Set | null; - #unsubscribeClient: (() => void) | null = null; + #subscription: MatrixSubscription | null = null; #userId: string | null = null; #webhookOptions: WebhookOptions | undefined; #isBeeperHomeserver: boolean; @@ -135,27 +137,22 @@ export class MatrixAdapter implements Adapter this.#handleClientEvent(event)); - const whoami = await this.#client.connect(); + await this.#subscription?.stop(); + this.#subscription = null; + const whoami = await this.#client.whoami(); this.#userId = whoami.userId; this.botUserId = whoami.userId; - if (this.#config.sync?.enabled !== false) { - const syncOptions = {}; - if (this.#config.sync?.retryDelayMs !== undefined) { - Object.assign(syncOptions, { retryDelayMs: this.#config.sync.retryDelayMs }); - } - if (this.#config.sync?.timeoutMs !== undefined) { - Object.assign(syncOptions, { timeoutMs: this.#config.sync.timeoutMs }); - } - await this.#client.sync.start(syncOptions); - } + this.#subscription = await this.#client.subscribe( + {}, + (event) => this.#handleClientEvent(event), + { live: this.#config.sync?.enabled !== false } + ); } async disconnect(): Promise { - this.#unsubscribeClient?.(); - this.#unsubscribeClient = null; + await this.#subscription?.stop(); + this.#subscription = null; await this.#client?.close(); this.#client = null; this.#chat = null; @@ -799,18 +796,15 @@ export class MatrixAdapter implements Adapter { + return typeof value === "object" && value !== null; +} + export function matrixLocalpart(userId: string): string { const withoutSigil = userId.startsWith("@") ? userId.slice(1) : userId; return withoutSigil.split(":")[0] || withoutSigil; diff --git a/packages/chat-adapter/src/types.ts b/packages/chat-adapter/src/types.ts index 7d5e008..08483ce 100644 --- a/packages/chat-adapter/src/types.ts +++ b/packages/chat-adapter/src/types.ts @@ -1,4 +1,5 @@ import type { + MatrixAccount, MatrixClient, MatrixStore, } from "better-matrix-js"; @@ -9,16 +10,13 @@ export interface MatrixChatThreadRef { } interface MatrixAdapterBaseConfig { + account?: MatrixAccount; beeper?: boolean; commandPrefix?: string; - deviceId?: string; homeserver?: string; - initialSync?: "persisted" | "latest" | "catchUp"; pickleKey?: string; sync?: { enabled?: boolean; - retryDelayMs?: number; - timeoutMs?: number; }; recoveryKey?: string; verifyRecoveryOnStart?: boolean; @@ -26,12 +24,10 @@ interface MatrixAdapterBaseConfig { inviterAllowlist?: string[]; }; roomAllowlist?: string[]; - since?: string; storePrefix?: string; store?: MatrixStore; token: string; typingTimeoutMs?: number; - userId?: string; wasmBytes?: BufferSource; wasmModule?: WebAssembly.Module; wasmUrl?: string | URL; diff --git a/packages/cloudflare/src/index.test.ts b/packages/cloudflare/src/index.test.ts index a30c6b6..cc55011 100644 --- a/packages/cloudflare/src/index.test.ts +++ b/packages/cloudflare/src/index.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it, vi } from "vitest"; +import { testMatrixStoreConformance } from "../../core/test/store-conformance"; import { createDurableObjectMatrixStore, decryptMatrixSyncWebhookEnvelope, @@ -43,6 +44,10 @@ class FakeDurableObjectStorage implements DurableObjectStorageLike { } describe("createDurableObjectMatrixStore", () => { + testMatrixStoreConformance("DurableObjectMatrixStore", () => + createDurableObjectMatrixStore(new FakeDurableObjectStorage(), { prefix: "matrix/" }) + ); + it("round-trips bytes through Durable Object storage", async () => { const storage = new FakeDurableObjectStorage(); const store = createDurableObjectMatrixStore(storage, { prefix: "matrix/" }); diff --git a/packages/cloudflare/src/index.ts b/packages/cloudflare/src/index.ts index f2da22d..005514b 100644 --- a/packages/cloudflare/src/index.ts +++ b/packages/cloudflare/src/index.ts @@ -133,7 +133,7 @@ export function createCloudflareKVMatrixStore( }, async get(key) { const value = await namespace.get(prefix + key, "arrayBuffer"); - return value ? new Uint8Array(value) : null; + return value ? copyBytes(new Uint8Array(value)) : null; }, async list(keyPrefix) { const keys: string[] = []; @@ -169,12 +169,12 @@ export function createDurableObjectMatrixStore( async get(key) { const value = await storage.get(prefix + key); if (value instanceof Uint8Array) { - return new Uint8Array(value); + return copyBytes(value); } if (value instanceof ArrayBuffer) { - return new Uint8Array(value); + return copyBytes(new Uint8Array(value)); } - return Array.isArray(value) ? new Uint8Array(value) : null; + return Array.isArray(value) ? copyBytes(new Uint8Array(value)) : null; }, async list(keyPrefix) { const values = await storage.list({ prefix: prefix + keyPrefix }); @@ -432,9 +432,15 @@ export class MatrixSyncDurableObject { } function copyToArrayBuffer(value: Uint8Array): ArrayBuffer { + const buffer = new ArrayBuffer(value.byteLength); + new Uint8Array(buffer).set(value); + return buffer; +} + +function copyBytes(value: Uint8Array): Uint8Array { const copy = new Uint8Array(value.byteLength); copy.set(value); - return copy.buffer; + return copy; } function errorMessage(error: unknown): string { diff --git a/packages/cloudflare/vitest.config.ts b/packages/cloudflare/vitest.config.ts index bdbea6f..61a25bb 100644 --- a/packages/cloudflare/vitest.config.ts +++ b/packages/cloudflare/vitest.config.ts @@ -1,6 +1,11 @@ import { defineProject } from "vitest/config"; export default defineProject({ + resolve: { + alias: { + "better-matrix-js": new URL("../core/src/index.ts", import.meta.url).pathname, + }, + }, test: { coverage: { include: ["src/**/*.ts"], diff --git a/packages/core/README.md b/packages/core/README.md index aa8d801..9b1f6a3 100644 --- a/packages/core/README.md +++ b/packages/core/README.md @@ -11,7 +11,7 @@ npm install better-matrix-js Use the Node entrypoint and a durable store. File or SQLite storage is enough for a single-process bot; production deployments can provide any `MatrixStore` implementation. ```ts -import { createMatrixClient } from "better-matrix-js/node"; +import { createMatrixClient, onMessage } from "better-matrix-js/node"; import { createFileMatrixStore } from "@better-matrix-js/state-file"; const client = createMatrixClient({ @@ -21,7 +21,7 @@ const client = createMatrixClient({ recoveryKey: process.env.MATRIX_RECOVERY_KEY, }); -client.events.onMessage(async (event) => { +await onMessage(client, undefined, async (event) => { if (event.sender.isMe) return; await client.messages.send({ roomId: event.roomId, @@ -29,9 +29,6 @@ client.events.onMessage(async (event) => { replyTo: event.eventId, }); }); - -await client.connect(); -await client.sync.start(); ``` Send directly through the explicit namespaces: @@ -62,7 +59,7 @@ const client = createMatrixClient({ wasmModule, }); -await client.connect(); +await client.boot(); ``` For sync, use `MatrixSyncDurableObject` from `@better-matrix-js/cloudflare` and forward the response into `client.sync.applyResponse({ response, since })`. See [`examples/cloudflare-worker`](https://github.com/batuhan/better-matrix-js/tree/main/examples/cloudflare-worker). @@ -110,7 +107,7 @@ Pass any of `wasmModule`, `wasmBytes`, or `wasmUrl` to `createMatrixClient()`, p ## Live sync vs serverless applyResponse -Use `client.sync.start()` when the same process can keep a long-lived `/sync` request open. In serverless runtimes, run `/sync` elsewhere and call `client.sync.applyResponse({ response, since })` for each response. Do not run both for the same account at the same time; only one component should advance a Matrix account cursor. +Use `client.subscribe(filter, handler)` when the same process can keep a long-lived `/sync` request open. In serverless runtimes, run `/sync` elsewhere and call `client.sync.applyResponse({ response, since })` for each response. Do not run both for the same account at the same time; only one component should advance a Matrix account cursor. ## E2EE storage and keys @@ -122,7 +119,7 @@ Recommended bot onboarding: 2. Pick a stable high-entropy `pickleKey` and store it with the bot secret material. 3. Pass a durable `store`, `userId`, `deviceId`, access token, and `pickleKey` on every boot. 4. Pass `recoveryKey` when the bot must decrypt historical encrypted messages from key backup. -5. Check `await client.crypto.status()` after `connect()` and alert on `keyBackupUnavailable`, `recoveryUnverified`, or a nonzero `pendingDecryptionCount`. +5. Check `await client.crypto.status()` after `boot()` and alert on `keyBackupUnavailable`, `recoveryUnverified`, or a nonzero `pendingDecryptionCount`. If `pickleKey` is omitted, the runtime currently falls back to the access token for compatibility with one-off bots. Treat that as development-only. Production encrypted bots should always set `pickleKey` explicitly so token rotation does not make local crypto state unreadable. diff --git a/packages/core/native/internal/core/core.go b/packages/core/native/internal/core/core.go index 8377ec6..fa6f3a2 100644 --- a/packages/core/native/internal/core/core.go +++ b/packages/core/native/internal/core/core.go @@ -76,10 +76,26 @@ func (c *Core) Handle(ctx context.Context, op string, payload []byte) ([]byte, e return c.handleInit(ctx, payload) case opWhoami: return c.handleWhoami(ctx) + case opLogout: + return c.handleLogout(ctx) case opGetCryptoStatus: return c.handleGetCryptoStatus() + case opRawRequest: + return c.handleRawRequest(ctx, payload) case opApplySyncResponse: return c.handleApplySyncResponse(ctx, payload) + case opGetAccountData: + return c.handleGetAccountData(ctx, payload) + case opSetAccountData: + return c.handleSetAccountData(ctx, payload) + case opGetRoomAccountData: + return c.handleGetRoomAccountData(ctx, payload) + case opSetRoomAccountData: + return c.handleSetRoomAccountData(ctx, payload) + case opSendToDevice: + return c.handleSendToDevice(ctx, payload) + case opSendReceipt: + return c.handleSendReceipt(ctx, payload) case opPostMessage: return c.handlePostMessage(ctx, payload) case opPostMediaMessage: diff --git a/packages/core/native/internal/core/crypto_status.go b/packages/core/native/internal/core/crypto_status.go index 8404831..7a92a02 100644 --- a/packages/core/native/internal/core/crypto_status.go +++ b/packages/core/native/internal/core/crypto_status.go @@ -7,7 +7,7 @@ type MatrixCryptoStatus struct { HasRecoveryKey bool `json:"hasRecoveryKey"` KeyBackupVersion string `json:"keyBackupVersion,omitempty"` PendingDecryptionCount int `json:"pendingDecryptionCount"` - State string `json:"state" tstype:"\"disabled\" | \"enabled\" | \"key_backup_unavailable\" | \"recovery_cache_unavailable\" | \"recovery_key_cached\" | \"recovery_key_loaded\" | \"recovery_restored\" | \"recovery_unverified\""` + State string `json:"state" tstype:"\"disabled\" | \"enabled\" | \"key_backup_updated\" | \"key_backup_unavailable\" | \"recovery_cache_unavailable\" | \"recovery_key_cached\" | \"recovery_key_loaded\" | \"recovery_restored\" | \"recovery_unverified\""` StoreBacked bool `json:"storeBacked"` UserID string `json:"userId,omitempty"` } diff --git a/packages/core/native/internal/core/decryption_queue.go b/packages/core/native/internal/core/decryption_queue.go index 1db7ee7..dbd36de 100644 --- a/packages/core/native/internal/core/decryption_queue.go +++ b/packages/core/native/internal/core/decryption_queue.go @@ -217,14 +217,21 @@ func (c *Core) restorePendingFromBackup(ctx context.Context, pending *pendingDec sessionID := id.SessionID(pending.SessionID) resp, err := mach.Client.GetKeyBackupForRoomAndSession(ctx, c.backupVersion, roomID, sessionID) if err != nil || resp == nil { + if err != nil { + c.emit(OutboundEvent{"type": "crypto_status", "status": "key_backup_unavailable", "error": err.Error()}) + } return false, true } sessionData, err := resp.SessionData.Decrypt(c.backupKey) if err != nil { + c.emit(OutboundEvent{"type": "crypto_status", "status": "key_backup_unavailable", "error": err.Error()}) return false, true } imported, err := mach.ImportRoomKeyFromBackup(ctx, c.backupVersion, roomID, sessionID, sessionData) - return err == nil && imported != nil && imported.Internal.FirstKnownIndex() <= uint32(pending.MinIndex), true + if err != nil { + c.emit(OutboundEvent{"type": "crypto_status", "status": "key_backup_unavailable", "error": err.Error()}) + } + return err == nil && imported != nil, true } func (c *Core) trimPendingDecryptions(pending []pendingDecryption) []pendingDecryption { diff --git a/packages/core/native/internal/core/init.go b/packages/core/native/internal/core/init.go index 150ae12..375c1f1 100644 --- a/packages/core/native/internal/core/init.go +++ b/packages/core/native/internal/core/init.go @@ -147,7 +147,7 @@ func resolveStartupSyncPlan(req MatrixCoreInitOptions, storedNextBatch string) s mode := req.InitialSyncMode if mode == "" { - mode = "persisted" + mode = "latest" } if req.CatchUpOnStart != nil { if *req.CatchUpOnStart { @@ -173,18 +173,10 @@ func resolveStartupSyncPlan(req MatrixCoreInitOptions, storedNextBatch string) s skipNextSync: true, } default: - if storedNextBatch != "" { - return startupSyncPlan{ - cursorSource: "stored", - loadPendingDecryptions: true, - nextBatch: storedNextBatch, - skipNextSync: false, - } - } return startupSyncPlan{ - cursorSource: "latest", + cursorSource: cursorSource(storedNextBatch, "stored_latest", "latest"), loadPendingDecryptions: false, - nextBatch: "", + nextBatch: storedNextBatch, skipNextSync: true, } } @@ -240,6 +232,15 @@ func (c *Core) handleWhoami(ctx context.Context) ([]byte, error) { return json.Marshal(MatrixWhoami{UserID: cli.UserID.String(), DeviceID: cli.DeviceID.String()}) } +func (c *Core) handleLogout(ctx context.Context) ([]byte, error) { + cli, err := c.requireClient() + if err != nil { + return nil, err + } + _, err = cli.Logout(ctx) + return c.emptyIfNil(err) +} + func (c *Core) setupCrypto(ctx context.Context, req MatrixCoreInitOptions) error { cli, err := c.requireClient() if err != nil { diff --git a/packages/core/native/internal/core/init_test.go b/packages/core/native/internal/core/init_test.go index f424fe7..a655715 100644 --- a/packages/core/native/internal/core/init_test.go +++ b/packages/core/native/internal/core/init_test.go @@ -21,16 +21,19 @@ func TestResolveStartupSyncPlanDefaultsToLiveCursorForFreshLogin(t *testing.T) { } } -func TestResolveStartupSyncPlanCatchesUpFromPersistedCursorByDefault(t *testing.T) { +func TestResolveStartupSyncPlanUsesPersistedCursorAsFutureOnlyByDefault(t *testing.T) { plan := resolveStartupSyncPlan(MatrixCoreInitOptions{}, "s123") - if plan.skipNextSync { - t.Fatal("persisted cursor should catch up timeline events by default") + if !plan.skipNextSync { + t.Fatal("persisted cursor should not catch up timeline events by default") } if plan.nextBatch != "s123" { t.Fatalf("expected stored cursor, got %q", plan.nextBatch) } - if !plan.loadPendingDecryptions { - t.Fatal("catch-up startup should load pending decryptions") + if plan.loadPendingDecryptions { + t.Fatal("future-only startup should not load stale pending decryptions") + } + if plan.cursorSource != "stored_latest" { + t.Fatalf("expected stored_latest source, got %q", plan.cursorSource) } } diff --git a/packages/core/native/internal/core/key_backup.go b/packages/core/native/internal/core/key_backup.go new file mode 100644 index 0000000..dbae810 --- /dev/null +++ b/packages/core/native/internal/core/key_backup.go @@ -0,0 +1,99 @@ +package core + +import ( + "context" + "encoding/json" + + "maunium.net/go/mautrix" + "maunium.net/go/mautrix/crypto/backup" + "maunium.net/go/mautrix/crypto/olm" + "maunium.net/go/mautrix/id" +) + +func (c *Core) backupOutboundMegolmSession(ctx context.Context, roomID id.RoomID) { + if c == nil || c.crypto == nil || c.backupKey == nil || roomID == "" { + return + } + mach := c.crypto.Machine() + if c.backupVersion == "" { + versionInfo, err := mach.GetAndVerifyLatestKeyBackupVersion(ctx, c.backupKey) + if err != nil || versionInfo == nil { + c.emit(OutboundEvent{"type": "crypto_status", "status": "key_backup_unavailable"}) + return + } + c.backupVersion = versionInfo.Version + _ = mach.SetKeyBackupVersion(ctx, versionInfo.Version) + } + session, err := mach.CryptoStore.GetOutboundGroupSession(ctx, roomID) + if err != nil || session == nil || !session.Shared { + return + } + content := session.ShareContent() + shareContent := content.AsRoomKey() + if shareContent == nil || shareContent.SessionID == "" || shareContent.SessionKey == "" { + return + } + inbound, err := olm.NewInboundGroupSession([]byte(shareContent.SessionKey)) + if err != nil { + c.emit(OutboundEvent{"type": "crypto_status", "status": "key_backup_unavailable", "error": err.Error()}) + return + } + exported, err := inbound.Export(inbound.FirstKnownIndex()) + if err != nil { + c.emit(OutboundEvent{"type": "crypto_status", "status": "key_backup_unavailable", "error": err.Error()}) + return + } + sessionData := backup.MegolmSessionData{ + Algorithm: id.AlgorithmMegolmV1, + ForwardingKeyChain: []string{}, + SenderClaimedKeys: backup.SenderClaimedKeys{Ed25519: mach.GetAccount().SigningKey()}, + SenderKey: mach.GetAccount().IdentityKey(), + SessionKey: string(exported), + } + encrypted, err := backup.EncryptSessionData(c.backupKey, sessionData) + if err != nil { + c.emit(OutboundEvent{"type": "crypto_status", "status": "key_backup_unavailable", "error": err.Error()}) + return + } + raw, err := json.Marshal(encrypted) + if err != nil { + c.emit(OutboundEvent{"type": "crypto_status", "status": "key_backup_unavailable", "error": err.Error()}) + return + } + req := mautrix.ReqKeyBackupData{ + FirstMessageIndex: 0, + ForwardedCount: 0, + IsVerified: true, + SessionData: raw, + } + _, err = mach.Client.PutKeysInBackupForRoom(ctx, c.backupVersion, roomID, &mautrix.ReqRoomKeyBackup{ + Sessions: map[id.SessionID]mautrix.ReqKeyBackupData{shareContent.SessionID: req}, + }) + if err != nil { + c.emit(OutboundEvent{"type": "crypto_status", "status": "key_backup_unavailable", "error": err.Error()}) + return + } + c.emit(OutboundEvent{"type": "crypto_status", "status": "key_backup_updated", "keyBackupVersion": c.backupVersion.String()}) +} + +func (c *Core) prepareOutboundMegolm(ctx context.Context, cli *mautrix.Client, roomID id.RoomID) error { + if err := ensureMegolmRecipients(ctx, cli, roomID); err != nil { + return err + } + if c == nil || c.crypto == nil || cli == nil || cli.StateStore == nil { + return nil + } + encrypted, err := cli.StateStore.IsEncrypted(ctx, roomID) + if err != nil || !encrypted { + return err + } + members, err := cli.StateStore.GetRoomJoinedOrInvitedMembers(ctx, roomID) + if err != nil { + return err + } + if err := c.crypto.Machine().ShareGroupSession(ctx, roomID, members); err != nil { + return err + } + c.backupOutboundMegolmSession(ctx, roomID) + return nil +} diff --git a/packages/core/native/internal/core/media.go b/packages/core/native/internal/core/media.go index 613e53a..6bbb2f1 100644 --- a/packages/core/native/internal/core/media.go +++ b/packages/core/native/internal/core/media.go @@ -127,7 +127,7 @@ func (c *Core) postMediaMessage(ctx context.Context, req MatrixSendMediaMessageO if content.Info.IsZero() { content.Info = nil } - if err := ensureMegolmRecipients(ctx, cli, id.RoomID(req.RoomID)); err != nil { + if err := c.prepareOutboundMegolm(ctx, cli, id.RoomID(req.RoomID)); err != nil { return nil, err } encrypted := false @@ -160,7 +160,7 @@ func (c *Core) postMediaMessage(ctx context.Context, req MatrixSendMediaMessageO content.RelatesTo = (&event.RelatesTo{}).SetThread(id.EventID(req.ThreadRootEventID), "") } resp, err := retryMatrix(ctx, func() (*mautrix.RespSendEvent, error) { - if err := ensureMegolmRecipients(ctx, cli, id.RoomID(req.RoomID)); err != nil { + if err := c.prepareOutboundMegolm(ctx, cli, id.RoomID(req.RoomID)); err != nil { return nil, err } return cli.SendMessageEvent(ctx, id.RoomID(req.RoomID), event.EventMessage, content) diff --git a/packages/core/native/internal/core/messages.go b/packages/core/native/internal/core/messages.go index f79ddd9..be058d6 100644 --- a/packages/core/native/internal/core/messages.go +++ b/packages/core/native/internal/core/messages.go @@ -60,7 +60,7 @@ func (c *Core) handlePostMessage(ctx context.Context, payload []byte) ([]byte, e contentMap["m.relates_to"] = content.RelatesTo } resp, err := retryMatrix(ctx, func() (*mautrix.RespSendEvent, error) { - if err := ensureMegolmRecipients(ctx, cli, id.RoomID(req.RoomID)); err != nil { + if err := c.prepareOutboundMegolm(ctx, cli, id.RoomID(req.RoomID)); err != nil { return nil, err } return cli.SendMessageEvent(ctx, id.RoomID(req.RoomID), event.EventMessage, contentMap) @@ -183,7 +183,7 @@ func (c *Core) handleEditMessage(ctx context.Context, payload []byte) ([]byte, e }, req.Content) content["m.new_content"] = newContentMap resp, err := retryMatrix(ctx, func() (*mautrix.RespSendEvent, error) { - if err := ensureMegolmRecipients(ctx, cli, id.RoomID(req.RoomID)); err != nil { + if err := c.prepareOutboundMegolm(ctx, cli, id.RoomID(req.RoomID)); err != nil { return nil, err } return cli.SendMessageEvent(ctx, id.RoomID(req.RoomID), event.EventMessage, content) @@ -238,6 +238,9 @@ func (c *Core) handleSendEphemeralEvent(ctx context.Context, payload []byte) ([] return nil, errors.New("eventType is required") } resp, err := retryMatrix(ctx, func() (*mautrix.RespSendEvent, error) { + if err := c.prepareOutboundMegolm(ctx, cli, id.RoomID(req.RoomID)); err != nil { + return nil, err + } return beeperSendEphemeralEvent(ctx, cli, id.RoomID(req.RoomID), event.Type{Type: req.EventType, Class: event.EphemeralEventType}, req.Content, req.TransactionID) }) if err != nil { diff --git a/packages/core/native/internal/core/operations.go b/packages/core/native/internal/core/operations.go index 066ea74..21db18f 100644 --- a/packages/core/native/internal/core/operations.go +++ b/packages/core/native/internal/core/operations.go @@ -11,10 +11,26 @@ const ( opInit = "init" // ts:operation whoami whoami - MatrixWhoami opWhoami = "whoami" + // ts:operation logout logout - void + opLogout = "logout" // ts:operation getCryptoStatus get_crypto_status - MatrixCryptoStatus opGetCryptoStatus = "get_crypto_status" + // ts:operation rawRequest raw_request MatrixRawRequestOptions MatrixRawRequestResult + opRawRequest = "raw_request" // ts:operation applySyncResponse apply_sync_response MatrixApplySyncResponseOptions void opApplySyncResponse = "apply_sync_response" + // ts:operation getAccountData get_account_data MatrixGetAccountDataOptions MatrixAccountDataResult + opGetAccountData = "get_account_data" + // ts:operation setAccountData set_account_data MatrixSetAccountDataOptions void + opSetAccountData = "set_account_data" + // ts:operation getRoomAccountData get_room_account_data MatrixGetRoomAccountDataOptions MatrixAccountDataResult + opGetRoomAccountData = "get_room_account_data" + // ts:operation setRoomAccountData set_room_account_data MatrixSetRoomAccountDataOptions void + opSetRoomAccountData = "set_room_account_data" + // ts:operation sendToDevice send_to_device MatrixSendToDeviceOptions MatrixSendToDeviceResult + opSendToDevice = "send_to_device" + // ts:operation sendReceipt send_receipt MatrixSendReceiptOptions void + opSendReceipt = "send_receipt" // ts:operation postMessage post_message MatrixSendMessageOptions MatrixRawMessage opPostMessage = "post_message" // ts:operation postMediaMessage post_media_message MatrixSendMediaMessageOptions MatrixRawMessage diff --git a/packages/core/native/internal/core/reactions.go b/packages/core/native/internal/core/reactions.go index b0fa770..315f0f8 100644 --- a/packages/core/native/internal/core/reactions.go +++ b/packages/core/native/internal/core/reactions.go @@ -37,7 +37,7 @@ func (c *Core) handleAddReaction(ctx context.Context, payload []byte) ([]byte, e RelatesTo: *(&event.RelatesTo{}).SetAnnotation(id.EventID(req.MessageID), req.Emoji), } resp, err := retryMatrix(ctx, func() (*mautrix.RespSendEvent, error) { - if err := ensureMegolmRecipients(ctx, cli, id.RoomID(req.RoomID)); err != nil { + if err := c.prepareOutboundMegolm(ctx, cli, id.RoomID(req.RoomID)); err != nil { return nil, err } return cli.SendMessageEvent(ctx, id.RoomID(req.RoomID), event.EventReaction, content) diff --git a/packages/core/native/internal/core/standard.go b/packages/core/native/internal/core/standard.go new file mode 100644 index 0000000..66d185a --- /dev/null +++ b/packages/core/native/internal/core/standard.go @@ -0,0 +1,292 @@ +package core + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/url" + "strings" + + "maunium.net/go/mautrix" + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" +) + +type MatrixAccountDataResult struct { + Content map[string]any `json:"content"` + Raw any `json:"raw"` + Type string `json:"type"` +} + +type MatrixGetAccountDataOptions struct { + EventType string `json:"eventType"` +} + +type MatrixSetAccountDataOptions struct { + Content map[string]any `json:"content"` + EventType string `json:"eventType"` +} + +type MatrixGetRoomAccountDataOptions struct { + EventType string `json:"eventType"` + RoomID string `json:"roomId"` +} + +type MatrixSetRoomAccountDataOptions struct { + Content map[string]any `json:"content"` + EventType string `json:"eventType"` + RoomID string `json:"roomId"` +} + +type MatrixSendToDeviceOptions struct { + Content map[string]any `json:"content,omitempty"` + EventType string `json:"eventType"` + Messages map[string]map[string]map[string]any `json:"messages,omitempty"` + TransactionID string `json:"transactionId,omitempty"` + UserID string `json:"userId,omitempty"` + DeviceID string `json:"deviceId,omitempty"` +} + +type MatrixSendToDeviceResult struct { + Raw any `json:"raw"` +} + +type MatrixSendReceiptOptions struct { + Content map[string]any `json:"content,omitempty"` + EventID string `json:"eventId"` + ReceiptType string `json:"receiptType,omitempty"` + RoomID string `json:"roomId"` + ThreadID string `json:"threadId,omitempty"` +} + +type MatrixRawRequestOptions struct { + Body any `json:"body,omitempty"` + Headers map[string]string `json:"headers,omitempty"` + Method string `json:"method,omitempty"` + Path string `json:"path"` + Query map[string]string `json:"query,omitempty"` +} + +type MatrixRawRequestResult struct { + Body any `json:"body,omitempty"` + Raw any `json:"raw,omitempty"` + Status int `json:"status"` + Header map[string]string `json:"headers,omitempty"` +} + +func (c *Core) handleGetAccountData(ctx context.Context, payload []byte) ([]byte, error) { + var req MatrixGetAccountDataOptions + if err := json.Unmarshal(payload, &req); err != nil { + return nil, err + } + if req.EventType == "" { + return nil, errors.New("eventType is required") + } + cli, err := c.requireClient() + if err != nil { + return nil, err + } + var content map[string]any + if err := cli.GetAccountData(ctx, req.EventType, &content); err != nil { + return nil, err + } + return json.Marshal(MatrixAccountDataResult{Content: content, Raw: content, Type: req.EventType}) +} + +func (c *Core) handleSetAccountData(ctx context.Context, payload []byte) ([]byte, error) { + var req MatrixSetAccountDataOptions + if err := json.Unmarshal(payload, &req); err != nil { + return nil, err + } + if req.EventType == "" { + return nil, errors.New("eventType is required") + } + cli, err := c.requireClient() + if err != nil { + return nil, err + } + return c.emptyIfNil(cli.SetAccountData(ctx, req.EventType, req.Content)) +} + +func (c *Core) handleGetRoomAccountData(ctx context.Context, payload []byte) ([]byte, error) { + var req MatrixGetRoomAccountDataOptions + if err := json.Unmarshal(payload, &req); err != nil { + return nil, err + } + if req.EventType == "" || req.RoomID == "" { + return nil, errors.New("roomId and eventType are required") + } + cli, err := c.requireClient() + if err != nil { + return nil, err + } + var content map[string]any + if err := cli.GetRoomAccountData(ctx, id.RoomID(req.RoomID), req.EventType, &content); err != nil { + return nil, err + } + return json.Marshal(MatrixAccountDataResult{Content: content, Raw: content, Type: req.EventType}) +} + +func (c *Core) handleSetRoomAccountData(ctx context.Context, payload []byte) ([]byte, error) { + var req MatrixSetRoomAccountDataOptions + if err := json.Unmarshal(payload, &req); err != nil { + return nil, err + } + if req.EventType == "" || req.RoomID == "" { + return nil, errors.New("roomId and eventType are required") + } + cli, err := c.requireClient() + if err != nil { + return nil, err + } + return c.emptyIfNil(cli.SetRoomAccountData(ctx, id.RoomID(req.RoomID), req.EventType, req.Content)) +} + +func (c *Core) handleSendToDevice(ctx context.Context, payload []byte) ([]byte, error) { + var req MatrixSendToDeviceOptions + if err := json.Unmarshal(payload, &req); err != nil { + return nil, err + } + if req.EventType == "" { + return nil, errors.New("eventType is required") + } + messages := req.Messages + if len(messages) == 0 && req.UserID != "" && req.DeviceID != "" { + messages = map[string]map[string]map[string]any{req.UserID: {req.DeviceID: req.Content}} + } + if len(messages) == 0 { + return nil, errors.New("messages or userId/deviceId/content are required") + } + cli, err := c.requireClient() + if err != nil { + return nil, err + } + converted := make(map[id.UserID]map[id.DeviceID]*event.Content, len(messages)) + for userID, devices := range messages { + converted[id.UserID(userID)] = make(map[id.DeviceID]*event.Content, len(devices)) + for deviceID, content := range devices { + converted[id.UserID(userID)][id.DeviceID(deviceID)] = &event.Content{Raw: content} + } + } + toDeviceReq := &mautrix.ReqSendToDevice{Messages: converted} + var resp *mautrix.RespSendToDevice + if req.TransactionID != "" { + urlPath := cli.BuildClientURL("v3", "sendToDevice", req.EventType, req.TransactionID) + _, err = cli.MakeRequest(ctx, http.MethodPut, urlPath, toDeviceReq, &resp) + } else { + resp, err = cli.SendToDevice(ctx, event.Type{Type: req.EventType}, toDeviceReq) + } + if err != nil { + return nil, err + } + return json.Marshal(MatrixSendToDeviceResult{Raw: resp}) +} + +func (c *Core) handleSendReceipt(ctx context.Context, payload []byte) ([]byte, error) { + var req MatrixSendReceiptOptions + if err := json.Unmarshal(payload, &req); err != nil { + return nil, err + } + if req.RoomID == "" || req.EventID == "" { + return nil, errors.New("roomId and eventId are required") + } + receiptType := event.ReceiptTypeRead + if req.ReceiptType != "" { + receiptType = event.ReceiptType(req.ReceiptType) + } + var content any = map[string]any{} + if len(req.Content) > 0 { + content = req.Content + } + if req.ThreadID != "" { + content = mautrix.ReqSendReceipt{ThreadID: req.ThreadID} + if len(req.Content) > 0 { + req.Content["thread_id"] = req.ThreadID + content = req.Content + } + } + cli, err := c.requireClient() + if err != nil { + return nil, err + } + return c.emptyIfNil(cli.SendReceipt(ctx, id.RoomID(req.RoomID), id.EventID(req.EventID), receiptType, content)) +} + +func (c *Core) handleRawRequest(ctx context.Context, payload []byte) ([]byte, error) { + var req MatrixRawRequestOptions + if err := json.Unmarshal(payload, &req); err != nil { + return nil, err + } + if req.Path == "" { + return nil, errors.New("path is required") + } + method := req.Method + if method == "" { + method = http.MethodGet + } + cli, err := c.requireClient() + if err != nil { + return nil, err + } + rawURL, err := rawClientURL(cli, req.Path, req.Query) + if err != nil { + return nil, err + } + headers := http.Header{} + for key, value := range req.Headers { + headers.Set(key, value) + } + var body any + data, resp, err := cli.MakeFullRequestWithResp(ctx, mautrix.FullRequest{ + Headers: headers, + Method: method, + RequestJSON: req.Body, + ResponseJSON: &body, + URL: rawURL, + }) + if err != nil { + return nil, err + } + result := MatrixRawRequestResult{Body: body, Raw: json.RawMessage(data), Status: resp.StatusCode} + if len(resp.Header) > 0 { + result.Header = make(map[string]string, len(resp.Header)) + for key, values := range resp.Header { + if len(values) > 0 { + result.Header[key] = values[0] + } + } + } + return json.Marshal(result) +} + +func rawClientURL(cli *mautrix.Client, path string, query map[string]string) (string, error) { + if strings.Contains(path, "://") { + return "", errors.New("raw request path must be relative to the homeserver") + } + path = "/" + strings.TrimPrefix(path, "/") + base := *cli.HomeserverURL + parsed, err := url.Parse(path) + if err != nil { + return "", err + } + base.Path = parsed.Path + base.RawPath = parsed.EscapedPath() + values := base.Query() + if parsed.RawQuery != "" { + parsedValues, err := url.ParseQuery(parsed.RawQuery) + if err != nil { + return "", err + } + for key, entries := range parsedValues { + for _, value := range entries { + values.Add(key, value) + } + } + } + for key, value := range query { + values.Set(key, value) + } + base.RawQuery = values.Encode() + return base.String(), nil +} diff --git a/packages/core/native/internal/core/standard_test.go b/packages/core/native/internal/core/standard_test.go new file mode 100644 index 0000000..badbf1f --- /dev/null +++ b/packages/core/native/internal/core/standard_test.go @@ -0,0 +1,151 @@ +package core + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "maunium.net/go/mautrix" + "maunium.net/go/mautrix/id" +) + +func TestStandardMatrixHelpersUseExpectedEndpoints(t *testing.T) { + var seen []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + seen = append(seen, r.Method+" "+r.URL.RequestURI()) + switch r.URL.Path { + case "/_matrix/client/v3/user/@alice:example/account_data/m.preference": + if r.Method == http.MethodGet { + _ = json.NewEncoder(w).Encode(map[string]any{"theme": "dark"}) + return + } + if r.Method == http.MethodPut { + _, _ = w.Write([]byte(`{}`)) + return + } + case "/_matrix/client/v3/user/@alice:example/rooms/!room:example/account_data/m.room.preference", + "/_matrix/client/v3/user/@alice:example/rooms/%21room:example/account_data/m.room.preference": + if r.Method == http.MethodGet { + _ = json.NewEncoder(w).Encode(map[string]any{"muted": true}) + return + } + if r.Method == http.MethodPut { + _, _ = w.Write([]byte(`{}`)) + return + } + case "/_matrix/client/v3/sendToDevice/m.test/txn": + if r.Method == http.MethodPut { + _ = json.NewEncoder(w).Encode(map[string]any{}) + return + } + case "/_matrix/client/v3/rooms/!room:example/receipt/m.read.private/$event", + "/_matrix/client/v3/rooms/%21room:example/receipt/m.read.private/%24event": + if r.Method == http.MethodPost { + _, _ = w.Write([]byte(`{}`)) + return + } + case "/_matrix/client/v3/custom": + if r.Method == http.MethodPost && r.URL.Query().Get("q") == "1" { + _ = json.NewEncoder(w).Encode(map[string]any{"ok": true}) + return + } + } + http.NotFound(w, r) + })) + defer server.Close() + + core := New(nil) + core.client, _ = mautrix.NewClient(server.URL, id.UserID("@alice:example"), "token") + core.client.DeviceID = id.DeviceID("DEVICE") + ctx := context.Background() + + raw, err := core.handleGetAccountData(ctx, []byte(`{"eventType":"m.preference"}`)) + if err != nil { + t.Fatal(err) + } + var accountData MatrixAccountDataResult + if err := json.Unmarshal(raw, &accountData); err != nil { + t.Fatal(err) + } + if accountData.Content["theme"] != "dark" { + t.Fatalf("unexpected account data: %#v", accountData.Content) + } + if _, err := core.handleSetAccountData(ctx, []byte(`{"eventType":"m.preference","content":{"theme":"light"}}`)); err != nil { + t.Fatal(err) + } + if _, err := core.handleGetRoomAccountData(ctx, []byte(`{"roomId":"!room:example","eventType":"m.room.preference"}`)); err != nil { + t.Fatal(err) + } + if _, err := core.handleSetRoomAccountData(ctx, []byte(`{"roomId":"!room:example","eventType":"m.room.preference","content":{"muted":false}}`)); err != nil { + t.Fatal(err) + } + if _, err := core.handleSendToDevice(ctx, []byte(`{"eventType":"m.test","userId":"@bob:example","deviceId":"BOB","content":{"hello":true},"transactionId":"txn"}`)); err != nil { + t.Fatal(err) + } + if _, err := core.handleSendReceipt(ctx, []byte(`{"roomId":"!room:example","eventId":"$event","receiptType":"m.read.private","threadId":"$thread"}`)); err != nil { + t.Fatal(err) + } + raw, err = core.handleRawRequest(ctx, []byte(`{"method":"POST","path":"/_matrix/client/v3/custom","query":{"q":"1"},"body":{"include":true}}`)) + if err != nil { + t.Fatal(err) + } + var rawResult MatrixRawRequestResult + if err := json.Unmarshal(raw, &rawResult); err != nil { + t.Fatal(err) + } + if rawResult.Status != http.StatusOK { + t.Fatalf("unexpected raw request status %d", rawResult.Status) + } + + expected := []string{ + "GET /_matrix/client/v3/user/@alice:example/account_data/m.preference", + "PUT /_matrix/client/v3/user/@alice:example/account_data/m.preference", + "GET /_matrix/client/v3/user/@alice:example/rooms/%21room:example/account_data/m.room.preference", + "PUT /_matrix/client/v3/user/@alice:example/rooms/%21room:example/account_data/m.room.preference", + "PUT /_matrix/client/v3/sendToDevice/m.test/txn", + "POST /_matrix/client/v3/rooms/%21room:example/receipt/m.read.private/$event", + "POST /_matrix/client/v3/custom?q=1", + } + if len(seen) != len(expected) { + t.Fatalf("expected %d requests, got %d: %#v", len(expected), len(seen), seen) + } + for index, want := range expected { + if seen[index] != want { + t.Fatalf("request %d: expected %q, got %q", index, want, seen[index]) + } + } +} + +func TestLogoutUsesMatrixLogoutEndpoint(t *testing.T) { + var seen string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + seen = r.Method + " " + r.URL.Path + if r.Method == http.MethodPost && r.URL.Path == "/_matrix/client/v3/logout" { + _, _ = w.Write([]byte(`{}`)) + return + } + http.NotFound(w, r) + })) + defer server.Close() + + core := New(nil) + core.client, _ = mautrix.NewClient(server.URL, id.UserID("@alice:example"), "token") + if _, err := core.handleLogout(context.Background()); err != nil { + t.Fatal(err) + } + if seen != "POST /_matrix/client/v3/logout" { + t.Fatalf("unexpected logout request %q", seen) + } +} + +func TestRawRequestRejectsAbsoluteURLs(t *testing.T) { + core := New(nil) + core.client, _ = mautrix.NewClient("https://example.com", id.UserID("@alice:example"), "token") + + _, err := core.handleRawRequest(context.Background(), []byte(`{"path":"https://evil.example/_matrix/client/v3/account/whoami"}`)) + if err == nil || err.Error() != "raw request path must be relative to the homeserver" { + t.Fatalf("expected relative path error, got %v", err) + } +} diff --git a/packages/core/native/internal/core/sync.go b/packages/core/native/internal/core/sync.go index 7f39df3..c19b079 100644 --- a/packages/core/native/internal/core/sync.go +++ b/packages/core/native/internal/core/sync.go @@ -7,6 +7,7 @@ import ( "time" "maunium.net/go/mautrix" + "maunium.net/go/mautrix/crypto" "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/id" ) @@ -18,6 +19,7 @@ const ( type MatrixSyncOnceOptions struct { BeeperStreaming bool `json:"beeperStreaming,omitempty"` + ReplayMissed bool `json:"replayMissed,omitempty"` TimeoutMS int `json:"timeoutMs,omitempty"` } @@ -42,7 +44,7 @@ func (c *Core) handleSyncOnce(ctx context.Context, payload []byte) ([]byte, erro if req.TimeoutMS <= 0 { req.TimeoutMS = defaultSyncTimeoutMS } - if err := c.syncOnce(ctx, req.TimeoutMS, req.BeeperStreaming, true); err != nil { + if err := c.syncOnce(ctx, req.TimeoutMS, req.BeeperStreaming, true, req.ReplayMissed); err != nil { return nil, err } return c.empty() @@ -100,7 +102,7 @@ func (c *Core) runSyncLoop(ctx context.Context, done chan struct{}, req MatrixSy failures := 0 for { c.syncMu.Lock() - err := c.syncOnce(ctx, req.TimeoutMS, req.BeeperStreaming, false) + err := c.syncOnce(ctx, req.TimeoutMS, req.BeeperStreaming, false, false) c.syncMu.Unlock() if err == nil { failures = 0 @@ -134,7 +136,13 @@ func (c *Core) runSyncLoop(ctx context.Context, done chan struct{}, req MatrixSy } } -func (c *Core) syncOnce(ctx context.Context, timeoutMS int, beeperStreaming bool, emitFailure bool) error { +func (c *Core) syncOnce( + ctx context.Context, + timeoutMS int, + beeperStreaming bool, + emitFailure bool, + replayMissed bool, +) error { started := time.Now() cli, err := c.requireClient() @@ -142,7 +150,7 @@ func (c *Core) syncOnce(ctx context.Context, timeoutMS int, beeperStreaming bool return err } filterID := liveSyncFilter - skipTimelines := c.skipNextSync + skipTimelines := c.skipNextSync && !replayMissed if skipTimelines { timeoutMS = 0 filterID = noHistorySyncFilter @@ -218,6 +226,16 @@ func (c *Core) handleApplySyncResponse(ctx context.Context, payload []byte) ([]b if err := json.Unmarshal(req.Response, &resp); err != nil { return nil, err } + if req.Since != "" && c.nextBatch != "" && req.Since != c.nextBatch { + c.emit(OutboundEvent{ + "type": "sync_status", + "status": "skipped", + "since": req.Since, + "nextBatch": c.nextBatch, + "remoteNext": resp.NextBatch, + }) + return c.empty() + } if err := c.processSyncResponse(ctx, &resp, req.Since); err != nil { return nil, err } @@ -236,6 +254,7 @@ func (c *Core) processSyncResponse(ctx context.Context, resp *mautrix.RespSync, if err != nil { return err } + c.processSyncMetadata(resp, since) c.processInvites(resp) c.processBeeperStreamSync(ctx, resp) if cli.Syncer != nil { @@ -252,6 +271,184 @@ func (c *Core) processSyncResponse(ctx context.Context, resp *mautrix.RespSync, return nil } +func (c *Core) processSyncMetadata(resp *mautrix.RespSync, since string) { + if resp == nil { + return + } + for _, evt := range resp.AccountData.Events { + c.emitSyncEvent("account_data", "accountData", "", evt, since, resp.NextBatch) + } + for _, evt := range resp.Presence.Events { + c.emitSyncEvent("presence", "presence", "", evt, since, resp.NextBatch) + } + for _, evt := range resp.ToDevice.Events { + c.emitSyncEvent("to_device", "toDevice", "", evt, since, resp.NextBatch) + } + if len(resp.DeviceLists.Changed) > 0 || len(resp.DeviceLists.Left) > 0 { + c.emitDeviceListEvent(resp.DeviceLists, since, resp.NextBatch) + } + for roomID, room := range resp.Rooms.Invite { + for _, evt := range room.State.Events { + c.emitClassifiedRoomEvent("invite_state", roomID, evt, since, resp.NextBatch) + } + } + for roomID, room := range resp.Rooms.Knock { + for _, evt := range room.State.Events { + c.emitClassifiedRoomEvent("knock_state", roomID, evt, since, resp.NextBatch) + } + } + for roomID, room := range resp.Rooms.Join { + for _, evt := range room.State.Events { + c.emitClassifiedRoomEvent("room_state", roomID, evt, since, resp.NextBatch) + } + if room.StateAfter != nil { + for _, evt := range room.StateAfter.Events { + c.emitClassifiedRoomEvent("room_state_after", roomID, evt, since, resp.NextBatch) + } + } + for _, evt := range room.Timeline.Events { + c.emitSyncEvent("room_timeline", "raw", roomID, evt, since, resp.NextBatch) + } + for _, evt := range room.Ephemeral.Events { + class := "ephemeral" + if evt != nil { + switch evt.Type { + case event.EphemeralEventReceipt: + class = "receipt" + case event.EphemeralEventTyping: + class = "typing" + } + } + c.emitSyncEvent("room_ephemeral", class, roomID, evt, since, resp.NextBatch) + } + for _, evt := range room.AccountData.Events { + c.emitSyncEvent("room_account_data", "accountData", roomID, evt, since, resp.NextBatch) + } + } + for roomID, room := range resp.Rooms.Leave { + for _, evt := range room.State.Events { + c.emitClassifiedRoomEvent("left_room_state", roomID, evt, since, resp.NextBatch) + } + for _, evt := range room.Timeline.Events { + c.emitClassifiedRoomEvent("left_room_timeline", roomID, evt, since, resp.NextBatch) + } + } +} + +func (c *Core) emitClassifiedRoomEvent(section string, roomID id.RoomID, evt *event.Event, since string, nextBatch string) { + class := "state" + if evt != nil { + switch evt.Type { + case event.StateMember: + class = "membership" + case event.EventRedaction: + class = "redaction" + } + } + c.emitSyncEvent(section, class, roomID, evt, since, nextBatch) +} + +func (c *Core) emitSyncEvent(section string, class string, roomID id.RoomID, evt *event.Event, since string, nextBatch string) { + if evt == nil { + return + } + if roomID != "" && evt.RoomID == "" { + evt.RoomID = roomID + } + syncEvent := c.toMatrixSyncEvent(section, class, evt, nextBatch) + c.emit(OutboundEvent{ + "type": "raw_event", + "event": syncEvent, + "since": since, + "nextBatch": nextBatch, + }) + switch class { + case "accountData": + c.emit(OutboundEvent{"type": "account_data", "event": syncEvent}) + case "toDevice": + c.emit(OutboundEvent{"type": "to_device", "event": syncEvent}) + case "receipt": + c.emit(OutboundEvent{"type": "receipt", "event": syncEvent}) + case "typing": + c.emit(OutboundEvent{"type": "typing", "event": syncEvent}) + case "presence": + c.emit(OutboundEvent{"type": "presence", "event": syncEvent}) + case "ephemeral": + c.emit(OutboundEvent{"type": "ephemeral", "event": syncEvent}) + case "membership": + c.emit(OutboundEvent{"type": "membership", "event": syncEvent}) + case "redaction": + c.emit(OutboundEvent{"type": "redaction", "event": syncEvent}) + case "state": + c.emit(OutboundEvent{"type": "room_state", "event": syncEvent}) + } +} + +func (c *Core) emitDeviceListEvent(lists mautrix.DeviceLists, since string, nextBatch string) { + changed := make([]string, 0, len(lists.Changed)) + for _, userID := range lists.Changed { + changed = append(changed, userID.String()) + } + left := make([]string, 0, len(lists.Left)) + for _, userID := range lists.Left { + left = append(left, userID.String()) + } + content := map[string]any{ + "changed": changed, + "left": left, + } + syncEvent := MatrixSyncEvent{ + Class: "deviceList", + Content: content, + NextBatch: optionalString(nextBatch), + Raw: lists, + Section: "device_lists", + Type: "m.device_list", + } + c.emit(OutboundEvent{ + "type": "raw_event", + "event": syncEvent, + "since": since, + "nextBatch": nextBatch, + }) + c.emit(OutboundEvent{"type": "device_list", "event": syncEvent}) +} + +func (c *Core) toMatrixSyncEvent(section string, class string, evt *event.Event, nextBatch string) MatrixSyncEvent { + content := evt.Content.Raw + if content == nil && len(evt.Content.VeryRaw) > 0 { + _ = json.Unmarshal(evt.Content.VeryRaw, &content) + } + if content == nil { + content = map[string]any{} + } + eventID := optionalString(evt.ID.String()) + roomID := optionalString(evt.RoomID.String()) + sender := optionalString(evt.Sender.String()) + stateKey := optionalString(evt.GetStateKey()) + var originServerTS *int64 + if evt.Timestamp != 0 { + originServerTS = &evt.Timestamp + } + encrypted := evt.Type == event.EventEncrypted + decrypted := encrypted && evt.Content.Raw["msgtype"] != nil + return MatrixSyncEvent{ + Class: class, + Content: content, + Decrypted: optionalBool(decrypted), + Encrypted: optionalBool(encrypted), + EventID: eventID, + NextBatch: optionalString(nextBatch), + OriginServerTS: originServerTS, + Raw: evt, + RoomID: roomID, + Section: section, + Sender: sender, + StateKey: stateKey, + Type: evt.Type.Type, + } +} + func (c *Core) processBeeperStreamSync(ctx context.Context, resp *mautrix.RespSync) { if c.beeperStream == nil || resp == nil { return @@ -374,6 +571,10 @@ func (c *Core) convertMaybeEncryptedMessageEvent(ctx context.Context, evt *event decrypted, err := c.decryptIfNeeded(ctx, evt) if err != nil { c.rememberPendingDecryption(ctx, evt) + if restored := c.restoreEventFromBackup(ctx, evt); restored != nil { + c.removePendingDecryption(ctx, evt.ID) + return c.convertMessageEvent(restored) + } eventData := OutboundEvent{} if evt != nil { eventData["eventId"] = evt.ID.String() @@ -394,6 +595,33 @@ func (c *Core) convertMaybeEncryptedMessageEvent(ctx context.Context, evt *event return c.convertMessageEvent(decrypted) } +func (c *Core) restoreEventFromBackup(ctx context.Context, evt *event.Event) *event.Event { + if evt == nil || evt.Type != event.EventEncrypted || c.crypto == nil || c.backupKey == nil { + return nil + } + _ = evt.Content.ParseRaw(evt.Type) + content := evt.Content.AsEncrypted() + minIndex, _ := crypto.ParseMegolmMessageIndex(content.MegolmCiphertext) + pending := &pendingDecryption{ + AddedAt: time.Now().UnixMilli(), + EventID: evt.ID.String(), + MinIndex: minIndex, + RoomID: evt.RoomID.String(), + Sender: evt.Sender.String(), + DeviceID: content.DeviceID.String(), + SenderKey: content.SenderKey.String(), + SessionID: content.SessionID.String(), + } + if restored, _ := c.restorePendingFromBackup(ctx, pending); !restored { + return nil + } + decrypted, err := c.decryptIfNeeded(ctx, evt) + if err != nil { + return nil + } + return decrypted +} + func (c *Core) decryptIfNeeded(ctx context.Context, evt *event.Event) (*event.Event, error) { if evt == nil { return nil, errors.New("matrix event is nil") diff --git a/packages/core/native/internal/core/sync_events_test.go b/packages/core/native/internal/core/sync_events_test.go new file mode 100644 index 0000000..8439efa --- /dev/null +++ b/packages/core/native/internal/core/sync_events_test.go @@ -0,0 +1,141 @@ +package core + +import ( + "context" + "encoding/json" + "testing" + + "maunium.net/go/mautrix" + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" +) + +func TestProcessSyncResponseEmitsGenericAndRawEvents(t *testing.T) { + var emitted []OutboundEvent + core := New(func(evt OutboundEvent) { + emitted = append(emitted, evt) + }) + core.client, _ = mautrix.NewClient("https://example.com", id.UserID("@alice:example"), "token") + stateKey := "@alice:example" + resp := &mautrix.RespSync{ + NextBatch: "s124", + AccountData: mautrix.SyncEventsList{Events: []*event.Event{syncTestEvent(event.AccountDataDirectChats, map[string]any{"@bob:example": []any{"!room:example"}})}}, + DeviceLists: mautrix.DeviceLists{ + Changed: []id.UserID{"@alice:example"}, + Left: []id.UserID{"@left:example"}, + }, + Presence: mautrix.SyncEventsList{Events: []*event.Event{syncTestEvent(event.EphemeralEventPresence, map[string]any{"presence": "online"})}}, + ToDevice: mautrix.SyncEventsList{Events: []*event.Event{syncTestEvent(event.ToDeviceRoomKey, map[string]any{"algorithm": "m.megolm.v1.aes-sha2"})}}, + Rooms: mautrix.RespSyncRooms{Join: map[id.RoomID]*mautrix.SyncJoinedRoom{ + "!room:example": { + AccountData: mautrix.SyncEventsList{Events: []*event.Event{syncTestEvent(event.Type{Type: "m.tag", Class: event.AccountDataEventType}, map[string]any{"tags": map[string]any{}})}}, + Ephemeral: mautrix.SyncEventsList{Events: []*event.Event{ + syncTestEvent(event.EphemeralEventReceipt, map[string]any{"$event": map[string]any{}}), + syncTestEvent(event.EphemeralEventTyping, map[string]any{"user_ids": []any{"@alice:example"}}), + }}, + State: mautrix.SyncEventsList{Events: []*event.Event{{ + Content: event.Content{Raw: map[string]any{"membership": "join"}}, + RoomID: id.RoomID("!room:example"), + Sender: id.UserID("@alice:example"), + StateKey: &stateKey, + Type: event.StateMember, + }}}, + Timeline: mautrix.SyncTimeline{ + SyncEventsList: mautrix.SyncEventsList{Events: []*event.Event{{ + Content: event.Content{Raw: map[string]any{"algorithm": "m.megolm.v1.aes-sha2", "ciphertext": "cipher"}}, + ID: id.EventID("$message"), + RoomID: id.RoomID("!room:example"), + Sender: id.UserID("@alice:example"), + Type: event.EventEncrypted, + }}}, + }, + }, + }}, + } + + if err := core.processSyncResponse(context.Background(), resp, "s123"); err != nil { + t.Fatal(err) + } + + counts := map[string]int{} + for _, evt := range emitted { + if eventType, ok := evt["type"].(string); ok { + counts[eventType]++ + } + } + for _, eventType := range []string{"raw_event", "account_data", "to_device", "receipt", "typing", "presence", "device_list", "membership"} { + if counts[eventType] == 0 { + t.Fatalf("expected %s event in %#v", eventType, counts) + } + } + if counts["raw_event"] < 9 { + t.Fatalf("expected raw event for each sync event, got %d", counts["raw_event"]) + } + foundTimelineRaw := false + for _, evt := range emitted { + syncEvent, ok := evt["event"].(MatrixSyncEvent) + if !ok || syncEvent.Section != "room_timeline" { + continue + } + if syncEvent.NextBatch == nil || *syncEvent.NextBatch != "s124" { + t.Fatalf("expected next batch on timeline raw event, got %#v", syncEvent.NextBatch) + } + if syncEvent.Encrypted == nil || !*syncEvent.Encrypted { + t.Fatalf("expected encrypted raw timeline status, got %#v", syncEvent.Encrypted) + } + foundTimelineRaw = true + } + if !foundTimelineRaw { + t.Fatal("expected room timeline raw event") + } +} + +func TestApplySyncResponseSkipsStaleWebhookReplay(t *testing.T) { + var emitted []OutboundEvent + core := New(func(evt OutboundEvent) { + emitted = append(emitted, evt) + }) + core.client, _ = mautrix.NewClient("https://example.com", id.UserID("@alice:example"), "token") + core.nextBatch = "s124" + + _, err := core.handleApplySyncResponse(context.Background(), []byte(`{ + "since":"s123", + "response":{ + "next_batch":"s124", + "rooms":{ + "join":{ + "!room:example":{ + "timeline":{ + "events":[{ + "content":{"body":"hello","msgtype":"m.text"}, + "event_id":"$message", + "sender":"@alice:example", + "type":"m.room.message" + }] + } + } + } + } + } + }`)) + if err != nil { + t.Fatal(err) + } + for _, evt := range emitted { + if evt["type"] == "message" || evt["type"] == "raw_event" { + t.Fatalf("stale apply should not replay events, got %#v", emitted) + } + } + if len(emitted) != 1 || emitted[0]["status"] != "skipped" { + t.Fatalf("expected skipped status, got %#v", emitted) + } +} + +func syncTestEvent(eventType event.Type, content map[string]any) *event.Event { + raw, _ := json.Marshal(content) + return &event.Event{ + Content: event.Content{Raw: content, VeryRaw: raw}, + Sender: id.UserID("@alice:example"), + Type: eventType, + } +} diff --git a/packages/core/native/internal/core/ts_contracts.go b/packages/core/native/internal/core/ts_contracts.go index 86e4aba..2456642 100644 --- a/packages/core/native/internal/core/ts_contracts.go +++ b/packages/core/native/internal/core/ts_contracts.go @@ -57,6 +57,13 @@ func optionalString(value string) *string { return &value } +func optionalBool(value bool) *bool { + if !value { + return nil + } + return &value +} + func stringValue(value *string) string { if value == nil { return "" @@ -96,6 +103,22 @@ type MatrixInviteEvent struct { RoomID string `json:"roomId"` } +type MatrixSyncEvent struct { + Class string `json:"class" tstype:"\"state\" | \"ephemeral\" | \"accountData\" | \"toDevice\" | \"membership\" | \"redaction\" | \"raw\" | string"` + Content map[string]any `json:"content"` + Decrypted *bool `json:"decrypted,omitempty"` + Encrypted *bool `json:"encrypted,omitempty"` + EventID *string `json:"eventId,omitempty"` + NextBatch *string `json:"nextBatch,omitempty"` + OriginServerTS *int64 `json:"originServerTs,omitempty"` + Raw any `json:"raw"` + RoomID *string `json:"roomId,omitempty"` + Section string `json:"section,omitempty"` + Sender *string `json:"sender,omitempty"` + StateKey *string `json:"stateKey,omitempty"` + Type string `json:"type"` +} + type MatrixRoomThreadSummary struct { LastReplyTS *int64 `json:"lastReplyTs,omitempty"` ReplyCount *int `json:"replyCount,omitempty"` diff --git a/packages/core/package.json b/packages/core/package.json index a818898..c2823be 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -20,10 +20,26 @@ "types": "./dist/index.d.ts", "import": "./dist/index.js" }, + "./helpers": { + "types": "./dist/helpers.d.ts", + "import": "./dist/helpers.js" + }, + "./beeper-login": { + "types": "./dist/beeper-login.d.ts", + "import": "./dist/beeper-login.js" + }, + "./login": { + "types": "./dist/login.d.ts", + "import": "./dist/login.js" + }, "./node": { "types": "./dist/node.d.ts", "import": "./dist/node.js" }, + "./types": { + "types": "./dist/types.d.ts", + "import": "./dist/types.js" + }, "./matrix-core.wasm": "./dist/matrix-core.wasm", "./wasm_exec.js": "./dist/wasm_exec.js" }, diff --git a/packages/core/src/beeper-login.test.ts b/packages/core/src/beeper-login.test.ts new file mode 100644 index 0000000..f6718bd --- /dev/null +++ b/packages/core/src/beeper-login.test.ts @@ -0,0 +1,86 @@ +import { describe, expect, it, vi } from "vitest"; +import { createBeeperLogin } from "./beeper-login"; + +describe("createBeeperLogin", () => { + it("uses Beeper homeserver and metadata for standard token login", async () => { + const fetchImpl = vi.fn(async () => Response.json({ + access_token: "access", + device_id: "DEVICE", + user_id: "@bot:beeper.com", + })); + const login = createBeeperLogin({ fetch: fetchImpl as typeof fetch }); + + await expect(login.token({ token: "jwt" })).resolves.toEqual({ + accessToken: "access", + deviceId: "DEVICE", + homeserver: "https://matrix.beeper.com", + metadata: { beeper: true }, + userId: "@bot:beeper.com", + }); + expect(String(fetchImpl.mock.calls[0]?.[0])).toBe("https://matrix.beeper.com/_matrix/client/v3/login"); + }); + + it("requests email registration tokens without fixed OTP assumptions", async () => { + const fetchImpl = vi.fn(async () => Response.json({ + sid: "sid", + submit_url: "https://matrix.beeper.com/submit", + })); + const login = createBeeperLogin({ + fetch: fetchImpl as typeof fetch, + homeserver: "https://matrix.example.com", + }); + + await expect(login.requestEmailToken({ + clientSecret: "secret", + email: "bot@example.com", + nextLink: "https://example.com/next", + sendAttempt: 1, + })).resolves.toEqual({ + raw: { sid: "sid", submit_url: "https://matrix.beeper.com/submit" }, + sid: "sid", + submitUrl: "https://matrix.beeper.com/submit", + }); + expect(String(fetchImpl.mock.calls[0]?.[0])).toBe("https://matrix.example.com/_matrix/client/v3/register/email/requestToken"); + expect(await requestBody(fetchImpl)).toEqual({ + client_secret: "secret", + email: "bot@example.com", + next_link: "https://example.com/next", + send_attempt: 1, + }); + }); + + it("registers through Matrix UI auth and returns an account when login is included", async () => { + const fetchImpl = vi.fn(async () => Response.json({ + access_token: "access", + device_id: "DEVICE", + user_id: "@bot:beeper.com", + })); + const login = createBeeperLogin({ fetch: fetchImpl as typeof fetch }); + + await expect(login.register({ + auth: { + session: "session", + threepid_creds: { sid: "sid" }, + type: "m.login.email.identity", + }, + password: "secret", + username: "bot", + })).resolves.toMatchObject({ + account: { + accessToken: "access", + deviceId: "DEVICE", + metadata: { beeper: true }, + userId: "@bot:beeper.com", + }, + accessToken: "access", + deviceId: "DEVICE", + userId: "@bot:beeper.com", + }); + expect(String(fetchImpl.mock.calls[0]?.[0])).toBe("https://matrix.beeper.com/_matrix/client/v3/register"); + }); +}); + +async function requestBody(fetchImpl: ReturnType) { + const init = fetchImpl.mock.calls[0]?.[1] as RequestInit; + return JSON.parse(String(init.body)); +} diff --git a/packages/core/src/beeper-login.ts b/packages/core/src/beeper-login.ts new file mode 100644 index 0000000..5221c1f --- /dev/null +++ b/packages/core/src/beeper-login.ts @@ -0,0 +1,144 @@ +import { createMatrixLogin, type MatrixLogin, type MatrixLoginOptions } from "./login"; +import type { MatrixAccount } from "./types"; + +const DEFAULT_BEEPER_HOMESERVER = "https://matrix.beeper.com"; + +export interface BeeperLoginOptions extends Omit { + homeserver?: string; +} + +export interface BeeperEmailTokenOptions { + clientSecret: string; + email: string; + nextLink?: string; + sendAttempt: number; +} + +export interface BeeperRegisterOptions { + auth?: Record; + inhibitLogin?: boolean; + password?: string; + username?: string; +} + +export interface BeeperEmailTokenResult { + raw: unknown; + sid: string; + submitUrl?: string; +} + +export interface BeeperRegisterResult { + account?: MatrixAccount; + accessToken?: string; + deviceId?: string; + raw: unknown; + userId?: string; +} + +export interface BeeperLogin extends MatrixLogin { + register(options: BeeperRegisterOptions): Promise; + requestEmailToken(options: BeeperEmailTokenOptions): Promise; +} + +export function createBeeperLogin(options: BeeperLoginOptions = {}): BeeperLogin { + const homeserver = options.homeserver ?? DEFAULT_BEEPER_HOMESERVER; + const fetchImpl = options.fetch ?? fetch; + const login = createMatrixLogin({ + ...options, + homeserver, + metadata: { ...options.metadata, beeper: true }, + }); + return { + ...login, + register: (registerOptions) => register(fetchImpl, homeserver, registerOptions), + requestEmailToken: (tokenOptions) => requestEmailToken(fetchImpl, homeserver, tokenOptions), + }; +} + +async function requestEmailToken( + fetchImpl: typeof fetch, + homeserver: string, + options: BeeperEmailTokenOptions +): Promise { + const body: Record = { + client_secret: options.clientSecret, + email: options.email, + send_attempt: options.sendAttempt, + }; + if (options.nextLink !== undefined) { + body.next_link = options.nextLink; + } + const raw = await matrixRequest(fetchImpl, homeserver, "/_matrix/client/v3/register/email/requestToken", body); + const result: BeeperEmailTokenResult = { + raw, + sid: readRequiredString(raw, "sid"), + }; + const submitUrl = readOptionalString(raw, "submit_url"); + if (submitUrl !== undefined) { + result.submitUrl = submitUrl; + } + return result; +} + +async function register( + fetchImpl: typeof fetch, + homeserver: string, + options: BeeperRegisterOptions +): Promise { + const body: Record = {}; + if (options.auth !== undefined) body.auth = options.auth; + if (options.inhibitLogin !== undefined) body.inhibit_login = options.inhibitLogin; + if (options.password !== undefined) body.password = options.password; + if (options.username !== undefined) body.username = options.username; + const raw = await matrixRequest(fetchImpl, homeserver, "/_matrix/client/v3/register", body); + const accessToken = readOptionalString(raw, "access_token"); + const deviceId = readOptionalString(raw, "device_id"); + const userId = readOptionalString(raw, "user_id"); + const result: BeeperRegisterResult = { raw }; + if (accessToken !== undefined) result.accessToken = accessToken; + if (deviceId !== undefined) result.deviceId = deviceId; + if (userId !== undefined) result.userId = userId; + if (accessToken && deviceId && userId) { + result.account = { + accessToken, + deviceId, + homeserver, + metadata: { beeper: true }, + userId, + }; + } + return result; +} + +async function matrixRequest( + fetchImpl: typeof fetch, + homeserver: string, + path: string, + body: Record +): Promise { + const response = await fetchImpl(new URL(path, homeserver), { + body: JSON.stringify(body), + headers: { "content-type": "application/json" }, + method: "POST", + }); + if (!response.ok) { + throw new Error(`Beeper request failed: ${response.status} ${await response.text()}`); + } + return response.json(); +} + +function readRequiredString(value: unknown, key: string): string { + const result = readOptionalString(value, key); + if (!result) { + throw new Error(`Beeper response is missing ${key}`); + } + return result; +} + +function readOptionalString(value: unknown, key: string): string | undefined { + if (!value || typeof value !== "object") { + return undefined; + } + const field = (value as Record)[key]; + return typeof field === "string" && field.length > 0 ? field : undefined; +} diff --git a/packages/core/src/client-types.ts b/packages/core/src/client-types.ts index 320969d..afc28a6 100644 --- a/packages/core/src/client-types.ts +++ b/packages/core/src/client-types.ts @@ -1,5 +1,7 @@ import type { ApplySyncResponseOptions, + AccountDataOptions, + AccountDataResult, BanUserOptions, CreateBeeperStreamOptions, CreateRoomOptions, @@ -29,8 +31,9 @@ import type { MarkReadOptions, MatrixClientEvent, MatrixCryptoStatus, - MatrixMessageEvent, - MatrixReactionEvent, + MatrixSubscribeFilter, + MatrixSubscribeOptions, + MatrixSubscription, MatrixWhoami, OpenDMOptions, OpenDMResult, @@ -42,6 +45,8 @@ import type { RegisterBeeperStreamOptions, ResolveRoomAliasOptions, ResolveRoomAliasResult, + RawRequestOptions, + RawRequestResult, RoomInfo, RoomPowerLevels, RoomStateEvent, @@ -49,12 +54,15 @@ import type { SendMatrixStreamOptions, SendMediaMessageOptions, SendMessageOptions, + SendReceiptOptions, SendRoomStateEventOptions, + SendToDeviceOptions, + SendToDeviceResult, SetOwnAvatarUrlOptions, SetOwnDisplayNameOptions, + SetAccountDataOptions, + SetRoomAccountDataOptions, SentEvent, - SyncOnceOptions, - SyncStartOptions, TypingOptions, UnbanUserOptions, UploadEncryptedMediaResult, @@ -65,21 +73,49 @@ import type { export interface MatrixClient { beeper: MatrixBeeper; + accountData: MatrixAccountData; + boot(): Promise; close(): Promise; - connect(options?: { signal?: AbortSignal }): Promise; crypto: MatrixCrypto; - events: MatrixEvents; media: MatrixMedia; messages: MatrixMessages; reactions: MatrixReactions; + raw: MatrixRaw; + receipts: MatrixReceipts; rooms: MatrixRooms; streams: MatrixStreams; + subscribe( + filter: MatrixSubscribeFilter, + handler: (event: MatrixClientEvent) => void | Promise, + options?: MatrixSubscribeOptions + ): Promise; sync: MatrixSync; typing: MatrixTyping; + toDevice: MatrixToDevice; users: MatrixUsers; + logout(): Promise; whoami(): Promise; } +export interface MatrixRaw { + request(options: RawRequestOptions): Promise; +} + +export interface MatrixAccountData { + get(options: AccountDataOptions): Promise; + getRoom(options: AccountDataOptions & { roomId: string }): Promise; + set(options: SetAccountDataOptions): Promise; + setRoom(options: SetRoomAccountDataOptions): Promise; +} + +export interface MatrixToDevice { + send(options: SendToDeviceOptions): Promise; +} + +export interface MatrixReceipts { + send(options: SendReceiptOptions): Promise; +} + export interface MatrixBeeper { ephemeral: { send(options: SendBeeperEphemeralOptions): Promise; @@ -99,12 +135,6 @@ export interface MatrixCrypto { status(): Promise; } -export interface MatrixEvents { - on(listener: (event: MatrixClientEvent) => void): () => void; - onMessage(listener: (event: MatrixMessageEvent) => void): () => void; - onReaction(listener: (event: MatrixReactionEvent) => void): () => void; -} - export interface MatrixMessages { edit(options: EditMessageOptions): Promise; get(options: FetchMessageOptions): Promise; @@ -165,7 +195,4 @@ export interface MatrixUsers { export interface MatrixSync { applyResponse(options: ApplySyncResponseOptions): Promise; - once(options?: SyncOnceOptions): Promise; - start(options?: SyncStartOptions): Promise; - stop(): Promise; } diff --git a/packages/core/src/client.test.ts b/packages/core/src/client.test.ts index b661400..61d9259 100644 --- a/packages/core/src/client.test.ts +++ b/packages/core/src/client.test.ts @@ -1,5 +1,6 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { createMatrixClient } from "./client"; +import { onInvite, onMessage, onRawEvent, onReaction } from "./helpers"; interface RuntimeCall { coreId: string; @@ -29,7 +30,6 @@ describe("createMatrixClient", () => { wasmModule: {} as WebAssembly.Module, }); - await client.connect(); await client.messages.send({ html: "Hello", mentions: { userIds: ["@alice:example.com"] }, @@ -78,8 +78,7 @@ describe("createMatrixClient", () => { wasmModule: {} as WebAssembly.Module, }); const listener = vi.fn(); - client.events.on(listener); - await client.connect(); + const sub = await client.subscribe({}, listener); globalThis.__matrixCoreEmit?.( "core-1", @@ -115,11 +114,322 @@ describe("createMatrixClient", () => { threadRoot: "$thread", }) ); + await sub.stop(); }); - it("delegates sync loop lifetime to the runtime", async () => { + it("maps raw and generic sync events through subscriptions", async () => { + installRuntime({ init: { deviceId: "DEVICE", userId: "@bot:example.com" }, start_sync: {}, stop_sync: {} }); + const client = createMatrixClient({ + homeserver: "https://matrix.example.com", + token: "token", + wasmModule: {} as WebAssembly.Module, + }); + const accountData = vi.fn(); + const deviceList = vi.fn(); + const presence = vi.fn(); + const raw = vi.fn(); + const typing = vi.fn(); + const accountSub = await client.subscribe({ kind: "accountData" }, accountData); + const deviceSub = await client.subscribe({ kind: "deviceList" }, deviceList); + const presenceSub = await client.subscribe({ kind: "presence" }, presence); + const rawSub = await onRawEvent(client, { roomId: "!room:example.com" }, raw); + const typingSub = await client.subscribe({ kind: "typing" }, typing); + + globalThis.__matrixCoreEmit?.( + "core-1", + JSON.stringify({ + event: { + class: "accountData", + content: { direct: {} }, + raw: { content: { direct: {} } }, + type: "m.direct", + }, + type: "account_data", + }) + ); + globalThis.__matrixCoreEmit?.( + "core-1", + JSON.stringify({ + event: { + class: "raw", + content: { body: "hello" }, + eventId: "$event", + raw: { event_id: "$event" }, + roomId: "!room:example.com", + section: "room_timeline", + sender: "@alice:example.com", + nextBatch: "s124", + type: "m.room.message", + }, + nextBatch: "s124", + since: "s123", + type: "raw_event", + }) + ); + globalThis.__matrixCoreEmit?.( + "core-1", + JSON.stringify({ + event: { + class: "typing", + content: { user_ids: ["@alice:example.com"] }, + raw: { content: { user_ids: ["@alice:example.com"] } }, + roomId: "!room:example.com", + section: "room_ephemeral", + type: "m.typing", + }, + type: "typing", + }) + ); + globalThis.__matrixCoreEmit?.( + "core-1", + JSON.stringify({ + event: { + class: "presence", + content: { presence: "online" }, + raw: { content: { presence: "online" } }, + section: "presence", + sender: "@alice:example.com", + type: "m.presence", + }, + type: "presence", + }) + ); + globalThis.__matrixCoreEmit?.( + "core-1", + JSON.stringify({ + event: { + class: "deviceList", + content: { changed: ["@alice:example.com"], left: [] }, + raw: { changed: ["@alice:example.com"], left: [] }, + section: "device_lists", + type: "m.device_list", + }, + type: "device_list", + }) + ); + + expect(accountData).toHaveBeenCalledWith(expect.objectContaining({ + content: { direct: {} }, + kind: "accountData", + type: "m.direct", + })); + expect(typing).toHaveBeenCalledWith(expect.objectContaining({ + content: { user_ids: ["@alice:example.com"] }, + kind: "typing", + roomId: "!room:example.com", + type: "m.typing", + })); + expect(presence).toHaveBeenCalledWith(expect.objectContaining({ + content: { presence: "online" }, + kind: "presence", + sender: { isMe: false, userId: "@alice:example.com" }, + type: "m.presence", + })); + expect(deviceList).toHaveBeenCalledWith(expect.objectContaining({ + content: { changed: ["@alice:example.com"], left: [] }, + kind: "deviceList", + type: "m.device_list", + })); + expect(raw).toHaveBeenCalledWith(expect.objectContaining({ + event: expect.objectContaining({ + eventId: "$event", + kind: "raw", + nextBatch: "s124", + roomId: "!room:example.com", + since: "s123", + }), + raw: { event_id: "$event" }, + source: { + kind: "raw", + roomId: "!room:example.com", + type: "m.room.message", + }, + })); + await accountSub.stop(); + await deviceSub.stop(); + await presenceSub.stop(); + await rawSub.stop(); + await typingSub.stop(); + }); + + it("keeps pure event helpers as thin subscription filters", async () => { + installRuntime({ init: { deviceId: "DEVICE", userId: "@bot:example.com" }, start_sync: {}, stop_sync: {} }); + const client = createMatrixClient({ + homeserver: "https://matrix.example.com", + token: "token", + wasmModule: {} as WebAssembly.Module, + }); + const message = vi.fn(); + const reaction = vi.fn(); + const invite = vi.fn(); + const messageSub = await onMessage(client, { roomId: "!room:example.com" }, message); + const reactionSub = await onReaction(client, { roomId: "!room:example.com" }, reaction); + const inviteSub = await onInvite(client, undefined, invite); + + globalThis.__matrixCoreEmit?.( + "core-1", + JSON.stringify({ + event: { + body: "Hello", + content: { body: "Hello", msgtype: "m.text" }, + eventId: "$message", + msgtype: "m.text", + raw: {}, + roomId: "!room:example.com", + sender: "@alice:example.com", + type: "m.room.message", + }, + type: "message", + }) + ); + globalThis.__matrixCoreEmit?.( + "core-1", + JSON.stringify({ + event: { + content: {}, + eventId: "$reaction", + key: "+1", + raw: {}, + relatesToEventId: "$message", + roomId: "!room:example.com", + sender: "@alice:example.com", + type: "m.reaction", + }, + type: "reaction", + }) + ); + globalThis.__matrixCoreEmit?.( + "core-1", + JSON.stringify({ + event: { raw: {}, roomId: "!invite:example.com" }, + type: "invite", + }) + ); + + expect(message).toHaveBeenCalledWith(expect.objectContaining({ eventId: "$message", kind: "message" })); + expect(reaction).toHaveBeenCalledWith(expect.objectContaining({ eventId: "$reaction", kind: "reaction" })); + expect(invite).toHaveBeenCalledWith(expect.objectContaining({ kind: "invite", roomId: "!invite:example.com" })); + await messageSub.stop(); + await reactionSub.stop(); + await inviteSub.stop(); + }); + + it("shares one sync runner across multiple subscribers", async () => { + const calls = installRuntime({ + init: { deviceId: "DEVICE", userId: "@bot:example.com" }, + start_sync: {}, + stop_sync: {}, + }); + const client = createMatrixClient({ + homeserver: "https://matrix.example.com", + token: "token", + wasmModule: {} as WebAssembly.Module, + }); + + const first = await client.subscribe({ kind: "message" }, () => undefined); + const second = await client.subscribe({ kind: "reaction" }, () => undefined); + await first.stop(); + await second.stop(); + + expect(calls.map((call) => call.operation)).toEqual(["init", "start_sync", "stop_sync"]); + await expect(first.done).resolves.toBeUndefined(); + await expect(second.done).resolves.toBeUndefined(); + }); + + it("boots without starting sync or delivering app events", async () => { + const calls = installRuntime({ + init: { deviceId: "DEVICE", userId: "@bot:example.com" }, + }); + const client = createMatrixClient({ + homeserver: "https://matrix.example.com", + token: "token", + wasmModule: {} as WebAssembly.Module, + }); + + await expect(client.boot()).resolves.toEqual({ + deviceId: "DEVICE", + userId: "@bot:example.com", + }); + globalThis.__matrixCoreEmit?.( + "core-1", + JSON.stringify({ + event: { + body: "Ignored", + content: { body: "Ignored", msgtype: "m.text" }, + eventId: "$ignored", + msgtype: "m.text", + raw: {}, + roomId: "!room:example.com", + sender: "@alice:example.com", + type: "m.room.message", + }, + type: "message", + }) + ); + + expect(calls.map((call) => call.operation)).toEqual(["init"]); + }); + + it("rejects subscription done when a handler fails", async () => { + installRuntime({ + init: { deviceId: "DEVICE", userId: "@bot:example.com" }, + start_sync: {}, + stop_sync: {}, + }); + const client = createMatrixClient({ + homeserver: "https://matrix.example.com", + token: "token", + wasmModule: {} as WebAssembly.Module, + }); + const failure = new Error("handler failed"); + const sub = await client.subscribe({ kind: "message" }, () => { + throw failure; + }); + + globalThis.__matrixCoreEmit?.( + "core-1", + JSON.stringify({ + event: { + body: "Hello", + content: { body: "Hello", msgtype: "m.text" }, + eventId: "$message", + msgtype: "m.text", + raw: {}, + roomId: "!room:example.com", + sender: "@alice:example.com", + type: "m.room.message", + }, + type: "message", + }) + ); + + await expect(sub.done).rejects.toThrow("handler failed"); + await sub.stop(); + }); + + it("rejects subscription done when the core emits an unrecoverable error", async () => { + installRuntime({ + init: { deviceId: "DEVICE", userId: "@bot:example.com" }, + start_sync: {}, + stop_sync: {}, + }); + const client = createMatrixClient({ + homeserver: "https://matrix.example.com", + token: "token", + wasmModule: {} as WebAssembly.Module, + }); + const sub = await client.subscribe({}, () => undefined); + + globalThis.__matrixCoreEmit?.("core-1", JSON.stringify({ error: "sync died", type: "error" })); + + await expect(sub.done).rejects.toThrow("sync died"); + await sub.stop(); + }); + + it("delegates subscription lifetime to the runtime", async () => { const calls = installRuntime({ init: { deviceId: "DEVICE", userId: "@bot:example.com" }, + sync_once: {}, start_sync: {}, stop_sync: {}, }); @@ -128,13 +438,255 @@ describe("createMatrixClient", () => { token: "token", wasmModule: {} as WebAssembly.Module, }); - await client.connect(); + const sub = await client.subscribe({ kind: "message" }, () => undefined); + await sub.catchUp(); + await sub.stop(); + + expect(calls.map((call) => call.operation)).toEqual(["init", "start_sync", "sync_once", "stop_sync"]); + expect(calls[1]?.payload).toEqual({}); + expect(calls[2]?.payload).toEqual({ replayMissed: true }); + }); - await client.sync.start({ retryDelayMs: 250, timeoutMs: 12_345 }); - await client.sync.stop(); + it("passes sync tuning through subscribe without exposing sync.start", async () => { + const calls = installRuntime({ + init: { deviceId: "DEVICE", userId: "@bot:example.com" }, + start_sync: {}, + stop_sync: {}, + }); + const client = createMatrixClient({ + homeserver: "https://matrix.example.com", + token: "token", + wasmModule: {} as WebAssembly.Module, + }); + + const sub = await client.subscribe({}, () => undefined, { + retryDelayMs: 250, + timeoutMs: 5000, + }); + await sub.stop(); expect(calls.map((call) => call.operation)).toEqual(["init", "start_sync", "stop_sync"]); - expect(calls[1]?.payload).toEqual({ retryDelayMs: 250, timeoutMs: 12_345 }); + expect(calls[1]?.payload).toEqual({ + retryDelayMs: 250, + timeoutMs: 5000, + }); + }); + + it("can subscribe to externally applied sync without starting live sync", async () => { + const calls = installRuntime({ + apply_sync_response: {}, + init: { deviceId: "DEVICE", userId: "@bot:example.com" }, + stop_sync: {}, + }); + const client = createMatrixClient({ + homeserver: "https://matrix.example.com", + token: "token", + wasmModule: {} as WebAssembly.Module, + }); + const listener = vi.fn(); + + const sub = await client.subscribe({}, listener, { live: false }); + await client.sync.applyResponse({ response: { next_batch: "s1" } }); + globalThis.__matrixCoreEmit?.( + "core-1", + JSON.stringify({ + event: { + body: "Webhook", + content: { body: "Webhook", msgtype: "m.text" }, + eventId: "$webhook", + raw: {}, + roomId: "!room:example.com", + sender: "@alice:example.com", + type: "m.room.message", + }, + type: "message", + }) + ); + await sub.stop(); + + expect(listener).toHaveBeenCalledWith(expect.objectContaining({ + eventId: "$webhook", + kind: "message", + })); + expect(calls.map((call) => call.operation)).toEqual(["init", "apply_sync_response", "stop_sync"]); + }); + + it("delivers catchUp events only to the calling subscription", async () => { + const calls = installRuntime({ + init: { deviceId: "DEVICE", userId: "@bot:example.com" }, + start_sync: {}, + stop_sync: {}, + sync_once: async () => { + globalThis.__matrixCoreEmit?.( + "core-1", + JSON.stringify({ + event: { + body: "Missed", + content: { body: "Missed", msgtype: "m.text" }, + eventId: "$missed", + msgtype: "m.text", + raw: {}, + roomId: "!room:example.com", + sender: "@alice:example.com", + type: "m.room.message", + }, + type: "message", + }) + ); + return {}; + }, + }); + const client = createMatrixClient({ + homeserver: "https://matrix.example.com", + token: "token", + wasmModule: {} as WebAssembly.Module, + }); + const first = vi.fn(); + const second = vi.fn(); + const firstSub = await client.subscribe({ kind: "message" }, first); + const secondSub = await client.subscribe({ kind: "message" }, second); + + await firstSub.catchUp(); + + expect(calls.map((call) => call.operation)).toEqual(["init", "start_sync", "sync_once"]); + expect(first).toHaveBeenCalledWith(expect.objectContaining({ eventId: "$missed" })); + expect(second).not.toHaveBeenCalled(); + await firstSub.stop(); + await secondSub.stop(); + }); + + it("filters events by sender, relation, and thread", async () => { + installRuntime({ + init: { deviceId: "DEVICE", userId: "@bot:example.com" }, + start_sync: {}, + stop_sync: {}, + }); + const client = createMatrixClient({ + homeserver: "https://matrix.example.com", + token: "token", + wasmModule: {} as WebAssembly.Module, + }); + const listener = vi.fn(); + const sub = await client.subscribe({ + kind: "message", + relationEventId: "$thread", + sender: "@alice:example.com", + threadRoot: "$thread", + }, listener); + + for (const [eventId, sender, threadRoot] of [ + ["$wrong-sender", "@bob:example.com", "$thread"], + ["$wrong-thread", "@alice:example.com", "$other"], + ["$match", "@alice:example.com", "$thread"], + ]) { + globalThis.__matrixCoreEmit?.( + "core-1", + JSON.stringify({ + event: { + body: "Hello", + content: { body: "Hello", msgtype: "m.text" }, + eventId, + msgtype: "m.text", + raw: {}, + relation: { eventId: threadRoot, type: "m.thread" }, + roomId: "!room:example.com", + sender, + threadRootEventId: threadRoot, + type: "m.room.message", + }, + type: "message", + }) + ); + } + + expect(listener).toHaveBeenCalledTimes(1); + expect(listener).toHaveBeenCalledWith(expect.objectContaining({ eventId: "$match" })); + await sub.stop(); + }); + + it("maps account data, to-device, receipts, and raw requests to core operations", async () => { + const calls = installRuntime({ + get_account_data: { content: { theme: "dark" }, raw: { theme: "dark" }, type: "m.preference" }, + get_room_account_data: { content: { muted: true }, raw: { muted: true }, type: "m.room.preference" }, + init: { deviceId: "DEVICE", userId: "@bot:example.com" }, + raw_request: { body: { ok: true }, headers: {}, raw: { ok: true }, status: 200 }, + send_receipt: {}, + send_to_device: { raw: {} }, + set_account_data: {}, + set_room_account_data: {}, + }); + const client = createMatrixClient({ + homeserver: "https://matrix.example.com", + token: "token", + wasmModule: {} as WebAssembly.Module, + }); + + await expect(client.accountData.get({ eventType: "m.preference" })).resolves.toEqual({ + content: { theme: "dark" }, + raw: { theme: "dark" }, + type: "m.preference", + }); + await client.accountData.set({ content: { theme: "light" }, eventType: "m.preference" }); + await expect(client.accountData.getRoom({ + eventType: "m.room.preference", + roomId: "!room:example.com", + })).resolves.toEqual({ + content: { muted: true }, + raw: { muted: true }, + type: "m.room.preference", + }); + await client.accountData.setRoom({ + content: { muted: false }, + eventType: "m.room.preference", + roomId: "!room:example.com", + }); + await client.toDevice.send({ + content: { hello: true }, + deviceId: "DEVICE2", + eventType: "m.test", + userId: "@alice:example.com", + }); + await client.receipts.send({ + eventId: "$event", + receiptType: "m.read.private", + roomId: "!room:example.com", + threadId: "$thread", + }); + await client.raw.request({ + body: { include: true }, + method: "POST", + path: "/_matrix/client/v3/custom", + query: { q: "1" }, + }); + + expect(calls.map((call) => call.operation)).toEqual([ + "init", + "get_account_data", + "set_account_data", + "get_room_account_data", + "set_room_account_data", + "send_to_device", + "send_receipt", + "raw_request", + ]); + expect(calls[5]?.payload).toEqual({ + content: { hello: true }, + deviceId: "DEVICE2", + eventType: "m.test", + userId: "@alice:example.com", + }); + expect(calls[6]?.payload).toEqual({ + eventId: "$event", + receiptType: "m.read.private", + roomId: "!room:example.com", + threadId: "$thread", + }); + expect(calls[7]?.payload).toEqual({ + body: { include: true }, + method: "POST", + path: "/_matrix/client/v3/custom", + query: { q: "1" }, + }); }); it("maps the public crypto status API to the runtime contract", async () => { @@ -156,8 +708,6 @@ describe("createMatrixClient", () => { wasmModule: {} as WebAssembly.Module, }); - await client.connect(); - await expect(client.crypto.status()).resolves.toEqual({ deviceId: "DEVICE", hasRecoveryKey: true, @@ -171,6 +721,22 @@ describe("createMatrixClient", () => { expect(calls[1]?.payload).toEqual({}); }); + it("maps logout to the runtime contract", async () => { + const calls = installRuntime({ + init: { deviceId: "DEVICE", userId: "@bot:example.com" }, + logout: {}, + }); + const client = createMatrixClient({ + homeserver: "https://matrix.example.com", + token: "token", + wasmModule: {} as WebAssembly.Module, + }); + + await client.logout(); + + expect(calls.map((call) => call.operation)).toEqual(["init", "logout"]); + }); + it("maps the public own profile API to the runtime contract", async () => { const calls = installRuntime({ get_own_avatar_url: { avatarUrl: "mxc://example/avatar" }, @@ -185,7 +751,6 @@ describe("createMatrixClient", () => { wasmModule: {} as WebAssembly.Module, }); - await client.connect(); await expect(client.users.getOwnDisplayName()).resolves.toEqual({ displayName: "Bot", raw: {} }); await client.users.setOwnDisplayName({ displayName: "New Bot" }); await expect(client.users.getOwnAvatarUrl()).resolves.toEqual({ avatarUrl: "mxc://example/avatar" }); @@ -213,8 +778,6 @@ describe("createMatrixClient", () => { token: "token", wasmModule: {} as WebAssembly.Module, }); - await client.connect(); - const sent = await client.streams.send({ roomId: "!room:example.com", stream: chunks("hel", "lo"), @@ -258,8 +821,6 @@ describe("createMatrixClient", () => { token: "token", wasmModule: {} as WebAssembly.Module, }); - await client.connect(); - const sent = await client.streams.send({ roomId: "!room:example.com", stream: chunks("hel", { text: "lo", type: "markdown_text" }), @@ -326,8 +887,6 @@ describe("createMatrixClient", () => { token: "token", wasmModule: {} as WebAssembly.Module, }); - await client.connect(); - await client.streams.send({ roomId: "!room:example.com", stream: chunks("hello"), @@ -336,6 +895,55 @@ describe("createMatrixClient", () => { expect(calls.map((call) => call.operation)).toContain("create_beeper_stream"); expect(calls.map((call) => call.operation)).toContain("publish_beeper_stream"); }); + + it("normalizes generic stream chunk shapes", async () => { + const calls = installRuntime({ + edit_message: { eventId: "$edit", raw: {}, roomId: "!room:example.com" }, + init: { deviceId: "DEVICE", userId: "@bot:example.com" }, + post_message: { eventId: "$message", raw: {}, roomId: "!room:example.com" }, + }); + const client = createMatrixClient({ + homeserver: "https://matrix.example.com", + token: "token", + wasmModule: {} as WebAssembly.Module, + }); + + await client.streams.send({ + roomId: "!room:example.com", + stream: chunks({ delta: "hel" }, { markdown: "lo" }, { ignored: true }), + }); + + expect(calls.map((call) => call.operation)).toEqual(["init", "post_message", "edit_message"]); + expect(calls[1]?.payload).toEqual({ + body: "hel", + roomId: "!room:example.com", + }); + expect(calls[2]?.payload).toEqual({ + body: "hello", + messageId: "$message", + roomId: "!room:example.com", + }); + }); + + it("sends placeholder text for empty streams", async () => { + const calls = installRuntime({ + init: { deviceId: "DEVICE", userId: "@bot:example.com" }, + post_message: { eventId: "$message", raw: {}, roomId: "!room:example.com" }, + }); + const client = createMatrixClient({ + homeserver: "https://matrix.example.com", + token: "token", + wasmModule: {} as WebAssembly.Module, + }); + + await client.streams.send({ roomId: "!room:example.com", stream: chunks() }); + + expect(calls.map((call) => call.operation)).toEqual(["init", "post_message"]); + expect(calls[1]?.payload).toEqual({ + body: "...", + roomId: "!room:example.com", + }); + }); }); async function* chunks( @@ -346,12 +954,13 @@ async function* chunks( } } -function installRuntime(responses: Record): RuntimeCall[] { +function installRuntime(responses: Record unknown | Promise)>): RuntimeCall[] { const calls: RuntimeCall[] = []; globalThis.__matrixCoreCreate = () => "core-1"; globalThis.__matrixCoreCall = async (coreId, operation, payload) => { calls.push({ coreId, operation, payload: JSON.parse(payload) as Record }); - return JSON.stringify(responses[operation] ?? {}); + const response = responses[operation]; + return JSON.stringify(typeof response === "function" ? await response() : response ?? {}); }; return calls; } diff --git a/packages/core/src/client.ts b/packages/core/src/client.ts index 2fd50b5..9961a72 100644 --- a/packages/core/src/client.ts +++ b/packages/core/src/client.ts @@ -1,15 +1,18 @@ import type { + MatrixAccountData, MatrixBeeper, MatrixClient, MatrixCrypto, - MatrixEvents, MatrixMedia, MatrixMessages, + MatrixRaw, + MatrixReceipts, MatrixReactions, MatrixRooms, MatrixStreams, MatrixSync, MatrixTyping, + MatrixToDevice, MatrixUsers, } from "./client-types"; import { toClientEvent, toCryptoStatusSnapshot, toMessageEvent } from "./events"; @@ -20,6 +23,9 @@ import { createMatrixStreams } from "./streams"; import type { MatrixClientEvent, MatrixClientOptions, + MatrixSubscribeFilter, + MatrixSubscribeOptions, + MatrixSubscription, MatrixThreadSummary, MatrixWhoami, } from "./types"; @@ -30,53 +36,60 @@ export function createMatrixClient(options: MatrixClientOptions): MatrixClient { } class DefaultMatrixClient implements MatrixClient { - readonly events: MatrixEvents; + readonly accountData: MatrixAccountData; readonly beeper: MatrixBeeper; readonly crypto: MatrixCrypto; readonly media: MatrixMedia; readonly messages: MatrixMessages; readonly reactions: MatrixReactions; + readonly raw: MatrixRaw; + readonly receipts: MatrixReceipts; readonly rooms: MatrixRooms; readonly streams: MatrixStreams; readonly sync: MatrixSync; readonly typing: MatrixTyping; + readonly toDevice: MatrixToDevice; readonly users: MatrixUsers; #core: MatrixCore | null = null; - #listeners = new Set<(event: MatrixClientEvent) => void>(); + #bootPromise: Promise | null = null; #options: MatrixClientOptions; - #syncAbort: (() => void) | null = null; + #syncDone: Promise | null = null; + #syncReject: ((error: unknown) => void) | null = null; + #syncResolve: (() => void) | null = null; + #subscriptions = new Set(); + #catchUpTarget: InternalSubscription | null = null; #unsubscribeCore: (() => void) | null = null; constructor(options: MatrixClientOptions) { this.#options = options; + this.accountData = { + get: (opts) => this.#withCore((core) => core.getAccountData(opts)), + getRoom: (opts) => this.#withCore((core) => core.getRoomAccountData(opts)), + set: (opts) => this.#withCore((core) => core.setAccountData(opts)), + setRoom: (opts) => this.#withCore((core) => core.setRoomAccountData(opts)), + }; this.beeper = { ephemeral: { send: (opts) => - this.#coreRequired().sendEphemeralEvent(stripUndefined({ + this.#withCore((core) => core.sendEphemeralEvent(stripUndefined({ content: opts.content ?? {}, eventType: opts.eventType ?? "m.room.message", roomId: opts.roomId, transactionId: opts.transactionId, - })), + }))), }, streams: { - create: (opts) => this.#coreRequired().createBeeperStream(opts), - publish: (opts) => this.#coreRequired().publishBeeperStream(opts), - register: (opts) => this.#coreRequired().registerBeeperStream(opts), + create: (opts) => this.#withCore((core) => core.createBeeperStream(opts)), + publish: (opts) => this.#withCore((core) => core.publishBeeperStream(opts)), + register: (opts) => this.#withCore((core) => core.registerBeeperStream(opts)), }, }; this.crypto = { - status: async () => toCryptoStatusSnapshot(await this.#coreRequired().getCryptoStatus()), - }; - this.events = { - on: (listener) => this.#on(listener), - onMessage: (listener) => this.#onKind("message", listener), - onReaction: (listener) => this.#onKind("reaction", listener), + status: async () => toCryptoStatusSnapshot(await this.#withCore((core) => core.getCryptoStatus())), }; this.messages = { - edit: (opts) => - this.#coreRequired().editMessage(stripUndefined({ + edit: (opts) => this.#withCore((core) => core.editMessage(stripUndefined({ body: opts.text, content: opts.content, formattedBody: opts.html, @@ -84,36 +97,34 @@ class DefaultMatrixClient implements MatrixClient { messageId: opts.eventId, msgtype: opts.messageType, roomId: opts.roomId, - })), + }))), get: async (opts) => { - const result = await this.#coreRequired().fetchMessage({ + const result = await this.#withCore((core) => core.fetchMessage({ messageId: opts.eventId, roomId: opts.roomId, - }); + })); return { message: result.message ? toMessageEvent(result.message) : null }; }, list: async (opts) => { - const result = await this.#coreRequired().fetchMessages(stripUndefined({ + const result = await this.#withCore((core) => core.fetchMessages(stripUndefined({ cursor: opts.cursor, direction: opts.direction, limit: opts.limit, roomId: opts.roomId, threadRootEventId: opts.threadRoot, - })); + }))); return stripUndefined({ messages: result.messages.map(toMessageEvent), nextCursor: result.nextCursor, }); }, - markRead: (opts) => this.#coreRequired().markRead(opts), - redact: (opts) => - this.#coreRequired().deleteMessage(stripUndefined({ + markRead: (opts) => this.#withCore((core) => core.markRead(opts)), + redact: (opts) => this.#withCore((core) => core.deleteMessage(stripUndefined({ messageId: opts.eventId, reason: opts.reason, roomId: opts.roomId, - })), - send: (opts) => - this.#coreRequired().postMessage(stripUndefined({ + }))), + send: (opts) => this.#withCore((core) => core.postMessage(stripUndefined({ body: opts.text, content: opts.content, formattedBody: opts.html, @@ -122,29 +133,31 @@ class DefaultMatrixClient implements MatrixClient { replyToEventId: opts.replyTo, roomId: opts.roomId, threadRootEventId: opts.threadRoot, - })), - sendMedia: (opts) => - postMediaMessageBytes(this.#coreRequired(), opts), + }))), + sendMedia: (opts) => this.#withCore((core) => postMediaMessageBytes(core, opts)), }; this.reactions = { - redact: (opts) => - this.#coreRequired().removeReaction({ + redact: (opts) => this.#withCore((core) => core.removeReaction({ emoji: opts.key, messageId: opts.eventId, roomId: opts.roomId, - }), - send: (opts) => - this.#coreRequired().addReaction({ + })), + send: (opts) => this.#withCore((core) => core.addReaction({ emoji: opts.key, messageId: opts.eventId, roomId: opts.roomId, - }), + })), + }; + this.raw = { + request: (opts) => this.#withCore((core) => core.rawRequest(opts)), + }; + this.receipts = { + send: (opts) => this.#withCore((core) => core.sendReceipt(opts)), }; - this.media = createMatrixMedia(() => this.#coreRequired()); + this.media = createMatrixMedia(() => this.#coreReady()); this.rooms = { - ban: (opts) => this.#coreRequired().banUser(opts), - create: (opts) => - this.#coreRequired().createRoom(stripUndefined({ + ban: (opts) => this.#withCore((core) => core.banUser(opts)), + create: (opts) => this.#withCore((core) => core.createRoom(stripUndefined({ creationContent: opts.creationContent, initialState: opts.initialState?.map((state) => ({ content: state.content, @@ -159,14 +172,14 @@ class DefaultMatrixClient implements MatrixClient { roomVersion: opts.roomVersion, topic: opts.topic, visibility: opts.visibility, - })), - get: (opts) => this.#coreRequired().fetchRoom(opts), + }))), + get: (opts) => this.#withCore((core) => core.fetchRoom(opts)), getPowerLevels: async (opts) => { - const event = await this.#coreRequired().fetchRoomStateEvent({ + const event = await this.#withCore((core) => core.fetchRoomStateEvent({ eventType: "m.room.power_levels", roomId: opts.roomId, stateKey: "", - }); + })); return stripUndefined({ ban: readNumber(event.content.ban), events: readNumberRecord(event.content.events), @@ -181,30 +194,30 @@ class DefaultMatrixClient implements MatrixClient { usersDefault: readNumber(event.content.users_default), }); }, - getState: (opts) => this.#coreRequired().fetchRoomState(opts), - getStateEvent: (opts) => this.#coreRequired().fetchRoomStateEvent(stripUndefined({ + getState: (opts) => this.#withCore((core) => core.fetchRoomState(opts)), + getStateEvent: (opts) => this.#withCore((core) => core.fetchRoomStateEvent(stripUndefined({ eventType: opts.eventType, roomId: opts.roomId, stateKey: opts.stateKey, - })), - invite: (opts) => this.#coreRequired().inviteUser(opts), - join: (opts) => this.#coreRequired().joinRoom(opts), - kick: (opts) => this.#coreRequired().kickUser(opts), - listPublic: (opts = {}) => this.#coreRequired().listPublicRooms(stripUndefined(opts)), - leave: (opts) => this.#coreRequired().leaveRoom(opts), - listMembers: (opts) => this.#coreRequired().fetchRoomMembers(stripUndefined(opts)), - listJoined: () => this.#coreRequired().fetchJoinedRooms(), - openDM: (opts) => this.#coreRequired().openDM(opts), - resolveAlias: (opts) => this.#coreRequired().resolveRoomAlias(opts), - sendStateEvent: (opts) => this.#coreRequired().sendRoomStateEvent(stripUndefined({ + }))), + invite: (opts) => this.#withCore((core) => core.inviteUser(opts)), + join: (opts) => this.#withCore((core) => core.joinRoom(opts)), + kick: (opts) => this.#withCore((core) => core.kickUser(opts)), + listPublic: (opts = {}) => this.#withCore((core) => core.listPublicRooms(stripUndefined(opts))), + leave: (opts) => this.#withCore((core) => core.leaveRoom(opts)), + listMembers: (opts) => this.#withCore((core) => core.fetchRoomMembers(stripUndefined(opts))), + listJoined: () => this.#withCore((core) => core.fetchJoinedRooms()), + openDM: (opts) => this.#withCore((core) => core.openDM(opts)), + resolveAlias: (opts) => this.#withCore((core) => core.resolveRoomAlias(opts)), + sendStateEvent: (opts) => this.#withCore((core) => core.sendRoomStateEvent(stripUndefined({ content: opts.content, eventType: opts.eventType, roomId: opts.roomId, stateKey: opts.stateKey, - })), + }))), threads: { list: async (opts) => { - const result = await this.#coreRequired().listRoomThreads(opts); + const result = await this.#withCore((core) => core.listRoomThreads(opts)); return stripUndefined({ nextCursor: result.nextCursor, threads: result.threads.map((thread): MatrixThreadSummary => ({ @@ -215,7 +228,7 @@ class DefaultMatrixClient implements MatrixClient { }); }, }, - unban: (opts) => this.#coreRequired().unbanUser(opts), + unban: (opts) => this.#withCore((core) => core.unbanUser(opts)), }; this.streams = createMatrixStreams({ beeper: this.beeper, @@ -223,50 +236,71 @@ class DefaultMatrixClient implements MatrixClient { messages: this.messages, }); this.sync = { - applyResponse: (opts) => this.#coreRequired().applySyncResponse(opts), - once: (opts = {}) => this.#coreRequired().syncOnce(stripUndefined({ - beeperStreaming: opts.beeper ?? this.#options.beeper, - timeoutMs: opts.timeoutMs, - })), - start: async (opts = {}) => { - if (this.#syncAbort) return; - if (opts.signal?.aborted) return; - await this.#coreRequired().startSync(stripUndefined({ - beeperStreaming: opts.beeper ?? this.#options.beeper, - retryDelayMs: opts.retryDelayMs, - timeoutMs: opts.timeoutMs, - })); - const abort = () => void this.sync.stop(); - opts.signal?.addEventListener("abort", abort, { once: true }); - this.#syncAbort = () => opts.signal?.removeEventListener("abort", abort); - }, - stop: async () => { - this.#syncAbort?.(); - this.#syncAbort = null; - await this.#core?.stopSync(); - }, + applyResponse: (opts) => this.#withCore((core) => core.applySyncResponse(opts)), }; this.typing = { - set: (opts) => this.#coreRequired().setTyping(opts), + set: (opts) => this.#withCore((core) => core.setTyping(opts)), + }; + this.toDevice = { + send: (opts) => this.#withCore((core) => core.sendToDevice(opts)), }; this.users = { - get: (opts) => this.#coreRequired().getUser(opts), - getOwnAvatarUrl: () => this.#coreRequired().getOwnAvatarURL(), - getOwnDisplayName: () => this.#coreRequired().getOwnDisplayName(), - setOwnAvatarUrl: (opts) => this.#coreRequired().setOwnAvatarURL(opts), - setOwnDisplayName: (opts) => this.#coreRequired().setOwnDisplayName(opts), + get: (opts) => this.#withCore((core) => core.getUser(opts)), + getOwnAvatarUrl: () => this.#withCore((core) => core.getOwnAvatarURL()), + getOwnDisplayName: () => this.#withCore((core) => core.getOwnDisplayName()), + setOwnAvatarUrl: (opts) => this.#withCore((core) => core.setOwnAvatarURL(opts)), + setOwnDisplayName: (opts) => this.#withCore((core) => core.setOwnDisplayName(opts)), }; + if (options.boot) void this.boot(); } async close(): Promise { - await this.sync.stop(); + await this.#stopSync(); this.#unsubscribeCore?.(); this.#unsubscribeCore = null; await this.#core?.close(); this.#core = null; + this.#bootPromise = null; } - async connect(): Promise { + boot(): Promise { + this.#bootPromise ??= this.#boot(); + return this.#bootPromise; + } + + async subscribe( + filter: MatrixSubscribeFilter, + handler: (event: MatrixClientEvent) => void | Promise, + options: MatrixSubscribeOptions = {} + ): Promise { + const core = await this.#coreReady(); + const subscription = createSubscription(filter, handler, async () => { + this.#subscriptions.delete(subscription); + if (this.#subscriptions.size === 0) { + await this.#stopSync(); + } + }); + this.#subscriptions.add(subscription); + if (options.live !== false) { + await this.#startSync(core, options); + } + return { + catchUp: () => this.#catchUp(subscription), + done: subscription.done, + stop: () => subscription.stop(), + }; + } + + whoami(): Promise { + return this.#withCore((core) => core.whoami()); + } + + logout(): Promise { + return this.#withCore((core) => core.logout()); + } + + async #boot(): Promise { + const account = this.#accountOptions(); if (!this.#core) { const loadOptions: LoadMatrixCoreOptions = stripUndefined({ host: this.#host(), @@ -277,37 +311,117 @@ class DefaultMatrixClient implements MatrixClient { this.#core = await loadMatrixCore(loadOptions); this.#unsubscribeCore = this.#core.onEvent((event) => this.#emit(event)); } - const initialSyncMode: "persisted" | "latest" | "catch_up" | undefined = - this.#options.initialSync === "catchUp" ? "catch_up" : this.#options.initialSync; return this.#core.init(stripUndefined({ - accessToken: this.#options.token, - deviceId: this.#options.deviceId, - homeserverUrl: this.#options.homeserver, - initialSyncMode, - initialSyncSince: this.#options.since, + accessToken: account.accessToken, + deviceId: account.deviceId, + homeserverUrl: account.homeserver, + initialSyncMode: "latest" as const, pickleKey: this.#options.pickleKey, recoveryKey: this.#options.recoveryKey, - userId: this.#options.userId, + userId: account.userId, verifyRecoveryOnStart: this.#options.verifyRecoveryOnStart, })); } - whoami(): Promise { - return this.#coreRequired().whoami(); + #accountOptions() { + const account = this.#options.account; + const homeserver = account?.homeserver ?? this.#options.homeserver; + const accessToken = account?.accessToken ?? this.#options.token; + if (!homeserver || !accessToken) { + throw new Error("Matrix client requires account or homeserver/token options."); + } + return { + accessToken, + deviceId: account?.deviceId, + homeserver, + userId: account?.userId, + }; } - #coreRequired(): MatrixCore { + async #coreReady(): Promise { + await this.boot(); if (!this.#core) { - throw new Error("Matrix client is not connected. Call connect() first."); + throw new Error("Matrix core failed to boot."); } return this.#core; } + async #withCore(fn: (core: MatrixCore) => Promise): Promise { + return fn(await this.#coreReady()); + } + + async #startSync(core: MatrixCore, options: MatrixSubscribeOptions): Promise { + if (this.#syncDone) return; + this.#syncDone = new Promise((resolve, reject) => { + this.#syncResolve = resolve; + this.#syncReject = reject; + }); + this.#syncDone.catch(() => undefined); + try { + await core.startSync(stripUndefined({ + beeperStreaming: this.#options.beeper, + retryDelayMs: options.retryDelayMs, + timeoutMs: options.timeoutMs, + })); + } catch (error) { + this.#syncReject?.(error); + this.#clearSyncDone(); + throw error; + } + } + + async #stopSync(): Promise { + for (const subscription of this.#subscriptions) { + subscription.close(); + } + this.#subscriptions.clear(); + try { + await this.#core?.stopSync(); + this.#syncResolve?.(); + } catch (error) { + this.#syncReject?.(error); + throw error; + } finally { + this.#clearSyncDone(); + } + } + + async #catchUp(subscription: InternalSubscription): Promise { + const core = await this.#coreReady(); + if (!this.#subscriptions.has(subscription)) return; + this.#catchUpTarget = subscription; + try { + await core.syncOnce(stripUndefined({ + beeperStreaming: this.#options.beeper, + replayMissed: true, + })); + } finally { + this.#catchUpTarget = null; + } + } + + #clearSyncDone(): void { + this.#syncDone = null; + this.#syncReject = null; + this.#syncResolve = null; + } + #emit(event: MatrixCoreEvent): void { const mapped = toClientEvent(event); if (!mapped) return; - for (const listener of this.#listeners) { - listener(mapped); + if (mapped.kind === "error") { + for (const subscription of this.#subscriptions) { + subscription.fail(new Error(mapped.error)); + } + this.#syncReject?.(new Error(mapped.error)); + return; + } + if (this.#catchUpTarget) { + this.#catchUpTarget.emit(mapped); + return; + } + for (const subscription of this.#subscriptions) { + subscription.emit(mapped); } } @@ -320,21 +434,85 @@ class DefaultMatrixClient implements MatrixClient { }); } - #on(listener: (event: MatrixClientEvent) => void): () => void { - this.#listeners.add(listener); - return () => this.#listeners.delete(listener); - } +} - #onKind( - kind: K, - listener: (event: Extract) => void - ): () => void { - return this.#on((event) => { - if (event.kind === kind) { - listener(event as Extract); +interface InternalSubscription { + close(): void; + done: Promise; + emit(event: MatrixClientEvent): void; + fail(error: unknown): void; + stop(): Promise; +} + +function createSubscription( + filter: MatrixSubscribeFilter, + handler: (event: MatrixClientEvent) => void | Promise, + onStop: () => Promise +): InternalSubscription { + let stopped = false; + let resolveDone!: () => void; + let rejectDone!: (error: unknown) => void; + const done = new Promise((resolve, reject) => { + resolveDone = resolve; + rejectDone = reject; + }); + done.catch(() => undefined); + return { + close: () => { + if (stopped) return; + stopped = true; + resolveDone(); + }, + done, + emit: (event) => { + if (stopped || !matchesFilter(filter, event)) return; + try { + void Promise.resolve(handler(event)).catch(rejectDone); + } catch (error) { + rejectDone(error); } - }); + }, + fail: (error) => rejectDone(error), + stop: async () => { + if (stopped) return; + stopped = true; + resolveDone(); + await onStop(); + }, + }; +} + +function matchesFilter(filter: MatrixSubscribeFilter, event: MatrixClientEvent): boolean { + if (!filter) return true; + return matchesValue(filter.kind, event.kind) + && matchesValue(filter.roomId, "roomId" in event ? event.roomId : undefined) + && matchesValue(filter.type, "type" in event ? event.type : undefined) + && matchesValue(filter.sender, eventSender(event)) + && matchesValue(filter.threadRoot, eventThreadRoot(event)) + && matchesValue(filter.relationEventId, eventRelationEventId(event)); +} + +function matchesValue(filter: string | string[] | undefined, value: string | undefined): boolean { + if (filter === undefined) return true; + if (value === undefined) return false; + return Array.isArray(filter) ? filter.includes(value) : filter === value; +} + +function eventSender(event: MatrixClientEvent): string | undefined { + if ("sender" in event) { + return typeof event.sender === "string" ? event.sender : event.sender?.userId; } + return undefined; +} + +function eventThreadRoot(event: MatrixClientEvent): string | undefined { + return "threadRoot" in event ? event.threadRoot : undefined; +} + +function eventRelationEventId(event: MatrixClientEvent): string | undefined { + if ("relation" in event) return event.relation?.eventId; + if ("relatesTo" in event) return event.relatesTo; + return undefined; } function readNumber(value: unknown): number | undefined { diff --git a/packages/core/src/events.test.ts b/packages/core/src/events.test.ts new file mode 100644 index 0000000..50b05ae --- /dev/null +++ b/packages/core/src/events.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from "vitest"; +import { toClientEvent } from "./events"; +import type { MatrixCoreEvent } from "./runtime-types"; + +describe("toClientEvent", () => { + it("maps every generic sync event kind", () => { + const cases: Array<[MatrixCoreEvent["type"], string]> = [ + ["account_data", "accountData"], + ["device_list", "deviceList"], + ["ephemeral", "ephemeral"], + ["membership", "membership"], + ["presence", "presence"], + ["raw_event", "raw"], + ["receipt", "receipt"], + ["redaction", "redaction"], + ["room_state", "roomState"], + ["to_device", "toDevice"], + ["typing", "typing"], + ]; + + for (const [type, kind] of cases) { + const mapped = toClientEvent({ + event: { + class: kind, + content: { ok: true }, + decrypted: true, + encrypted: true, + eventId: "$event", + nextBatch: "s2", + originServerTs: 1, + raw: { raw: true }, + roomId: "!room:example.com", + section: "section", + sender: "@alice:example.com", + stateKey: "@alice:example.com", + type: "m.test", + }, + nextBatch: "s2", + since: "s1", + type, + } as Extract); + + expect(mapped).toMatchObject({ + content: { ok: true }, + decrypted: true, + encrypted: true, + eventId: "$event", + kind, + nextBatch: "s2", + raw: { raw: true }, + roomId: "!room:example.com", + section: "section", + sender: { isMe: false, userId: "@alice:example.com" }, + type: "m.test", + }); + if (kind === "raw") { + expect(mapped).toMatchObject({ since: "s1" }); + } + } + }); +}); diff --git a/packages/core/src/events.ts b/packages/core/src/events.ts index b6228da..75ea4cf 100644 --- a/packages/core/src/events.ts +++ b/packages/core/src/events.ts @@ -9,6 +9,7 @@ import type { MatrixClientEvent, MatrixCryptoStatus, MatrixCryptoStatusEvent, + MatrixGenericEvent, MatrixMessageEvent, MatrixReactionEvent, MatrixSyncStatusEvent, @@ -18,6 +19,17 @@ export function toClientEvent(event: MatrixCoreEvent): MatrixClientEvent | null if (event.type === "message") return toMessageEvent(event.event); if (event.type === "reaction") return toReactionEvent(event.event); if (event.type === "invite") return { kind: "invite", ...event.event }; + if (event.type === "raw_event") return toGenericEvent(event.event, "raw", event.since, event.nextBatch); + if (event.type === "account_data") return toGenericEvent(event.event, "accountData"); + if (event.type === "to_device") return toGenericEvent(event.event, "toDevice"); + if (event.type === "receipt") return toGenericEvent(event.event, "receipt"); + if (event.type === "typing") return toGenericEvent(event.event, "typing"); + if (event.type === "presence") return toGenericEvent(event.event, "presence"); + if (event.type === "device_list") return toGenericEvent(event.event, "deviceList"); + if (event.type === "ephemeral") return toGenericEvent(event.event, "ephemeral"); + if (event.type === "membership") return toGenericEvent(event.event, "membership"); + if (event.type === "redaction") return toGenericEvent(event.event, "redaction"); + if (event.type === "room_state") return toGenericEvent(event.event, "roomState"); if (event.type === "sync_status") return toSyncEvent(event); if (event.type === "crypto_status") return toCryptoEvent(event); if (event.type === "decryption_error") { @@ -33,6 +45,31 @@ export function toClientEvent(event: MatrixCoreEvent): MatrixClientEvent | null return null; } +function toGenericEvent( + event: import("./runtime-types").MatrixSyncEvent, + kind: MatrixGenericEvent["kind"], + since?: string, + nextBatch?: string +): MatrixGenericEvent { + return stripUndefined({ + class: event.class === "raw" ? "unknown" : event.class, + content: event.content, + decrypted: event.decrypted, + encrypted: event.encrypted, + eventId: event.eventId, + kind, + nextBatch: event.nextBatch ?? nextBatch, + raw: event.raw, + roomId: event.roomId, + section: event.section, + sender: event.sender ? { isMe: false, userId: event.sender } : undefined, + since, + stateKey: event.stateKey, + timestamp: event.originServerTs, + type: event.type, + }) as MatrixGenericEvent; +} + export function toMessageEvent(event: RuntimeMessageEvent): MatrixMessageEvent { return stripUndefined({ attachments: (event.attachments ?? []).map(toAttachment), @@ -96,6 +133,7 @@ function toSyncEvent(event: Extract): init_step: "initStep", initialized: "initialized", retrying: "retrying", + skipped: "skipped", stopped: "stopped", synced: "synced", syncing: "syncing", @@ -144,6 +182,7 @@ function toCryptoState( const states = { disabled: "disabled", enabled: "enabled", + key_backup_updated: "keyBackupUpdated", key_backup_unavailable: "keyBackupUnavailable", recovery_cache_unavailable: "recoveryCacheUnavailable", recovery_key_cached: "recoveryKeyCached", diff --git a/packages/core/src/generated-runtime-operations.ts b/packages/core/src/generated-runtime-operations.ts index f8b0caa..2d8a850 100644 --- a/packages/core/src/generated-runtime-operations.ts +++ b/packages/core/src/generated-runtime-operations.ts @@ -1,6 +1,7 @@ // Code generated by go run ./native/cmd/matrix-ts-types; DO NOT EDIT. import type { + MatrixAccountDataResult, MatrixApplySyncResponseOptions, MatrixBanUserOptions, MatrixBeeperStreamOptions, @@ -25,6 +26,8 @@ import type { MatrixFetchRoomStateEventOptions, MatrixFetchRoomStateOptions, MatrixFetchRoomStateResult, + MatrixGetAccountDataOptions, + MatrixGetRoomAccountDataOptions, MatrixGetUserOptions, MatrixInviteUserOptions, MatrixJoinRoomOptions, @@ -42,6 +45,8 @@ import type { MatrixOwnAvatarURLResult, MatrixOwnDisplayNameResult, MatrixRawMessage, + MatrixRawRequestOptions, + MatrixRawRequestResult, MatrixReactionOptions, MatrixRegisterBeeperStreamOptions, MatrixResolveRoomAliasOptions, @@ -52,9 +57,14 @@ import type { MatrixSendEphemeralEventOptions, MatrixSendMediaMessageOptions, MatrixSendMessageOptions, + MatrixSendReceiptOptions, MatrixSendRoomStateEventOptions, + MatrixSendToDeviceOptions, + MatrixSendToDeviceResult, + MatrixSetAccountDataOptions, MatrixSetOwnAvatarURLOptions, MatrixSetOwnDisplayNameOptions, + MatrixSetRoomAccountDataOptions, MatrixSyncOnceOptions, MatrixSyncStartOptions, MatrixTypingOptions, @@ -72,8 +82,16 @@ export interface MatrixCoreOperations { stopSync(): Promise; init(options: MatrixCoreInitOptions): Promise; whoami(): Promise; + logout(): Promise; getCryptoStatus(): Promise; + rawRequest(options: MatrixRawRequestOptions): Promise; applySyncResponse(options: MatrixApplySyncResponseOptions): Promise; + getAccountData(options: MatrixGetAccountDataOptions): Promise; + setAccountData(options: MatrixSetAccountDataOptions): Promise; + getRoomAccountData(options: MatrixGetRoomAccountDataOptions): Promise; + setRoomAccountData(options: MatrixSetRoomAccountDataOptions): Promise; + sendToDevice(options: MatrixSendToDeviceOptions): Promise; + sendReceipt(options: MatrixSendReceiptOptions): Promise; postMessage(options: MatrixSendMessageOptions): Promise; postMediaMessage(options: MatrixSendMediaMessageOptions): Promise; editMessage(options: MatrixEditMessageOptions): Promise; @@ -142,14 +160,46 @@ export abstract class MatrixCoreOperationCaller implements MatrixCoreOperations return this.call("whoami"); } + logout(): Promise { + return this.call("logout"); + } + getCryptoStatus(): Promise { return this.call("get_crypto_status"); } + rawRequest(options: MatrixRawRequestOptions): Promise { + return this.call("raw_request", options); + } + applySyncResponse(options: MatrixApplySyncResponseOptions): Promise { return this.call("apply_sync_response", options); } + getAccountData(options: MatrixGetAccountDataOptions): Promise { + return this.call("get_account_data", options); + } + + setAccountData(options: MatrixSetAccountDataOptions): Promise { + return this.call("set_account_data", options); + } + + getRoomAccountData(options: MatrixGetRoomAccountDataOptions): Promise { + return this.call("get_room_account_data", options); + } + + setRoomAccountData(options: MatrixSetRoomAccountDataOptions): Promise { + return this.call("set_room_account_data", options); + } + + sendToDevice(options: MatrixSendToDeviceOptions): Promise { + return this.call("send_to_device", options); + } + + sendReceipt(options: MatrixSendReceiptOptions): Promise { + return this.call("send_receipt", options); + } + postMessage(options: MatrixSendMessageOptions): Promise { return this.call("post_message", options); } diff --git a/packages/core/src/generated-runtime-types.ts b/packages/core/src/generated-runtime-types.ts index 66e1aad..c204021 100644 --- a/packages/core/src/generated-runtime-types.ts +++ b/packages/core/src/generated-runtime-types.ts @@ -21,7 +21,7 @@ export interface MatrixCryptoStatus { hasRecoveryKey: boolean; keyBackupVersion?: string; pendingDecryptionCount: number /* int */; - state: "disabled" | "enabled" | "key_backup_unavailable" | "recovery_cache_unavailable" | "recovery_key_cached" | "recovery_key_loaded" | "recovery_restored" | "recovery_unverified"; + state: "disabled" | "enabled" | "key_backup_updated" | "key_backup_unavailable" | "recovery_cache_unavailable" | "recovery_key_cached" | "recovery_key_loaded" | "recovery_restored" | "recovery_unverified"; storeBacked: boolean; userId?: string; } @@ -303,8 +303,61 @@ export interface MatrixUnbanUserOptions { roomId: string; userId: string; } +export interface MatrixAccountDataResult { + content: { [key: string]: unknown}; + raw: unknown; + type: string; +} +export interface MatrixGetAccountDataOptions { + eventType: string; +} +export interface MatrixSetAccountDataOptions { + content: { [key: string]: unknown}; + eventType: string; +} +export interface MatrixGetRoomAccountDataOptions { + eventType: string; + roomId: string; +} +export interface MatrixSetRoomAccountDataOptions { + content: { [key: string]: unknown}; + eventType: string; + roomId: string; +} +export interface MatrixSendToDeviceOptions { + content?: { [key: string]: unknown}; + eventType: string; + messages?: { [key: string]: { [key: string]: { [key: string]: unknown}}}; + transactionId?: string; + userId?: string; + deviceId?: string; +} +export interface MatrixSendToDeviceResult { + raw: unknown; +} +export interface MatrixSendReceiptOptions { + content?: { [key: string]: unknown}; + eventId: string; + receiptType?: string; + roomId: string; + threadId?: string; +} +export interface MatrixRawRequestOptions { + body?: unknown; + headers?: { [key: string]: string}; + method?: string; + path: string; + query?: { [key: string]: string}; +} +export interface MatrixRawRequestResult { + body?: unknown; + raw?: unknown; + status: number /* int */; + headers?: { [key: string]: string}; +} export interface MatrixSyncOnceOptions { beeperStreaming?: boolean; + replayMissed?: boolean; timeoutMs?: number /* int */; } export interface MatrixSyncStartOptions { @@ -375,6 +428,21 @@ export interface MatrixInviteEvent { raw: unknown; roomId: string; } +export interface MatrixSyncEvent { + class: "state" | "ephemeral" | "accountData" | "toDevice" | "membership" | "redaction" | "raw" | string; + content: { [key: string]: unknown}; + decrypted?: boolean; + encrypted?: boolean; + eventId?: string; + nextBatch?: string; + originServerTs?: number /* int64 */; + raw: unknown; + roomId?: string; + section?: string; + sender?: string; + stateKey?: string; + type: string; +} export interface MatrixRoomThreadSummary { lastReplyTs?: number /* int64 */; replyCount?: number /* int */; diff --git a/packages/core/src/helpers.ts b/packages/core/src/helpers.ts new file mode 100644 index 0000000..4ab14f0 --- /dev/null +++ b/packages/core/src/helpers.ts @@ -0,0 +1,57 @@ +import type { MatrixClient } from "./client-types"; +import { stripUndefined } from "./object"; +import type { + MatrixClientEvent, + MatrixInviteEvent, + MatrixMessageEvent, + MatrixRawEventEnvelope, + MatrixReactionEvent, + MatrixSubscribeFilter, + MatrixSubscription, +} from "./types"; + +type Handler = (event: T) => void | Promise; + +export function onMessage( + client: MatrixClient, + options: Omit, "kind"> | undefined, + handler: Handler +): Promise { + return client.subscribe({ ...options, kind: "message" }, handler as Handler); +} + +export function onReaction( + client: MatrixClient, + options: Omit, "kind"> | undefined, + handler: Handler +): Promise { + return client.subscribe({ ...options, kind: "reaction" }, handler as Handler); +} + +export function onInvite( + client: MatrixClient, + options: Omit, "kind"> | undefined, + handler: Handler +): Promise { + return client.subscribe({ ...options, kind: "invite" }, handler as Handler); +} + +export function onRawEvent( + client: MatrixClient, + options: MatrixSubscribeFilter, + handler: Handler +): Promise { + return client.subscribe({ ...options, kind: "raw" }, (event) => { + if (event.kind !== "raw") return; + return handler({ + event, + kind: "raw", + raw: event.raw, + source: stripUndefined({ + kind: event.kind, + roomId: event.roomId, + type: event.type, + }), + }); + }); +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 9c615f9..607714c 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1,17 +1,30 @@ export { copyBytes } from "./bytes"; export { createMatrixClient } from "./client"; +export { onInvite, onMessage, onRawEvent, onReaction } from "./helpers"; +export { createBeeperLogin } from "./beeper-login"; export { createMatrixLogin } from "./login"; +export type { + BeeperEmailTokenOptions, + BeeperEmailTokenResult, + BeeperLogin, + BeeperLoginOptions, + BeeperRegisterOptions, + BeeperRegisterResult, +} from "./beeper-login"; export type { MatrixClient, + MatrixAccountData, MatrixBeeper, - MatrixEvents, MatrixMedia, MatrixMessages, + MatrixRaw, + MatrixReceipts, MatrixReactions, MatrixRooms, MatrixStreams, MatrixSync, MatrixTyping, + MatrixToDevice, MatrixUsers, } from "./client-types"; export type { @@ -22,6 +35,8 @@ export type { } from "./login"; export type { ApplySyncResponseOptions, + AccountDataOptions, + AccountDataResult, BanUserOptions, CreateBeeperStreamOptions, CreateRoomOptions, @@ -51,6 +66,7 @@ export type { MarkReadOptions, MatrixAttachment, MatrixBeeperStreamDescriptor, + MatrixAccount, MatrixBaseEvent, MatrixClientEvent, MatrixClientOptions, @@ -70,6 +86,10 @@ export type { MatrixSession, MatrixStore, MatrixStream, + MatrixRawEventEnvelope, + MatrixSubscribeFilter, + MatrixSubscribeOptions, + MatrixSubscription, MatrixSyncStatusEvent, MatrixThreadSummary, MatrixWhoami, @@ -83,6 +103,8 @@ export type { RegisterBeeperStreamOptions, ResolveRoomAliasOptions, ResolveRoomAliasResult, + RawRequestOptions, + RawRequestResult, RoomMember, RoomInfo, RoomPowerLevels, @@ -93,12 +115,15 @@ export type { SendMatrixStreamOptions, SendMediaMessageOptions, SendMessageOptions, + SendReceiptOptions, SendRoomStateEventOptions, + SendToDeviceOptions, + SendToDeviceResult, SetOwnAvatarUrlOptions, SetOwnDisplayNameOptions, + SetAccountDataOptions, + SetRoomAccountDataOptions, SentEvent, - SyncOnceOptions, - SyncStartOptions, TypingOptions, UnbanUserOptions, UploadEncryptedMediaResult, diff --git a/packages/core/src/login.test.ts b/packages/core/src/login.test.ts new file mode 100644 index 0000000..07e2df2 --- /dev/null +++ b/packages/core/src/login.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it, vi } from "vitest"; +import { createMatrixLogin } from "./login"; + +describe("createMatrixLogin", () => { + it("returns MatrixAccount from token login", async () => { + const fetchImpl = vi.fn(async () => Response.json({ + access_token: "access", + device_id: "DEVICE", + user_id: "@bot:example.com", + })); + const login = createMatrixLogin({ + fetch: fetchImpl as typeof fetch, + homeserver: "https://matrix.example.com", + initialDeviceDisplayName: "Bot", + metadata: { label: "cached-beeper-bot" }, + }); + + await expect(login.token({ token: "jwt", type: "org.matrix.login.jwt" })).resolves.toEqual({ + accessToken: "access", + deviceId: "DEVICE", + homeserver: "https://matrix.example.com", + metadata: { label: "cached-beeper-bot" }, + userId: "@bot:example.com", + }); + expect(await requestBody(fetchImpl)).toEqual({ + initial_device_display_name: "Bot", + token: "jwt", + type: "org.matrix.login.jwt", + }); + }); + + it("returns MatrixAccount from password login", async () => { + const fetchImpl = vi.fn(async () => Response.json({ + access_token: "access", + device_id: "DEVICE", + user_id: "@bot:example.com", + })); + const login = createMatrixLogin({ + fetch: fetchImpl as typeof fetch, + homeserver: "https://matrix.example.com", + }); + + await expect(login.password({ password: "secret", username: "bot" })).resolves.toMatchObject({ + accessToken: "access", + deviceId: "DEVICE", + }); + expect(await requestBody(fetchImpl)).toEqual({ + identifier: { type: "m.id.user", user: "bot" }, + initial_device_display_name: "Matrix", + password: "secret", + type: "m.login.password", + }); + }); +}); + +async function requestBody(fetchImpl: ReturnType) { + const init = fetchImpl.mock.calls[0]?.[1] as RequestInit; + return JSON.parse(String(init.body)); +} diff --git a/packages/core/src/login.ts b/packages/core/src/login.ts index 8e64add..9c9bf42 100644 --- a/packages/core/src/login.ts +++ b/packages/core/src/login.ts @@ -1,10 +1,10 @@ import type { MatrixSession } from "./types"; export interface MatrixLoginOptions { - deviceId?: string; fetch?: typeof fetch; homeserver: string; initialDeviceDisplayName?: string; + metadata?: Record; } export interface MatrixPasswordLoginOptions { @@ -26,19 +26,17 @@ export function createMatrixLogin(options: MatrixLoginOptions): MatrixLogin { const fetchImpl = options.fetch ?? fetch; return { password: (login) => - matrixLoginRequest(fetchImpl, options.homeserver, { + matrixLoginRequest(fetchImpl, options.homeserver, options.metadata, { identifier: { type: "m.id.user", user: login.username, }, - ...(options.deviceId ? { device_id: options.deviceId } : {}), initial_device_display_name: options.initialDeviceDisplayName ?? "Matrix", password: login.password, type: "m.login.password", }), token: (login) => - matrixLoginRequest(fetchImpl, options.homeserver, { - ...(options.deviceId ? { device_id: options.deviceId } : {}), + matrixLoginRequest(fetchImpl, options.homeserver, options.metadata, { initial_device_display_name: options.initialDeviceDisplayName ?? "Matrix", token: login.token, type: login.type ?? "m.login.token", @@ -49,6 +47,7 @@ export function createMatrixLogin(options: MatrixLoginOptions): MatrixLogin { async function matrixLoginRequest( fetchImpl: typeof fetch, homeserver: string, + metadata: Record | undefined, body: Record ): Promise { const response = await fetchImpl(new URL("/_matrix/client/v3/login", homeserver), { @@ -67,10 +66,14 @@ async function matrixLoginRequest( user_id: string; }; - return { + const session: MatrixSession = { accessToken: data.access_token, deviceId: data.device_id, homeserver, userId: data.user_id, }; + if (metadata !== undefined) { + session.metadata = { ...metadata }; + } + return session; } diff --git a/packages/core/src/media.ts b/packages/core/src/media.ts index 1a110d6..f51765e 100644 --- a/packages/core/src/media.ts +++ b/packages/core/src/media.ts @@ -10,10 +10,10 @@ import type { UploadMediaResult, } from "./types"; -export function createMatrixMedia(core: () => MatrixCore): MatrixMedia { +export function createMatrixMedia(core: () => MatrixCore | Promise): MatrixMedia { return { download: async (opts) => { - const runtime = core(); + const runtime = await core(); if (runtime.callBytesResult && runtime.supportsByteCalls?.()) { return { bytes: await runtime.callBytesResult("download_media_bytes", opts) }; } @@ -21,7 +21,7 @@ export function createMatrixMedia(core: () => MatrixCore): MatrixMedia { return { bytes: base64ToBytes(result.bytesBase64) }; }, downloadThumbnail: async (opts) => { - const runtime = core(); + const runtime = await core(); if (runtime.callBytesResult && runtime.supportsByteCalls?.()) { return { bytes: await runtime.callBytesResult("download_media_thumbnail_bytes", opts) }; } @@ -29,15 +29,15 @@ export function createMatrixMedia(core: () => MatrixCore): MatrixMedia { return { bytes: base64ToBytes(result.bytesBase64) }; }, downloadEncrypted: async (opts) => { - const runtime = core(); + const runtime = await core(); if (runtime.callBytesResult && runtime.supportsByteCalls?.()) { return { bytes: await runtime.callBytesResult("download_encrypted_media_bytes", opts) }; } const result = await runtime.downloadEncryptedMedia(opts); return { bytes: base64ToBytes(result.bytesBase64) }; }, - upload: (opts) => uploadMediaBytes(core(), opts), - uploadEncrypted: (opts) => uploadEncryptedMediaBytes(core(), opts), + upload: async (opts) => uploadMediaBytes(await core(), opts), + uploadEncrypted: async (opts) => uploadEncryptedMediaBytes(await core(), opts), }; } diff --git a/packages/core/src/node.ts b/packages/core/src/node.ts index 2e8b4a6..bed0af0 100644 --- a/packages/core/src/node.ts +++ b/packages/core/src/node.ts @@ -3,8 +3,9 @@ import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { runInThisContext } from "node:vm"; import { createMatrixClient as createRuntimeMatrixClient } from "./client"; +export { onInvite, onMessage, onRawEvent, onReaction } from "./helpers"; import type { MatrixClient } from "./client-types"; -import type { MatrixClientOptions } from "./types"; +import type { MatrixClientEvent, MatrixClientOptions, MatrixSubscribeFilter } from "./types"; import { loadMatrixCore, type LoadMatrixCoreOptions, type MatrixWasmCore } from "./wasm"; interface LoadMatrixCoreFromNodeOptions extends Omit { @@ -24,110 +25,141 @@ export function createMatrixClient(options: NodeMatrixClientOptions): MatrixClie class NodeMatrixClient implements MatrixClient { readonly #options: NodeMatrixClientOptions; #client: MatrixClient | null = null; - #eventListeners = new Set[0]>(); + #clientPromise: Promise | null = null; constructor(options: NodeMatrixClientOptions) { this.#options = options; } - get events() { - return { - on: (listener: Parameters[0]) => { - this.#eventListeners.add(listener); - const unsubscribe = this.#client?.events.on(listener); - return () => { - this.#eventListeners.delete(listener); - unsubscribe?.(); - }; - }, - onMessage: (listener: Parameters[0]) => - this.events.on((event) => { - if (event.kind === "message") listener(event); - }), - onReaction: (listener: Parameters[0]) => - this.events.on((event) => { - if (event.kind === "reaction") listener(event); - }), - }; + get accountData() { + return this.#namespace("accountData"); } get beeper() { - return this.#clientRequired().beeper; + return this.#namespace("beeper"); } get crypto() { - return this.#clientRequired().crypto; + return this.#namespace("crypto"); } get media() { - return this.#clientRequired().media; + return this.#namespace("media"); } get messages() { - return this.#clientRequired().messages; + return this.#namespace("messages"); } get reactions() { - return this.#clientRequired().reactions; + return this.#namespace("reactions"); + } + + get raw() { + return this.#namespace("raw"); + } + + get receipts() { + return this.#namespace("receipts"); } get rooms() { - return this.#clientRequired().rooms; + return this.#namespace("rooms"); } get streams() { - return this.#clientRequired().streams; + return this.#namespace("streams"); } get sync() { - return this.#clientRequired().sync; + return this.#namespace("sync"); } get typing() { - return this.#clientRequired().typing; + return this.#namespace("typing"); + } + + get toDevice() { + return this.#namespace("toDevice"); } get users() { - return this.#clientRequired().users; + return this.#namespace("users"); } async close(): Promise { await this.#client?.close(); this.#client = null; + this.#clientPromise = null; } - async connect(options?: { signal?: AbortSignal }) { - if (!this.#client) { - const { wasmExecPath, wasmPath, ...clientOptions } = this.#options; - const distDir = dirname(fileURLToPath(import.meta.url)); - if (!clientOptions.wasmBytes && !clientOptions.wasmModule) { - clientOptions.wasmBytes = await readFile(wasmPath ?? join(distDir, "matrix-core.wasm")); - } - if (!clientOptions.wasmBytes && !clientOptions.wasmModule) { - throw new Error("Matrix WASM bytes are missing"); - } - if (!globalThis.Go) { - const runtimePath = wasmExecPath ?? join(distDir, "wasm_exec.js"); - runInThisContext(await readFile(runtimePath, "utf8"), { filename: runtimePath }); - } - this.#client = createRuntimeMatrixClient(clientOptions); - for (const listener of this.#eventListeners) { - this.#client.events.on(listener); - } - } - return this.#client.connect(options); + async boot() { + return (await this.#runtime()).boot(); + } + + async subscribe( + filter: MatrixSubscribeFilter, + handler: (event: MatrixClientEvent) => void | Promise, + options?: import("./types").MatrixSubscribeOptions + ) { + return (await this.#runtime()).subscribe(filter, handler, options); + } + + async whoami() { + return (await this.#runtime()).whoami(); } - whoami() { - return this.#clientRequired().whoami(); + async logout() { + return (await this.#runtime()).logout(); } - #clientRequired(): MatrixClient { + async #runtime(): Promise { if (!this.#client) { - throw new Error("Matrix client is not connected. Call connect() first."); + this.#clientPromise ??= this.#createRuntime(); + this.#client = await this.#clientPromise; } return this.#client; } + + async #createRuntime(): Promise { + const { wasmExecPath, wasmPath, ...clientOptions } = this.#options; + const distDir = dirname(fileURLToPath(import.meta.url)); + if (!clientOptions.wasmBytes && !clientOptions.wasmModule) { + clientOptions.wasmBytes = await readFile(wasmPath ?? join(distDir, "matrix-core.wasm")); + } + if (!clientOptions.wasmBytes && !clientOptions.wasmModule) { + throw new Error("Matrix WASM bytes are missing"); + } + if (!globalThis.Go) { + const runtimePath = wasmExecPath ?? join(distDir, "wasm_exec.js"); + runInThisContext(await readFile(runtimePath, "utf8"), { filename: runtimePath }); + } + return createRuntimeMatrixClient(clientOptions); + } + + #namespace(name: K): MatrixClient[K] { + return createAsyncNamespace(async () => (await this.#runtime())[name]) as MatrixClient[K]; + } +} + +function createAsyncNamespace(load: () => Promise): T { + const build = (path: string[]): unknown => + new Proxy(async () => undefined, { + apply: async (_target, _thisArg, args) => { + let parent: unknown = await load(); + for (const key of path.slice(0, -1)) { + parent = (parent as Record)[key]; + } + const value = (parent as Record)[path[path.length - 1] ?? ""]; + if (typeof value !== "function") return value; + return value.apply(parent, args); + }, + get: (_target, prop) => { + if (typeof prop !== "string") return undefined; + return build([...path, prop]); + }, + }); + return build([]) as T; } async function loadMatrixCoreFromNodePackage( diff --git a/packages/core/src/runtime-types.ts b/packages/core/src/runtime-types.ts index c43b0b9..9a0ff4f 100644 --- a/packages/core/src/runtime-types.ts +++ b/packages/core/src/runtime-types.ts @@ -3,10 +3,12 @@ import type { MatrixMessageEvent, MatrixRawEvent, MatrixReactionEvent, + MatrixSyncEvent, } from "./generated-runtime-types"; import type { MatrixCoreOperations } from "./generated-runtime-operations"; export type { + MatrixAccountDataResult, MatrixApplySyncResponseOptions, MatrixBanUserOptions, MatrixBeeperStreamOptions, @@ -33,6 +35,8 @@ export type { MatrixFetchRoomStateEventOptions, MatrixFetchRoomStateOptions, MatrixFetchRoomStateResult, + MatrixGetAccountDataOptions, + MatrixGetRoomAccountDataOptions, MatrixGetUserOptions, MatrixInviteEvent, MatrixInviteUserOptions, @@ -56,6 +60,8 @@ export type { MatrixOwnDisplayNameResult, MatrixRawEvent, MatrixRawMessage, + MatrixRawRequestOptions, + MatrixRawRequestResult, MatrixReactionEvent, MatrixReactionOptions, MatrixRegisterBeeperStreamOptions, @@ -70,10 +76,16 @@ export type { MatrixSendEphemeralEventOptions, MatrixSendMediaMessageOptions, MatrixSendMessageOptions, + MatrixSendReceiptOptions, MatrixSendRoomStateEventOptions, + MatrixSendToDeviceOptions, + MatrixSendToDeviceResult, MatrixSetOwnAvatarURLOptions, MatrixSetOwnDisplayNameOptions, + MatrixSetAccountDataOptions, + MatrixSetRoomAccountDataOptions, MatrixSyncOnceOptions, + MatrixSyncEvent, MatrixSyncStartOptions, MatrixTypingOptions, MatrixUnbanUserOptions, @@ -88,6 +100,23 @@ export type MatrixCoreEvent = | { event: MatrixMessageEvent; type: "message" } | { event: MatrixReactionEvent; type: "reaction" } | { event: MatrixInviteEvent; type: "invite" } + | { + event: MatrixSyncEvent; + nextBatch?: string; + since?: string; + type: + | "account_data" + | "device_list" + | "ephemeral" + | "membership" + | "presence" + | "raw_event" + | "receipt" + | "redaction" + | "room_state" + | "typing" + | "to_device"; + } | { event: { content?: Record; @@ -104,6 +133,7 @@ export type MatrixCoreEvent = keyId?: string; status: | "enabled" + | "key_backup_updated" | "key_backup_unavailable" | "recovery_cache_unavailable" | "recovery_key_cached" @@ -123,7 +153,7 @@ export type MatrixCoreEvent = error?: string; failures?: number; nextRetryMs?: number; - status: "initialized" | "init_step" | "syncing" | "synced" | "retrying" | "stopped"; + status: "initialized" | "init_step" | "syncing" | "synced" | "retrying" | "skipped" | "stopped"; step?: string; type: "sync_status"; }; diff --git a/packages/core/src/streams.ts b/packages/core/src/streams.ts index 9ab1d4d..8424917 100644 --- a/packages/core/src/streams.ts +++ b/packages/core/src/streams.ts @@ -212,7 +212,8 @@ function isBeeperHomeserver(homeserverUrl: string): boolean { } function supportsBeeperFeatures(options: MatrixClientOptions): boolean { - return options.beeper ?? isBeeperHomeserver(options.homeserver); + const homeserver = options.account?.homeserver ?? options.homeserver; + return options.beeper ?? (homeserver ? isBeeperHomeserver(homeserver) : false); } function streamChunkText(chunk: string | Record): string { diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index 3a3ac6b..114b7cb 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -10,19 +10,17 @@ export interface MatrixLogger { } export interface MatrixClientOptions { + account?: MatrixAccount; beeper?: boolean; - deviceId?: string; + boot?: boolean; fetch?: typeof fetch; - homeserver: string; - initialSync?: "persisted" | "latest" | "catchUp"; + homeserver?: string; logger?: MatrixLogger; pickleKey?: string; randomBytes?: (length: number) => Uint8Array; recoveryKey?: string; - since?: string; store?: MatrixStore; - token: string; - userId?: string; + token?: string; verifyRecoveryOnStart?: boolean; wasmBytes?: BufferSource; wasmModule?: WebAssembly.Module; @@ -68,13 +66,16 @@ export interface SendBeeperEphemeralOptions { transactionId?: string; } -export interface MatrixSession { +export interface MatrixAccount { accessToken: string; deviceId: string; homeserver: string; + metadata?: Record; userId: string; } +export type MatrixSession = MatrixAccount; + export interface MatrixWhoami { deviceId: string; userId: string; @@ -175,13 +176,33 @@ export interface MatrixInviteEvent { roomId: string; } +export interface MatrixGenericEvent extends MatrixBaseEvent { + decrypted?: boolean; + encrypted?: boolean; + kind: + | "accountData" + | "deviceList" + | "ephemeral" + | "membership" + | "presence" + | "raw" + | "receipt" + | "redaction" + | "roomState" + | "typing" + | "toDevice"; + nextBatch?: string; + section?: string; + since?: string; +} + export interface MatrixSyncStatusEvent { durationMs?: number; error?: string; failures?: number; kind: "sync"; nextRetryMs?: number; - state: "initialized" | "initStep" | "syncing" | "synced" | "retrying" | "stopped"; + state: "initialized" | "initStep" | "syncing" | "synced" | "retrying" | "skipped" | "stopped"; step?: string; } @@ -192,6 +213,7 @@ export interface MatrixCryptoStatusEvent { kind: "crypto"; state: | "enabled" + | "keyBackupUpdated" | "keyBackupUnavailable" | "recoveryCacheUnavailable" | "recoveryKeyCached" @@ -208,6 +230,7 @@ export interface MatrixCryptoStatus { state: | "disabled" | "enabled" + | "keyBackupUpdated" | "keyBackupUnavailable" | "recoveryCacheUnavailable" | "recoveryKeyCached" @@ -218,6 +241,64 @@ export interface MatrixCryptoStatus { userId?: string; } +export interface RawRequestOptions { + body?: unknown; + headers?: Record; + method?: "DELETE" | "GET" | "PATCH" | "POST" | "PUT" | string; + path: string; + query?: Record; +} + +export interface RawRequestResult { + body?: unknown; + headers?: Record; + raw?: unknown; + status: number; +} + +export interface AccountDataResult { + content: Record; + raw: unknown; + type: string; +} + +export interface AccountDataOptions { + eventType: string; +} + +export interface SetAccountDataOptions extends AccountDataOptions { + content: Record; +} + +export interface RoomAccountDataOptions extends AccountDataOptions { + roomId: string; +} + +export interface SetRoomAccountDataOptions extends RoomAccountDataOptions { + content: Record; +} + +export interface SendToDeviceOptions { + content?: Record; + deviceId?: string; + eventType: string; + messages?: Record>>; + transactionId?: string; + userId?: string; +} + +export interface SendToDeviceResult { + raw: unknown; +} + +export interface SendReceiptOptions { + content?: Record; + eventId: string; + receiptType?: "m.read" | "m.read.private" | "m.fully_read" | string; + roomId: string; + threadId?: string; +} + export interface MatrixDecryptionErrorEvent { error: string; event?: Pick & { senderId?: string }; @@ -233,11 +314,46 @@ export type MatrixClientEvent = | MatrixMessageEvent | MatrixReactionEvent | MatrixInviteEvent + | MatrixGenericEvent | MatrixSyncStatusEvent | MatrixCryptoStatusEvent | MatrixDecryptionErrorEvent | MatrixErrorEvent; +export type MatrixSubscribeFilter = + | { + kind?: MatrixClientEvent["kind"] | MatrixClientEvent["kind"][]; + relationEventId?: string | string[]; + roomId?: string | string[]; + sender?: string | string[]; + threadRoot?: string | string[]; + type?: string | string[]; + } + | undefined; + +export interface MatrixSubscribeOptions { + live?: boolean; + retryDelayMs?: number; + timeoutMs?: number; +} + +export interface MatrixSubscription { + catchUp(): Promise; + done: Promise; + stop(): Promise; +} + +export interface MatrixRawEventEnvelope { + event: MatrixGenericEvent; + kind: "raw"; + raw: unknown; + source: { + kind: MatrixClientEvent["kind"]; + roomId?: string; + type?: string; + }; +} + export interface SendMessageOptions { content?: Record; html?: string; @@ -582,18 +698,6 @@ export interface ListThreadsResult { threads: MatrixThreadSummary[]; } -export interface SyncStartOptions { - beeper?: boolean; - retryDelayMs?: number; - signal?: AbortSignal; - timeoutMs?: number; -} - -export interface SyncOnceOptions { - beeper?: boolean; - timeoutMs?: number; -} - export interface ApplySyncResponseOptions { response: unknown; since?: string; diff --git a/packages/core/test/store-conformance.ts b/packages/core/test/store-conformance.ts new file mode 100644 index 0000000..ca11dc0 --- /dev/null +++ b/packages/core/test/store-conformance.ts @@ -0,0 +1,30 @@ +import { expect, it } from "vitest"; +import type { MatrixStore } from "../src/types"; + +export function testMatrixStoreConformance( + name: string, + createStore: () => MatrixStore | Promise +): void { + it(`${name} conforms to MatrixStore`, async () => { + const store = await createStore(); + const original = new Uint8Array([1, 2, 3]); + + await store.set("crypto/account", original); + await store.set("sync/next", new Uint8Array([4])); + original[0] = 9; + + expect([...(await store.get("crypto/account"))!]).toEqual([1, 2, 3]); + expect(await store.get("missing")).toBeNull(); + expect(await store.list("crypto/")).toEqual(["crypto/account"]); + expect(await store.list("sync/")).toEqual(["sync/next"]); + + const fetched = (await store.get("crypto/account"))!; + fetched[1] = 9; + expect([...(await store.get("crypto/account"))!]).toEqual([1, 2, 3]); + + await store.delete("crypto/account"); + expect(await store.get("crypto/account")).toBeNull(); + expect(await store.list("crypto/")).toEqual([]); + expect(await store.list("")).toEqual(["sync/next"]); + }); +} diff --git a/packages/core/tsup.config.ts b/packages/core/tsup.config.ts index 8b09969..e5b493d 100644 --- a/packages/core/tsup.config.ts +++ b/packages/core/tsup.config.ts @@ -1,7 +1,7 @@ import { defineConfig } from "tsup"; export default defineConfig({ - entry: ["src/index.ts", "src/node.ts"], + entry: ["src/index.ts", "src/beeper-login.ts", "src/helpers.ts", "src/login.ts", "src/node.ts", "src/types.ts"], format: ["esm"], dts: true, clean: true, diff --git a/packages/state-file/src/index.test.ts b/packages/state-file/src/index.test.ts index 9b300f1..42ab706 100644 --- a/packages/state-file/src/index.test.ts +++ b/packages/state-file/src/index.test.ts @@ -2,9 +2,15 @@ import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; +import { testMatrixStoreConformance } from "../../core/test/store-conformance"; import { createFileMatrixStore } from "./index"; describe("FileMatrixStore", () => { + testMatrixStoreConformance("FileMatrixStore", async () => { + const dir = await mkdtemp(join(tmpdir(), "matrix-store-conformance-")); + return createFileMatrixStore(dir); + }); + it("round-trips bytes and lists by prefix", async () => { const dir = await mkdtemp(join(tmpdir(), "matrix-store-")); try { diff --git a/packages/state-file/vitest.config.ts b/packages/state-file/vitest.config.ts index bdbea6f..61a25bb 100644 --- a/packages/state-file/vitest.config.ts +++ b/packages/state-file/vitest.config.ts @@ -1,6 +1,11 @@ import { defineProject } from "vitest/config"; export default defineProject({ + resolve: { + alias: { + "better-matrix-js": new URL("../core/src/index.ts", import.meta.url).pathname, + }, + }, test: { coverage: { include: ["src/**/*.ts"], diff --git a/packages/state-indexeddb/src/index.test.ts b/packages/state-indexeddb/src/index.test.ts index 40e7ea6..7c5f705 100644 --- a/packages/state-indexeddb/src/index.test.ts +++ b/packages/state-indexeddb/src/index.test.ts @@ -1,8 +1,14 @@ import { IDBFactory } from "fake-indexeddb"; import { describe, expect, it } from "vitest"; +import { testMatrixStoreConformance } from "../../core/test/store-conformance"; import { createIndexedDBMatrixStore } from "./index"; describe("IndexedDBMatrixStore", () => { + testMatrixStoreConformance("IndexedDBMatrixStore", () => createIndexedDBMatrixStore({ + databaseName: `matrix-store-conformance-${crypto.randomUUID()}`, + indexedDB: new IDBFactory(), + })); + it("round-trips bytes and lists by prefix", async () => { const store = createIndexedDBMatrixStore({ databaseName: `matrix-store-${crypto.randomUUID()}`, diff --git a/packages/state-indexeddb/vitest.config.ts b/packages/state-indexeddb/vitest.config.ts index bdbea6f..61a25bb 100644 --- a/packages/state-indexeddb/vitest.config.ts +++ b/packages/state-indexeddb/vitest.config.ts @@ -1,6 +1,11 @@ import { defineProject } from "vitest/config"; export default defineProject({ + resolve: { + alias: { + "better-matrix-js": new URL("../core/src/index.ts", import.meta.url).pathname, + }, + }, test: { coverage: { include: ["src/**/*.ts"], diff --git a/packages/state-memory/src/index.test.ts b/packages/state-memory/src/index.test.ts index fb7de36..40f42e6 100644 --- a/packages/state-memory/src/index.test.ts +++ b/packages/state-memory/src/index.test.ts @@ -1,7 +1,10 @@ import { describe, expect, it } from "vitest"; +import { testMatrixStoreConformance } from "../../core/test/store-conformance"; import { createMemoryMatrixStore } from "./index"; describe("MemoryMatrixStore", () => { + testMatrixStoreConformance("MemoryMatrixStore", () => createMemoryMatrixStore()); + it("copies bytes on set/get", async () => { const store = createMemoryMatrixStore(); const original = new Uint8Array([1, 2, 3]); diff --git a/packages/state-memory/vitest.config.ts b/packages/state-memory/vitest.config.ts index bdbea6f..61a25bb 100644 --- a/packages/state-memory/vitest.config.ts +++ b/packages/state-memory/vitest.config.ts @@ -1,6 +1,11 @@ import { defineProject } from "vitest/config"; export default defineProject({ + resolve: { + alias: { + "better-matrix-js": new URL("../core/src/index.ts", import.meta.url).pathname, + }, + }, test: { coverage: { include: ["src/**/*.ts"], diff --git a/packages/state-simple/src/index.test.ts b/packages/state-simple/src/index.test.ts index 2e534ce..63b1956 100644 --- a/packages/state-simple/src/index.test.ts +++ b/packages/state-simple/src/index.test.ts @@ -1,7 +1,23 @@ import { describe, expect, it } from "vitest"; +import { testMatrixStoreConformance } from "../../core/test/store-conformance"; import { createMatrixStore } from "./index"; describe("createMatrixStore", () => { + testMatrixStoreConformance("SimpleMatrixStore", () => { + const values = new Map(); + return createMatrixStore({ + async delete(key) { + values.delete(key); + }, + async get(key) { + return values.get(key) ?? null; + }, + async set(key, value) { + values.set(key, value); + }, + }); + }); + it("adapts simple get/set stores and maintains an index for list()", async () => { const values = new Map(); const store = createMatrixStore({ diff --git a/packages/state-simple/vitest.config.ts b/packages/state-simple/vitest.config.ts index bdbea6f..61a25bb 100644 --- a/packages/state-simple/vitest.config.ts +++ b/packages/state-simple/vitest.config.ts @@ -1,6 +1,11 @@ import { defineProject } from "vitest/config"; export default defineProject({ + resolve: { + alias: { + "better-matrix-js": new URL("../core/src/index.ts", import.meta.url).pathname, + }, + }, test: { coverage: { include: ["src/**/*.ts"], diff --git a/packages/state-sqlite/src/index.test.ts b/packages/state-sqlite/src/index.test.ts index aa5dbb6..98312cf 100644 --- a/packages/state-sqlite/src/index.test.ts +++ b/packages/state-sqlite/src/index.test.ts @@ -2,9 +2,15 @@ import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; +import { testMatrixStoreConformance } from "../../core/test/store-conformance"; import { createSQLiteMatrixStore } from "./index"; describe("SQLiteMatrixStore", () => { + testMatrixStoreConformance("SQLiteMatrixStore", async () => { + const dir = await mkdtemp(join(tmpdir(), "matrix-sqlite-conformance-")); + return createSQLiteMatrixStore(join(dir, "matrix.db")); + }); + it("round-trips bytes and lists by prefix", async () => { const dir = await mkdtemp(join(tmpdir(), "matrix-sqlite-store-")); try { diff --git a/packages/state-sqlite/vitest.config.ts b/packages/state-sqlite/vitest.config.ts index bdbea6f..61a25bb 100644 --- a/packages/state-sqlite/vitest.config.ts +++ b/packages/state-sqlite/vitest.config.ts @@ -1,6 +1,11 @@ import { defineProject } from "vitest/config"; export default defineProject({ + resolve: { + alias: { + "better-matrix-js": new URL("../core/src/index.ts", import.meta.url).pathname, + }, + }, test: { coverage: { include: ["src/**/*.ts"], diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d86d192..c76bc21 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -52,6 +52,15 @@ importers: specifier: ^4.85.0 version: 4.87.0 + examples/dummybridge-bot: + dependencies: + '@better-matrix-js/state-file': + specifier: workspace:* + version: link:../../packages/state-file + better-matrix-js: + specifier: workspace:* + version: link:../../packages/core + packages/ai-sdk: devDependencies: '@better-matrix-js/chat-adapter': diff --git a/scripts/audit-package-surface.mjs b/scripts/audit-package-surface.mjs new file mode 100644 index 0000000..c0d6aef --- /dev/null +++ b/scripts/audit-package-surface.mjs @@ -0,0 +1,48 @@ +import { readFile, readdir } from "node:fs/promises"; +import { join, relative } from "node:path"; + +const root = new URL("..", import.meta.url).pathname; +const packagesDir = join(root, "packages"); +const packages = await readdir(packagesDir, { withFileTypes: true }); +const failures = []; + +for (const entry of packages) { + if (!entry.isDirectory()) { + continue; + } + const packageDir = join(packagesDir, entry.name); + const packageJson = JSON.parse(await readFile(join(packageDir, "package.json"), "utf8")); + const sourceDir = join(packageDir, "src"); + for (const file of await sourceFiles(sourceDir)) { + const source = await readFile(file, "utf8"); + const imports = [...source.matchAll(/\bfrom\s+["']([^"']+)["']/g)].map((match) => match[1]); + for (const specifier of imports) { + if (specifier === "./index" || specifier === `${packageJson.name}`) { + failures.push(`${relative(root, file)} imports ${specifier}`); + } + } + } +} + +const aiPackage = JSON.parse(await readFile(join(packagesDir, "ai-sdk/package.json"), "utf8")); +if (aiPackage.dependencies?.ai || aiPackage.peerDependencies?.ai) { + failures.push("@better-matrix-js/ai-sdk must not require the AI SDK at runtime"); +} + +if (failures.length > 0) { + console.error(failures.join("\n")); + process.exit(1); +} + +async function sourceFiles(dir) { + const result = []; + for (const entry of await readdir(dir, { withFileTypes: true })) { + const file = join(dir, entry.name); + if (entry.isDirectory()) { + result.push(...await sourceFiles(file)); + } else if (entry.name.endsWith(".ts") && !entry.name.endsWith(".test.ts")) { + result.push(file); + } + } + return result; +} diff --git a/scripts/docker-redis-store-smoke.mjs b/scripts/docker-redis-store-smoke.mjs new file mode 100644 index 0000000..8219ef0 --- /dev/null +++ b/scripts/docker-redis-store-smoke.mjs @@ -0,0 +1,202 @@ +import assert from "node:assert/strict"; +import { Socket } from "node:net"; +import { createMatrixStore } from "../packages/state-simple/dist/index.js"; + +const host = process.env.BMJS_E2E_REDIS_HOST ?? "127.0.0.1"; +const port = Number(process.env.BMJS_E2E_REDIS_PORT ?? 6379); +const prefix = process.env.BMJS_E2E_REDIS_PREFIX ?? `bmjs:e2e:${Date.now()}:`; + +async function eventually(label, fn, timeoutMs = 30000) { + const started = Date.now(); + let lastError; + while (Date.now() - started < timeoutMs) { + try { + const result = await fn(); + if (result) { + return result; + } + } catch (error) { + lastError = error; + } + await new Promise((resolve) => setTimeout(resolve, 250)); + } + throw new Error(`${label} timed out${lastError ? `: ${lastError.message}` : ""}`); +} + +class RedisConnection { + static connect(options) { + const socket = new Socket(); + const connection = new RedisConnection(socket); + return new Promise((resolve, reject) => { + socket.once("error", reject); + socket.connect(options.port, options.host, () => { + socket.off("error", reject); + socket.on("data", (chunk) => connection.push(chunk)); + socket.on("error", (error) => connection.fail(error)); + resolve(connection); + }); + }); + } + + constructor(socket) { + this.buffer = Buffer.alloc(0); + this.pending = []; + this.socket = socket; + } + + command(...args) { + return new Promise((resolve, reject) => { + this.pending.push({ reject, resolve }); + this.socket.write(encodeCommand(args)); + }); + } + + close() { + this.socket.destroy(); + } + + fail(error) { + for (const pending of this.pending.splice(0)) { + pending.reject(error); + } + } + + push(chunk) { + this.buffer = Buffer.concat([this.buffer, chunk]); + while (this.pending.length > 0) { + const parsed = parseReply(this.buffer, 0); + if (!parsed) { + return; + } + this.buffer = this.buffer.subarray(parsed.offset); + const pending = this.pending.shift(); + if (parsed.error) { + pending.reject(parsed.error); + } else { + pending.resolve(parsed.value); + } + } + } +} + +function encodeCommand(args) { + const chunks = [Buffer.from(`*${args.length}\r\n`)]; + for (const arg of args) { + const value = arg instanceof Uint8Array ? Buffer.from(arg) : Buffer.from(String(arg)); + chunks.push(Buffer.from(`$${value.length}\r\n`), value, Buffer.from("\r\n")); + } + return Buffer.concat(chunks); +} + +function parseReply(buffer, offset) { + if (offset >= buffer.length) { + return null; + } + const type = String.fromCharCode(buffer[offset]); + if (type === "+") { + const line = readLine(buffer, offset + 1); + return line && { offset: line.offset, value: line.value }; + } + if (type === "-") { + const line = readLine(buffer, offset + 1); + return line && { error: new Error(line.value), offset: line.offset }; + } + if (type === ":") { + const line = readLine(buffer, offset + 1); + return line && { offset: line.offset, value: Number(line.value) }; + } + if (type === "$") { + const line = readLine(buffer, offset + 1); + if (!line) { + return null; + } + const length = Number(line.value); + if (length < 0) { + return { offset: line.offset, value: null }; + } + const end = line.offset + length; + if (buffer.length < end + 2) { + return null; + } + return { offset: end + 2, value: new Uint8Array(buffer.subarray(line.offset, end)) }; + } + if (type === "*") { + const line = readLine(buffer, offset + 1); + if (!line) { + return null; + } + const length = Number(line.value); + if (length < 0) { + return { offset: line.offset, value: null }; + } + const values = []; + let cursor = line.offset; + for (let index = 0; index < length; index += 1) { + const parsed = parseReply(buffer, cursor); + if (!parsed) { + return null; + } + values.push(parsed.value instanceof Uint8Array ? new TextDecoder().decode(parsed.value) : parsed.value); + cursor = parsed.offset; + } + return { offset: cursor, value: values }; + } + throw new Error(`Unsupported Redis reply type ${type}`); +} + +function readLine(buffer, offset) { + const end = buffer.indexOf("\r\n", offset); + if (end === -1) { + return null; + } + return { + offset: end + 2, + value: buffer.subarray(offset, end).toString("utf8"), + }; +} + +const redis = await RedisConnection.connect({ host, port }); + +try { + await eventually("redis ping", async () => (await redis.command("PING")) === "PONG"); + const store = createMatrixStore({ + delete: (key) => redis.command("DEL", prefix + key), + get: async (key) => { + const value = await redis.command("GET", prefix + key); + return value instanceof Uint8Array ? value : null; + }, + list: async (keyPrefix) => { + const keys = await redis.command("KEYS", prefix + keyPrefix + "*"); + assert.ok(Array.isArray(keys)); + return keys.map((key) => key.slice(prefix.length)).sort(); + }, + set: (key, value) => redis.command("SET", prefix + key, value), + }); + + const first = new Uint8Array([0, 1, 2, 253, 254, 255]); + const second = new TextEncoder().encode("second value"); + await store.set("crypto/account", first); + await store.set("crypto/session/1", second); + await store.set("state/room", new Uint8Array([42])); + + assert.deepEqual(await store.get("crypto/account"), first); + assert.deepEqual(await store.get("crypto/session/1"), second); + assert.deepEqual(await store.list("crypto/"), ["crypto/account", "crypto/session/1"]); + + const fetched = await store.get("crypto/account"); + assert.ok(fetched); + fetched[0] = 99; + assert.deepEqual(await store.get("crypto/account"), first, "store reads must be defensive copies"); + + await store.delete("crypto/account"); + assert.equal(await store.get("crypto/account"), null); + assert.deepEqual(await store.list("crypto/"), ["crypto/session/1"]); + + console.log("redis store smoke passed"); +} finally { + const keys = await redis.command("KEYS", prefix + "*").catch(() => []); + if (Array.isArray(keys) && keys.length > 0) { + await redis.command("DEL", ...keys); + } + redis.close(); +} diff --git a/scripts/live-e2e.mjs b/scripts/live-e2e.mjs index dbe112f..b96f9cf 100644 --- a/scripts/live-e2e.mjs +++ b/scripts/live-e2e.mjs @@ -36,15 +36,9 @@ async function createAccount(role, runDir) { store: createFileMatrixStore(join(runDir, role.toLowerCase())), token: accessToken, }); - client.events.on((event) => events.push(event)); - const whoami = await client.connect(); - return { client, events, role, userId: whoami.userId }; -} - -async function sync(client, count = 1, timeoutMs = 3_000) { - for (let index = 0; index < count; index += 1) { - await client.sync.once({ timeoutMs }); - } + const whoami = await client.whoami(); + const subscription = await client.subscribe({}, (event) => events.push(event)); + return { client, events, role, subscription, userId: whoami.userId }; } async function retry(label, fn, timeoutMs = 30_000) { @@ -74,11 +68,11 @@ async function joinRoomIfNeeded(client, roomId) { async function syncUntil(label, account, predicate, timeoutMs = 45_000) { const started = Date.now(); while (Date.now() - started < timeoutMs) { - await sync(account.client); const match = predicate(); if (match) { return match; } + await new Promise((resolve) => setTimeout(resolve, 500)); } throw new Error(`${label} timed out`); } @@ -126,10 +120,8 @@ async function main() { createAccount("PEER", runDir), ]); - await Promise.all([sync(bot.client), sync(peer.client)]); const dm = await bot.client.rooms.openDM({ userId: peer.userId }); await retry("peer join", async () => joinRoomIfNeeded(peer.client, dm.roomId), timeoutMs); - await Promise.all([sync(bot.client, 5), sync(peer.client, 5)]); const room = await bot.client.rooms.get({ roomId: dm.roomId }); if (!room.encrypted) { @@ -145,8 +137,6 @@ async function main() { if (!seenByPeer.encrypted) { throw new Error("Peer received bot message, but it was not marked encrypted"); } - await Promise.all([sync(bot.client, 5), sync(peer.client, 5)]); - const peerText = `hello from peer ${Date.now()}`; const peerMessage = await peer.client.messages.send({ text: peerText, roomId: dm.roomId }); const seenByBot = await syncUntil("bot receives peer message", bot, () => diff --git a/scripts/package-consumer-smoke.mjs b/scripts/package-consumer-smoke.mjs index 30ba3e2..a5fd7a2 100644 --- a/scripts/package-consumer-smoke.mjs +++ b/scripts/package-consumer-smoke.mjs @@ -82,12 +82,18 @@ try { "--eval", ` import * as core from "better-matrix-js"; + import * as beeperLogin from "better-matrix-js/beeper-login"; + import * as helpers from "better-matrix-js/helpers"; + import * as login from "better-matrix-js/login"; import * as node from "better-matrix-js/node"; import * as cloudflare from "@better-matrix-js/cloudflare"; import * as adapter from "@better-matrix-js/chat-adapter"; const checks = { core: ["createMatrixClient", "createMatrixLogin"].every((key) => key in core), + beeperLogin: ["createBeeperLogin"].every((key) => key in beeperLogin), + helpers: ["onMessage", "onReaction", "onInvite", "onRawEvent"].every((key) => key in helpers), + login: ["createMatrixLogin"].every((key) => key in login), node: ["createMatrixClient"].every((key) => key in node), cloudflare: ["createCloudflareKVMatrixStore", "createDurableObjectMatrixStore", "MatrixSyncDurableObject"].every((key) => key in cloudflare), adapter: ["createMatrixAdapter", "MatrixAdapter", "MatrixFormatConverter"].every((key) => key in adapter),