Initial import of LogDarioSeq baseline
Establishes version control for the personal Logseq DB fork before starting the server-client architecture migration and domain features described in docs/agent-guide and the approved implementation plan. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
5
deps/db-sync/.carve/config.edn
vendored
Normal file
5
deps/db-sync/.carve/config.edn
vendored
Normal file
@@ -0,0 +1,5 @@
|
||||
{:paths ["src"]
|
||||
:api-namespaces [;; CLI entry points
|
||||
logseq.db-sync.node.find-checksum-drift
|
||||
logseq.db-sync.node.recompute-log-checksum]
|
||||
:report {:format :ignore}}
|
||||
38
deps/db-sync/.carve/ignore
vendored
Normal file
38
deps/db-sync/.carve/ignore
vendored
Normal file
@@ -0,0 +1,38 @@
|
||||
;; API
|
||||
logseq.db-sync.cycle/fix-cycle!
|
||||
;; API
|
||||
logseq.db-sync.node.entry/main
|
||||
;; API (CLI entrypoint)
|
||||
logseq.db-sync.node.show-checksum/main
|
||||
;; API
|
||||
logseq.db-sync.order/fix-duplicate-orders!
|
||||
;; TODO: is this ns needed?
|
||||
logseq.db-sync.platform.cloudflare/response
|
||||
logseq.db-sync.platform.cloudflare/request
|
||||
logseq.db-sync.platform.cloudflare/request-url
|
||||
;; testing
|
||||
logseq.db-sync.protocol/tx->transit
|
||||
;; debugging
|
||||
logseq.db-sync.protocol/datoms->wire
|
||||
;; debugging
|
||||
logseq.db-sync.protocol/datoms->wire
|
||||
;; debugging
|
||||
logseq.db-sync.sentry.node/capture-exception!
|
||||
;; API
|
||||
logseq.db-sync.snapshot/framed-length
|
||||
logseq.db-sync.snapshot/finalize-datoms-jsonl-buffer
|
||||
;; API
|
||||
logseq.db-sync.worker/worker
|
||||
;; debugging
|
||||
logseq.db-sync.worker.timing/summary
|
||||
;; API
|
||||
logseq.db-sync.snapshot/finalize-datoms-jsonl-buffer
|
||||
;; API
|
||||
logseq.db-sync.checksum/recompute-checksum-diagnostics
|
||||
;; API (test runner entrypoint)
|
||||
logseq.db-sync.test-runner/main
|
||||
;; Disabled FIXME tests in node_adapter_test.cljs
|
||||
logseq.db-sync.node-adapter-test/post-json
|
||||
logseq.db-sync.node-adapter-test/get-json
|
||||
logseq.db-sync.node-adapter-test/parse-json
|
||||
logseq.db-sync.node-adapter-test/start-test-server
|
||||
28
deps/db-sync/.clj-condo/config.edn
vendored
Normal file
28
deps/db-sync/.clj-condo/config.edn
vendored
Normal file
@@ -0,0 +1,28 @@
|
||||
{:linters
|
||||
{:aliased-namespace-symbol {:level :warning}
|
||||
:namespace-name-mismatch {:level :warning}
|
||||
:used-underscored-binding {:level :warning}
|
||||
:shadowed-var {:level :warning
|
||||
:exclude [meta name key keys uuid type]}
|
||||
|
||||
:consistent-alias
|
||||
{:aliases {clojure.pprint pprint
|
||||
clojure.string string
|
||||
datascript.core d
|
||||
datascript.transit dt
|
||||
logseq.db-sync.common common
|
||||
logseq.db-sync.config config
|
||||
logseq.db-sync.node.routes node-routes
|
||||
logseq.db-sync.platform.core platform
|
||||
logseq.db-sync.protocol protocol
|
||||
logseq.db-sync.storage storage
|
||||
logseq.db-sync.worker.auth auth
|
||||
logseq.db-sync.worker.handler.sync sync-handler
|
||||
logseq.db-sync.worker.handler.ws ws-handler
|
||||
logseq.db-sync.worker.http http
|
||||
logseq.db-sync.worker.routes.index routes
|
||||
logseq.db-sync.worker.ws ws}}}
|
||||
:lint-as {promesa.core/let clojure.core/let
|
||||
shadow.cljs.modern/defclass clj-kondo.lint-as/def-catch-all}
|
||||
:skip-comments true
|
||||
:output {:progress true}}
|
||||
151
deps/db-sync/README.md
vendored
Normal file
151
deps/db-sync/README.md
vendored
Normal file
@@ -0,0 +1,151 @@
|
||||
# Logseq DB Sync (deps/db-sync)
|
||||
|
||||
This package contains the DB sync server code and tests used by Logseq.
|
||||
It includes the Cloudflare Worker implementation and a Node.js adapter for self-hosting.
|
||||
|
||||
## Requirements
|
||||
- Node.js (see repo root for required version)
|
||||
- Clojure (for shadow-cljs builds)
|
||||
|
||||
## Build and Test
|
||||
|
||||
### Cloudflare Worker
|
||||
|
||||
```bash
|
||||
cd deps/db-sync
|
||||
pnpm watch
|
||||
|
||||
# open another terminal
|
||||
cd deps/db-sync/worker
|
||||
wrangler dev
|
||||
```
|
||||
|
||||
### D1 Schema (Worker)
|
||||
|
||||
The worker no longer initializes schema at request time. Apply the D1 schema
|
||||
via migrations during deployment/CI.
|
||||
|
||||
```bash
|
||||
cd deps/db-sync/worker
|
||||
wrangler d1 migrations apply logseq-sync-graph-meta-staging --env staging
|
||||
wrangler d1 migrations apply logseq-sync-graphs-prod --env prod
|
||||
```
|
||||
|
||||
For local development, run `wrangler d1 migrations apply DB --local`.
|
||||
|
||||
### Production Graph Lookup
|
||||
|
||||
Show the graphs available to a production user by `username` or `user id`:
|
||||
|
||||
```bash
|
||||
cd deps/db-sync
|
||||
pnpm show-graphs-for-user --username alice
|
||||
pnpm show-graphs-for-user --user-id us-east-1:example-user-id
|
||||
```
|
||||
|
||||
The script uses `worker/wrangler.toml`, runs against the remote D1 binding `DB`,
|
||||
defaults to `--env prod`, and prints JSON when `--json` is added.
|
||||
|
||||
Show usage totals (total users, total graphs, users created today, graphs created today):
|
||||
|
||||
```bash
|
||||
cd deps/db-sync
|
||||
pnpm show-usage-stats
|
||||
pnpm show-usage-stats --days 7
|
||||
pnpm show-usage-stats --json
|
||||
```
|
||||
|
||||
`created today` uses UTC day boundaries from D1 (`date('now')`).
|
||||
`active_*_last_n_days` uses deduplicated UTC-day activity rows from
|
||||
`daily_active_entities` with the provided `--days` window.
|
||||
|
||||
Download a graph snapshot into a local sqlite debug file matching local graph DB schema (`kvs` table only):
|
||||
|
||||
```bash
|
||||
cd deps/db-sync
|
||||
pnpm run download-graph-db -- --graph-id 6f2d7f6f-xxxx-xxxx-xxxx-xxxxxxxxxxxx --admin-token <admin-token>
|
||||
```
|
||||
|
||||
You can also pass `--admin-token <token>` or set `DB_SYNC_ADMIN_TOKEN`. The output defaults to
|
||||
`tmp/graph-<graph-id>.snapshot.sqlite` and can be changed with `--output`.
|
||||
|
||||
Show stored and recomputed checksum for a local sqlite graph db:
|
||||
|
||||
```bash
|
||||
cd deps/db-sync
|
||||
pnpm run show-sqlite-checksum -- --db ~/Downloads/test.sqlite
|
||||
```
|
||||
|
||||
Delete the graphs owned by a production user after an explicit confirmation:
|
||||
|
||||
```bash
|
||||
cd deps/db-sync
|
||||
pnpm delete-graphs-for-user --username alice
|
||||
pnpm delete-graphs-for-user --user-id us-east-1:example-user-id
|
||||
```
|
||||
|
||||
The delete script shows the owned graphs first and requires typing `DELETE`
|
||||
before it calls the worker delete endpoint for each graph. Set
|
||||
`DB_SYNC_BASE_URL` and `DB_SYNC_ADMIN_TOKEN` or pass `--base-url` and
|
||||
`--admin-token` when running it.
|
||||
|
||||
Delete a user completely (owned graphs, memberships, keys, and user row):
|
||||
|
||||
```bash
|
||||
cd deps/db-sync
|
||||
pnpm run delete-user-totally -- --username alice
|
||||
pnpm run delete-user-totally -- --user-id us-east-1:example-user-id
|
||||
```
|
||||
|
||||
The script prints all linked graphs first, deletes owned graphs through the
|
||||
admin graph delete endpoint, then removes the user's remaining D1 references.
|
||||
It requires typing `DELETE` as confirmation.
|
||||
|
||||
### Node.js Adapter (self-hosted)
|
||||
|
||||
Build the adapter:
|
||||
|
||||
```bash
|
||||
cd deps/db-sync
|
||||
pnpm build:node-adapter
|
||||
```
|
||||
|
||||
Run the adapter with Cognito auth:
|
||||
|
||||
```bash
|
||||
DB_SYNC_PORT=8787 \
|
||||
COGNITO_ISSUER=https://cognito-idp.us-east-1.amazonaws.com/us-east-1_dtagLnju8 \
|
||||
COGNITO_CLIENT_ID=69cs1lgme7p8kbgld8n5kseii6 \
|
||||
COGNITO_JWKS_URL=https://cognito-idp.us-east-1.amazonaws.com/us-east-1_dtagLnju8/.well-known/jwks.json \
|
||||
node worker/dist/node-adapter.js
|
||||
```
|
||||
|
||||
### Tests
|
||||
|
||||
Run db-sync tests (includes Node adapter tests):
|
||||
|
||||
```bash
|
||||
cd deps/db-sync
|
||||
pnpm test:node-adapter
|
||||
```
|
||||
|
||||
## Environment Variables
|
||||
|
||||
| Variable | Purpose |
|
||||
| --- | --- |
|
||||
| DB_SYNC_PORT | HTTP server port |
|
||||
| DB_SYNC_BASE_URL | External base URL for asset links |
|
||||
| DB_SYNC_ADMIN_TOKEN | Admin-only token for operator graph deletion endpoints |
|
||||
| DB_SYNC_DATA_DIR | Data directory for sqlite + assets |
|
||||
| DB_SYNC_STORAGE_DRIVER | Storage backend selection (sqlite) |
|
||||
| DB_SYNC_ASSETS_DRIVER | Assets backend selection (filesystem) |
|
||||
| SENTRY_DSN | Sentry DSN |
|
||||
| SENTRY_RELEASE | Release identifier for Sentry events and sourcemaps |
|
||||
| SENTRY_ENVIRONMENT | Sentry environment name (prod, staging, etc.) |
|
||||
| SENTRY_TRACES_SAMPLE_RATE | Traces sample rate (0.0 - 1.0) |
|
||||
| COGNITO_ISSUER | Cognito issuer URL |
|
||||
| COGNITO_CLIENT_ID | Cognito client id |
|
||||
| COGNITO_JWKS_URL | Cognito JWKS URL |
|
||||
|
||||
## Notes
|
||||
- Runtime and db-sync engineering guidance is consolidated in `../../docs/agent-guide/implemented/architecture/2026-08-24-logseq-runtime-and-engineering-guide.md`; current protocol and route definitions live under `src/logseq/db_sync/worker/`.
|
||||
28
deps/db-sync/bb.edn
vendored
Normal file
28
deps/db-sync/bb.edn
vendored
Normal file
@@ -0,0 +1,28 @@
|
||||
{:min-bb-version "1.0.168"
|
||||
:deps
|
||||
{logseq/bb-tasks
|
||||
#_{:local/root "../../../bb-tasks"}
|
||||
{:git/url "https://github.com/logseq/bb-tasks"
|
||||
:git/sha "7cc218286c03f093996539b8195de56a0ec5df6a"}}
|
||||
|
||||
:pods
|
||||
{clj-kondo/clj-kondo {:version "2026.04.15"}}
|
||||
|
||||
:tasks
|
||||
{lint:large-vars
|
||||
logseq.bb-tasks.lint.large-vars/-main
|
||||
|
||||
lint:carve
|
||||
logseq.bb-tasks.lint.carve/-main
|
||||
|
||||
lint:ns-docstrings
|
||||
logseq.bb-tasks.lint.ns-docstrings/-main
|
||||
|
||||
lint:minimize-public-vars
|
||||
logseq.bb-tasks.lint.minimize-public-vars/-main}
|
||||
|
||||
:tasks/config
|
||||
{:large-vars
|
||||
{:metadata-exceptions #{:large-vars/cleanup-todo}
|
||||
;; AI generated code has its tradeoffs
|
||||
:max-lines-count 110}}}
|
||||
26
deps/db-sync/deps.edn
vendored
Normal file
26
deps/db-sync/deps.edn
vendored
Normal file
@@ -0,0 +1,26 @@
|
||||
{:paths ["src" "test" "../../resources"]
|
||||
:deps
|
||||
{org.clojure/clojure {:mvn/version "1.12.4"}
|
||||
datascript/datascript {:git/url "https://github.com/logseq/datascript"
|
||||
:sha "3f141af97b70e1f14c65eaa119acd822ebece37e"}
|
||||
datascript-transit/datascript-transit {:mvn/version "0.3.0"
|
||||
:exclusions [datascript/datascript]}
|
||||
com.cognitect/transit-cljs {:mvn/version "0.8.280"}
|
||||
cljs-bean/cljs-bean {:mvn/version "1.9.0"}
|
||||
thheller/shadow-cljs {:mvn/version "3.4.4"}
|
||||
com.lambdaisland/glogi {:git/url "https://github.com/lambdaisland/glogi"
|
||||
:git/sha "30328a045141717aadbbb693465aed55f0904976"}
|
||||
metosin/reitit {:mvn/version "0.10.1"
|
||||
:exclusions [metosin/malli]}
|
||||
logseq/common {:local/root "../common"}
|
||||
logseq/db {:local/root "../db"}
|
||||
logseq/outliner {:local/root "../outliner"}}
|
||||
:aliases
|
||||
{:cljs {:extra-deps {org.clojure/tools.namespace {:mvn/version "1.5.0"}
|
||||
cider/cider-nrepl {:mvn/version "0.59.0"}
|
||||
org.clojars.knubie/cljs-run-test {:mvn/version "1.0.1"}
|
||||
tortue/spy {:mvn/version "2.15.0"}}
|
||||
:main-opts ["-m" "shadow.cljs.devtools.cli"]}
|
||||
:clj-kondo
|
||||
{:replace-deps {clj-kondo/clj-kondo {:mvn/version "2026.04.15"}}
|
||||
:main-opts ["-m" "clj-kondo.main"]}}}
|
||||
57
deps/db-sync/package.json
vendored
Normal file
57
deps/db-sync/package.json
vendored
Normal file
@@ -0,0 +1,57 @@
|
||||
{
|
||||
"name": "@logseq/db-sync",
|
||||
"version": "1.0.0",
|
||||
"packageManager": "pnpm@10.33.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "cd ./worker && pnpm exec wrangler dev",
|
||||
"watch": "clojure -M:cljs watch db-sync db-sync-node",
|
||||
"release": "clojure -M:cljs release db-sync",
|
||||
"delete-graphs-for-user": "node worker/scripts/delete_graphs_for_user.js",
|
||||
"delete-user-totally": "node worker/scripts/delete_user_totally.js",
|
||||
"download-graph-db": "node worker/scripts/download_graph_db.js",
|
||||
"show-graphs-for-user": "node worker/scripts/show_graphs_for_user.js",
|
||||
"show-usage-stats": "node worker/scripts/show_usage_stats.js",
|
||||
"build:node-adapter": "clojure -M:cljs release db-sync-node",
|
||||
"dev:node-adapter": "clojure -M:cljs watch db-sync-node",
|
||||
"start:node-adapter": "node worker/dist/node-adapter.js",
|
||||
"show-sqlite-checksum": "clojure -M:cljs compile db-sync-show-checksum && node worker/dist/show-sqlite-checksum.js",
|
||||
"recompute-log-checksum": "clojure -M:cljs compile db-sync-recompute-log-checksum && node worker/dist/recompute-log-checksum.js",
|
||||
"migrate:local": "cd ./worker && wrangler d1 migrations apply DB --local",
|
||||
"migrate:staging": "cd ./worker && wrangler d1 migrations apply logseq-sync-graph-meta-staging --env staging --remote",
|
||||
"migrate:prod": "cd ./worker && wrangler d1 migrations apply logseq-sync-graphs-prod --env prod --remote",
|
||||
"test:node-adapter": "clojure -M:cljs compile db-sync-test && node worker/dist/worker-test.js",
|
||||
"test:chatgpt-app": "node --test worker/entry_test.mjs worker/asset_image_test.mjs",
|
||||
"build:api-docs": "export COGNITO_ISSUER=https://cognito-idp.us-east-1.amazonaws.com/us-east-1_dtagLnju8 && clojure -M:cljs release db-sync-api-docs && node worker/dist/api-docs-spec.js && redocly build-docs worker/dist/openapi.json --output=worker/dist/api-docs.html --title=\"Logseq Server API\" --disableGoogleFont && node worker/scripts/embed_api_docs.mjs",
|
||||
"test:api-docs": "pnpm build:api-docs && node --test worker/api_docs_test.mjs worker/api_docs_build_test.mjs",
|
||||
"test:large-op-128m": "clojure -M:cljs release db-sync-large-op-memory-test && node --max-old-space-size=128 worker/dist/large-op-memory-test.js",
|
||||
"test": "clojure -M:cljs compile db-sync-test && node worker/dist/worker-test.js",
|
||||
"test:mcp": "pnpm clean && pnpm release && pnpm build:api-docs && cd worker && pnpm exec wrangler deploy --dry-run --env=\"\"",
|
||||
"clean": "rm -rf ./worker/dist/",
|
||||
"sentry:sourcemaps": "sentry-cli sourcemaps upload --release $SENTRY_RELEASE --rewrite --strip-prefix worker/dist/worker --url-prefix \"~/\" worker/dist/worker",
|
||||
"deploy-prod": "export SENTRY_RELEASE=$(git rev-parse HEAD) && pnpm clean && pnpm release && pnpm build:api-docs && pnpm sentry:sourcemaps && cd ./worker && pnpm exec wrangler deploy --env prod",
|
||||
"deploy-staging": "export SENTRY_RELEASE=$(git rev-parse HEAD) && pnpm clean && pnpm release && pnpm build:api-docs && pnpm sentry:sourcemaps && cd ./worker && pnpm exec wrangler deploy --env staging"
|
||||
},
|
||||
"dependencies": {
|
||||
"@cfworker/json-schema": "^4.1.1",
|
||||
"@cloudflare/codemode": "^0.4.2",
|
||||
"@modelcontextprotocol/sdk": "^1.29.0",
|
||||
"@sentry/cloudflare": "^10.45.0",
|
||||
"@sentry/node": "^10.45.0",
|
||||
"better-sqlite3": "^12.8.0",
|
||||
"mldoc": "^1.5.9",
|
||||
"shadow-cljs": "^3.4.4",
|
||||
"transit-js": "^0.8.874",
|
||||
"ws": "^8.19.0",
|
||||
"zod": "^4.4.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@redocly/cli": "^2.38.0",
|
||||
"@sentry/cli": "^3.3.3"
|
||||
},
|
||||
"pnpm": {
|
||||
"onlyBuiltDependencies": [
|
||||
"better-sqlite3"
|
||||
]
|
||||
}
|
||||
}
|
||||
2529
deps/db-sync/pnpm-lock.yaml
generated
vendored
Normal file
2529
deps/db-sync/pnpm-lock.yaml
generated
vendored
Normal file
File diff suppressed because it is too large
Load Diff
62
deps/db-sync/shadow-cljs.edn
vendored
Normal file
62
deps/db-sync/shadow-cljs.edn
vendored
Normal file
@@ -0,0 +1,62 @@
|
||||
;; shadow-cljs configuration
|
||||
{:deps true
|
||||
:http {:port 9632}
|
||||
:nrepl {:port 8703}
|
||||
:builds
|
||||
{:db-sync {:target :esm
|
||||
:output-dir "worker/dist/worker"
|
||||
:compiler-options {:source-map true
|
||||
:warnings {:fn-deprecated false
|
||||
:redef false}}
|
||||
:modules {:main {:exports {default logseq.db-sync.worker/worker
|
||||
SyncDO logseq.db-sync.worker/SyncDO}}}
|
||||
:js-options {:js-provider :import}
|
||||
:closure-defines {shadow.cljs.devtools.client.env/enabled false
|
||||
goog.debug.LOGGING_ENABLED true}
|
||||
:devtools {:enabled false}}
|
||||
:db-sync-node {:target :node-script
|
||||
:output-to "worker/dist/node-adapter.js"
|
||||
:main logseq.db-sync.node.entry/main
|
||||
:compiler-options {:source-map true
|
||||
:warnings {:fn-deprecated false
|
||||
:redef false}}
|
||||
:devtools {:enabled false}
|
||||
:closure-defines {goog.debug.LOGGING_ENABLED true}}
|
||||
:db-sync-show-checksum {:target :node-script
|
||||
:output-to "worker/dist/show-sqlite-checksum.js"
|
||||
:main logseq.db-sync.node.show-checksum/main
|
||||
:compiler-options {:source-map true
|
||||
:warnings {:fn-deprecated false
|
||||
:redef false}}
|
||||
:devtools {:enabled false}}
|
||||
:db-sync-find-checksum-drift {:target :node-script
|
||||
:output-to "worker/dist/find-checksum-drift.js"
|
||||
:main logseq.db-sync.node.find-checksum-drift/main
|
||||
:compiler-options {:source-map true
|
||||
:warnings {:fn-deprecated false
|
||||
:redef false}}
|
||||
:devtools {:enabled false}}
|
||||
:db-sync-recompute-log-checksum {:target :node-script
|
||||
:output-to "worker/dist/recompute-log-checksum.js"
|
||||
:main logseq.db-sync.node.recompute-log-checksum/main
|
||||
:compiler-options {:source-map true
|
||||
:warnings {:fn-deprecated false
|
||||
:redef false}}
|
||||
:devtools {:enabled false}}
|
||||
:db-sync-api-docs {:target :node-script
|
||||
:output-to "worker/dist/api-docs-spec.js"
|
||||
:main logseq.db-sync.node.api-docs/main
|
||||
:compiler-options {:warnings {:fn-deprecated false
|
||||
:redef false}}
|
||||
:devtools {:enabled false}}
|
||||
:db-sync-test {:target :node-test
|
||||
:output-to "worker/dist/worker-test.js"
|
||||
:devtools {:enabled false}
|
||||
:main logseq.db-sync.test-runner/main}
|
||||
:db-sync-large-op-memory-test {:target :node-script
|
||||
:output-to "worker/dist/large-op-memory-test.js"
|
||||
:main logseq.db-sync.worker-large-op-memory-test/main
|
||||
:compiler-options {:source-map true
|
||||
:warnings {:fn-deprecated false
|
||||
:redef false}}
|
||||
:devtools {:enabled false}}}}
|
||||
18
deps/db-sync/src/logseq/db_sync/batch.cljs
vendored
Normal file
18
deps/db-sync/src/logseq/db_sync/batch.cljs
vendored
Normal file
@@ -0,0 +1,18 @@
|
||||
(ns logseq.db-sync.batch
|
||||
(:require [clojure.string :as string]))
|
||||
|
||||
(def max-sql-params 99)
|
||||
(def ^:private row-param-count 3)
|
||||
|
||||
(defn rows->insert-batches
|
||||
[table rows {:keys [max-params] :or {max-params max-sql-params}}]
|
||||
(if (seq rows)
|
||||
(let [max-rows (max 1 (js/Math.floor (/ max-params row-param-count)))
|
||||
batches (partition-all max-rows rows)]
|
||||
(mapv (fn [batch]
|
||||
(let [placeholders (string/join "," (repeat (count batch) "(?, ?, ?)"))
|
||||
sql (str "insert or replace into " table " (addr, content, addresses) values " placeholders)
|
||||
args (vec (mapcat identity batch))]
|
||||
{:sql sql :args args}))
|
||||
batches))
|
||||
[]))
|
||||
429
deps/db-sync/src/logseq/db_sync/checksum.cljs
vendored
Normal file
429
deps/db-sync/src/logseq/db_sync/checksum.cljs
vendored
Normal file
@@ -0,0 +1,429 @@
|
||||
(ns logseq.db-sync.checksum
|
||||
(:require [clojure.set :as set]
|
||||
[datascript.core :as d]
|
||||
[logseq.db :as ldb]))
|
||||
|
||||
(def ^:private fnv-offset 2166136261)
|
||||
(def ^:private djb-offset 5381)
|
||||
(def ^:private field-separator 31)
|
||||
|
||||
(defn- fnv-step
|
||||
[h code]
|
||||
(bit-or (js/Math.imul (bit-xor h code) 16777619) 0))
|
||||
|
||||
(defn- djb-step
|
||||
[h code]
|
||||
(bit-or (+ (js/Math.imul h 33) code) 0))
|
||||
|
||||
(defn- add-step
|
||||
[acc value]
|
||||
(bit-or (+ acc value) 0))
|
||||
|
||||
(defn- sub-step
|
||||
[acc value]
|
||||
(bit-or (- acc value) 0))
|
||||
|
||||
(defn- hash-code
|
||||
[[fnv djb] code]
|
||||
[(fnv-step fnv code)
|
||||
(djb-step djb code)])
|
||||
|
||||
(defn- digest-string
|
||||
[state value]
|
||||
(let [value (or value "")]
|
||||
(loop [idx 0
|
||||
state state]
|
||||
(if (< idx (count value))
|
||||
(recur (inc idx)
|
||||
(hash-code state (.charCodeAt value idx)))
|
||||
state))))
|
||||
|
||||
(defn- unsigned-hex
|
||||
[n]
|
||||
(-> (unsigned-bit-shift-right n 0)
|
||||
(.toString 16)
|
||||
(.padStart 8 "0")))
|
||||
|
||||
(defn- parse-hex32
|
||||
[s]
|
||||
(when (= 8 (count s))
|
||||
(bit-or (js/parseInt s 16) 0)))
|
||||
|
||||
(defn- checksum->state
|
||||
[checksum]
|
||||
(if (and (string? checksum) (= 16 (count checksum)))
|
||||
[(or (parse-hex32 (subs checksum 0 8)) 0)
|
||||
(or (parse-hex32 (subs checksum 8 16)) 0)]
|
||||
[0 0]))
|
||||
|
||||
(defn- valid-checksum?
|
||||
[checksum]
|
||||
(boolean
|
||||
(and (string? checksum)
|
||||
(re-matches #"[0-9a-fA-F]{16}" checksum))))
|
||||
|
||||
(defn- state->checksum
|
||||
[[fnv djb]]
|
||||
(str (unsigned-hex fnv)
|
||||
(unsigned-hex djb)))
|
||||
|
||||
(defn- relevant-attrs
|
||||
[e2ee?]
|
||||
(cond-> #{:block/uuid :block/parent :block/page :block/order}
|
||||
(not e2ee?) (into #{:block/title :block/name})))
|
||||
|
||||
(defn- get-block-uuid
|
||||
[db eid]
|
||||
(:block/uuid (d/entity db eid)))
|
||||
|
||||
(defn- parse-uuid-string
|
||||
[value]
|
||||
(when (string? value)
|
||||
(try
|
||||
(uuid value)
|
||||
(catch :default _
|
||||
nil))))
|
||||
|
||||
(defn- entids
|
||||
[db-before db-after value]
|
||||
(let [lookup-ref (fn [db value]
|
||||
(when value
|
||||
(try
|
||||
(d/entid db value)
|
||||
(catch :default _
|
||||
nil))))
|
||||
uuid-value (cond
|
||||
(uuid? value) value
|
||||
(string? value) (parse-uuid-string value)
|
||||
:else nil)]
|
||||
(cond-> #{}
|
||||
(integer? value) (conj value)
|
||||
(lookup-ref db-before value) (conj (lookup-ref db-before value))
|
||||
(lookup-ref db-after value) (conj (lookup-ref db-after value))
|
||||
(and uuid-value (lookup-ref db-before [:block/uuid uuid-value]))
|
||||
(conj (lookup-ref db-before [:block/uuid uuid-value]))
|
||||
(and uuid-value (lookup-ref db-after [:block/uuid uuid-value]))
|
||||
(conj (lookup-ref db-after [:block/uuid uuid-value])))))
|
||||
|
||||
(defn- normalize-checksum-value
|
||||
[db attr value]
|
||||
(case attr
|
||||
:block/parent (get-block-uuid db value)
|
||||
:block/page (get-block-uuid db value)
|
||||
value))
|
||||
|
||||
(defn- entity-values
|
||||
[db eid e2ee?]
|
||||
(let [attrs (relevant-attrs e2ee?)
|
||||
datoms (d/datoms db :eavt eid)]
|
||||
(reduce (fn [acc datom]
|
||||
(let [attr (:a datom)]
|
||||
(if (contains? attrs attr)
|
||||
(case attr
|
||||
:block/uuid (assoc acc :block/uuid (:v datom))
|
||||
:block/order (assoc acc :block/order (:v datom))
|
||||
:block/title (assoc acc :block/title (:v datom))
|
||||
:block/name (assoc acc :block/name (:v datom))
|
||||
:block/parent (assoc acc :block/parent (get-block-uuid db (:v datom)))
|
||||
:block/page (assoc acc :block/page (get-block-uuid db (:v datom)))
|
||||
acc)
|
||||
acc)))
|
||||
{}
|
||||
datoms)))
|
||||
|
||||
(defn- checksum-eligible-entity?
|
||||
[db eid]
|
||||
(when-let [ent (d/entity db eid)]
|
||||
(and (uuid? (:block/uuid ent))
|
||||
(not (ldb/built-in? ent))
|
||||
(or (ldb/page? ent)
|
||||
(some? (:block/page ent))
|
||||
(some? (:block/name ent))))))
|
||||
|
||||
(defn- entity-checksum-tuples
|
||||
[db eid e2ee?]
|
||||
(when-let [entity-uuid (get-block-uuid db eid)]
|
||||
(let [attrs (relevant-attrs e2ee?)]
|
||||
(->> (d/datoms db :eavt eid)
|
||||
(keep (fn [{:keys [a v]}]
|
||||
(when (contains? attrs a)
|
||||
[entity-uuid
|
||||
a
|
||||
(normalize-checksum-value db a v)])))
|
||||
set))))
|
||||
|
||||
(defn- tuple-digest
|
||||
[[entity-uuid attr value]]
|
||||
(-> [fnv-offset djb-offset]
|
||||
(digest-string (str entity-uuid))
|
||||
(hash-code field-separator)
|
||||
(digest-string (str attr))
|
||||
(hash-code field-separator)
|
||||
(digest-string (some-> value str))))
|
||||
|
||||
(defn- subtract-digest
|
||||
[[sum-fnv sum-djb] [fnv djb]]
|
||||
[(sub-step sum-fnv fnv)
|
||||
(sub-step sum-djb djb)])
|
||||
|
||||
(defn- add-digest
|
||||
[[sum-fnv sum-djb] [fnv djb]]
|
||||
[(add-step sum-fnv fnv)
|
||||
(add-step sum-djb djb)])
|
||||
|
||||
(defn- db-checksum-tuples
|
||||
[db e2ee?]
|
||||
(->> (d/datoms db :avet :block/uuid)
|
||||
(mapcat (fn [{:keys [e]}]
|
||||
(when (checksum-eligible-entity? db e)
|
||||
(entity-checksum-tuples db e e2ee?))))))
|
||||
|
||||
(defn- tx-item-eids
|
||||
[db-before db-after tx-item]
|
||||
(if-let [eid (:e tx-item)]
|
||||
#{eid}
|
||||
(if (vector? tx-item)
|
||||
(let [[op entity attr value] tx-item
|
||||
entity-ids (entids db-before db-after entity)]
|
||||
(cond-> entity-ids
|
||||
(and (= op :db/add) (= attr :block/uuid))
|
||||
(into (entids db-before db-after value))
|
||||
(and (= op :db/retract) (= attr :block/uuid))
|
||||
(into (entids db-before db-after value))))
|
||||
#{})))
|
||||
|
||||
(defn- block-uuid-tx-item?
|
||||
[tx-item]
|
||||
(= :block/uuid
|
||||
(if (vector? tx-item)
|
||||
(nth tx-item 2 nil)
|
||||
(:a tx-item))))
|
||||
|
||||
(defn- touched-base-eids
|
||||
[db-before db-after tx-data]
|
||||
(let [before-eligible-cache (volatile! {})
|
||||
after-eligible-cache (volatile! {})
|
||||
cached-eligible? (fn [db cache eid]
|
||||
(if-some [cached (find @cache eid)]
|
||||
(val cached)
|
||||
(let [eligible? (boolean (checksum-eligible-entity? db eid))]
|
||||
(vswap! cache assoc eid eligible?)
|
||||
eligible?)))]
|
||||
(reduce (fn [result tx-item]
|
||||
(let [block-uuid-change? (block-uuid-tx-item? tx-item)]
|
||||
(reduce (fn [eids eid]
|
||||
(cond-> eids
|
||||
(or block-uuid-change?
|
||||
(not= (get-block-uuid db-before eid)
|
||||
(get-block-uuid db-after eid))
|
||||
(cached-eligible? db-before before-eligible-cache eid)
|
||||
(cached-eligible? db-after after-eligible-cache eid))
|
||||
(conj eid)))
|
||||
result
|
||||
(tx-item-eids db-before db-after tx-item))))
|
||||
#{}
|
||||
tx-data)))
|
||||
|
||||
(defn- touched-checksum-uuids
|
||||
[db-before db-after eids]
|
||||
(->> eids
|
||||
(mapcat (fn [eid]
|
||||
[(get-block-uuid db-before eid)
|
||||
(get-block-uuid db-after eid)]))
|
||||
(remove nil?)
|
||||
set))
|
||||
|
||||
(defn- eids-with-changed-block-uuid
|
||||
[db-before db-after eids]
|
||||
(->> eids
|
||||
(filter (fn [eid]
|
||||
(not= (get-block-uuid db-before eid)
|
||||
(get-block-uuid db-after eid))))
|
||||
set))
|
||||
|
||||
(defn- referrer-eids-by-target
|
||||
[db target-eid]
|
||||
(->> (concat (d/datoms db :avet :block/parent target-eid)
|
||||
(d/datoms db :avet :block/page target-eid))
|
||||
(map :e)
|
||||
set))
|
||||
|
||||
(defn- impacted-referrer-eids
|
||||
[db-before db-after target-eids]
|
||||
(->> target-eids
|
||||
(mapcat (fn [target-eid]
|
||||
(concat (referrer-eids-by-target db-before target-eid)
|
||||
(referrer-eids-by-target db-after target-eid))))
|
||||
set))
|
||||
|
||||
(defn- eids-by-block-uuid
|
||||
[db block-uuid]
|
||||
(->> (d/datoms db :avet :block/uuid block-uuid)
|
||||
(map :e)
|
||||
set))
|
||||
|
||||
(defn- block-uuid-datom-count
|
||||
[db eid]
|
||||
(count (d/datoms db :eavt eid :block/uuid)))
|
||||
|
||||
(defn- duplicate-block-uuid?
|
||||
[db-before db-after block-uuids]
|
||||
(some (fn [uuid]
|
||||
(or (> (count (d/datoms db-before :avet :block/uuid uuid)) 1)
|
||||
(> (count (d/datoms db-after :avet :block/uuid uuid)) 1)))
|
||||
block-uuids))
|
||||
|
||||
(defn- tuple-set-for-eids
|
||||
[db eids e2ee?]
|
||||
(reduce (fn [tuples eid]
|
||||
(if (checksum-eligible-entity? db eid)
|
||||
(into tuples (or (entity-checksum-tuples db eid e2ee?) #{}))
|
||||
tuples))
|
||||
#{}
|
||||
eids))
|
||||
|
||||
(defn- tuple-counts-for-eids
|
||||
[db eids e2ee?]
|
||||
(reduce
|
||||
(fn [counts eid]
|
||||
(let [datom-count (block-uuid-datom-count db eid)]
|
||||
(if (and (pos? datom-count)
|
||||
(checksum-eligible-entity? db eid))
|
||||
(reduce (fn [acc tuple]
|
||||
(update acc tuple (fnil + 0) datom-count))
|
||||
counts
|
||||
(or (entity-checksum-tuples db eid e2ee?) #{}))
|
||||
counts)))
|
||||
{}
|
||||
eids))
|
||||
|
||||
(defn- net-tuple-delta
|
||||
[db-before db-after e2ee? tx-data]
|
||||
(let [base-eids (touched-base-eids db-before db-after tx-data)]
|
||||
(if (empty? base-eids)
|
||||
{:removed {}
|
||||
:added {}}
|
||||
(let [uuid-changed-eids (eids-with-changed-block-uuid db-before db-after base-eids)
|
||||
dependent-eids (when (seq uuid-changed-eids)
|
||||
(->> (impacted-referrer-eids db-before db-after uuid-changed-eids)
|
||||
(filter (fn [eid]
|
||||
(or (checksum-eligible-entity? db-before eid)
|
||||
(checksum-eligible-entity? db-after eid))))
|
||||
set))
|
||||
effective-eids (if (seq dependent-eids)
|
||||
(set/union base-eids dependent-eids)
|
||||
base-eids)
|
||||
touched-uuids (touched-checksum-uuids db-before db-after effective-eids)]
|
||||
(if (duplicate-block-uuid? db-before db-after touched-uuids)
|
||||
(let [peer-eids (->> touched-uuids
|
||||
(mapcat (fn [uuid]
|
||||
(concat (eids-by-block-uuid db-before uuid)
|
||||
(eids-by-block-uuid db-after uuid))))
|
||||
(filter (fn [eid]
|
||||
(or (checksum-eligible-entity? db-before eid)
|
||||
(checksum-eligible-entity? db-after eid))))
|
||||
set)
|
||||
touched-eids (set/union effective-eids peer-eids)
|
||||
before-counts (tuple-counts-for-eids db-before touched-eids e2ee?)
|
||||
after-counts (tuple-counts-for-eids db-after touched-eids e2ee?)
|
||||
all-tuples (set/union (set (keys before-counts))
|
||||
(set (keys after-counts)))]
|
||||
(reduce
|
||||
(fn [{:keys [removed added]} tuple]
|
||||
(let [before-count (get before-counts tuple 0)
|
||||
after-count (get after-counts tuple 0)]
|
||||
(cond
|
||||
(> before-count after-count)
|
||||
{:removed (assoc removed tuple (- before-count after-count))
|
||||
:added added}
|
||||
|
||||
(> after-count before-count)
|
||||
{:removed removed
|
||||
:added (assoc added tuple (- after-count before-count))}
|
||||
|
||||
:else
|
||||
{:removed removed
|
||||
:added added})))
|
||||
{:removed {}
|
||||
:added {}}
|
||||
all-tuples))
|
||||
(let [before-tuples (tuple-set-for-eids db-before effective-eids e2ee?)
|
||||
after-tuples (tuple-set-for-eids db-after effective-eids e2ee?)
|
||||
removed (set/difference before-tuples after-tuples)
|
||||
added (set/difference after-tuples before-tuples)]
|
||||
{:removed (into {} (map (fn [tuple] [tuple 1]) removed))
|
||||
:added (into {} (map (fn [tuple] [tuple 1]) added))}))))))
|
||||
|
||||
(defn- apply-digest-n
|
||||
[checksum-state tuple count op]
|
||||
(let [digest (tuple-digest tuple)]
|
||||
(loop [n count
|
||||
state checksum-state]
|
||||
(if (pos? n)
|
||||
(recur (dec n) (op state digest))
|
||||
state))))
|
||||
|
||||
(defn recompute-checksum
|
||||
[db]
|
||||
(let [e2ee? (ldb/get-graph-rtc-e2ee? db)
|
||||
tuples (db-checksum-tuples db e2ee?)]
|
||||
(->> tuples
|
||||
(reduce (fn [checksum-state tuple]
|
||||
(add-digest checksum-state (tuple-digest tuple)))
|
||||
[0 0])
|
||||
state->checksum)))
|
||||
|
||||
(defn recompute-checksum-diagnostics
|
||||
[db]
|
||||
(let [e2ee? (boolean (ldb/get-graph-rtc-e2ee? db))
|
||||
attrs (relevant-attrs e2ee?)
|
||||
eids (->> (d/datoms db :eavt)
|
||||
(keep (fn [datom]
|
||||
(when (contains? attrs (:a datom))
|
||||
(:e datom))))
|
||||
distinct)
|
||||
blocks (->> eids
|
||||
(keep (fn [eid]
|
||||
(when (checksum-eligible-entity? db eid)
|
||||
(let [{:block/keys [uuid title name parent page order]}
|
||||
(entity-values db eid e2ee?)]
|
||||
(cond-> {:block/uuid uuid
|
||||
:block/parent parent
|
||||
:block/page page
|
||||
:block/order order}
|
||||
(not e2ee?) (assoc :block/title title
|
||||
:block/name name))))))
|
||||
(sort-by (comp str :block/uuid))
|
||||
vec)]
|
||||
{:checksum (recompute-checksum db)
|
||||
:e2ee? e2ee?
|
||||
:attrs (->> attrs (sort-by str) vec)
|
||||
:blocks blocks}))
|
||||
|
||||
(defn update-checksum
|
||||
[checksum {:keys [db-before db-after tx-data]}]
|
||||
(let [before-e2ee? (ldb/get-graph-rtc-e2ee? db-before)
|
||||
after-e2ee? (ldb/get-graph-rtc-e2ee? db-after)
|
||||
tx-data (or tx-data [])]
|
||||
(cond
|
||||
(not= before-e2ee? after-e2ee?)
|
||||
;; E2EE mode changes the global digest semantics, so incremental deltas are invalid.
|
||||
(recompute-checksum db-after)
|
||||
|
||||
(empty? tx-data)
|
||||
checksum
|
||||
|
||||
:else
|
||||
(let [initial-state (if (valid-checksum? checksum)
|
||||
(checksum->state checksum)
|
||||
(checksum->state (recompute-checksum db-before)))
|
||||
{:keys [removed added]} (net-tuple-delta db-before db-after after-e2ee? tx-data)
|
||||
state-after-removals (reduce-kv (fn [checksum-state tuple count]
|
||||
(apply-digest-n checksum-state tuple count subtract-digest))
|
||||
initial-state
|
||||
removed)
|
||||
state-after-additions (reduce-kv (fn [checksum-state tuple count]
|
||||
(apply-digest-n checksum-state tuple count add-digest))
|
||||
state-after-removals
|
||||
added)]
|
||||
(state->checksum state-after-additions)))))
|
||||
92
deps/db-sync/src/logseq/db_sync/common.cljs
vendored
Normal file
92
deps/db-sync/src/logseq/db_sync/common.cljs
vendored
Normal file
@@ -0,0 +1,92 @@
|
||||
(ns logseq.db-sync.common
|
||||
(:require [clojure.string :as string]
|
||||
[logseq.db-sync.platform.core :as platform]
|
||||
[logseq.db.sqlite.util :as sqlite-util]
|
||||
[promesa.core :as p]))
|
||||
|
||||
(defn cors-headers []
|
||||
#js {"Access-Control-Allow-Origin" "*"
|
||||
"Access-Control-Allow-Headers" "content-type,content-encoding,authorization,x-amz-meta-checksum,x-amz-meta-type"
|
||||
"Access-Control-Allow-Methods" "GET,POST,PUT,DELETE,OPTIONS,HEAD"
|
||||
"Access-Control-Expose-Headers" "content-type,content-encoding,content-length,cache-control,x-asset-type,x-asset-size,x-snapshot-row-count"})
|
||||
|
||||
(defn json-response
|
||||
([data] (json-response data 200))
|
||||
([data status]
|
||||
(platform/response
|
||||
(js/JSON.stringify (clj->js data))
|
||||
#js {:status status
|
||||
:headers (js/Object.assign
|
||||
#js {"content-type" "application/json"}
|
||||
(cors-headers))})))
|
||||
|
||||
(defn options-response []
|
||||
(platform/response nil #js {:status 204 :headers (cors-headers)}))
|
||||
|
||||
(defn get-sql-rows [^js result]
|
||||
(let [iter-fn (when result (aget result js/Symbol.iterator))]
|
||||
(cond
|
||||
(nil? result) []
|
||||
(fn? (.-toArray result)) (.toArray result)
|
||||
(fn? iter-fn) (vec (js/Array.from result))
|
||||
(array? (.-results result)) (.-results result)
|
||||
(array? (.-rows result)) (.-rows result)
|
||||
(array? result) (if (empty? result)
|
||||
[]
|
||||
(let [first-row (first result)]
|
||||
(cond
|
||||
(array? (.-results first-row)) (.-results first-row)
|
||||
(array? (.-rows first-row)) (.-rows first-row)
|
||||
:else result)))
|
||||
:else [])))
|
||||
|
||||
(defn sql-exec
|
||||
[sql sql-str & args]
|
||||
(.apply (.-exec sql) sql (to-array (cons sql-str args))))
|
||||
|
||||
(defn <d1-run
|
||||
[^js db sql-str & args]
|
||||
(p/let [^js stmt (.prepare db sql-str)
|
||||
^js stmt (if (seq args)
|
||||
(.apply (.-bind stmt) stmt (to-array args))
|
||||
stmt)]
|
||||
(.run stmt)))
|
||||
|
||||
(defn- with-session-db [^js db session-mode]
|
||||
(let [with-session (.-withSession db)]
|
||||
(if (fn? with-session)
|
||||
(if (some? session-mode)
|
||||
(.withSession db session-mode)
|
||||
(.withSession db))
|
||||
db)))
|
||||
|
||||
(defn <d1-all
|
||||
[^js db sql-or-opts & more]
|
||||
(let [[opts sql-str args] (if (map? sql-or-opts)
|
||||
[sql-or-opts (first more) (rest more)]
|
||||
[nil sql-or-opts more])
|
||||
session (with-session-db db (:session opts))]
|
||||
(p/let [^js stmt (.prepare session sql-str)
|
||||
^js stmt (if (seq args)
|
||||
(.apply (.-bind stmt) stmt (to-array args))
|
||||
stmt)]
|
||||
(.all stmt))))
|
||||
|
||||
(defn read-json [request]
|
||||
(p/let [body (.text request)]
|
||||
(when (seq body)
|
||||
(js/JSON.parse body))))
|
||||
|
||||
(defn read-transit [value]
|
||||
(when (string? value)
|
||||
(sqlite-util/read-transit-str value)))
|
||||
|
||||
(defn write-transit [value]
|
||||
(sqlite-util/write-transit-str value))
|
||||
|
||||
(defn now-ms []
|
||||
(.now js/Date))
|
||||
|
||||
(defn upgrade-request? [request]
|
||||
(= "websocket"
|
||||
(string/lower-case (or (.get (.-headers request) "upgrade") ""))))
|
||||
372
deps/db-sync/src/logseq/db_sync/cycle.cljs
vendored
Normal file
372
deps/db-sync/src/logseq/db_sync/cycle.cljs
vendored
Normal file
@@ -0,0 +1,372 @@
|
||||
(ns logseq.db-sync.cycle
|
||||
"Generic cycle repair utilities for DataScript graphs.
|
||||
|
||||
Supports:
|
||||
- :block/parent (cardinality-one)
|
||||
- :logseq.property.class/extends (cardinality-many)
|
||||
|
||||
Runs AFTER rebase:
|
||||
- Detect cycles reachable from touched entities and break enough edges until
|
||||
the graph becomes acyclic (within a bounded number of iterations).
|
||||
|
||||
Notes:
|
||||
- Repairs are plain datoms: [:db/retract e attr v] (and optional add).
|
||||
- For cardinality-many attrs, we retract ONE edge on the detected cycle path
|
||||
per iteration; we iterate to handle multiple/overlapping cycles."
|
||||
(:refer-clojure :exclude [cycle])
|
||||
(:require
|
||||
[datascript.core :as d]
|
||||
[logseq.db :as ldb]))
|
||||
|
||||
;; -----------------------------------------------------------------------------
|
||||
;; Config
|
||||
;; -----------------------------------------------------------------------------
|
||||
|
||||
(defn- entid
|
||||
"Resolve to an entity id.
|
||||
Accepts eid, entity map, or keyword ident."
|
||||
[db x]
|
||||
(cond
|
||||
(nil? x) nil
|
||||
(number? x) x
|
||||
(map? x) (:db/id x)
|
||||
(keyword? x) (d/entid db x)
|
||||
:else x))
|
||||
|
||||
(defn- safe-target-for-block-parent
|
||||
"Default safe target for :block/parent.
|
||||
If your outliner requires a page-root BLOCK instead of the page entity,
|
||||
replace this to return that block eid."
|
||||
[db e _attr _bad-v]
|
||||
(some-> (d/entity db e) :block/page :db/id))
|
||||
|
||||
(defn- safe-target-for-class-extends
|
||||
"Default safe target for :logseq.property.class/extends.
|
||||
Must resolve to an existing entity. If you prefer 'retract only', return nil."
|
||||
[db _e _attr _bad-v]
|
||||
(d/entid db :logseq.class/Root))
|
||||
|
||||
(defn- ancestor?
|
||||
"Return true if `ancestor-eid` appears in the parent chain of `start-eid`."
|
||||
[db ancestor-eid start-eid]
|
||||
(loop [current start-eid
|
||||
seen #{}]
|
||||
(cond
|
||||
(nil? current) false
|
||||
(= current ancestor-eid) true
|
||||
(contains? seen current) false
|
||||
:else
|
||||
(let [parent (some-> (d/entity db current) :block/parent :db/id)]
|
||||
(recur parent (conj seen current))))))
|
||||
|
||||
(def ^:private default-attr-opts
|
||||
{;; Cardinality-one
|
||||
:block/parent
|
||||
{:cardinality :one
|
||||
:safe-target-fn safe-target-for-block-parent}
|
||||
|
||||
;; Cardinality-many
|
||||
:logseq.property.class/extends
|
||||
{:cardinality :many
|
||||
:safe-target-fn safe-target-for-class-extends}})
|
||||
|
||||
(def ^:private repair-attrs
|
||||
(set (keys default-attr-opts)))
|
||||
|
||||
;; -----------------------------------------------------------------------------
|
||||
;; Basics
|
||||
;; -----------------------------------------------------------------------------
|
||||
|
||||
(defn- ref-eids
|
||||
"Read a ref attribute as a set of eids.
|
||||
|
||||
Supports:
|
||||
- cardinality-one: returns #{v} or #{}
|
||||
- cardinality-many: returns #{v1 v2 ...} or #{}"
|
||||
[db e attr {:keys [cardinality] :or {cardinality :one}}]
|
||||
(let [v (some-> (d/entity db e) (get attr))]
|
||||
(case cardinality
|
||||
:many (cond
|
||||
(nil? v) #{}
|
||||
(set? v) (into #{} (keep #(entid db %)) v)
|
||||
(sequential? v) (into #{} (keep #(entid db %)) v)
|
||||
:else (let [id (entid db v)] (if id #{id} #{})))
|
||||
;; :one
|
||||
(let [id (entid db v)] (if id #{id} #{})))))
|
||||
|
||||
(defn- touched-eids
|
||||
"Collect entity ids whose `attr` was added/changed (added=true) in tx-data."
|
||||
[tx-data attr]
|
||||
(->> tx-data
|
||||
(keep (fn [[e a _v _t added]]
|
||||
(when (and added (= a attr)) e)))
|
||||
distinct))
|
||||
|
||||
(defn- touched-eids-many
|
||||
"Collect touched entity ids for repair attrs.
|
||||
Returns {attr #{eid ...}}"
|
||||
[tx-data]
|
||||
(reduce (fn [m attr]
|
||||
(let [xs (touched-eids tx-data attr)]
|
||||
(if (seq xs) (assoc m attr (set xs)) m)))
|
||||
{}
|
||||
repair-attrs))
|
||||
|
||||
;; -----------------------------------------------------------------------------
|
||||
;; Cycle detection (DFS; cardinality-one and cardinality-many)
|
||||
;; -----------------------------------------------------------------------------
|
||||
|
||||
(defn- reachable-cycle
|
||||
"Detect a ref-cycle reachable from `start-eid` following edges of `attr`.
|
||||
|
||||
Returns a vector like [a b c a] or nil.
|
||||
For cardinality-many attrs, we DFS across ALL outgoing edges.
|
||||
|
||||
attr-opts:
|
||||
- :cardinality :one|:many
|
||||
- :skip? (fn [db from attr to] -> boolean) ; ignore a specific edge"
|
||||
[db start-eid attr {:keys [skip?] :as attr-opts}]
|
||||
(let [visited (volatile! #{})
|
||||
stack (volatile! [])
|
||||
in-stack (volatile! #{})
|
||||
cycle (volatile! nil)]
|
||||
(letfn [(neighbors [e]
|
||||
(->> (ref-eids db e attr attr-opts)
|
||||
(remove nil?)
|
||||
(remove (fn [to] (and skip? (skip? db e attr to))))
|
||||
vec))
|
||||
(record-cycle! [e]
|
||||
(let [stk @stack
|
||||
idx (.indexOf stk e)]
|
||||
(when (>= idx 0)
|
||||
(vreset! cycle (conj (subvec stk idx) e)))))
|
||||
(dfs! [e]
|
||||
(when-not @cycle
|
||||
(cond
|
||||
(contains? @in-stack e) (record-cycle! e)
|
||||
(contains? @visited e) nil
|
||||
:else
|
||||
(do
|
||||
(vswap! visited conj e)
|
||||
(vswap! in-stack conj e)
|
||||
(vswap! stack conj e)
|
||||
(doseq [n (neighbors e)]
|
||||
(when-not @cycle
|
||||
(dfs! n)))
|
||||
(vswap! stack pop)
|
||||
(vswap! in-stack disj e)))))]
|
||||
(dfs! start-eid)
|
||||
@cycle)))
|
||||
|
||||
;; -----------------------------------------------------------------------------
|
||||
;; Breaking cycles
|
||||
;; -----------------------------------------------------------------------------
|
||||
|
||||
(defn- cycle-edges
|
||||
"Convert cycle path [a b c a] into directed edges [[a b] [b c] [c a]]."
|
||||
[cycle]
|
||||
(when (and (vector? cycle) (>= (count cycle) 2))
|
||||
(mapv (fn [i] [(nth cycle i) (nth cycle (inc i))])
|
||||
(range (dec (count cycle))))))
|
||||
|
||||
(defn- pick-victim
|
||||
"Pick which node in the cycle to detach.
|
||||
|
||||
Preference:
|
||||
1) touched by local rebase (prefer remote value)
|
||||
2) touched by remote (detach the remote-changed edge)
|
||||
3) untouched by remote/local
|
||||
4) first node"
|
||||
[cycle {:keys [local-touched remote-touched]}]
|
||||
(let [nodes (vec (distinct (butlast cycle)))
|
||||
local? (fn [e] (contains? (or local-touched #{}) e))
|
||||
remote? (fn [e] (contains? (or remote-touched #{}) e))
|
||||
untouched? (fn [e] (and (not (local? e)) (not (remote? e))))]
|
||||
(or (some (fn [e] (when (local? e) e)) nodes)
|
||||
(some (fn [e] (when (remote? e) e)) nodes)
|
||||
(some (fn [e] (when (untouched? e) e)) nodes)
|
||||
(first nodes))))
|
||||
|
||||
(defn- break-cycle-edge!
|
||||
"Given a detected cycle, retract ONE edge on the cycle path.
|
||||
|
||||
Returns a tx vector (possibly with an add), or nil."
|
||||
[db cycle attr {:keys [safe-target-fn skip?] :as attr-opts} touched]
|
||||
(let [edges (cycle-edges cycle)
|
||||
cycle-nodes (set (distinct (butlast cycle)))
|
||||
remote-parent-fn (:remote-parent-fn attr-opts)
|
||||
remote-ref-set-fn (:remote-ref-set-fn attr-opts)
|
||||
remote-parent-candidates (when (and (= :block/parent attr) remote-parent-fn)
|
||||
(keep (fn [[from to]]
|
||||
(let [remote-parent (remote-parent-fn db from to)
|
||||
remote-parent (entid db remote-parent)
|
||||
to-id (entid db to)]
|
||||
(when (and remote-parent
|
||||
(not= remote-parent to-id))
|
||||
{:victim from
|
||||
:bad-v to-id
|
||||
:safe remote-parent
|
||||
:outside? (not (contains? cycle-nodes remote-parent))})))
|
||||
edges))
|
||||
remote-ref-candidates (when (and (= :logseq.property.class/extends attr) remote-ref-set-fn)
|
||||
(keep (fn [[from to]]
|
||||
(let [to-id (entid db to)
|
||||
remote-set (remote-ref-set-fn db from)
|
||||
remote-set (set (keep #(entid db %) remote-set))]
|
||||
(when (and (seq remote-set)
|
||||
(not (contains? remote-set to-id)))
|
||||
{:victim from
|
||||
:bad-v to-id})))
|
||||
edges))
|
||||
remote-choice (cond
|
||||
(seq remote-parent-candidates)
|
||||
(or (some #(when (:outside? %) %) remote-parent-candidates)
|
||||
(first remote-parent-candidates))
|
||||
(seq remote-ref-candidates)
|
||||
(first remote-ref-candidates)
|
||||
:else
|
||||
nil)
|
||||
victim (or (:victim remote-choice) (pick-victim cycle touched))
|
||||
[_from bad-v] (or (when (and remote-choice (:bad-v remote-choice))
|
||||
[victim (:bad-v remote-choice)])
|
||||
(some (fn [[from _to :as e]]
|
||||
(when (= from victim) e))
|
||||
edges))
|
||||
bad-v (entid db bad-v)]
|
||||
(when (and victim bad-v)
|
||||
(when-not (and skip? (skip? db victim attr bad-v))
|
||||
;; Ensure the edge still exists in current db.
|
||||
(when (contains? (ref-eids db victim attr attr-opts) bad-v)
|
||||
(let [safe (or (:safe remote-choice)
|
||||
(when safe-target-fn (safe-target-fn db victim attr bad-v)))
|
||||
safe (if (and (= :block/parent attr)
|
||||
(number? safe)
|
||||
(nil? remote-choice)
|
||||
(ancestor? db victim safe))
|
||||
(safe-target-for-block-parent db victim attr bad-v)
|
||||
safe)]
|
||||
(cond
|
||||
(and safe (not= safe bad-v))
|
||||
[[:db/retract victim attr bad-v]
|
||||
[:db/add victim attr safe]]
|
||||
|
||||
:else
|
||||
[[:db/retract victim attr bad-v]])))))))
|
||||
|
||||
;; -----------------------------------------------------------------------------
|
||||
;; Iterative repair (THIS is the key fix)
|
||||
;; -----------------------------------------------------------------------------
|
||||
|
||||
(def ^:private fix-cycle-tx-meta
|
||||
{:outliner-op :fix-cycle
|
||||
:gen-undo-ops? false
|
||||
:persist-op? false})
|
||||
|
||||
(defn- apply-cycle-repairs!
|
||||
"Detect & break cycles AFTER rebase, iterating until stable.
|
||||
|
||||
Why iteration:
|
||||
- With cardinality-many (extends), you can end up with multiple disjoint or
|
||||
overlapping cycles after merging remote + local edges.
|
||||
- Breaking one cycle edge may still leave other cycles (e.g. 1<->2 and 2<->3).
|
||||
|
||||
Inputs:
|
||||
- candidates-by-attr: {attr #{eid ...}}
|
||||
- touched-by-attr: {attr {:local-touched #{...} :remote-touched #{...}}}
|
||||
- attr-opts: {attr {:cardinality :one|:many :safe-target-fn ... :skip? ...}}
|
||||
|
||||
We cap iterations to avoid infinite loops if something keeps reintroducing cycles."
|
||||
[transact! temp-conn candidates-by-attr touched-by-attr attr-opts tx-meta]
|
||||
(let [max-iterations 16]
|
||||
(loop [it 0
|
||||
;; Track edges we already retracted to avoid repeating identical work.
|
||||
;; Edge key is [e attr v].
|
||||
seen-edges #{}]
|
||||
(when (< it max-iterations)
|
||||
(let [db @temp-conn
|
||||
;; Collect a set of edge-breaking txs for this iteration.
|
||||
;; We dedupe by edge key, not by tx vector.
|
||||
{tx :tx
|
||||
seen-edges' :seen}
|
||||
(reduce
|
||||
(fn [acc [attr es]]
|
||||
(let [opts (merge {:cardinality :one} (get attr-opts attr {}))
|
||||
touched (get touched-by-attr attr {})]
|
||||
(reduce
|
||||
(fn [{:keys [tx seen] :as acc2} e]
|
||||
(if-let [cycle (reachable-cycle db e attr opts)]
|
||||
(if-let [t (break-cycle-edge! db cycle attr opts touched)]
|
||||
(let [[_op e2 a2 v2] (first t)
|
||||
edge [e2 a2 v2]]
|
||||
;; Skip if we've already retracted this exact edge.
|
||||
(if (contains? seen edge)
|
||||
acc2
|
||||
{:tx (into tx t)
|
||||
:seen (conj seen edge)}))
|
||||
acc2)
|
||||
acc2))
|
||||
acc
|
||||
es)))
|
||||
{:tx [] :seen seen-edges}
|
||||
candidates-by-attr)]
|
||||
|
||||
(if (seq tx)
|
||||
(do
|
||||
(transact! temp-conn tx (merge tx-meta fix-cycle-tx-meta))
|
||||
(recur (inc it) seen-edges'))
|
||||
;; No more cycles detected from these candidates => done
|
||||
nil))))))
|
||||
|
||||
(defn- union-candidates
|
||||
"Union remote + local candidates: {attr #{...}}"
|
||||
[remote-by-attr local-by-attr]
|
||||
(reduce
|
||||
(fn [m attr]
|
||||
(let [r (get remote-by-attr attr #{})
|
||||
l (get local-by-attr attr #{})
|
||||
u (into (set r) l)]
|
||||
(if (seq u) (assoc m attr u) m)))
|
||||
{}
|
||||
(distinct (concat (keys remote-by-attr) (keys local-by-attr)))))
|
||||
|
||||
(defn- touched-info-by-attr
|
||||
"Build {attr {:remote-touched #{...} :local-touched #{...}}}."
|
||||
[remote-by-attr local-by-attr]
|
||||
(reduce
|
||||
(fn [m attr]
|
||||
(let [r (get remote-by-attr attr #{})
|
||||
l (get local-by-attr attr #{})]
|
||||
(assoc m attr {:remote-touched r :local-touched l})))
|
||||
{}
|
||||
(distinct (concat (keys remote-by-attr) (keys local-by-attr)))))
|
||||
|
||||
(defn fix-cycle!
|
||||
"Entry point: call AFTER rebase.
|
||||
|
||||
- Computes candidates from remote + rebase tx reports for configured attrs.
|
||||
- Iteratively breaks cycles until stable."
|
||||
[temp-conn remote-tx-report rebase-tx-report & {:keys [transact! tx-meta]
|
||||
:or {transact! ldb/transact!}}]
|
||||
(let [remote-db (:db-after remote-tx-report)
|
||||
attr-opts (cond-> default-attr-opts
|
||||
remote-db
|
||||
(assoc-in [:block/parent :remote-parent-fn]
|
||||
(fn [_db e _bad-v]
|
||||
(some-> (d/entity remote-db e) :block/parent :db/id)))
|
||||
remote-db
|
||||
(assoc-in [:block/parent :safe-target-fn]
|
||||
(fn [db e attr bad-v]
|
||||
(let [remote-parent (some-> (d/entity remote-db e) :block/parent :db/id)
|
||||
remote-parent (when (and remote-parent (not= remote-parent bad-v)) remote-parent)]
|
||||
(or remote-parent (safe-target-for-block-parent db e attr bad-v)))))
|
||||
remote-db
|
||||
(assoc-in [:logseq.property.class/extends :remote-ref-set-fn]
|
||||
(fn [_db e]
|
||||
(some->> (d/entity remote-db e)
|
||||
:logseq.property.class/extends))))
|
||||
remote-touched-by-attr (touched-eids-many (:tx-data remote-tx-report))
|
||||
local-touched-by-attr (touched-eids-many (:tx-data rebase-tx-report))
|
||||
candidates-by-attr (union-candidates remote-touched-by-attr local-touched-by-attr)
|
||||
touched-info (touched-info-by-attr remote-touched-by-attr local-touched-by-attr)]
|
||||
(when (seq candidates-by-attr)
|
||||
(apply-cycle-repairs! transact! temp-conn candidates-by-attr touched-info attr-opts tx-meta))))
|
||||
593
deps/db-sync/src/logseq/db_sync/index.cljs
vendored
Normal file
593
deps/db-sync/src/logseq/db_sync/index.cljs
vendored
Normal file
@@ -0,0 +1,593 @@
|
||||
(ns logseq.db-sync.index
|
||||
(:require [clojure.string :as string]
|
||||
[logseq.db-sync.common :as common]
|
||||
[promesa.core :as p]))
|
||||
|
||||
(def ^:private user-upsert-cache-ttl-ms (* 60 60 1000))
|
||||
(def ^:private user-upsert-cache-max 1024)
|
||||
(defonce ^:private *user-upsert-cache (atom {}))
|
||||
(def ^:private activity-touch-cache-max 8192)
|
||||
(defonce ^:private *activity-touch-cache (atom {}))
|
||||
|
||||
(defn- prune-user-upsert-cache! [now-ms]
|
||||
(swap! *user-upsert-cache
|
||||
(fn [cache]
|
||||
(let [cache (into {}
|
||||
(remove (fn [[_ {:keys [cached-at]}]]
|
||||
(>= (- now-ms cached-at) user-upsert-cache-ttl-ms)))
|
||||
cache)
|
||||
count-cache (count cache)]
|
||||
(if (> count-cache user-upsert-cache-max)
|
||||
(into {}
|
||||
(drop (- count-cache user-upsert-cache-max)
|
||||
(sort-by (comp :cached-at val) cache)))
|
||||
cache)))))
|
||||
|
||||
(defn- cache-user-upsert! [user-id email email-verified username now-ms]
|
||||
(swap! *user-upsert-cache assoc user-id {:email email
|
||||
:email-verified email-verified
|
||||
:username username
|
||||
:cached-at now-ms})
|
||||
(prune-user-upsert-cache! now-ms))
|
||||
|
||||
(defn- graph-e2ee-sql->bool
|
||||
[v]
|
||||
(cond
|
||||
(nil? v) true
|
||||
(or (= 1 v) (= "1" v)) true
|
||||
(or (= 0 v) (= "0" v)) false
|
||||
:else (true? v)))
|
||||
|
||||
(defn- graph-e2ee-bool->sql
|
||||
[v]
|
||||
(if (false? v) 0 1))
|
||||
|
||||
(defn- graph-ready-for-use-sql->bool
|
||||
[v]
|
||||
(cond
|
||||
(nil? v) true
|
||||
(or (= 1 v) (= "1" v)) true
|
||||
(or (= 0 v) (= "0" v)) false
|
||||
:else (true? v)))
|
||||
|
||||
(defn- graph-ready-for-use-bool->sql
|
||||
[v]
|
||||
(if (false? v) 0 1))
|
||||
|
||||
(def ^:private graph-e2ee-migration-sql
|
||||
"alter table graphs add column graph_e2ee INTEGER DEFAULT 1")
|
||||
(def ^:private graph-ready-for-use-migration-sql
|
||||
"alter table graphs add column graph_ready_for_use integer default 1")
|
||||
(def ^:private user-created-at-migration-sql
|
||||
"alter table users add column created_at integer")
|
||||
(def ^:private daily-active-entities-create-table-sql
|
||||
(str "create table if not exists daily_active_entities ("
|
||||
"day_utc TEXT,"
|
||||
"entity_type TEXT,"
|
||||
"entity_id TEXT,"
|
||||
"first_seen_at INTEGER,"
|
||||
"primary key (day_utc, entity_type, entity_id),"
|
||||
"check (entity_type in ('user', 'graph'))"
|
||||
");"))
|
||||
(def ^:private daily-active-entities-create-index-sql
|
||||
"create index if not exists idx_daily_active_entities_type_day on daily_active_entities (entity_type, day_utc)")
|
||||
|
||||
(defn- duplicate-column-error?
|
||||
[error column-name]
|
||||
(let [message (-> (or (ex-message error) (some-> error .-message) (str error))
|
||||
string/lower-case)]
|
||||
(and (string/includes? message "duplicate column")
|
||||
(string/includes? message (string/lower-case column-name)))))
|
||||
|
||||
(defn- <ensure-graph-e2ee-column!
|
||||
[db]
|
||||
(letfn [(<run-migration! []
|
||||
(-> (common/<d1-run db graph-e2ee-migration-sql)
|
||||
(p/catch (fn [error]
|
||||
(if (duplicate-column-error? error "graph_e2ee")
|
||||
nil
|
||||
(p/rejected error))))))]
|
||||
(-> (p/let [result (common/<d1-all db
|
||||
"select name from pragma_table_info('graphs') where name = 'graph_e2ee'")
|
||||
rows (common/get-sql-rows result)]
|
||||
(when (empty? rows)
|
||||
(<run-migration!)))
|
||||
(p/catch (fn [_]
|
||||
(<run-migration!))))))
|
||||
|
||||
(defn- <ensure-graph-ready-for-use-column!
|
||||
[db]
|
||||
(letfn [(<run-migration! []
|
||||
(-> (common/<d1-run db graph-ready-for-use-migration-sql)
|
||||
(p/catch (fn [error]
|
||||
(if (duplicate-column-error? error "graph_ready_for_use")
|
||||
nil
|
||||
(p/rejected error))))))]
|
||||
(-> (p/let [result (common/<d1-all db
|
||||
"select name from pragma_table_info('graphs') where name = 'graph_ready_for_use'")
|
||||
rows (common/get-sql-rows result)]
|
||||
(when (empty? rows)
|
||||
(<run-migration!)))
|
||||
(p/catch (fn [_]
|
||||
(<run-migration!))))))
|
||||
|
||||
(defn- <ensure-user-created-at-column!
|
||||
[db]
|
||||
(letfn [(<run-migration! []
|
||||
(-> (common/<d1-run db user-created-at-migration-sql)
|
||||
(p/catch (fn [error]
|
||||
(if (duplicate-column-error? error "created_at")
|
||||
nil
|
||||
(p/rejected error))))))]
|
||||
(-> (p/let [result (common/<d1-all db
|
||||
"select name from pragma_table_info('users') where name = 'created_at'")
|
||||
rows (common/get-sql-rows result)]
|
||||
(when (empty? rows)
|
||||
(<run-migration!)))
|
||||
(p/catch (fn [_]
|
||||
(<run-migration!))))))
|
||||
|
||||
(defn <index-init! [db]
|
||||
(p/do!
|
||||
(common/<d1-run db
|
||||
(str "create table if not exists graphs ("
|
||||
"graph_id TEXT primary key,"
|
||||
"graph_name TEXT,"
|
||||
"user_id TEXT,"
|
||||
"schema_version TEXT,"
|
||||
"graph_e2ee INTEGER DEFAULT 1,"
|
||||
"graph_ready_for_use INTEGER DEFAULT 1,"
|
||||
"created_at INTEGER,"
|
||||
"updated_at INTEGER"
|
||||
");"))
|
||||
(<ensure-graph-e2ee-column! db)
|
||||
(<ensure-graph-ready-for-use-column! db)
|
||||
(common/<d1-run db
|
||||
(str "create table if not exists users ("
|
||||
"id TEXT primary key,"
|
||||
"email TEXT,"
|
||||
"email_verified INTEGER,"
|
||||
"username TEXT,"
|
||||
"created_at INTEGER"
|
||||
");"))
|
||||
(<ensure-user-created-at-column! db)
|
||||
(common/<d1-run db
|
||||
(str "create table if not exists user_rsa_keys ("
|
||||
"user_id TEXT primary key,"
|
||||
"public_key TEXT,"
|
||||
"encrypted_private_key TEXT,"
|
||||
"created_at INTEGER,"
|
||||
"updated_at INTEGER"
|
||||
");"))
|
||||
(common/<d1-run db
|
||||
(str "create table if not exists graph_members ("
|
||||
"user_id TEXT,"
|
||||
"graph_id TEXT,"
|
||||
"role TEXT,"
|
||||
"invited_by TEXT,"
|
||||
"created_at INTEGER,"
|
||||
"primary key (user_id, graph_id),"
|
||||
"check (role in ('manager', 'member'))"
|
||||
");"))
|
||||
(common/<d1-run db
|
||||
(str "create table if not exists graph_aes_keys ("
|
||||
"graph_id TEXT,"
|
||||
"user_id TEXT,"
|
||||
"encrypted_aes_key TEXT,"
|
||||
"created_at INTEGER,"
|
||||
"updated_at INTEGER,"
|
||||
"primary key (graph_id, user_id)"
|
||||
");"))
|
||||
(common/<d1-run db daily-active-entities-create-table-sql)
|
||||
(common/<d1-run db
|
||||
"create index if not exists idx_graph_members_graph_id_created_at on graph_members (graph_id, created_at)")
|
||||
(common/<d1-run db
|
||||
"create index if not exists idx_graphs_user_id_updated_at on graphs (user_id, updated_at desc)")
|
||||
(common/<d1-run db
|
||||
"create index if not exists idx_users_email on users (email)")
|
||||
(common/<d1-run db daily-active-entities-create-index-sql)))
|
||||
|
||||
(defn- utc-day-str
|
||||
[timestamp-ms]
|
||||
(-> (.toISOString (js/Date. timestamp-ms))
|
||||
(subs 0 10)))
|
||||
|
||||
(defn- prune-activity-touch-cache!
|
||||
[cache]
|
||||
(if (<= (count cache) activity-touch-cache-max)
|
||||
cache
|
||||
(let [drop-count (- (count cache) activity-touch-cache-max)]
|
||||
(->> cache
|
||||
(sort-by val)
|
||||
(drop drop-count)
|
||||
(into {})))))
|
||||
|
||||
(defn- cache-activity-touch!
|
||||
[cache-key now-ms]
|
||||
(swap! *activity-touch-cache
|
||||
(fn [cache]
|
||||
(-> cache
|
||||
(assoc cache-key now-ms)
|
||||
(prune-activity-touch-cache!)))))
|
||||
|
||||
(defn- activity-touch-cached?
|
||||
[cache-key]
|
||||
(contains? @*activity-touch-cache cache-key))
|
||||
|
||||
(defn- <activity-touch!
|
||||
[db entity-type entity-id]
|
||||
(if (and (string? entity-id) (contains? #{"user" "graph"} entity-type))
|
||||
(let [now (common/now-ms)
|
||||
day-utc (utc-day-str now)
|
||||
cache-key [day-utc entity-type entity-id]]
|
||||
(if (activity-touch-cached? cache-key)
|
||||
(p/resolved nil)
|
||||
(p/let [_ (common/<d1-run db
|
||||
(str "insert into daily_active_entities (day_utc, entity_type, entity_id, first_seen_at) "
|
||||
"values (?, ?, ?, ?) "
|
||||
"on conflict(day_utc, entity_type, entity_id) do nothing")
|
||||
day-utc
|
||||
entity-type
|
||||
entity-id
|
||||
now)]
|
||||
(cache-activity-touch! cache-key now)
|
||||
nil)))
|
||||
(p/resolved nil)))
|
||||
|
||||
(defn <user-activity-touch!
|
||||
[db user-id]
|
||||
(<activity-touch! db "user" user-id))
|
||||
|
||||
(defn <graph-activity-touch!
|
||||
[db graph-id]
|
||||
(<activity-touch! db "graph" graph-id))
|
||||
|
||||
(defn <index-list [db user-id]
|
||||
(if (string? user-id)
|
||||
(p/let [result (common/<d1-all db
|
||||
(str "select g.graph_id, g.graph_name, g.schema_version, g.graph_e2ee, g.graph_ready_for_use, g.created_at, g.updated_at, "
|
||||
"m.role, m.invited_by "
|
||||
"from graphs g "
|
||||
"left join graph_members m on g.graph_id = m.graph_id and m.user_id = ? "
|
||||
"where g.user_id = ? or m.user_id = ? "
|
||||
"order by g.updated_at desc")
|
||||
user-id
|
||||
user-id
|
||||
user-id)
|
||||
rows (common/get-sql-rows result)]
|
||||
(mapv (fn [row]
|
||||
{:graph-id (aget row "graph_id")
|
||||
:graph-name (aget row "graph_name")
|
||||
:schema-version (aget row "schema_version")
|
||||
:graph-e2ee? (graph-e2ee-sql->bool (aget row "graph_e2ee"))
|
||||
:graph-ready-for-use? (graph-ready-for-use-sql->bool (aget row "graph_ready_for_use"))
|
||||
:role (aget row "role")
|
||||
:invited-by (aget row "invited_by")
|
||||
:created-at (aget row "created_at")
|
||||
:updated-at (aget row "updated_at")})
|
||||
rows))
|
||||
[]))
|
||||
|
||||
(defn decode-semantic-graph-cursor [cursor]
|
||||
(when (string? cursor)
|
||||
(try
|
||||
(let [value (js->clj (js/JSON.parse (js/atob cursor)))]
|
||||
(when (and (vector? value)
|
||||
(= 2 (count value))
|
||||
(number? (first value))
|
||||
(string? (second value)))
|
||||
value))
|
||||
(catch :default _
|
||||
nil))))
|
||||
|
||||
(defn- encode-semantic-graph-cursor [updated-at graph-id]
|
||||
(js/btoa (js/JSON.stringify (clj->js [updated-at graph-id]))))
|
||||
|
||||
(defn <semantic-graphs-list
|
||||
[db user-id {:keys [name limit cursor]}]
|
||||
(let [cursor-key (when cursor (decode-semantic-graph-cursor cursor))
|
||||
[cursor-updated-at cursor-graph-id] cursor-key
|
||||
conditions (cond-> [(str "(g.user_id = ? or m.user_id = ?) ")
|
||||
"g.graph_e2ee = 0"
|
||||
"g.graph_ready_for_use = 1"]
|
||||
(string? name) (conj "lower(g.graph_name) = lower(?)")
|
||||
cursor-key (conj "(g.updated_at < ? or (g.updated_at = ? and g.graph_id > ?))"))
|
||||
args (cond-> [user-id user-id user-id]
|
||||
(string? name) (conj name)
|
||||
cursor-key (conj cursor-updated-at cursor-updated-at cursor-graph-id)
|
||||
true (conj (inc limit)))
|
||||
sql (str "select g.graph_id, g.graph_name, g.schema_version, g.graph_ready_for_use, "
|
||||
"g.created_at, g.updated_at, m.role, m.invited_by "
|
||||
"from graphs g "
|
||||
"left join graph_members m on g.graph_id = m.graph_id and m.user_id = ? "
|
||||
"where " (string/join " and " conditions) " "
|
||||
"order by g.updated_at desc, g.graph_id asc limit ?")]
|
||||
(p/let [result (apply common/<d1-all db sql args)
|
||||
rows (vec (common/get-sql-rows result))
|
||||
more? (> (count rows) limit)
|
||||
selected (take limit rows)
|
||||
graphs (mapv (fn [row]
|
||||
{:graph-id (aget row "graph_id")
|
||||
:graph-name (aget row "graph_name")
|
||||
:schema-version (aget row "schema_version")
|
||||
:graph-e2ee? false
|
||||
:graph-ready-for-use? (graph-ready-for-use-sql->bool (aget row "graph_ready_for_use"))
|
||||
:role (aget row "role")
|
||||
:invited-by (aget row "invited_by")
|
||||
:created-at (aget row "created_at")
|
||||
:updated-at (aget row "updated_at")})
|
||||
selected)]
|
||||
(cond-> {:graphs graphs}
|
||||
more? (assoc :next-cursor
|
||||
(let [row (nth rows (dec limit))]
|
||||
(encode-semantic-graph-cursor (aget row "updated_at")
|
||||
(aget row "graph_id"))))))))
|
||||
|
||||
(defn <index-upsert!
|
||||
([db graph-id graph-name user-id schema-version graph-e2ee?]
|
||||
(<index-upsert! db graph-id graph-name user-id schema-version graph-e2ee? true))
|
||||
([db graph-id graph-name user-id schema-version graph-e2ee? graph-ready-for-use?]
|
||||
(p/let [now (common/now-ms)
|
||||
graph-e2ee? (graph-e2ee-bool->sql graph-e2ee?)
|
||||
graph-ready-for-use? (graph-ready-for-use-bool->sql graph-ready-for-use?)
|
||||
result (common/<d1-run db
|
||||
(str "insert into graphs (graph_id, graph_name, user_id, schema_version, graph_e2ee, graph_ready_for_use, created_at, updated_at) "
|
||||
"values (?, ?, ?, ?, ?, ?, ?, ?) "
|
||||
"on conflict(graph_id) do update set "
|
||||
"graph_name = excluded.graph_name, "
|
||||
"user_id = excluded.user_id, "
|
||||
"schema_version = excluded.schema_version, "
|
||||
"graph_e2ee = excluded.graph_e2ee, "
|
||||
"graph_ready_for_use = excluded.graph_ready_for_use, "
|
||||
"updated_at = excluded.updated_at")
|
||||
graph-id
|
||||
graph-name
|
||||
user-id
|
||||
schema-version
|
||||
graph-e2ee?
|
||||
graph-ready-for-use?
|
||||
now
|
||||
now)]
|
||||
result)))
|
||||
|
||||
(defn <graph-ready-for-use?
|
||||
[db graph-id]
|
||||
(when (string? graph-id)
|
||||
(p/let [result (common/<d1-all db
|
||||
"select graph_ready_for_use from graphs where graph_id = ?"
|
||||
graph-id)
|
||||
rows (common/get-sql-rows result)
|
||||
row (first rows)]
|
||||
(graph-ready-for-use-sql->bool (some-> row (aget "graph_ready_for_use"))))))
|
||||
|
||||
(defn <graph-e2ee?
|
||||
[db graph-id]
|
||||
(when (string? graph-id)
|
||||
(p/let [result (common/<d1-all db
|
||||
"select graph_e2ee from graphs where graph_id = ?"
|
||||
graph-id)
|
||||
row (first (common/get-sql-rows result))]
|
||||
(graph-e2ee-sql->bool (some-> row (aget "graph_e2ee"))))))
|
||||
|
||||
(defn <graph-ready-for-use-set!
|
||||
[db graph-id graph-ready-for-use?]
|
||||
(when (string? graph-id)
|
||||
(common/<d1-run db
|
||||
"update graphs set graph_ready_for_use = ?, updated_at = ? where graph_id = ?"
|
||||
(graph-ready-for-use-bool->sql graph-ready-for-use?)
|
||||
(common/now-ms)
|
||||
graph-id)))
|
||||
|
||||
(defn <graph-delete-metadata! [db graph-id]
|
||||
(p/do!
|
||||
(common/<d1-run db "delete from graph_aes_keys where graph_id = ?" graph-id)
|
||||
(common/<d1-run db "delete from graph_members where graph_id = ?" graph-id)))
|
||||
|
||||
(defn <graph-delete-index-entry! [db graph-id]
|
||||
(common/<d1-run db "delete from graphs where graph_id = ?" graph-id))
|
||||
|
||||
(defn <graph-name-exists?
|
||||
[db graph-name user-id]
|
||||
(when (and (string? graph-name) (string? user-id))
|
||||
(p/let [result (common/<d1-all db
|
||||
"select graph_id from graphs where graph_name = ? and user_id = ?"
|
||||
graph-name
|
||||
user-id)
|
||||
rows (common/get-sql-rows result)]
|
||||
(boolean (seq rows)))))
|
||||
|
||||
(defn <user-upsert! [db claims]
|
||||
(let [user-id (aget claims "sub")]
|
||||
(when (string? user-id)
|
||||
(let [email (aget claims "email")
|
||||
email-verified (aget claims "email_verified")
|
||||
username (aget claims "cognito:username")
|
||||
email-verified (cond
|
||||
(true? email-verified) 1
|
||||
(false? email-verified) 0
|
||||
:else nil)
|
||||
now (common/now-ms)
|
||||
cached (get @*user-upsert-cache user-id)]
|
||||
(if (and cached
|
||||
(= email (:email cached))
|
||||
(= email-verified (:email-verified cached))
|
||||
(= username (:username cached))
|
||||
(< (- now (:cached-at cached)) user-upsert-cache-ttl-ms))
|
||||
(cache-user-upsert! user-id email email-verified username now)
|
||||
(p/let [result (common/<d1-run db
|
||||
(str "insert into users (id, email, email_verified, username, created_at) "
|
||||
"values (?, ?, ?, ?, ?) "
|
||||
"on conflict(id) do update set "
|
||||
"email = excluded.email, "
|
||||
"email_verified = excluded.email_verified, "
|
||||
"username = excluded.username, "
|
||||
"created_at = coalesce(users.created_at, excluded.created_at)")
|
||||
user-id
|
||||
email
|
||||
email-verified
|
||||
username
|
||||
now)]
|
||||
(cache-user-upsert! user-id email email-verified username now)
|
||||
result))))))
|
||||
|
||||
(defn <user-id-by-email [db email]
|
||||
(when (string? email)
|
||||
(p/let [result (common/<d1-all db {:session "first-primary"}
|
||||
"select id from users where email = ?"
|
||||
email)
|
||||
rows (common/get-sql-rows result)
|
||||
row (first rows)]
|
||||
(when row
|
||||
(aget row "id")))))
|
||||
|
||||
(defn <user-rsa-key-pair-upsert!
|
||||
[db user-id public-key encrypted-private-key]
|
||||
(when (string? user-id)
|
||||
(let [now (common/now-ms)]
|
||||
(common/<d1-run db
|
||||
(str "insert into user_rsa_keys (user_id, public_key, encrypted_private_key, created_at, updated_at) "
|
||||
"values (?, ?, ?, ?, ?) "
|
||||
"on conflict(user_id) do update set "
|
||||
"public_key = excluded.public_key, "
|
||||
"encrypted_private_key = excluded.encrypted_private_key, "
|
||||
"updated_at = excluded.updated_at")
|
||||
user-id
|
||||
public-key
|
||||
encrypted-private-key
|
||||
now
|
||||
now))))
|
||||
|
||||
(defn <user-rsa-key-pair
|
||||
[db user-id]
|
||||
(when (string? user-id)
|
||||
(p/let [result (common/<d1-all db {:session "first-primary"}
|
||||
"select public_key, encrypted_private_key from user_rsa_keys where user_id = ?"
|
||||
user-id)
|
||||
rows (common/get-sql-rows result)
|
||||
row (first rows)]
|
||||
(when row
|
||||
{:public-key (aget row "public_key")
|
||||
:encrypted-private-key (aget row "encrypted_private_key")}))))
|
||||
|
||||
(defn <user-rsa-public-key-by-email
|
||||
[db email]
|
||||
(when (string? email)
|
||||
(p/let [result (common/<d1-all db {:session "first-primary"}
|
||||
(str "select k.public_key from user_rsa_keys k "
|
||||
"left join users u on k.user_id = u.id "
|
||||
"where u.email = ?")
|
||||
email)
|
||||
rows (common/get-sql-rows result)
|
||||
row (first rows)]
|
||||
(when row
|
||||
(aget row "public_key")))))
|
||||
|
||||
(defn <graph-encrypted-aes-key-upsert!
|
||||
[db graph-id user-id encrypted-aes-key]
|
||||
(when (and (string? graph-id) (string? user-id))
|
||||
(let [now (common/now-ms)]
|
||||
(common/<d1-run db
|
||||
(str "insert into graph_aes_keys (graph_id, user_id, encrypted_aes_key, created_at, updated_at) "
|
||||
"values (?, ?, ?, ?, ?) "
|
||||
"on conflict(graph_id, user_id) do update set "
|
||||
"encrypted_aes_key = excluded.encrypted_aes_key, "
|
||||
"updated_at = excluded.updated_at")
|
||||
graph-id
|
||||
user-id
|
||||
encrypted-aes-key
|
||||
now
|
||||
now))))
|
||||
|
||||
(defn <graph-encrypted-aes-key
|
||||
[db graph-id user-id]
|
||||
(when (and (string? graph-id) (string? user-id))
|
||||
(p/let [result (common/<d1-all db
|
||||
"select encrypted_aes_key from graph_aes_keys where graph_id = ? and user_id = ?"
|
||||
graph-id
|
||||
user-id)
|
||||
rows (common/get-sql-rows result)
|
||||
row (first rows)]
|
||||
(when row
|
||||
(aget row "encrypted_aes_key")))))
|
||||
|
||||
(defn <graph-member-upsert! [db graph-id user-id role invited-by]
|
||||
(let [now (common/now-ms)]
|
||||
(common/<d1-run db
|
||||
(str "insert into graph_members (user_id, graph_id, role, invited_by, created_at) "
|
||||
"values (?, ?, ?, ?, ?) "
|
||||
"on conflict(user_id, graph_id) do update set "
|
||||
"role = excluded.role, "
|
||||
"invited_by = excluded.invited_by")
|
||||
user-id
|
||||
graph-id
|
||||
role
|
||||
invited-by
|
||||
now)))
|
||||
|
||||
(defn <graph-members-list [db graph-id]
|
||||
(p/let [result (common/<d1-all db {:session "first-primary"}
|
||||
(str "select m.user_id, m.graph_id, m.role, m.invited_by, m.created_at, "
|
||||
"u.email, u.username "
|
||||
"from graph_members m "
|
||||
"left join users u on m.user_id = u.id "
|
||||
"where m.graph_id = ? order by m.created_at asc")
|
||||
graph-id)
|
||||
rows (common/get-sql-rows result)]
|
||||
(mapv (fn [row]
|
||||
{:user-id (aget row "user_id")
|
||||
:graph-id (aget row "graph_id")
|
||||
:role (aget row "role")
|
||||
:invited-by (aget row "invited_by")
|
||||
:created-at (aget row "created_at")
|
||||
:email (aget row "email")
|
||||
:username (aget row "username")})
|
||||
rows)))
|
||||
|
||||
(defn <graph-member-update-role! [db graph-id user-id role]
|
||||
(common/<d1-run db
|
||||
(str "update graph_members set role = ? "
|
||||
"where graph_id = ? and user_id = ?")
|
||||
role
|
||||
graph-id
|
||||
user-id))
|
||||
|
||||
(defn <graph-member-delete! [db graph-id user-id]
|
||||
(common/<d1-run db
|
||||
"delete from graph_members where graph_id = ? and user_id = ?"
|
||||
graph-id
|
||||
user-id))
|
||||
|
||||
(defn <graph-member-role [db graph-id user-id]
|
||||
(when (and (string? graph-id) (string? user-id))
|
||||
(p/let [result (common/<d1-all db
|
||||
"select role from graph_members where graph_id = ? and user_id = ?"
|
||||
graph-id
|
||||
user-id)
|
||||
rows (common/get-sql-rows result)
|
||||
row (first rows)]
|
||||
(when row
|
||||
(aget row "role")))))
|
||||
|
||||
(defn <user-has-access-to-graph? [db graph-id user-id]
|
||||
(when (and (string? graph-id) (string? user-id))
|
||||
(p/let [result (common/<d1-all db
|
||||
(str "select graph_id from graphs where graph_id = ? and user_id = ? "
|
||||
"union select graph_id from graph_members where graph_id = ? and user_id = ?")
|
||||
graph-id
|
||||
user-id
|
||||
graph-id
|
||||
user-id)
|
||||
rows (common/get-sql-rows result)]
|
||||
(boolean (seq rows)))))
|
||||
|
||||
(defn <user-is-manager? [db graph-id user-id]
|
||||
(when (and (string? graph-id) (string? user-id))
|
||||
(p/let [result (common/<d1-all db
|
||||
(str "select graph_id from graphs where graph_id = ? and user_id = ? "
|
||||
"union select graph_id from graph_members where graph_id = ? and user_id = ? and role = 'manager'")
|
||||
graph-id
|
||||
user-id
|
||||
graph-id
|
||||
user-id)
|
||||
rows (common/get-sql-rows result)]
|
||||
(boolean (seq rows)))))
|
||||
9
deps/db-sync/src/logseq/db_sync/logging.cljs
vendored
Normal file
9
deps/db-sync/src/logseq/db_sync/logging.cljs
vendored
Normal file
@@ -0,0 +1,9 @@
|
||||
(ns logseq.db-sync.logging
|
||||
(:require [lambdaisland.glogi :as log]
|
||||
[lambdaisland.glogi.console :as glogi-console]))
|
||||
|
||||
(defn install!
|
||||
"Installs console logging and sets the root logger level to `:info`."
|
||||
[]
|
||||
(glogi-console/install!)
|
||||
(log/set-level :glogi/root :info))
|
||||
306
deps/db-sync/src/logseq/db_sync/malli_schema.cljs
vendored
Normal file
306
deps/db-sync/src/logseq/db_sync/malli_schema.cljs
vendored
Normal file
@@ -0,0 +1,306 @@
|
||||
(ns logseq.db-sync.malli-schema
|
||||
(:require [malli.core :as ma]
|
||||
[malli.transform :as mt]))
|
||||
|
||||
(def tx-entry-schema
|
||||
[:map
|
||||
[:tx-id {:optional true} :uuid]
|
||||
[:tx :string]
|
||||
[:outliner-op {:optional true} [:maybe :keyword]]])
|
||||
|
||||
(def tx-log-entry-schema
|
||||
[:map
|
||||
[:t :int]
|
||||
[:tx :string]
|
||||
[:outliner-op {:optional true} [:maybe :keyword]]])
|
||||
|
||||
(def client-revision-entry
|
||||
[:client-revision {:optional true} :string])
|
||||
|
||||
(def ws-client-message-schema
|
||||
[:multi {:dispatch :type}
|
||||
["hello"
|
||||
[:map
|
||||
[:type [:= "hello"]]
|
||||
[:client :string]]]
|
||||
["presence"
|
||||
[:map
|
||||
[:type [:= "presence"]]
|
||||
[:editing-block-uuid {:optional true} [:maybe :string]]]]
|
||||
["pull"
|
||||
[:map
|
||||
[:type [:= "pull"]]
|
||||
[:since {:optional true} :int]]]
|
||||
["tx/batch"
|
||||
[:map
|
||||
[:type [:= "tx/batch"]]
|
||||
client-revision-entry
|
||||
[:t-before :int]
|
||||
[:txs [:sequential tx-entry-schema]]]]
|
||||
["ping"
|
||||
[:map
|
||||
[:type [:= "ping"]]]]])
|
||||
|
||||
(def tx-reject-reason-schema
|
||||
[:enum "stale"
|
||||
"empty tx data"
|
||||
"invalid tx"
|
||||
"invalid t-before"
|
||||
"db transact failed"
|
||||
"snapshot upload in progress"])
|
||||
|
||||
(def tx-reject-schema
|
||||
[:map
|
||||
[:type [:= "tx/reject"]]
|
||||
[:reason tx-reject-reason-schema]
|
||||
[:t {:optional true} :int]
|
||||
[:success-tx-ids {:optional true} [:sequential :uuid]]
|
||||
[:failed-tx-id {:optional true} :uuid]
|
||||
[:missing-block-uuids {:optional true} [:sequential :uuid]]
|
||||
[:error-detail {:optional true} :string]
|
||||
[:data {:optional true} :string]])
|
||||
|
||||
(def user-presence-schema
|
||||
[:map
|
||||
[:user-id :string]
|
||||
[:email {:optional true} [:maybe :string]]
|
||||
[:username {:optional true} [:maybe :string]]
|
||||
[:name {:optional true} [:maybe :string]]])
|
||||
|
||||
(def online-users-schema
|
||||
[:map
|
||||
[:type [:= "online-users"]]
|
||||
[:online-users [:sequential user-presence-schema]]])
|
||||
|
||||
(def pull-ok-schema
|
||||
[:map
|
||||
[:type [:= "pull/ok"]]
|
||||
[:t :int]
|
||||
[:checksum {:optional true} :string]
|
||||
[:txs [:sequential tx-log-entry-schema]]])
|
||||
|
||||
(def tx-batch-ok-schema
|
||||
[:map
|
||||
[:type [:= "tx/batch/ok"]]
|
||||
[:t :int]
|
||||
[:checksum {:optional true} :string]])
|
||||
|
||||
(def ws-server-message-schema
|
||||
[:multi {:dispatch :type}
|
||||
["hello"
|
||||
[:map
|
||||
[:type [:= "hello"]]
|
||||
[:t :int]
|
||||
[:checksum {:optional true} :string]]]
|
||||
["online-users" online-users-schema]
|
||||
["presence"
|
||||
[:map
|
||||
[:type [:= "presence"]]
|
||||
[:user-id :string]
|
||||
[:editing-block-uuid [:maybe :string]]]]
|
||||
["pull/ok" pull-ok-schema]
|
||||
["tx/batch/ok" tx-batch-ok-schema]
|
||||
["changed"
|
||||
[:map
|
||||
[:type [:= "changed"]]
|
||||
[:t :int]]]
|
||||
["tx/reject" tx-reject-schema]
|
||||
["pong"
|
||||
[:map
|
||||
[:type [:= "pong"]]]]
|
||||
["error"
|
||||
[:map
|
||||
[:type [:= "error"]]
|
||||
[:message :string]]]])
|
||||
|
||||
(def http-error-response-schema
|
||||
[:map
|
||||
[:error :string]])
|
||||
|
||||
(def http-ok-response-schema
|
||||
[:map
|
||||
[:ok :boolean]])
|
||||
|
||||
(def graph-member-role-schema
|
||||
[:enum "manager" "member"])
|
||||
|
||||
(def graph-info-schema
|
||||
[:map
|
||||
[:graph-id :string]
|
||||
[:graph-name :string]
|
||||
[:schema-version {:optional true} [:maybe :string]]
|
||||
[:graph-e2ee? {:optional true} :boolean]
|
||||
[:graph-ready-for-use? {:optional true} :boolean]
|
||||
[:role {:optional true} [:maybe graph-member-role-schema]]
|
||||
[:invited-by {:optional true} [:maybe :string]]
|
||||
[:created-at :int]
|
||||
[:updated-at :int]])
|
||||
|
||||
(def graph-member-info-schema
|
||||
[:map
|
||||
[:user-id :string]
|
||||
[:graph-id :string]
|
||||
[:role graph-member-role-schema]
|
||||
[:invited-by {:optional true} [:maybe :string]]
|
||||
[:created-at :int]
|
||||
[:email {:optional true} [:maybe :string]]
|
||||
[:username {:optional true} [:maybe :string]]])
|
||||
|
||||
(def graph-members-list-response-schema
|
||||
[:map
|
||||
[:members [:sequential graph-member-info-schema]]])
|
||||
|
||||
(def graphs-list-response-schema
|
||||
[:map
|
||||
[:graphs [:sequential graph-info-schema]]
|
||||
[:user-rsa-keys-exists? {:optional true} :boolean]])
|
||||
|
||||
(def graph-create-request-schema
|
||||
[:map
|
||||
[:graph-name :string]
|
||||
[:schema-version {:optional true} [:maybe :string]]
|
||||
[:graph-e2ee? {:optional true} :boolean]
|
||||
[:graph-ready-for-use? {:optional true} :boolean]])
|
||||
|
||||
(def graph-create-response-schema
|
||||
[:map
|
||||
[:graph-id :string]
|
||||
[:graph-e2ee? {:optional true} :boolean]
|
||||
[:graph-ready-for-use? {:optional true} :boolean]])
|
||||
|
||||
(def graph-access-response-schema http-ok-response-schema)
|
||||
|
||||
(def graph-delete-response-schema
|
||||
[:map
|
||||
[:graph-id :string]
|
||||
[:deleted :boolean]])
|
||||
|
||||
(def graph-member-create-request-schema
|
||||
[:or
|
||||
[:map
|
||||
[:user-id :string]
|
||||
[:role {:optional true} graph-member-role-schema]]
|
||||
[:map
|
||||
[:email :string]
|
||||
[:role {:optional true} graph-member-role-schema]]])
|
||||
|
||||
(def graph-member-update-request-schema
|
||||
[:map
|
||||
[:role graph-member-role-schema]])
|
||||
|
||||
(def tx-batch-request-schema
|
||||
[:map
|
||||
client-revision-entry
|
||||
[:t-before :int]
|
||||
[:txs [:sequential tx-entry-schema]]])
|
||||
|
||||
(def e2ee-user-key-request-schema
|
||||
[:map
|
||||
[:public-key :string]
|
||||
[:encrypted-private-key :string]
|
||||
[:reset-private-key {:optional true} :boolean]])
|
||||
|
||||
(def e2ee-user-key-response-schema
|
||||
[:map
|
||||
[:public-key {:optional true} [:maybe :string]]
|
||||
[:encrypted-private-key {:optional true} [:maybe :string]]])
|
||||
|
||||
(def e2ee-user-public-key-response-schema
|
||||
[:map
|
||||
[:public-key {:optional true} [:maybe :string]]])
|
||||
|
||||
(def e2ee-graph-aes-key-request-schema
|
||||
[:map
|
||||
[:encrypted-aes-key :string]])
|
||||
|
||||
(def e2ee-graph-aes-key-response-schema
|
||||
[:map
|
||||
[:encrypted-aes-key {:optional true} [:maybe :string]]])
|
||||
|
||||
(def e2ee-grant-access-entry-schema
|
||||
[:map
|
||||
[:email :string]
|
||||
[:encrypted-aes-key :string]])
|
||||
|
||||
(def e2ee-grant-access-request-schema
|
||||
[:map
|
||||
[:target-user-email+encrypted-aes-key-coll [:sequential e2ee-grant-access-entry-schema]]])
|
||||
|
||||
(def e2ee-grant-access-response-schema
|
||||
[:map
|
||||
[:ok :boolean]
|
||||
[:missing-users {:optional true} [:sequential :string]]])
|
||||
|
||||
(def snapshot-download-response-schema
|
||||
[:map
|
||||
[:ok :boolean]
|
||||
[:key :string]
|
||||
[:url :string]
|
||||
[:content-encoding {:optional true} [:maybe :string]]])
|
||||
|
||||
(def snapshot-upload-response-schema
|
||||
[:map
|
||||
[:ok :boolean]
|
||||
[:count :int]])
|
||||
|
||||
(def asset-get-response-schema
|
||||
[:or
|
||||
:any
|
||||
http-error-response-schema])
|
||||
|
||||
(def http-request-schemas
|
||||
{:graphs/create graph-create-request-schema
|
||||
:graph-members/create graph-member-create-request-schema
|
||||
:graph-members/update graph-member-update-request-schema
|
||||
:sync/tx-batch tx-batch-request-schema
|
||||
:e2ee/user-keys e2ee-user-key-request-schema
|
||||
:e2ee/graph-aes-key e2ee-graph-aes-key-request-schema
|
||||
:e2ee/grant-access e2ee-grant-access-request-schema})
|
||||
|
||||
(def http-response-schemas
|
||||
{:graphs/list graphs-list-response-schema
|
||||
:graphs/create graph-create-response-schema
|
||||
:graphs/access graph-access-response-schema
|
||||
:graphs/delete graph-delete-response-schema
|
||||
:graph-members/list graph-members-list-response-schema
|
||||
:graph-members/create http-ok-response-schema
|
||||
:graph-members/update http-ok-response-schema
|
||||
:graph-members/delete http-ok-response-schema
|
||||
:worker/health http-ok-response-schema
|
||||
:sync/health http-ok-response-schema
|
||||
:sync/pull pull-ok-schema
|
||||
:sync/tx-batch [:or tx-batch-ok-schema tx-reject-schema http-error-response-schema]
|
||||
:sync/snapshot-download snapshot-download-response-schema
|
||||
:sync/snapshot-upload snapshot-upload-response-schema
|
||||
:sync/admin-reset http-ok-response-schema
|
||||
:e2ee/user-keys e2ee-user-key-response-schema
|
||||
:e2ee/user-public-key e2ee-user-public-key-response-schema
|
||||
:e2ee/graph-aes-key e2ee-graph-aes-key-response-schema
|
||||
:e2ee/grant-access e2ee-grant-access-response-schema
|
||||
:assets/get asset-get-response-schema
|
||||
:assets/put http-ok-response-schema
|
||||
:assets/delete http-ok-response-schema
|
||||
:error http-error-response-schema})
|
||||
|
||||
(def ^:private json-transformer
|
||||
(mt/transformer
|
||||
{:name :db-sync/json}
|
||||
mt/json-transformer))
|
||||
|
||||
(defn- ->coercer [schema]
|
||||
(ma/coercer schema json-transformer nil
|
||||
#(ma/-fail! ::db-sync-malli-coerce
|
||||
(select-keys % [:value]))))
|
||||
|
||||
(def ws-client-message-coercer (->coercer ws-client-message-schema))
|
||||
(def ws-server-message-coercer (->coercer ws-server-message-schema))
|
||||
|
||||
(def http-request-coercers
|
||||
(into {}
|
||||
(map (fn [[k schema]] [k (->coercer schema)]))
|
||||
http-request-schemas))
|
||||
|
||||
(def http-response-coercers
|
||||
(into {}
|
||||
(map (fn [[k schema]] [k (->coercer schema)]))
|
||||
http-response-schemas))
|
||||
10
deps/db-sync/src/logseq/db_sync/node/api_docs.cljs
vendored
Normal file
10
deps/db-sync/src/logseq/db_sync/node/api_docs.cljs
vendored
Normal file
@@ -0,0 +1,10 @@
|
||||
(ns logseq.db-sync.node.api-docs
|
||||
(:require ["node:fs" :as fs]
|
||||
[logseq.db-sync.worker.routes.semantic :as semantic-routes]))
|
||||
|
||||
(defn ^:export main []
|
||||
(let [issuer (or (aget js/process.env "COGNITO_ISSUER") "")
|
||||
document (semantic-routes/openapi-document issuer)]
|
||||
(.mkdirSync fs "worker/dist" #js {:recursive true})
|
||||
(.writeFileSync fs "worker/dist/openapi.json"
|
||||
(js/JSON.stringify (clj->js document) nil 2))))
|
||||
58
deps/db-sync/src/logseq/db_sync/node/assets.cljs
vendored
Normal file
58
deps/db-sync/src/logseq/db_sync/node/assets.cljs
vendored
Normal file
@@ -0,0 +1,58 @@
|
||||
(ns logseq.db-sync.node.assets
|
||||
(:require ["fs" :as fs]
|
||||
["path" :as node-path]
|
||||
[promesa.core :as p]))
|
||||
|
||||
(defn- ensure-dir! [dir]
|
||||
(.mkdirSync fs dir #js {:recursive true}))
|
||||
|
||||
(defn- meta-path [path]
|
||||
(str path ".meta.json"))
|
||||
|
||||
(defn- read-meta [path]
|
||||
(p/let [content (.readFile (.-promises fs) path "utf8")]
|
||||
(js/JSON.parse content)))
|
||||
|
||||
(defn- write-meta! [path meta]
|
||||
(.writeFile (.-promises fs) path (js/JSON.stringify meta) "utf8"))
|
||||
|
||||
(defn- normalize-bytes [data]
|
||||
(cond
|
||||
(instance? js/Uint8Array data) data
|
||||
(instance? js/ArrayBuffer data) (js/Uint8Array. data)
|
||||
:else (js/Uint8Array. data)))
|
||||
|
||||
(defn make-bucket [base-dir]
|
||||
(ensure-dir! base-dir)
|
||||
#js {:get (fn [key]
|
||||
(let [file-path (node-path/join base-dir key)
|
||||
meta-file (meta-path file-path)]
|
||||
(p/catch
|
||||
(p/let [buf (.readFile (.-promises fs) file-path)
|
||||
meta (p/catch (read-meta meta-file) (fn [_] #js {}))]
|
||||
#js {:body buf
|
||||
:httpMetadata #js {:contentType (aget meta "contentType")
|
||||
:contentEncoding (aget meta "contentEncoding")
|
||||
:cacheControl (aget meta "cacheControl")}
|
||||
:customMetadata (aget meta "customMetadata")})
|
||||
(fn [_] nil))))
|
||||
:put (fn [key body opts]
|
||||
(let [file-path (node-path/join base-dir key)
|
||||
meta-file (meta-path file-path)
|
||||
dir (node-path/dirname file-path)
|
||||
data (normalize-bytes body)
|
||||
metadata (or (aget opts "httpMetadata") #js {})
|
||||
custom (or (aget opts "customMetadata") #js {})]
|
||||
(ensure-dir! dir)
|
||||
(p/let [_ (.writeFile (.-promises fs) file-path data)
|
||||
_ (write-meta! meta-file #js {:contentType (aget metadata "contentType")
|
||||
:contentEncoding (aget metadata "contentEncoding")
|
||||
:cacheControl (aget metadata "cacheControl")
|
||||
:customMetadata custom})]
|
||||
#js {:ok true})))
|
||||
:delete (fn [key]
|
||||
(let [file-path (node-path/join base-dir key)
|
||||
meta-file (meta-path file-path)]
|
||||
(p/let [_ (p/catch (.unlink (.-promises fs) file-path) (fn [_] nil))
|
||||
_ (p/catch (.unlink (.-promises fs) meta-file) (fn [_] nil))]
|
||||
#js {:ok true})))})
|
||||
44
deps/db-sync/src/logseq/db_sync/node/config.cljs
vendored
Normal file
44
deps/db-sync/src/logseq/db_sync/node/config.cljs
vendored
Normal file
@@ -0,0 +1,44 @@
|
||||
(ns logseq.db-sync.node.config
|
||||
(:require [clojure.string :as string]))
|
||||
|
||||
(defn- env-value [^js env k]
|
||||
(let [v (aget env k)]
|
||||
(when (seq v) v)))
|
||||
|
||||
(defn- parse-int [v default]
|
||||
(let [n (js/parseInt v 10)]
|
||||
(if (js/isNaN n) default n)))
|
||||
|
||||
(defn config-from-env []
|
||||
(let [env (.-env js/process)]
|
||||
{:port (when-let [v (env-value env "DB_SYNC_PORT")] (parse-int v 8080))
|
||||
:base-url (env-value env "DB_SYNC_BASE_URL")
|
||||
:data-dir (or (env-value env "DB_SYNC_DATA_DIR") "data/db-sync")
|
||||
:storage-driver (or (env-value env "DB_SYNC_STORAGE_DRIVER") "sqlite")
|
||||
:assets-driver (or (env-value env "DB_SYNC_ASSETS_DRIVER") "filesystem")
|
||||
:log-level (or (env-value env "DB_SYNC_LOG_LEVEL") "info")
|
||||
:cognito-issuer (env-value env "COGNITO_ISSUER")
|
||||
:cognito-client-id (env-value env "COGNITO_CLIENT_ID")
|
||||
:cognito-jwks-url (env-value env "COGNITO_JWKS_URL")}))
|
||||
|
||||
(def ^:private allowed-config-keys
|
||||
[:port :base-url :data-dir :storage-driver :assets-driver :log-level
|
||||
:cognito-issuer :cognito-client-id :cognito-jwks-url])
|
||||
|
||||
(defn normalize-config [overrides]
|
||||
(let [defaults {:port 8080
|
||||
:data-dir "data/db-sync"
|
||||
:storage-driver "sqlite"
|
||||
:assets-driver "filesystem"
|
||||
:log-level "info"}
|
||||
merged (merge defaults (config-from-env) overrides)
|
||||
storage-driver (string/lower-case (:storage-driver merged))
|
||||
assets-driver (string/lower-case (:assets-driver merged))]
|
||||
(when-not (#{"sqlite"} storage-driver)
|
||||
(throw (js/Error. (str "unsupported storage driver: " storage-driver))))
|
||||
(when-not (#{"filesystem"} assets-driver)
|
||||
(throw (js/Error. (str "unsupported assets driver: " assets-driver))))
|
||||
(-> merged
|
||||
(select-keys allowed-config-keys)
|
||||
(assoc :storage-driver storage-driver
|
||||
:assets-driver assets-driver))))
|
||||
76
deps/db-sync/src/logseq/db_sync/node/dispatch.cljs
vendored
Normal file
76
deps/db-sync/src/logseq/db_sync/node/dispatch.cljs
vendored
Normal file
@@ -0,0 +1,76 @@
|
||||
(ns logseq.db-sync.node.dispatch
|
||||
(:require [clojure.string :as string]
|
||||
[logseq.db-sync.common :as common]
|
||||
[logseq.db-sync.node.graph :as graph]
|
||||
[logseq.db-sync.node.routes :as node-routes]
|
||||
[logseq.db-sync.platform.core :as platform]
|
||||
[logseq.db-sync.worker.handler.assets :as assets-handler]
|
||||
[logseq.db-sync.worker.handler.index :as index-handler]
|
||||
[logseq.db-sync.worker.handler.sync :as sync-handler]
|
||||
[logseq.db-sync.worker.http :as http]
|
||||
[promesa.core :as p]))
|
||||
|
||||
(defn- admin-token-valid?
|
||||
[request env]
|
||||
(let [expected (aget env "DB_SYNC_ADMIN_TOKEN")
|
||||
actual (.get (.-headers request) "x-db-sync-admin-token")]
|
||||
(and (string? expected)
|
||||
(seq expected)
|
||||
(= expected actual))))
|
||||
|
||||
(defn handle-node-fetch
|
||||
[{:keys [request env registry deps]}]
|
||||
(let [url (platform/request-url request)
|
||||
path (.-pathname url)
|
||||
method (.-method request)
|
||||
index-self #js {:env env :d1 (aget env "DB")}]
|
||||
(cond
|
||||
(= path "/health")
|
||||
(http/json-response :worker/health {:ok true})
|
||||
|
||||
(or (= path "/graphs")
|
||||
(string/starts-with? path "/graphs/"))
|
||||
(index-handler/handle-fetch index-self request)
|
||||
|
||||
(string/starts-with? path "/e2ee")
|
||||
(index-handler/handle-fetch index-self request)
|
||||
|
||||
(string/starts-with? path "/assets/")
|
||||
(if (= method "OPTIONS")
|
||||
(assets-handler/handle request env)
|
||||
(if-let [{:keys [graph-id]} (assets-handler/parse-asset-path path)]
|
||||
(if (admin-token-valid? request env)
|
||||
(assets-handler/handle request env)
|
||||
(p/let [access-resp (index-handler/graph-access-response request env graph-id)]
|
||||
(if (.-ok access-resp)
|
||||
(assets-handler/handle request env)
|
||||
access-resp)))
|
||||
(http/bad-request "invalid asset path")))
|
||||
|
||||
(= method "OPTIONS")
|
||||
(common/options-response)
|
||||
|
||||
(string/starts-with? path "/sync/")
|
||||
(if-let [{:keys [graph-id tail]} (node-routes/parse-sync-path path)]
|
||||
(if (seq graph-id)
|
||||
(if (= method "OPTIONS")
|
||||
(common/options-response)
|
||||
(if (admin-token-valid? request env)
|
||||
(let [ctx (graph/get-or-create-graph registry deps graph-id)
|
||||
new-url (js/URL. (str (.-origin url) tail (.-search url)))]
|
||||
(.set (.-searchParams new-url) "graph-id" graph-id)
|
||||
(let [rewritten (platform/request (.toString new-url) request)]
|
||||
(sync-handler/handle-http ctx rewritten)))
|
||||
(p/let [access-resp (index-handler/graph-access-response request env graph-id)]
|
||||
(if (.-ok access-resp)
|
||||
(let [ctx (graph/get-or-create-graph registry deps graph-id)
|
||||
new-url (js/URL. (str (.-origin url) tail (.-search url)))]
|
||||
(.set (.-searchParams new-url) "graph-id" graph-id)
|
||||
(let [rewritten (platform/request (.toString new-url) request)]
|
||||
(sync-handler/handle-http ctx rewritten)))
|
||||
access-resp))))
|
||||
(http/bad-request "missing graph id"))
|
||||
(http/bad-request "missing graph id"))
|
||||
|
||||
:else
|
||||
(http/not-found))))
|
||||
18
deps/db-sync/src/logseq/db_sync/node/entry.cljs
vendored
Normal file
18
deps/db-sync/src/logseq/db_sync/node/entry.cljs
vendored
Normal file
@@ -0,0 +1,18 @@
|
||||
(ns logseq.db-sync.node.entry
|
||||
(:require [logseq.db-sync.node.config :as config]
|
||||
[logseq.db-sync.node.server :as server]
|
||||
[logseq.db-sync.sentry.node :as sentry]
|
||||
[promesa.core :as p]))
|
||||
|
||||
(defonce ^:private *server (atom nil))
|
||||
|
||||
(defn main [& _args]
|
||||
(sentry/init!)
|
||||
(let [cfg (config/normalize-config {})]
|
||||
(js/console.log "Starting Logseq sync...")
|
||||
(-> (server/start! cfg)
|
||||
(p/then (fn [result]
|
||||
(reset! *server result)
|
||||
(js/console.log (str "Logseq sync listening on port " (:port result)))))
|
||||
(p/catch (fn [error]
|
||||
(js/console.error "Logseq sync failed to start" error))))))
|
||||
199
deps/db-sync/src/logseq/db_sync/node/find_checksum_drift.cljs
vendored
Normal file
199
deps/db-sync/src/logseq/db_sync/node/find_checksum_drift.cljs
vendored
Normal file
@@ -0,0 +1,199 @@
|
||||
(ns logseq.db-sync.node.find-checksum-drift
|
||||
(:require ["better-sqlite3" :as sqlite3]
|
||||
["fs" :as fs]
|
||||
["path" :as node-path]
|
||||
[clojure.string :as string]
|
||||
[datascript.core :as d]
|
||||
[logseq.db-sync.checksum :as sync-checksum]
|
||||
[logseq.db-sync.common :as common]
|
||||
[logseq.db-sync.storage :as storage]
|
||||
[logseq.db.frontend.schema :as db-schema]))
|
||||
|
||||
(def sqlite (if (find-ns 'nbb.core) (aget sqlite3 "default") sqlite3))
|
||||
|
||||
(defn- fail!
|
||||
[msg]
|
||||
(binding [*print-fn* *print-err-fn*]
|
||||
(println (str "Error: " msg)))
|
||||
(js/process.exit 1))
|
||||
|
||||
(defn- parse-args
|
||||
[argv]
|
||||
(loop [m {}
|
||||
xs (vec argv)]
|
||||
(if-let [x (first xs)]
|
||||
(cond
|
||||
(some #{x} ["-h" "--help"])
|
||||
(recur (assoc m :help? true) (next xs))
|
||||
|
||||
(= "--db" x)
|
||||
(if-let [db (second xs)]
|
||||
(recur (assoc m :db db) (nnext xs))
|
||||
(fail! "Missing value for --db"))
|
||||
|
||||
(= "--diff-out" x)
|
||||
(if-let [diff-out (second xs)]
|
||||
(recur (assoc m :diff-out diff-out) (nnext xs))
|
||||
(fail! "Missing value for --diff-out"))
|
||||
|
||||
:else
|
||||
(fail! (str "Unknown argument: " x)))
|
||||
m)))
|
||||
|
||||
(defn- print-help!
|
||||
[]
|
||||
(println "Find first tx_log checksum drift in a sqlite graph db.")
|
||||
(println "")
|
||||
(println "Usage:")
|
||||
(println " node worker/dist/find-checksum-drift.js --db <path/to/db.sqlite>")
|
||||
(println " node worker/dist/find-checksum-drift.js --db <path/to/db.sqlite> --diff-out <path/to/diff.edn>")
|
||||
(println "")
|
||||
(println "Output:")
|
||||
(println " - stored/recomputed checksum for current db")
|
||||
(println " - first tx_log t where incremental != recomputed during replay")
|
||||
(println " - optional full current-vs-replayed block diff EDN file"))
|
||||
|
||||
(defn- normalize-sql
|
||||
[sql]
|
||||
(-> sql string/trim string/lower-case))
|
||||
|
||||
(defn- select-sql?
|
||||
[sql]
|
||||
(string/starts-with? (normalize-sql sql) "select"))
|
||||
|
||||
(defn- exec-with-args [^js stmt args]
|
||||
(.apply (.-run stmt) stmt (to-array args)))
|
||||
|
||||
(defn- all-with-args [^js stmt args]
|
||||
(.apply (.-all stmt) stmt (to-array args)))
|
||||
|
||||
(defn- wrap-db [^js db]
|
||||
#js {:exec (fn [sql & args]
|
||||
(if (seq args)
|
||||
(let [stmt (.prepare db sql)]
|
||||
(if (select-sql? sql)
|
||||
(all-with-args stmt args)
|
||||
(do
|
||||
(exec-with-args stmt args)
|
||||
nil)))
|
||||
(if (select-sql? sql)
|
||||
(.all (.prepare db sql))
|
||||
(.exec db sql))))
|
||||
:close (fn [] (.close db))
|
||||
:_db db})
|
||||
|
||||
(defn- short-preview
|
||||
[tx-data]
|
||||
(->> tx-data
|
||||
(take 8)
|
||||
vec))
|
||||
|
||||
(defn- tx-rows
|
||||
[^js db]
|
||||
(.all (.prepare db "select t, tx, outliner_op from tx_log order by t asc")))
|
||||
|
||||
(defn- replay-find-first-mismatch
|
||||
[rows]
|
||||
(let [conn (d/create-conn db-schema/schema)]
|
||||
(loop [idx 0
|
||||
checksum nil]
|
||||
(if (>= idx (count rows))
|
||||
{:ok? true
|
||||
:idx idx
|
||||
:checksum checksum
|
||||
:recomputed (sync-checksum/recompute-checksum @conn)
|
||||
:db @conn}
|
||||
(let [row (nth rows idx)
|
||||
t (aget row "t")
|
||||
outliner-op (aget row "outliner_op")
|
||||
tx-str (aget row "tx")
|
||||
tx-data (common/read-transit tx-str)
|
||||
tx-report (d/transact! conn tx-data)
|
||||
incremental (sync-checksum/update-checksum checksum tx-report)
|
||||
recomputed (sync-checksum/recompute-checksum (:db-after tx-report))]
|
||||
(if (= incremental recomputed)
|
||||
(recur (inc idx) incremental)
|
||||
{:ok? false
|
||||
:idx idx
|
||||
:t t
|
||||
:outliner-op outliner-op
|
||||
:prev-checksum checksum
|
||||
:incremental incremental
|
||||
:recomputed recomputed
|
||||
:tx-size (count tx-data)
|
||||
:tx-preview (short-preview tx-data)
|
||||
:db (:db-after tx-report)}))))))
|
||||
|
||||
(defn- block-map-by-uuid
|
||||
[blocks]
|
||||
(into {} (map (juxt :block/uuid identity) blocks)))
|
||||
|
||||
(defn- diff-blocks
|
||||
[blocks-a blocks-b]
|
||||
(let [a-by-id (block-map-by-uuid blocks-a)
|
||||
b-by-id (block-map-by-uuid blocks-b)
|
||||
all-ids (set (concat (keys a-by-id) (keys b-by-id)))]
|
||||
(->> all-ids
|
||||
(keep (fn [id]
|
||||
(let [a (get a-by-id id)
|
||||
b (get b-by-id id)]
|
||||
(when (not= a b)
|
||||
{:block/uuid id
|
||||
:current a
|
||||
:replayed b}))))
|
||||
vec)))
|
||||
|
||||
(defn- sorted-block-diffs
|
||||
[block-diffs]
|
||||
(sort-by (fn [m]
|
||||
(some-> (:block/uuid m) str))
|
||||
block-diffs))
|
||||
|
||||
(defn main
|
||||
[& argv]
|
||||
(let [{:keys [help? db diff-out]} (parse-args argv)]
|
||||
(when help?
|
||||
(print-help!)
|
||||
(js/process.exit 0))
|
||||
(when-not db
|
||||
(fail! "Missing required --db <path> argument"))
|
||||
(let [db-path (node-path/resolve db)]
|
||||
(when-not (.existsSync fs db-path)
|
||||
(fail! (str "SQLite file not found: " db-path)))
|
||||
(let [sqlite-db (new sqlite db-path nil)
|
||||
sql (wrap-db sqlite-db)]
|
||||
(try
|
||||
(storage/init-schema! sql)
|
||||
(let [conn (storage/open-conn sql)
|
||||
db' @conn
|
||||
stored (storage/get-checksum sql)
|
||||
recomputed (sync-checksum/recompute-checksum db')
|
||||
rows (tx-rows sqlite-db)
|
||||
replay-result (replay-find-first-mismatch rows)
|
||||
current-diag (sync-checksum/recompute-checksum-diagnostics db')
|
||||
replay-diag (sync-checksum/recompute-checksum-diagnostics (:db replay-result))
|
||||
block-diffs (-> (diff-blocks (:blocks current-diag) (:blocks replay-diag))
|
||||
sorted-block-diffs)]
|
||||
(println (str "db: " db-path))
|
||||
(println (str "stored-checksum: " stored))
|
||||
(println (str "recomputed-checksum: " recomputed))
|
||||
(println (str "tx-log-count: " (count rows)))
|
||||
(println (str "current-blocks: " (count (:blocks current-diag))
|
||||
", replayed-blocks: " (count (:blocks replay-diag))
|
||||
", different-blocks: " (count block-diffs)))
|
||||
(when (seq block-diffs)
|
||||
(println "different-blocks-sample:")
|
||||
(println (pr-str (take 5 block-diffs))))
|
||||
(when (seq diff-out)
|
||||
(let [diff-out-path (node-path/resolve diff-out)]
|
||||
(.writeFileSync fs diff-out-path
|
||||
(str (pr-str block-diffs) "\n")
|
||||
"utf8")
|
||||
(println (str "diff-out: " diff-out-path))))
|
||||
(if (:ok? replay-result)
|
||||
(println (pr-str (assoc replay-result
|
||||
:stored-checksum stored
|
||||
:final-recomputed recomputed)))
|
||||
(println (pr-str replay-result))))
|
||||
(finally
|
||||
(.close sql)))))))
|
||||
57
deps/db-sync/src/logseq/db_sync/node/graph.cljs
vendored
Normal file
57
deps/db-sync/src/logseq/db_sync/node/graph.cljs
vendored
Normal file
@@ -0,0 +1,57 @@
|
||||
(ns logseq.db-sync.node.graph
|
||||
(:require [logseq.db-sync.node.storage :as storage]))
|
||||
|
||||
(defn- make-state []
|
||||
(let [sockets (atom #{})]
|
||||
#js {:getWebSockets (fn [] (to-array @sockets))
|
||||
:addWebSocket (fn [ws] (swap! sockets conj ws))
|
||||
:removeWebSocket (fn [ws] (swap! sockets disj ws))}))
|
||||
|
||||
(defn- env-object [cfg index-db assets-bucket]
|
||||
(let [allow-unverified-jwt-claims (some-> js/process .-env (aget "DB_SYNC_ALLOW_UNVERIFIED_JWT_CLAIMS"))
|
||||
env (doto (js-obj)
|
||||
(aset "DB" index-db)
|
||||
(aset "LOGSEQ_SYNC_ASSETS" assets-bucket)
|
||||
;; Keep node-adapter snapshot stream uncompressed.
|
||||
(aset "DB_SYNC_SNAPSHOT_STREAM_GZIP" "false")
|
||||
(aset "COGNITO_ISSUER" (:cognito-issuer cfg))
|
||||
(aset "COGNITO_CLIENT_ID" (:cognito-client-id cfg))
|
||||
(aset "COGNITO_JWKS_URL" (:cognito-jwks-url cfg)))]
|
||||
(when (some? allow-unverified-jwt-claims)
|
||||
(aset env "DB_SYNC_ALLOW_UNVERIFIED_JWT_CLAIMS" allow-unverified-jwt-claims))
|
||||
env))
|
||||
|
||||
(defn graph-context
|
||||
[{:keys [config index-db assets-bucket]} graph-id]
|
||||
(let [{:keys [sql]} (storage/open-graph-db (:data-dir config) graph-id)
|
||||
state (make-state)
|
||||
env (env-object config index-db assets-bucket)]
|
||||
#js {:state state
|
||||
:env env
|
||||
:sql sql
|
||||
:conn nil
|
||||
:schema-ready false}))
|
||||
|
||||
(defn get-or-create-graph
|
||||
[registry deps graph-id]
|
||||
(or (get @registry graph-id)
|
||||
(let [ctx (graph-context deps graph-id)]
|
||||
(swap! registry assoc graph-id ctx)
|
||||
ctx)))
|
||||
|
||||
(defn- close-graph-context!
|
||||
[^js ctx]
|
||||
(when-let [^js sql (.-sql ctx)]
|
||||
(when-let [close (.-close sql)]
|
||||
(close))))
|
||||
|
||||
(defn delete-graph!
|
||||
[registry deps graph-id]
|
||||
(when-let [^js ctx (get @registry graph-id)]
|
||||
(close-graph-context! ctx)
|
||||
(swap! registry dissoc graph-id))
|
||||
(storage/delete-graph-db! (get-in deps [:config :data-dir]) graph-id))
|
||||
|
||||
(defn close-graphs! [registry]
|
||||
(doseq [[_ ^js ctx] @registry]
|
||||
(close-graph-context! ctx)))
|
||||
119
deps/db-sync/src/logseq/db_sync/node/recompute_log_checksum.cljs
vendored
Normal file
119
deps/db-sync/src/logseq/db_sync/node/recompute_log_checksum.cljs
vendored
Normal file
@@ -0,0 +1,119 @@
|
||||
(ns logseq.db-sync.node.recompute-log-checksum
|
||||
(:require ["fs" :as fs]
|
||||
["path" :as node-path]
|
||||
[cljs.reader :as reader]
|
||||
[clojure.string :as string]
|
||||
[datascript.core :as d]
|
||||
[logseq.db :as ldb]
|
||||
[logseq.db-sync.checksum :as sync-checksum]))
|
||||
|
||||
(defn- fail!
|
||||
[msg]
|
||||
(binding [*print-fn* *print-err-fn*]
|
||||
(println (str "Error: " msg)))
|
||||
(js/process.exit 1))
|
||||
|
||||
(defn- parse-args
|
||||
[argv]
|
||||
(loop [m {}
|
||||
xs (vec argv)]
|
||||
(if-let [x (first xs)]
|
||||
(cond
|
||||
(some #{x} ["-h" "--help"])
|
||||
(recur (assoc m :help? true) (next xs))
|
||||
|
||||
(= "--log" x)
|
||||
(if-let [log-file (second xs)]
|
||||
(recur (assoc m :log-file log-file) (nnext xs))
|
||||
(fail! "Missing value for --log"))
|
||||
|
||||
:else
|
||||
(fail! (str "Unknown argument: " x)))
|
||||
m)))
|
||||
|
||||
(defn- print-help!
|
||||
[]
|
||||
(println "Recompute checksum from a :before-checksum-error server log line.")
|
||||
(println "")
|
||||
(println "Usage:")
|
||||
(println " node worker/dist/recompute-log-checksum.js --log /tmp/server1.log")
|
||||
(println "")
|
||||
(println "Output:")
|
||||
(println " - logged incremental and recomputed checksums")
|
||||
(println " - replayed incremental and recomputed checksums")
|
||||
(println " - equality checks"))
|
||||
|
||||
(defn- parse-log-payload
|
||||
[line]
|
||||
(let [prefix ":debug :before-checksum-error "]
|
||||
(when-not (string/starts-with? line prefix)
|
||||
(fail! (str "Log does not start with expected prefix: " prefix)))
|
||||
(reader/read-string (subs line (count prefix)))))
|
||||
|
||||
(defn- ensure-tx-data
|
||||
[tx-data]
|
||||
(cond
|
||||
(string? tx-data)
|
||||
(let [decoded (ldb/read-transit-str tx-data)]
|
||||
(if (sequential? decoded)
|
||||
decoded
|
||||
(fail! "Decoded transit tx-data is not sequential")))
|
||||
|
||||
(sequential? tx-data)
|
||||
tx-data
|
||||
|
||||
:else
|
||||
(fail! (str "Unsupported tx-data shape: " (type tx-data)))))
|
||||
|
||||
(defn main
|
||||
[& argv]
|
||||
(let [{:keys [help? log-file]} (parse-args argv)]
|
||||
(when help?
|
||||
(print-help!)
|
||||
(js/process.exit 0))
|
||||
(when-not log-file
|
||||
(fail! "Missing required --log <path> argument"))
|
||||
(let [log-path (node-path/resolve log-file)]
|
||||
(when-not (.existsSync fs log-path)
|
||||
(fail! (str "Log file not found: " log-path)))
|
||||
(let [line (string/trim (.readFileSync fs log-path "utf8"))
|
||||
payload (parse-log-payload line)
|
||||
db-before (ldb/read-transit-str (:db-before payload))
|
||||
tx-data (vec (ensure-tx-data (:tx-data payload)))
|
||||
tx-report (d/with db-before tx-data)
|
||||
input-checksum (or (:prev-checksum payload)
|
||||
(:current-checksum payload))
|
||||
logged-prev-full (:prev-full-checksum payload)
|
||||
logged-full-after (or (:recomputed-after-checksum payload)
|
||||
(:full-checksum payload))
|
||||
replayed-prev-full (sync-checksum/recompute-checksum db-before)
|
||||
replayed-incremental-from-full-before
|
||||
(sync-checksum/update-checksum replayed-prev-full tx-report)
|
||||
replayed-incremental-from-input
|
||||
(sync-checksum/update-checksum input-checksum tx-report)
|
||||
replayed-recomputed (sync-checksum/recompute-checksum (:db-after tx-report))
|
||||
result {:log-file log-path
|
||||
:prev-tx (:prev-tx payload)
|
||||
:tx-meta (:tx-meta payload)
|
||||
:tx-count (count tx-data)
|
||||
:input-checksum input-checksum
|
||||
:logged-prev-full-checksum logged-prev-full
|
||||
:replayed-prev-full-checksum replayed-prev-full
|
||||
:input-checksum-eq-replayed-prev-full?
|
||||
(= input-checksum replayed-prev-full)
|
||||
:match-logged-prev-full?
|
||||
(= logged-prev-full replayed-prev-full)
|
||||
:logged-new-checksum (:new-checksum payload)
|
||||
:replayed-incremental replayed-incremental-from-input
|
||||
:replayed-incremental-from-full-before replayed-incremental-from-full-before
|
||||
:logged-full-after-checksum logged-full-after
|
||||
:replayed-recomputed-after replayed-recomputed
|
||||
:match-logged-new?
|
||||
(= replayed-incremental-from-input (:new-checksum payload))
|
||||
:match-logged-full-after?
|
||||
(= replayed-recomputed logged-full-after)
|
||||
:incremental-eq-full?
|
||||
(= replayed-incremental-from-input replayed-recomputed)
|
||||
:incremental-from-full-before-eq-full?
|
||||
(= replayed-incremental-from-full-before replayed-recomputed)}]
|
||||
(println (pr-str result))))))
|
||||
16
deps/db-sync/src/logseq/db_sync/node/routes.cljs
vendored
Normal file
16
deps/db-sync/src/logseq/db_sync/node/routes.cljs
vendored
Normal file
@@ -0,0 +1,16 @@
|
||||
(ns logseq.db-sync.node.routes
|
||||
(:require [clojure.string :as string]))
|
||||
|
||||
(defn parse-sync-path [path]
|
||||
(when (string/starts-with? path "/sync/")
|
||||
(let [rest-path (subs path (count "/sync/"))
|
||||
rest-path (if (string/starts-with? rest-path "/")
|
||||
(subs rest-path 1)
|
||||
rest-path)
|
||||
slash-idx (or (string/index-of rest-path "/") -1)
|
||||
graph-id (if (neg? slash-idx) rest-path (subs rest-path 0 slash-idx))
|
||||
tail (if (neg? slash-idx)
|
||||
"/"
|
||||
(subs rest-path slash-idx))]
|
||||
{:graph-id graph-id
|
||||
:tail tail})))
|
||||
181
deps/db-sync/src/logseq/db_sync/node/server.cljs
vendored
Normal file
181
deps/db-sync/src/logseq/db_sync/node/server.cljs
vendored
Normal file
@@ -0,0 +1,181 @@
|
||||
(ns logseq.db-sync.node.server
|
||||
(:require ["http" :as http]
|
||||
["path" :as node-path]
|
||||
["ws" :as ws]
|
||||
[clojure.string :as string]
|
||||
[lambdaisland.glogi :as log]
|
||||
[logseq.db-sync.index :as index]
|
||||
[logseq.db-sync.logging :as logging]
|
||||
[logseq.db-sync.node.assets :as assets]
|
||||
[logseq.db-sync.node.config :as config]
|
||||
[logseq.db-sync.node.dispatch :as dispatch]
|
||||
[logseq.db-sync.node.graph :as graph]
|
||||
[logseq.db-sync.node.routes :as node-routes]
|
||||
[logseq.db-sync.node.storage :as storage]
|
||||
[logseq.db-sync.platform.core :as platform]
|
||||
[logseq.db-sync.platform.node :as platform-node]
|
||||
[logseq.db-sync.worker.auth :as auth]
|
||||
[logseq.db-sync.worker.handler.sync :as sync-handler]
|
||||
[logseq.db-sync.worker.handler.ws :as ws-handler]
|
||||
[logseq.db-sync.worker.http :as worker-http]
|
||||
[logseq.db-sync.worker.presence :as presence]
|
||||
[promesa.core :as p]))
|
||||
|
||||
(logging/install!)
|
||||
|
||||
(defn- make-env [cfg index-db assets-bucket]
|
||||
(let [allow-unverified-jwt-claims (some-> js/process .-env (aget "DB_SYNC_ALLOW_UNVERIFIED_JWT_CLAIMS"))
|
||||
env (doto (js-obj)
|
||||
(aset "DB" index-db)
|
||||
(aset "LOGSEQ_SYNC_ASSETS" assets-bucket)
|
||||
;; Node adapter serves snapshot transit stream without gzip to avoid
|
||||
;; browser/adapter content-encoding mismatches during graph download.
|
||||
(aset "DB_SYNC_SNAPSHOT_STREAM_GZIP" "false")
|
||||
(aset "COGNITO_ISSUER" (:cognito-issuer cfg))
|
||||
(aset "COGNITO_CLIENT_ID" (:cognito-client-id cfg))
|
||||
(aset "COGNITO_JWKS_URL" (:cognito-jwks-url cfg)))]
|
||||
(when (some? allow-unverified-jwt-claims)
|
||||
(aset env "DB_SYNC_ALLOW_UNVERIFIED_JWT_CLAIMS" allow-unverified-jwt-claims))
|
||||
env))
|
||||
|
||||
(defn- request-origin-opts
|
||||
[cfg]
|
||||
(if-let [base-url (:base-url cfg)]
|
||||
(let [url (js/URL. base-url)
|
||||
protocol (.-protocol url)
|
||||
scheme (if (string/ends-with? protocol ":")
|
||||
(subs protocol 0 (dec (count protocol)))
|
||||
protocol)]
|
||||
{:scheme scheme
|
||||
:host (.-host url)})
|
||||
{:scheme "http"}))
|
||||
|
||||
(defn- access-allowed?
|
||||
[env graph-id request]
|
||||
(p/let [claims (auth/auth-claims request env)
|
||||
user-id (when claims (aget claims "sub"))
|
||||
db (aget env "DB")]
|
||||
(if (string? user-id)
|
||||
(index/<user-has-access-to-graph? db graph-id user-id)
|
||||
false)))
|
||||
|
||||
(defn- attach-ws! [^js ctx ^js socket]
|
||||
(let [state (.-state ctx)]
|
||||
(when-let [add-ws (.-addWebSocket state)]
|
||||
(add-ws socket))
|
||||
(set! (.-serializeAttachment socket) (fn [_] nil))
|
||||
(set! (.-deserializeAttachment socket) (fn [] nil))))
|
||||
|
||||
(defn- detach-ws! [^js ctx ^js socket]
|
||||
(let [state (.-state ctx)]
|
||||
(when-let [remove-ws (.-removeWebSocket state)]
|
||||
(remove-ws socket))))
|
||||
|
||||
(defn- reject-ws-upgrade!
|
||||
[^js socket status reason]
|
||||
(.write socket
|
||||
(str "HTTP/1.1 " status " Conflict\r\n"
|
||||
"Connection: close\r\n"
|
||||
"Content-Type: application/json\r\n\r\n"
|
||||
"{\"error\":\"" reason "\"}"))
|
||||
(.destroy socket))
|
||||
|
||||
(defn- handle-ws-connection
|
||||
[ctx env request ^js socket]
|
||||
(p/let [claims (auth/auth-claims request env)
|
||||
user (presence/claims->user claims)]
|
||||
(when user
|
||||
(presence/add-presence! ctx socket user))
|
||||
(presence/broadcast-online-users! ctx))
|
||||
(.on socket "message"
|
||||
(fn [data]
|
||||
(let [text (if (string? data) data (.toString data))]
|
||||
(try
|
||||
(ws-handler/handle-ws-message! ctx socket text)
|
||||
(catch :default e
|
||||
(log/error :db-sync/ws-error e)
|
||||
(.send socket (js/JSON.stringify #js {:type "error" :message "server error"})))))))
|
||||
(.on socket "close"
|
||||
(fn []
|
||||
(presence/remove-presence! ctx socket)
|
||||
(presence/broadcast-online-users! ctx)
|
||||
(detach-ws! ctx socket)))
|
||||
(.on socket "error"
|
||||
(fn [error]
|
||||
(presence/remove-presence! ctx socket)
|
||||
(presence/broadcast-online-users! ctx)
|
||||
(detach-ws! ctx socket)
|
||||
(log/error :db-sync/ws-error error))))
|
||||
|
||||
(defn start!
|
||||
[overrides]
|
||||
(let [cfg (config/normalize-config overrides)
|
||||
request-origin (request-origin-opts cfg)
|
||||
index-db (storage/open-index-db (:data-dir cfg))
|
||||
assets-bucket (assets/make-bucket (node-path/join (:data-dir cfg) "assets"))
|
||||
registry (atom {})
|
||||
deps {:config cfg
|
||||
:index-db index-db
|
||||
:assets-bucket assets-bucket}
|
||||
env (doto (make-env cfg index-db assets-bucket)
|
||||
(aset "DB_SYNC_DELETE_GRAPH"
|
||||
(fn [graph-id]
|
||||
(graph/delete-graph! registry deps graph-id))))
|
||||
server (.createServer http
|
||||
(fn [req res]
|
||||
(-> (p/let [request (platform-node/request-from-node req request-origin)
|
||||
response (dispatch/handle-node-fetch {:request request
|
||||
:env env
|
||||
:registry registry
|
||||
:deps deps})]
|
||||
(platform-node/send-response! res response))
|
||||
(p/catch
|
||||
(fn [e]
|
||||
(log/error :db-sync/node-request-failed {:error e})
|
||||
(js/console.error ":db-sync/node-request-failed" e)
|
||||
(platform-node/send-response! res (worker-http/error-response "server error" 500)))))))
|
||||
WSS (or (.-WebSocketServer ws) (.-Server ws))
|
||||
^js wss (new WSS #js {:noServer true})]
|
||||
(.on server "error" (fn [error] (log/error :db-sync/node-server-error {:error error})))
|
||||
(.on wss "error" (fn [error] (log/error :db-sync/node-ws-error {:error error})))
|
||||
(p/let [_ (index/<index-init! index-db)]
|
||||
(.on server "upgrade"
|
||||
(fn [req ^js socket head]
|
||||
(let [request (platform-node/request-from-node req request-origin)
|
||||
url (platform/request-url request)
|
||||
path (.-pathname url)
|
||||
parsed (node-routes/parse-sync-path path)
|
||||
graph-id (:graph-id parsed)]
|
||||
(if (and graph-id (seq graph-id))
|
||||
(p/let [allowed? (access-allowed? env graph-id request)]
|
||||
(if allowed?
|
||||
(let [ctx (graph/get-or-create-graph registry deps graph-id)]
|
||||
(p/let [ready-for-sync? (sync-handler/<ready-for-sync? ctx graph-id)]
|
||||
(if ready-for-sync?
|
||||
(.handleUpgrade wss req socket head
|
||||
(fn [ws-socket]
|
||||
(attach-ws! ctx ws-socket)
|
||||
(handle-ws-connection ctx env request ws-socket)))
|
||||
(reject-ws-upgrade! socket 409 "graph not ready"))))
|
||||
(.destroy socket)))
|
||||
(.destroy socket)))))
|
||||
(p/let [_ (js/Promise.
|
||||
(fn [resolve]
|
||||
(.listen server (:port cfg)
|
||||
(fn [] (resolve nil)))))
|
||||
address (.address server)
|
||||
port (if (number? address) address (.-port address))
|
||||
base-url (or (:base-url cfg) (str "http://localhost:" port))]
|
||||
{:server server
|
||||
:wss wss
|
||||
:env env
|
||||
:registry registry
|
||||
:port port
|
||||
:base-url base-url
|
||||
:stop! (fn []
|
||||
(graph/close-graphs! registry)
|
||||
(when-let [close (.-close index-db)]
|
||||
(close))
|
||||
(js/Promise.
|
||||
(fn [resolve]
|
||||
(.close server (fn [] (resolve nil))))))}))))
|
||||
105
deps/db-sync/src/logseq/db_sync/node/show_checksum.cljs
vendored
Normal file
105
deps/db-sync/src/logseq/db_sync/node/show_checksum.cljs
vendored
Normal file
@@ -0,0 +1,105 @@
|
||||
(ns logseq.db-sync.node.show-checksum
|
||||
(:require ["better-sqlite3" :as sqlite3]
|
||||
["fs" :as fs]
|
||||
["path" :as node-path]
|
||||
[clojure.string :as string]
|
||||
[datascript.core :as d]
|
||||
[logseq.db-sync.checksum :as sync-checksum]
|
||||
[logseq.db-sync.storage :as storage]))
|
||||
|
||||
(def sqlite (if (find-ns 'nbb.core) (aget sqlite3 "default") sqlite3))
|
||||
|
||||
(defn- fail!
|
||||
[msg]
|
||||
(binding [*print-fn* *print-err-fn*]
|
||||
(println (str "Error: " msg)))
|
||||
(js/process.exit 1))
|
||||
|
||||
(defn- parse-args
|
||||
[argv]
|
||||
(let [args (vec argv)]
|
||||
(cond
|
||||
(some #{"-h" "--help"} args)
|
||||
{:help? true}
|
||||
|
||||
:else
|
||||
(loop [m {}
|
||||
xs args]
|
||||
(if-let [x (first xs)]
|
||||
(cond
|
||||
(= "--db" x)
|
||||
(if-let [db (second xs)]
|
||||
(recur (assoc m :db db) (nnext xs))
|
||||
(fail! "Missing value for --db"))
|
||||
|
||||
:else
|
||||
(fail! (str "Unknown argument: " x)))
|
||||
m)))))
|
||||
|
||||
(defn- print-help!
|
||||
[]
|
||||
(println "Show checksums for a sqlite db.")
|
||||
(println "")
|
||||
(println "Usage:")
|
||||
(println " node worker/dist/show-sqlite-checksum.js --db <path/to/db.sqlite>")
|
||||
(println "")
|
||||
(println "Output:")
|
||||
(println " - stored checksum (sync_meta.checksum, if present)")
|
||||
(println " - recomputed checksum (from datascript graph)"))
|
||||
|
||||
(defn- normalize-sql
|
||||
[sql]
|
||||
(-> sql string/trim string/lower-case))
|
||||
|
||||
(defn- select-sql?
|
||||
[sql]
|
||||
(string/starts-with? (normalize-sql sql) "select"))
|
||||
|
||||
(defn- exec-with-args [^js stmt args]
|
||||
(.apply (.-run stmt) stmt (to-array args)))
|
||||
|
||||
(defn- all-with-args [^js stmt args]
|
||||
(.apply (.-all stmt) stmt (to-array args)))
|
||||
|
||||
(defn- wrap-db [^js db]
|
||||
#js {:exec (fn [sql & args]
|
||||
(if (seq args)
|
||||
(let [stmt (.prepare db sql)]
|
||||
(if (select-sql? sql)
|
||||
(all-with-args stmt args)
|
||||
(do
|
||||
(exec-with-args stmt args)
|
||||
nil)))
|
||||
(if (select-sql? sql)
|
||||
(.all (.prepare db sql))
|
||||
(.exec db sql))))
|
||||
:close (fn [] (.close db))
|
||||
:_db db})
|
||||
|
||||
(defn- datom-count
|
||||
[db]
|
||||
(count (d/datoms db :eavt)))
|
||||
|
||||
(defn main
|
||||
[& argv]
|
||||
(let [{:keys [help? db]} (parse-args argv)]
|
||||
(when help?
|
||||
(print-help!)
|
||||
(js/process.exit 0))
|
||||
(when-not db
|
||||
(fail! "Missing required --db <path> argument"))
|
||||
(let [db-path (node-path/resolve db)]
|
||||
(when-not (.existsSync fs db-path)
|
||||
(fail! (str "SQLite file not found: " db-path)))
|
||||
(let [sqlite-db (new sqlite db-path nil)
|
||||
sql (wrap-db sqlite-db)]
|
||||
(try
|
||||
(storage/init-schema! sql)
|
||||
(let [conn (storage/open-conn sql)
|
||||
db' @conn
|
||||
recomputed-checksum (sync-checksum/recompute-checksum db')]
|
||||
(println (str "db: " db-path))
|
||||
(println (str "datoms: " (datom-count db')))
|
||||
(println (str "recomputed-checksum: " recomputed-checksum)))
|
||||
(finally
|
||||
(.close sql)))))))
|
||||
58
deps/db-sync/src/logseq/db_sync/node/storage.cljs
vendored
Normal file
58
deps/db-sync/src/logseq/db_sync/node/storage.cljs
vendored
Normal file
@@ -0,0 +1,58 @@
|
||||
(ns logseq.db-sync.node.storage
|
||||
(:require ["better-sqlite3" :as sqlite3]
|
||||
["fs" :as fs]
|
||||
["path" :as node-path]
|
||||
[clojure.string :as string]
|
||||
[logseq.db.common.sqlite :as common-sqlite]))
|
||||
|
||||
(def sqlite (if (find-ns 'nbb.core) (aget sqlite3 "default") sqlite3))
|
||||
|
||||
(defn- ensure-dir! [dir]
|
||||
(.mkdirSync fs dir #js {:recursive true}))
|
||||
|
||||
(defn- normalize-sql [sql]
|
||||
(-> sql string/trim string/lower-case))
|
||||
|
||||
(defn- select-sql? [sql]
|
||||
(string/starts-with? (normalize-sql sql) "select"))
|
||||
|
||||
(defn- exec-with-args [^js stmt args]
|
||||
(.apply (.-run stmt) stmt (to-array args)))
|
||||
|
||||
(defn- all-with-args [^js stmt args]
|
||||
(.apply (.-all stmt) stmt (to-array args)))
|
||||
|
||||
(defn- wrap-db [^js db]
|
||||
#js {:exec (fn [sql & args]
|
||||
(if (seq args)
|
||||
(let [stmt (.prepare db sql)]
|
||||
(if (select-sql? sql)
|
||||
(all-with-args stmt args)
|
||||
(do
|
||||
(exec-with-args stmt args)
|
||||
nil)))
|
||||
(if (select-sql? sql)
|
||||
(.all (.prepare db sql))
|
||||
(.exec db sql))))
|
||||
:prepare (fn [sql] (.prepare db sql))
|
||||
:close (fn [] (.close db))
|
||||
:_db db})
|
||||
|
||||
(defn open-index-db [data-dir]
|
||||
(let [db-path (node-path/join data-dir "index.sqlite")]
|
||||
(ensure-dir! (node-path/dirname db-path))
|
||||
(wrap-db (new sqlite db-path nil))))
|
||||
|
||||
(defn open-graph-db [data-dir graph-id]
|
||||
(let [[graph-name db-path] (common-sqlite/get-db-full-path (node-path/join data-dir "graphs") graph-id)]
|
||||
(ensure-dir! (node-path/dirname db-path))
|
||||
{:graph-name graph-name
|
||||
:sql (wrap-db (new sqlite db-path nil))}))
|
||||
|
||||
(defn delete-graph-db!
|
||||
[data-dir graph-id]
|
||||
(let [[_graph-name db-path] (common-sqlite/get-db-full-path (node-path/join data-dir "graphs") graph-id)
|
||||
graph-dir (node-path/dirname db-path)]
|
||||
(when (.existsSync fs graph-dir)
|
||||
(.rmSync fs graph-dir #js {:recursive true
|
||||
:force true}))))
|
||||
46
deps/db-sync/src/logseq/db_sync/order.cljs
vendored
Normal file
46
deps/db-sync/src/logseq/db_sync/order.cljs
vendored
Normal file
@@ -0,0 +1,46 @@
|
||||
(ns logseq.db-sync.order
|
||||
(:require [datascript.core :as d]
|
||||
[logseq.db.common.order :as db-order]))
|
||||
|
||||
(defn fix-duplicate-orders! [conn tx-data tx-meta]
|
||||
(let [db @conn
|
||||
updates (->> tx-data
|
||||
(filter (fn [[e a v _tx added]]
|
||||
(and (= a :block/order) added e v
|
||||
(:block/uuid (d/entity db e))))))
|
||||
groups (group-by (fn [{:keys [e v]}]
|
||||
(let [parent (:block/parent (d/entity db e))]
|
||||
[(:db/id parent) v]))
|
||||
updates)
|
||||
fixes (reduce-kv
|
||||
(fn [acc [parent-eid value] _group]
|
||||
(if (and parent-eid value)
|
||||
(let [siblings (d/datoms db :avet :block/parent parent-eid)
|
||||
existing-same (->> siblings
|
||||
(keep (fn [datom]
|
||||
(let [sib-eid (:e datom)]
|
||||
(when (= value (:block/order (d/entity db sib-eid)))
|
||||
sib-eid)))))
|
||||
same-order-siblings (->> (map (fn [id] (d/entity db id)) existing-same)
|
||||
(sort-by :block/uuid))
|
||||
need-fix? (> (count same-order-siblings) 1)]
|
||||
(if need-fix?
|
||||
(let [orders (->> siblings
|
||||
(keep (fn [d]
|
||||
(let [sib-eid (:e d)]
|
||||
(:block/order (d/entity db sib-eid))))))
|
||||
end (some (fn [order]
|
||||
(when (> (compare order value) 0)
|
||||
order))
|
||||
orders)
|
||||
new-orders (db-order/gen-n-keys (count same-order-siblings) value end)]
|
||||
(into acc
|
||||
(map-indexed (fn [idx id]
|
||||
[:db/add id :block/order (nth new-orders idx)])
|
||||
(map :db/id same-order-siblings))))
|
||||
acc))
|
||||
acc))
|
||||
[]
|
||||
groups)]
|
||||
(when (seq fixes)
|
||||
(d/transact! conn fixes (merge tx-meta {:op :fix-duplicate-order})))))
|
||||
6
deps/db-sync/src/logseq/db_sync/platform/cloudflare.cljs
vendored
Normal file
6
deps/db-sync/src/logseq/db_sync/platform/cloudflare.cljs
vendored
Normal file
@@ -0,0 +1,6 @@
|
||||
(ns logseq.db-sync.platform.cloudflare
|
||||
(:require [logseq.db-sync.platform.core :as core]))
|
||||
|
||||
(def response core/response)
|
||||
(def request core/request)
|
||||
(def request-url core/request-url)
|
||||
13
deps/db-sync/src/logseq/db_sync/platform/core.cljs
vendored
Normal file
13
deps/db-sync/src/logseq/db_sync/platform/core.cljs
vendored
Normal file
@@ -0,0 +1,13 @@
|
||||
(ns logseq.db-sync.platform.core)
|
||||
|
||||
(defn response
|
||||
[body init]
|
||||
(js/Response. body init))
|
||||
|
||||
(defn request
|
||||
[url init]
|
||||
(js/Request. url init))
|
||||
|
||||
(defn request-url
|
||||
[request']
|
||||
(js/URL. (.-url request')))
|
||||
50
deps/db-sync/src/logseq/db_sync/platform/node.cljs
vendored
Normal file
50
deps/db-sync/src/logseq/db_sync/platform/node.cljs
vendored
Normal file
@@ -0,0 +1,50 @@
|
||||
(ns logseq.db-sync.platform.node
|
||||
(:require [clojure.string :as string]
|
||||
[logseq.db-sync.platform.core :as core]
|
||||
[promesa.core :as p]))
|
||||
|
||||
(defn- headers->object [headers]
|
||||
(let [out (js-obj)]
|
||||
(.forEach headers (fn [value key] (aset out key value)))
|
||||
out))
|
||||
|
||||
(defn request-from-node
|
||||
[^js req {:keys [scheme host]}]
|
||||
(let [headers (js/Headers.)
|
||||
node-headers (.-headers req)
|
||||
header-keys (js/Object.keys node-headers)
|
||||
_ (doseq [k header-keys]
|
||||
(let [value (aget node-headers k)]
|
||||
(when (some? value)
|
||||
(.set headers (string/lower-case k) value))))
|
||||
method (or (.-method req) "GET")
|
||||
host (or host (aget node-headers "host") "localhost")
|
||||
scheme (or scheme "http")
|
||||
url (str scheme "://" host (.-url req))
|
||||
init #js {:method method
|
||||
:headers headers}]
|
||||
(when-not (or (= method "GET") (= method "HEAD"))
|
||||
(aset init "body" req)
|
||||
(aset init "duplex" "half"))
|
||||
(core/request url init)))
|
||||
|
||||
(defn send-response!
|
||||
[^js res ^js response]
|
||||
(let [headers (headers->object (.-headers response))
|
||||
status (.-status response)]
|
||||
(.writeHead res status headers)
|
||||
(if-let [body (.-body response)]
|
||||
(let [^js stream (try
|
||||
(let [Readable (.-Readable (js/require "stream"))]
|
||||
(when (and Readable (.-fromWeb Readable))
|
||||
(.fromWeb Readable body)))
|
||||
(catch :default _ nil))]
|
||||
(if stream
|
||||
(do
|
||||
(.pipe stream res)
|
||||
(js/Promise.resolve nil))
|
||||
(p/let [buf (.arrayBuffer response)]
|
||||
(.end res (js/Buffer.from buf)))))
|
||||
(do
|
||||
(.end res)
|
||||
(js/Promise.resolve nil)))))
|
||||
27
deps/db-sync/src/logseq/db_sync/protocol.cljs
vendored
Normal file
27
deps/db-sync/src/logseq/db_sync/protocol.cljs
vendored
Normal file
@@ -0,0 +1,27 @@
|
||||
(ns logseq.db-sync.protocol
|
||||
(:require [clojure.walk :as walk]
|
||||
[logseq.db-sync.common :as common]))
|
||||
|
||||
(defn- stringify-uuid
|
||||
[value]
|
||||
(if (uuid? value)
|
||||
(str value)
|
||||
value))
|
||||
|
||||
(defn parse-message [raw]
|
||||
(try
|
||||
(let [data (js->clj (js/JSON.parse raw) :keywordize-keys true)]
|
||||
(when (map? data)
|
||||
data))
|
||||
(catch :default _
|
||||
nil)))
|
||||
|
||||
(defn encode-message [m]
|
||||
(js/JSON.stringify
|
||||
(clj->js (walk/postwalk stringify-uuid m))))
|
||||
|
||||
(defn transit->tx [value]
|
||||
(common/read-transit value))
|
||||
|
||||
(defn tx->transit [tx-data]
|
||||
(common/write-transit tx-data))
|
||||
19
deps/db-sync/src/logseq/db_sync/sentry.cljs
vendored
Normal file
19
deps/db-sync/src/logseq/db_sync/sentry.cljs
vendored
Normal file
@@ -0,0 +1,19 @@
|
||||
(ns logseq.db-sync.sentry)
|
||||
|
||||
(defn- not-blank [v]
|
||||
(when (and (string? v) (seq v)) v))
|
||||
|
||||
(defn- parse-sample-rate [v]
|
||||
(let [n (js/parseFloat v)]
|
||||
(when (and (not (js/isNaN n)) (<= 0 n 1)) n)))
|
||||
|
||||
(defn options-from-env [^js env]
|
||||
(let [dsn (not-blank (aget env "SENTRY_DSN"))
|
||||
release (not-blank (aget env "SENTRY_RELEASE"))
|
||||
environment (not-blank (aget env "SENTRY_ENVIRONMENT"))
|
||||
traces-sample-rate (parse-sample-rate (aget env "SENTRY_TRACES_SAMPLE_RATE"))]
|
||||
(when dsn
|
||||
(cond-> {:dsn dsn}
|
||||
release (assoc :release release)
|
||||
environment (assoc :environment environment)
|
||||
traces-sample-rate (assoc :tracesSampleRate traces-sample-rate)))))
|
||||
16
deps/db-sync/src/logseq/db_sync/sentry/node.cljs
vendored
Normal file
16
deps/db-sync/src/logseq/db_sync/sentry/node.cljs
vendored
Normal file
@@ -0,0 +1,16 @@
|
||||
(ns logseq.db-sync.sentry.node
|
||||
(:require [logseq.db-sync.sentry :as sentry-config]))
|
||||
|
||||
(defonce ^:private *sentry (atom nil))
|
||||
|
||||
(defn- sentry! []
|
||||
(or @*sentry
|
||||
(reset! *sentry (js/require "@sentry/node"))))
|
||||
|
||||
(defn init! []
|
||||
(when-let [opts (sentry-config/options-from-env (.-env js/process))]
|
||||
(.init (sentry!) (clj->js opts))))
|
||||
|
||||
(defn capture-exception! [error]
|
||||
(when-let [^js sentry @*sentry]
|
||||
(.captureException sentry error)))
|
||||
11
deps/db-sync/src/logseq/db_sync/sentry/worker.cljs
vendored
Normal file
11
deps/db-sync/src/logseq/db_sync/sentry/worker.cljs
vendored
Normal file
@@ -0,0 +1,11 @@
|
||||
(ns logseq.db-sync.sentry.worker
|
||||
(:require ["@sentry/cloudflare" :as sentry]
|
||||
[logseq.db-sync.sentry :as sentry-config]))
|
||||
|
||||
(defn wrap-handler [handler]
|
||||
(sentry/withSentry (fn [^js env]
|
||||
(clj->js (or (sentry-config/options-from-env env) {})))
|
||||
handler))
|
||||
|
||||
(defn capture-exception! [error]
|
||||
(sentry/captureException error))
|
||||
62
deps/db-sync/src/logseq/db_sync/snapshot.cljs
vendored
Normal file
62
deps/db-sync/src/logseq/db_sync/snapshot.cljs
vendored
Normal file
@@ -0,0 +1,62 @@
|
||||
(ns logseq.db-sync.snapshot
|
||||
(:require [cognitect.transit :as transit]))
|
||||
|
||||
(def ^:private transit-w (transit/writer :json))
|
||||
(def ^:private transit-r (transit/reader :json))
|
||||
(def ^:private text-decoder (js/TextDecoder.))
|
||||
|
||||
(defn- ->uint8
|
||||
[data]
|
||||
(cond
|
||||
(instance? js/Uint8Array data) data
|
||||
(instance? js/ArrayBuffer data) (js/Uint8Array. data)
|
||||
(string? data) (.encode (js/TextEncoder.) data)
|
||||
:else (js/Uint8Array. data)))
|
||||
|
||||
(defn encode-rows
|
||||
[rows]
|
||||
(->uint8 (transit/write transit-w rows)))
|
||||
|
||||
(defn decode-rows
|
||||
[payload]
|
||||
(transit/read transit-r (.decode text-decoder (->uint8 payload))))
|
||||
|
||||
(defn concat-bytes
|
||||
[^js a ^js b]
|
||||
(cond
|
||||
(nil? a) b
|
||||
(nil? b) a
|
||||
:else
|
||||
(let [out (js/Uint8Array. (+ (.-byteLength a) (.-byteLength b)))]
|
||||
(.set out a 0)
|
||||
(.set out b (.-byteLength a))
|
||||
out)))
|
||||
|
||||
(defn parse-framed-chunk
|
||||
[buffer chunk]
|
||||
(let [data (concat-bytes buffer chunk)
|
||||
total (.-byteLength data)]
|
||||
(loop [offset 0
|
||||
rows []]
|
||||
(if (< (- total offset) 4)
|
||||
{:rows rows
|
||||
:buffer (when (< offset total)
|
||||
(.slice data offset total))}
|
||||
(let [view (js/DataView. (.-buffer data) offset 4)
|
||||
len (.getUint32 view 0 false)
|
||||
next-offset (+ offset 4 len)]
|
||||
(if (<= next-offset total)
|
||||
(let [payload (.slice data (+ offset 4) next-offset)
|
||||
decoded (decode-rows payload)]
|
||||
(recur next-offset (into rows decoded)))
|
||||
{:rows rows
|
||||
:buffer (.slice data offset total)}))))))
|
||||
|
||||
(defn finalize-framed-buffer
|
||||
[buffer]
|
||||
(if (or (nil? buffer) (zero? (.-byteLength buffer)))
|
||||
[]
|
||||
(let [{:keys [rows buffer]} (parse-framed-chunk nil buffer)]
|
||||
(if (and (seq rows) (or (nil? buffer) (zero? (.-byteLength buffer))))
|
||||
rows
|
||||
(throw (ex-info "incomplete framed buffer" {:buffer buffer :rows rows}))))))
|
||||
218
deps/db-sync/src/logseq/db_sync/storage.cljs
vendored
Normal file
218
deps/db-sync/src/logseq/db_sync/storage.cljs
vendored
Normal file
@@ -0,0 +1,218 @@
|
||||
(ns logseq.db-sync.storage
|
||||
(:require
|
||||
[cljs-bean.core :as bean]
|
||||
[clojure.string :as string]
|
||||
[datascript.core :as d]
|
||||
[datascript.storage :refer [IStorage]]
|
||||
[logseq.db-sync.checksum :as sync-checksum]
|
||||
[logseq.db-sync.common :as common]
|
||||
[logseq.db.common.normalize :as db-normalize]
|
||||
[logseq.db.common.sqlite :as common-sqlite]
|
||||
[logseq.db.frontend.schema :as db-schema]))
|
||||
|
||||
(def ^:private tx-log-outliner-op-migration-sql
|
||||
"alter table tx_log add column outliner_op TEXT")
|
||||
|
||||
(defn- duplicate-column-error?
|
||||
[error column-name]
|
||||
(let [message (-> (or (ex-message error) (some-> error .-message) (str error))
|
||||
string/lower-case)]
|
||||
(and (string/includes? message "duplicate column")
|
||||
(string/includes? message (string/lower-case column-name)))))
|
||||
|
||||
(defn- ensure-tx-log-outliner-op-column!
|
||||
[sql]
|
||||
(try
|
||||
(common/sql-exec sql tx-log-outliner-op-migration-sql)
|
||||
(catch :default error
|
||||
(when-not (duplicate-column-error? error "outliner_op")
|
||||
(throw error)))))
|
||||
|
||||
;; TODO: GC kvs table
|
||||
|
||||
(defn init-schema! [sql]
|
||||
(common/sql-exec sql "create table if not exists kvs (addr INTEGER primary key, content TEXT, addresses JSON)")
|
||||
(common/sql-exec sql
|
||||
(str "create table if not exists tx_log ("
|
||||
"t INTEGER primary key,"
|
||||
"tx TEXT not null,"
|
||||
"created_at INTEGER"
|
||||
");"))
|
||||
(ensure-tx-log-outliner-op-column! sql)
|
||||
(common/sql-exec sql
|
||||
(str "create table if not exists sync_meta ("
|
||||
"key TEXT primary key,"
|
||||
"value TEXT"
|
||||
");")))
|
||||
|
||||
(defn- select-one [sql sql-str & args]
|
||||
(first (common/get-sql-rows (apply common/sql-exec sql sql-str args))))
|
||||
|
||||
(defn get-meta [sql k]
|
||||
(when-let [row (select-one sql "select value from sync_meta where key = ?" (name k))]
|
||||
(aget row "value")))
|
||||
|
||||
(defn set-meta! [sql k v]
|
||||
(common/sql-exec sql
|
||||
(str "insert into sync_meta (key, value) values (?, ?)"
|
||||
" on conflict(key) do update set value = excluded.value")
|
||||
(name k)
|
||||
(str v)))
|
||||
|
||||
(defn get-checksum [sql]
|
||||
(get-meta sql :checksum))
|
||||
|
||||
(defn set-checksum! [sql checksum]
|
||||
(set-meta! sql :checksum checksum))
|
||||
|
||||
(defn get-t [sql]
|
||||
(let [value (get-meta sql :t)]
|
||||
(if (string? value)
|
||||
(js/parseInt value 10)
|
||||
0)))
|
||||
|
||||
(defn set-t! [sql t]
|
||||
(set-meta! sql :t t))
|
||||
|
||||
(def ^:dynamic *in-sql-transaction?* false)
|
||||
|
||||
(defn with-sql-transaction!
|
||||
[sql f]
|
||||
(if *in-sql-transaction?*
|
||||
(f)
|
||||
(let [f' (fn []
|
||||
(binding [*in-sql-transaction?* true]
|
||||
(f)))]
|
||||
(if-let [db (aget sql "_db")]
|
||||
(let [transaction (.-transaction db)]
|
||||
(if (fn? transaction)
|
||||
(let [tx-fn (.call transaction db f')]
|
||||
(tx-fn))
|
||||
(f')))
|
||||
(f')))))
|
||||
|
||||
(defn set-initial-checksum! [sql checksum]
|
||||
(with-sql-transaction!
|
||||
sql
|
||||
(fn []
|
||||
(let [existing-checksum (get-checksum sql)
|
||||
current-t (get-t sql)]
|
||||
(cond
|
||||
(and (some? existing-checksum)
|
||||
(not= existing-checksum checksum))
|
||||
(throw (ex-info "Cannot overwrite existing checksum with snapshot checksum"
|
||||
{:type :db-sync/stale-snapshot-checksum
|
||||
:existing-checksum existing-checksum
|
||||
:snapshot-checksum checksum}))
|
||||
|
||||
(and (nil? existing-checksum)
|
||||
(pos? current-t))
|
||||
(throw (ex-info "Cannot initialize checksum after tx history advanced"
|
||||
{:type :db-sync/snapshot-checksum-after-tx-log
|
||||
:t current-t
|
||||
:snapshot-checksum checksum}))
|
||||
|
||||
(nil? existing-checksum)
|
||||
(set-checksum! sql checksum))))))
|
||||
|
||||
(defn- outliner-op->sql [outliner-op]
|
||||
(cond
|
||||
(keyword? outliner-op) (name outliner-op)
|
||||
(string? outliner-op) outliner-op
|
||||
:else nil))
|
||||
|
||||
(defn- sql->outliner-op [value]
|
||||
(when (string? value)
|
||||
(keyword value)))
|
||||
|
||||
(defn append-tx! [sql t tx-str created-at outliner-op]
|
||||
(common/sql-exec sql
|
||||
(str "insert into tx_log (t, tx, created_at, outliner_op) values (?, ?, ?, ?)"
|
||||
" on conflict(t) do update set tx = excluded.tx, created_at = excluded.created_at, outliner_op = excluded.outliner_op")
|
||||
t
|
||||
tx-str
|
||||
created-at
|
||||
(outliner-op->sql outliner-op)))
|
||||
|
||||
(defn fetch-tx-since [sql since-t]
|
||||
(let [rows (common/get-sql-rows
|
||||
(common/sql-exec sql
|
||||
"select t, tx, outliner_op from tx_log where t > ? order by t asc"
|
||||
since-t))]
|
||||
(mapv (fn [row]
|
||||
{:t (aget row "t")
|
||||
:tx (aget row "tx")
|
||||
:outliner-op (sql->outliner-op (aget row "outliner_op"))})
|
||||
rows)))
|
||||
|
||||
(defn- upsert-addr-content! [sql data]
|
||||
(doseq [item data]
|
||||
(common/sql-exec sql
|
||||
(str "insert into kvs (addr, content, addresses) values (?, ?, ?)"
|
||||
" on conflict(addr) do update set content = excluded.content, addresses = excluded.addresses")
|
||||
(aget item "addr")
|
||||
(aget item "content")
|
||||
(aget item "addresses"))))
|
||||
|
||||
(defn- restore-data-from-addr [sql addr]
|
||||
(when-let [row (select-one sql "select content, addresses from kvs where addr = ?" addr)]
|
||||
(let [{:keys [content addresses]} (bean/->clj row)
|
||||
addresses (when addresses (js/JSON.parse addresses))
|
||||
data (common/read-transit content)]
|
||||
(if (and addresses (map? data))
|
||||
(assoc data :addresses addresses)
|
||||
data))))
|
||||
|
||||
(defn new-sqlite-storage [sql]
|
||||
(reify IStorage
|
||||
(-store [_ addr+data-seq _delete-addrs]
|
||||
(let [data (map
|
||||
(fn [[addr data]]
|
||||
(let [data' (if (map? data) (dissoc data :addresses) data)
|
||||
addresses (when (map? data)
|
||||
(when-let [addresses (:addresses data)]
|
||||
(js/JSON.stringify (bean/->js addresses))))]
|
||||
#js {"addr" addr
|
||||
"content" (common/write-transit data')
|
||||
"addresses" addresses}))
|
||||
addr+data-seq)]
|
||||
(upsert-addr-content! sql data)))
|
||||
(-restore [_ addr]
|
||||
(restore-data-from-addr sql addr))))
|
||||
|
||||
(defn- append-tx-for-tx-report
|
||||
[sql {:keys [db-after db-before tx-data tx-meta] :as tx-report}]
|
||||
(when-not (or (:db-sync/skip-tx-log? tx-meta)
|
||||
(empty? tx-data))
|
||||
(let [created-at (common/now-ms)
|
||||
normalized-data (->> tx-data
|
||||
(db-normalize/normalize-tx-data db-after db-before)
|
||||
vec)
|
||||
tx-str (common/write-transit normalized-data)]
|
||||
(with-sql-transaction!
|
||||
sql
|
||||
(fn []
|
||||
(let [new-t (inc (get-t sql))]
|
||||
(append-tx! sql new-t tx-str created-at (:outliner-op tx-meta))
|
||||
(set-t! sql new-t)
|
||||
(when-not (:db-sync/skip-checksum-update? tx-meta)
|
||||
(let [prev-checksum (get-checksum sql)
|
||||
checksum (sync-checksum/update-checksum
|
||||
prev-checksum
|
||||
(assoc tx-report :tx-data normalized-data))]
|
||||
(set-checksum! sql checksum)))))))))
|
||||
|
||||
(defn- listen-db-updates!
|
||||
[sql conn]
|
||||
(d/listen! conn ::listen-db-updates
|
||||
(fn [tx-report]
|
||||
(append-tx-for-tx-report sql tx-report))))
|
||||
|
||||
(defn open-conn
|
||||
[sql]
|
||||
(init-schema! sql)
|
||||
(let [storage (new-sqlite-storage sql)
|
||||
schema db-schema/schema
|
||||
conn (common-sqlite/get-storage-conn storage schema)]
|
||||
(listen-db-updates! sql conn)
|
||||
conn))
|
||||
208
deps/db-sync/src/logseq/db_sync/tx_sanitize.cljs
vendored
Normal file
208
deps/db-sync/src/logseq/db_sync/tx_sanitize.cljs
vendored
Normal file
@@ -0,0 +1,208 @@
|
||||
(ns logseq.db-sync.tx-sanitize
|
||||
(:require [clojure.set :as set]
|
||||
[datascript.core :as d]
|
||||
[logseq.db :as ldb]
|
||||
[logseq.db.frontend.kv-entity :as kv-entity]))
|
||||
|
||||
(def ^:private retract-entity-ops
|
||||
#{:db/retractEntity :db.fn/retractEntity})
|
||||
|
||||
(defn- retract-entity-op?
|
||||
[item]
|
||||
(and (vector? item)
|
||||
(= 2 (count item))
|
||||
(contains? retract-entity-ops (first item))))
|
||||
|
||||
(defn- entity-ref->eid
|
||||
[db entity-ref]
|
||||
(cond
|
||||
(and (number? entity-ref) (neg? entity-ref))
|
||||
nil
|
||||
|
||||
:else
|
||||
(try
|
||||
(some-> (d/entity db entity-ref) :db/id)
|
||||
(catch :default _
|
||||
nil))))
|
||||
|
||||
(def ^:private entity-op-kinds
|
||||
#{:db/add :db/retract :db/cas :db.fn/cas})
|
||||
|
||||
(defn- entity-op?
|
||||
[item]
|
||||
(and (vector? item)
|
||||
(<= 4 (count item))
|
||||
(contains? entity-op-kinds (first item))))
|
||||
|
||||
(def ^:private ignored-kv-entities
|
||||
(into #{}
|
||||
(keep (fn [[kw config]]
|
||||
(when (get-in config [:rtc :rtc/ignore-entity-when-init-upload])
|
||||
kw)))
|
||||
kv-entity/kv-entities))
|
||||
|
||||
(defn- tx-ignored-kv-entity-refs
|
||||
[tx-data]
|
||||
(into ignored-kv-entities
|
||||
(keep (fn [item]
|
||||
(cond
|
||||
(and (map? item)
|
||||
(contains? ignored-kv-entities (:db/ident item)))
|
||||
(:db/id item)
|
||||
|
||||
(and (entity-op? item)
|
||||
(= :db/ident (nth item 2))
|
||||
(contains? ignored-kv-entities (nth item 3)))
|
||||
(second item))))
|
||||
tx-data))
|
||||
|
||||
(defn- ignored-kv-entity-ref?
|
||||
[db ignored-entity-refs entity-ref]
|
||||
(or (contains? ignored-entity-refs entity-ref)
|
||||
(when-let [eid (entity-ref->eid db entity-ref)]
|
||||
(contains? ignored-kv-entities (:db/ident (d/entity db eid))))))
|
||||
|
||||
(defn- ignored-kv-entity-op?
|
||||
[db ignored-entity-refs item]
|
||||
(cond
|
||||
(and (map? item) (contains? item :db/id))
|
||||
(ignored-kv-entity-ref? db ignored-entity-refs (:db/id item))
|
||||
|
||||
(retract-entity-op? item)
|
||||
(ignored-kv-entity-ref? db ignored-entity-refs (second item))
|
||||
|
||||
(entity-op? item)
|
||||
(ignored-kv-entity-ref? db ignored-entity-refs (second item))
|
||||
|
||||
:else
|
||||
false))
|
||||
|
||||
(defn- strip-ignored-kv-entity-ops
|
||||
[db tx-data]
|
||||
(let [ignored-entity-refs (tx-ignored-kv-entity-refs tx-data)]
|
||||
(remove (partial ignored-kv-entity-op? db ignored-entity-refs) tx-data)))
|
||||
|
||||
(defn- drop-ops-targeting-retracted-entities
|
||||
[db tx-data]
|
||||
(let [retract-eids (->> tx-data
|
||||
(keep (fn [item]
|
||||
(when (retract-entity-op? item)
|
||||
(entity-ref->eid db (second item)))))
|
||||
set)]
|
||||
(if (empty? retract-eids)
|
||||
tx-data
|
||||
(remove (fn [item]
|
||||
(and (entity-op? item)
|
||||
(contains? retract-eids
|
||||
(entity-ref->eid db (second item)))))
|
||||
tx-data))))
|
||||
|
||||
(def ^:private migration-deleted-attrs
|
||||
#{:block/path-refs
|
||||
:block/pre-block?
|
||||
:logseq.property.embedding/hnsw-label
|
||||
:logseq.property.embedding/hnsw-label-updated-at})
|
||||
|
||||
(defn- strip-migration-deleted-attrs
|
||||
[tx-data]
|
||||
(keep (fn [item]
|
||||
(when-not (and (vector? item)
|
||||
(<= 4 (count item))
|
||||
(entity-op? item)
|
||||
(contains? migration-deleted-attrs (nth item 2)))
|
||||
item))
|
||||
tx-data))
|
||||
|
||||
(def ^:private encrypted-attrs
|
||||
#{:block/title :block/name})
|
||||
|
||||
(defn- drop-conflicted-encrypted-retracts
|
||||
"When encrypted tx data is decrypted, old/new ciphertexts can collapse to the
|
||||
same plaintext value. A valid pair like
|
||||
[:db/retract e :block/title old-cipher] + [:db/add e :block/title new-cipher]
|
||||
may become a same-key add/retract pair. Keep the add, drop the retract."
|
||||
[tx-data]
|
||||
(let [conflicted-keys (->> tx-data
|
||||
(keep (fn [item]
|
||||
(when (and (vector? item)
|
||||
(<= 4 (count item))
|
||||
(contains? entity-op-kinds (first item))
|
||||
(contains? encrypted-attrs (nth item 2)))
|
||||
{:op (first item)
|
||||
:k [(second item) (nth item 2) (nth item 3)]})))
|
||||
(group-by :k)
|
||||
(keep (fn [[k entries]]
|
||||
(let [ops (set (map :op entries))]
|
||||
(when (and (contains? ops :db/add)
|
||||
(contains? ops :db/retract))
|
||||
k))))
|
||||
set)]
|
||||
(if (empty? conflicted-keys)
|
||||
tx-data
|
||||
(remove (fn [item]
|
||||
(and (vector? item)
|
||||
(<= 4 (count item))
|
||||
(= :db/retract (first item))
|
||||
(contains? conflicted-keys
|
||||
[(second item) (nth item 2) (nth item 3)])))
|
||||
tx-data))))
|
||||
|
||||
(defn- touched-entity-eid
|
||||
[db item]
|
||||
(cond
|
||||
(and (map? item) (contains? item :db/id))
|
||||
(entity-ref->eid db (:db/id item))
|
||||
|
||||
(and (map? item) (contains? item :block/uuid))
|
||||
(entity-ref->eid db [:block/uuid (:block/uuid item)])
|
||||
|
||||
(and (vector? item)
|
||||
(contains? entity-op-kinds (first item))
|
||||
(<= 4 (count item)))
|
||||
(entity-ref->eid db (second item))
|
||||
|
||||
:else
|
||||
nil))
|
||||
|
||||
(defn sanitize-tx
|
||||
([db tx-data]
|
||||
(sanitize-tx db tx-data nil))
|
||||
([db tx-data {:keys [drop-missing-retract-ops?
|
||||
drop-ops-targeting-retracted-entities?
|
||||
retract-touched-descendants?]
|
||||
:or {drop-missing-retract-ops? false
|
||||
drop-ops-targeting-retracted-entities? false
|
||||
retract-touched-descendants? false}}]
|
||||
(let [tx-data* (cond->> (strip-migration-deleted-attrs tx-data)
|
||||
(seq ignored-kv-entities)
|
||||
(strip-ignored-kv-entity-ops db)
|
||||
drop-missing-retract-ops?
|
||||
(remove (fn [item]
|
||||
(and (retract-entity-op? item)
|
||||
(nil? (entity-ref->eid db (second item))))))
|
||||
drop-ops-targeting-retracted-entities?
|
||||
(drop-ops-targeting-retracted-entities db))
|
||||
tx-data* (drop-conflicted-encrypted-retracts tx-data*)
|
||||
tx-data* (vec tx-data*)
|
||||
retract-eids (->> tx-data*
|
||||
(keep (fn [item]
|
||||
(when (retract-entity-op? item)
|
||||
(entity-ref->eid db (second item)))))
|
||||
set)
|
||||
touched-eids (->> tx-data*
|
||||
(remove retract-entity-op?)
|
||||
(keep (partial touched-entity-eid db))
|
||||
set)
|
||||
descendant-retract-eids (->> retract-eids
|
||||
(mapcat (fn [eid]
|
||||
(let [entity (d/entity db eid)]
|
||||
(when (:block/uuid entity)
|
||||
(ldb/get-block-full-children-ids db eid)))))
|
||||
set)
|
||||
preserved-eids (cond-> retract-eids
|
||||
(not retract-touched-descendants?)
|
||||
(set/union touched-eids))
|
||||
missing-retract-eids (sort (set/difference descendant-retract-eids preserved-eids))]
|
||||
(cond-> tx-data*
|
||||
(seq missing-retract-eids)
|
||||
(into (map (fn [eid] [:db/retractEntity eid]) missing-retract-eids))))))
|
||||
77
deps/db-sync/src/logseq/db_sync/worker.cljs
vendored
Normal file
77
deps/db-sync/src/logseq/db_sync/worker.cljs
vendored
Normal file
@@ -0,0 +1,77 @@
|
||||
(ns logseq.db-sync.worker
|
||||
;; Turn off false defclass errors
|
||||
{:clj-kondo/config {:linters {:unresolved-symbol {:level :off}}}}
|
||||
(:require ["cloudflare:workers" :refer [DurableObject]]
|
||||
[lambdaisland.glogi :as log]
|
||||
[logseq.db-sync.common :as common]
|
||||
[logseq.db-sync.logging :as logging]
|
||||
[logseq.db-sync.sentry.worker :as sentry]
|
||||
[logseq.db-sync.worker.dispatch :as dispatch]
|
||||
[logseq.db-sync.worker.handler.sync :as sync-handler]
|
||||
[logseq.db-sync.worker.handler.ws :as ws-handler]
|
||||
[logseq.db-sync.worker.presence :as presence]
|
||||
[logseq.db-sync.worker.ws :as ws]
|
||||
[promesa.core :as p]
|
||||
[shadow.cljs.modern :refer (defclass)]))
|
||||
|
||||
(logging/install!)
|
||||
|
||||
(def worker
|
||||
(sentry/wrap-handler
|
||||
#js {:fetch (fn [request env _ctx]
|
||||
(dispatch/handle-worker-fetch request env))}))
|
||||
|
||||
(defclass SyncDO
|
||||
(extends DurableObject)
|
||||
|
||||
(constructor [this ^js state env]
|
||||
(super state env)
|
||||
(set! (.-state this) state)
|
||||
(set! (.-env this) env)
|
||||
(set! (.-sql this) (.-sql ^js (.-storage state)))
|
||||
(set! (.-conn this) nil)
|
||||
(set! (.-schema-ready this) false)
|
||||
(let [presence (presence/presence* this)
|
||||
sockets (.getWebSockets state)]
|
||||
(doseq [^js ws sockets]
|
||||
(when-let [attachment (.deserializeAttachment ws)]
|
||||
(when-let [user (presence/attachment->user attachment)]
|
||||
(swap! presence assoc ws user))))
|
||||
(.setWebSocketAutoResponse
|
||||
state
|
||||
(js/WebSocketRequestResponsePair. "ping" "pong"))))
|
||||
|
||||
Object
|
||||
(fetch [this request]
|
||||
(->
|
||||
(p/do
|
||||
(if (common/upgrade-request? request)
|
||||
(ws-handler/handle-ws this request)
|
||||
(sync-handler/handle-http this request)))
|
||||
(p/catch (fn [error]
|
||||
(let [message (cond
|
||||
(instance? ExceptionInfo error) (str (.-message error) " | " (pr-str (ex-data error)))
|
||||
(instance? js/Error error) (.-message error)
|
||||
:else (pr-str error))
|
||||
stack (when (instance? js/Error error) (.-stack error))]
|
||||
(common/json-response
|
||||
{:error "DO internal error"
|
||||
:debug-message message
|
||||
:debug-stack stack}
|
||||
500))))))
|
||||
(webSocketMessage [this ws message]
|
||||
(try
|
||||
(ws-handler/handle-ws-message! this ws message)
|
||||
(catch :default e
|
||||
(sentry/capture-exception! e)
|
||||
(log/error :db-sync/ws-error e)
|
||||
(js/console.error e)
|
||||
(ws/send! ws {:type "error" :message "server error"}))))
|
||||
(webSocketClose [this ws _code _reason]
|
||||
(presence/remove-presence! this ws)
|
||||
(presence/broadcast-online-users! this))
|
||||
(webSocketError [this ws error]
|
||||
(presence/remove-presence! this ws)
|
||||
(presence/broadcast-online-users! this)
|
||||
(sentry/capture-exception! error)
|
||||
(log/error :db-sync/ws-error error)))
|
||||
58
deps/db-sync/src/logseq/db_sync/worker/asset_link.cljs
vendored
Normal file
58
deps/db-sync/src/logseq/db_sync/worker/asset_link.cljs
vendored
Normal file
@@ -0,0 +1,58 @@
|
||||
(ns logseq.db-sync.worker.asset-link
|
||||
(:require [logseq.db-sync.worker.handler.assets :as assets-handler]
|
||||
[promesa.core :as p]))
|
||||
|
||||
(def ^:private ttl-seconds 300)
|
||||
|
||||
(defn- secret-key [^js env usages]
|
||||
(let [secret (aget env "ASSET_LINK_SECRET")]
|
||||
(if-not (and (string? secret) (seq secret))
|
||||
(p/rejected (ex-info "missing ASSET_LINK_SECRET" {:binding "ASSET_LINK_SECRET"}))
|
||||
(.importKey (.-subtle js/crypto)
|
||||
"raw"
|
||||
(.encode (js/TextEncoder.) secret)
|
||||
#js {:name "HMAC" :hash "SHA-256"}
|
||||
false
|
||||
(clj->js usages)))))
|
||||
|
||||
(defn- signature-hex [buffer]
|
||||
(->> (js/Uint8Array. buffer)
|
||||
(map #(-> (.toString % 16) (.padStart 2 "0")))
|
||||
(apply str)))
|
||||
|
||||
(defn- hex-bytes [value]
|
||||
(when (and (string? value) (zero? (mod (count value) 2)))
|
||||
(let [payload-bytes (js/Uint8Array. (/ (count value) 2))]
|
||||
(doseq [index (range (.-length payload-bytes))]
|
||||
(aset payload-bytes index (js/parseInt (subs value (* index 2) (+ (* index 2) 2)) 16)))
|
||||
payload-bytes)))
|
||||
|
||||
(defn- payload [graph-id file expires]
|
||||
(str graph-id "/" file ":" expires))
|
||||
|
||||
(defn <temporary-url [^js request ^js env graph-id asset-uuid asset-type]
|
||||
(let [expires (+ (js/Math.floor (/ (.now js/Date) 1000)) ttl-seconds)
|
||||
file (str asset-uuid "." asset-type)
|
||||
data (.encode (js/TextEncoder.) (payload graph-id file expires))]
|
||||
(p/let [key (secret-key env ["sign"])
|
||||
signature (.sign (.-subtle js/crypto) "HMAC" key data)
|
||||
url (js/URL. (str "/assets/" graph-id "/" file) (.-url request))]
|
||||
(.set (.-searchParams url) "expires" (str expires))
|
||||
(.set (.-searchParams url) "signature" (signature-hex signature))
|
||||
{:url (str url) :expires-at expires})))
|
||||
|
||||
(defn <valid-request? [^js request ^js env]
|
||||
(let [url (js/URL. (.-url request))
|
||||
parsed (assets-handler/parse-asset-path (.-pathname url))
|
||||
expires-raw (.get (.-searchParams url) "expires")
|
||||
signature-raw (.get (.-searchParams url) "signature")
|
||||
expires (when expires-raw (js/parseInt expires-raw 10))
|
||||
signature (hex-bytes signature-raw)
|
||||
now (js/Math.floor (/ (.now js/Date) 1000))]
|
||||
(if-not (and parsed (number? expires) (not (js/isNaN expires)) signature
|
||||
(> expires now) (<= expires (+ now ttl-seconds)))
|
||||
(p/resolved false)
|
||||
(let [file (str (:asset-uuid parsed) "." (:asset-type parsed))
|
||||
data (.encode (js/TextEncoder.) (payload (:graph-id parsed) file expires))]
|
||||
(p/let [key (secret-key env ["verify"])]
|
||||
(.verify (.-subtle js/crypto) "HMAC" key signature data))))))
|
||||
83
deps/db-sync/src/logseq/db_sync/worker/auth.cljs
vendored
Normal file
83
deps/db-sync/src/logseq/db_sync/worker/auth.cljs
vendored
Normal file
@@ -0,0 +1,83 @@
|
||||
(ns logseq.db-sync.worker.auth
|
||||
(:require [clojure.string :as string]
|
||||
[logseq.common.authorization :as authorization]
|
||||
[promesa.core :as p]))
|
||||
|
||||
(defn- bearer-token [auth-header]
|
||||
(when (and (string? auth-header) (string/starts-with? auth-header "Bearer "))
|
||||
(subs auth-header 7)))
|
||||
|
||||
(defn token-from-request [request]
|
||||
(or (bearer-token (.get (.-headers request) "authorization"))
|
||||
(let [url (js/URL. (.-url request))]
|
||||
(.get (.-searchParams url) "token"))))
|
||||
|
||||
(defn- decode-jwt-part [part]
|
||||
(let [pad (if (pos? (mod (count part) 4))
|
||||
(apply str (repeat (- 4 (mod (count part) 4)) "="))
|
||||
"")
|
||||
base64 (-> (str part pad)
|
||||
(string/replace "-" "+")
|
||||
(string/replace "_" "/"))
|
||||
raw (js/atob base64)]
|
||||
(js/JSON.parse raw)))
|
||||
|
||||
(defn unsafe-jwt-claims [token]
|
||||
(try
|
||||
(when (string? token)
|
||||
(let [parts (string/split token #"\.")]
|
||||
(when (= 3 (count parts))
|
||||
(decode-jwt-part (nth parts 1)))))
|
||||
(catch :default _
|
||||
nil)))
|
||||
|
||||
(def ^:private recoverable-auth-errors
|
||||
#{"invalid" "iss not found" "aud not found" "exp" "kid"})
|
||||
|
||||
(def ^:private truthy-env-values
|
||||
#{"1" "true" "yes" "on"})
|
||||
|
||||
(defn- recoverable-auth-error?
|
||||
[error]
|
||||
(when error
|
||||
(let [message (or (ex-message error) (some-> error .-message))]
|
||||
(contains? recoverable-auth-errors message))))
|
||||
|
||||
(defn- env-flag-enabled?
|
||||
[env k]
|
||||
(let [v (some-> env (aget k))]
|
||||
(cond
|
||||
(true? v) true
|
||||
(false? v) false
|
||||
(string? v) (contains? truthy-env-values (string/lower-case v))
|
||||
:else false)))
|
||||
|
||||
(defn- allow-unverified-jwt-claims?
|
||||
[env]
|
||||
(env-flag-enabled? env "DB_SYNC_ALLOW_UNVERIFIED_JWT_CLAIMS"))
|
||||
|
||||
(defn- expired-token?
|
||||
[token]
|
||||
(when-let [claims (unsafe-jwt-claims token)]
|
||||
(let [exp (aget claims "exp")
|
||||
now-s (js/Math.floor (/ (.now js/Date) 1000))]
|
||||
(and (number? exp)
|
||||
(<= exp now-s)))))
|
||||
|
||||
(defn auth-claims [request env]
|
||||
(let [token (token-from-request request)]
|
||||
(if (string? token)
|
||||
(if (expired-token? token)
|
||||
(p/resolved nil)
|
||||
(-> (authorization/verify-jwt token env)
|
||||
(p/catch (fn [error]
|
||||
(cond
|
||||
(recoverable-auth-error? error)
|
||||
nil
|
||||
|
||||
(allow-unverified-jwt-claims? env)
|
||||
(unsafe-jwt-claims token)
|
||||
|
||||
:else
|
||||
(p/rejected error))))))
|
||||
(p/resolved nil))))
|
||||
13
deps/db-sync/src/logseq/db_sync/worker/coerce.cljs
vendored
Normal file
13
deps/db-sync/src/logseq/db_sync/worker/coerce.cljs
vendored
Normal file
@@ -0,0 +1,13 @@
|
||||
(ns logseq.db-sync.worker.coerce
|
||||
(:require [lambdaisland.glogi :as log]))
|
||||
|
||||
(def invalid-coerce ::invalid-coerce)
|
||||
|
||||
(defn coerce
|
||||
[coercer value context]
|
||||
(try
|
||||
(coercer value)
|
||||
(catch :default e
|
||||
(js/console.error "DEBUG coerce FAILED:" (pr-str context) "value:" (pr-str value) "error:" e)
|
||||
(log/error :db-sync/malli-coerce-failed (merge context {:error e :value value}))
|
||||
invalid-coerce)))
|
||||
239
deps/db-sync/src/logseq/db_sync/worker/dispatch.cljs
vendored
Normal file
239
deps/db-sync/src/logseq/db_sync/worker/dispatch.cljs
vendored
Normal file
@@ -0,0 +1,239 @@
|
||||
(ns logseq.db-sync.worker.dispatch
|
||||
(:require [clojure.string :as string]
|
||||
[lambdaisland.glogi :as log]
|
||||
[logseq.db-sync.common :as common]
|
||||
[logseq.db-sync.index :as index]
|
||||
[logseq.db-sync.platform.core :as platform]
|
||||
[logseq.db-sync.worker.auth :as auth]
|
||||
[logseq.db-sync.worker.asset-link :as asset-link]
|
||||
[logseq.db-sync.worker.handler.assets :as assets-handler]
|
||||
[logseq.db-sync.worker.handler.index :as index-handler]
|
||||
[logseq.db-sync.worker.http :as http]
|
||||
[logseq.db-sync.worker.routes.semantic :as semantic-routes]
|
||||
[promesa.core :as p]))
|
||||
|
||||
(defn- admin-token-valid?
|
||||
[request ^js env]
|
||||
(let [expected (aget env "DB_SYNC_ADMIN_TOKEN")
|
||||
actual (.get (.-headers request) "x-db-sync-admin-token")]
|
||||
(and (string? expected)
|
||||
(seq expected)
|
||||
(= expected actual))))
|
||||
|
||||
(defn- forward-sync-request
|
||||
[request ^js env graph-id ^js new-url]
|
||||
(let [^js namespace (.-LOGSEQ_SYNC_DO env)
|
||||
do-id (.idFromName namespace graph-id)
|
||||
stub (.get namespace do-id)]
|
||||
(if (common/upgrade-request? request)
|
||||
(.fetch stub request)
|
||||
(do
|
||||
(.set (.-searchParams new-url) "graph-id" graph-id)
|
||||
(let [rewritten (platform/request (.toString new-url) request)]
|
||||
(.fetch stub rewritten))))))
|
||||
|
||||
(defn- scopes [claims]
|
||||
(-> (or (some-> claims (aget "scope")) "")
|
||||
(string/split #"\s+")
|
||||
set))
|
||||
|
||||
(defn- forward-semantic-request [request ^js env {:keys [internal-path path-params]} ^js url]
|
||||
(let [graph-id (:graph-id path-params)
|
||||
path (reduce-kv (fn [result k value]
|
||||
(string/replace result (str ":" (name k)) value))
|
||||
internal-path
|
||||
path-params)
|
||||
target (js/URL. (str (.-origin url) path (.-search url)))]
|
||||
(.set (.-searchParams target) "graph-id" graph-id)
|
||||
(forward-sync-request request env graph-id target)))
|
||||
|
||||
(defn- rate-limit-response []
|
||||
(js/Response. (js/JSON.stringify #js {:error "rate limit exceeded"})
|
||||
#js {:status 429
|
||||
:headers #js {"content-type" "application/json"
|
||||
"retry-after" "60"}}))
|
||||
|
||||
(defn- semantic-limit [^js url]
|
||||
(let [raw (.get (.-searchParams url) "limit")
|
||||
parsed (when raw (js/parseInt raw 10))]
|
||||
(cond
|
||||
(nil? raw) 50
|
||||
(or (js/isNaN parsed) (not (pos? parsed))) nil
|
||||
:else (min parsed 200))))
|
||||
|
||||
(defn- handle-semantic-graphs-list [^js env ^js url claims operation]
|
||||
(let [limit (semantic-limit url)
|
||||
cursor (.get (.-searchParams url) "cursor")
|
||||
name (.get (.-searchParams url) "name")]
|
||||
(cond
|
||||
(nil? limit) (http/bad-request "invalid limit")
|
||||
(and cursor (nil? (index/decode-semantic-graph-cursor cursor))) (http/bad-request "invalid cursor")
|
||||
:else
|
||||
(let [^js limiter (aget env "SEMANTIC_READ_RATE_LIMITER")]
|
||||
(if-not limiter
|
||||
(http/error-response "rate limiter unavailable" 503)
|
||||
(p/let [result (.limit limiter #js {:key (str (aget claims "sub") ":"
|
||||
(:operation-id operation))})]
|
||||
(if (false? (aget result "success"))
|
||||
(rate-limit-response)
|
||||
(p/let [result (index/<semantic-graphs-list
|
||||
(aget env "DB")
|
||||
(aget claims "sub")
|
||||
{:name name :limit limit :cursor cursor})]
|
||||
(http/json-response nil result)))))))))
|
||||
|
||||
(defn- handle-semantic-request [request ^js env ^js url operation]
|
||||
(p/let [claims (auth/auth-claims request env)]
|
||||
(cond
|
||||
(nil? claims) (http/unauthorized)
|
||||
(not (contains? (scopes claims) (:scope operation))) (http/error-response "insufficient scope" 403)
|
||||
(= :semantic/graphs-list (:handler operation))
|
||||
(handle-semantic-graphs-list env url claims operation)
|
||||
:else
|
||||
(let [graph-id (get-in operation [:path-params :graph-id])]
|
||||
(p/let [access (index-handler/graph-access-response request env graph-id)]
|
||||
(if-not (.-ok access)
|
||||
access
|
||||
(p/let [e2ee? (index/<graph-e2ee? (aget env "DB") graph-id)]
|
||||
(cond
|
||||
(nil? e2ee?) (http/not-found)
|
||||
e2ee? (http/error-response "semantic-api-unavailable-for-e2ee" 409)
|
||||
:else
|
||||
(let [binding-name (if (= :read (:rate-class operation))
|
||||
"SEMANTIC_READ_RATE_LIMITER"
|
||||
"SEMANTIC_WRITE_RATE_LIMITER")
|
||||
^js limiter (aget env binding-name)]
|
||||
(if-not limiter
|
||||
(http/error-response "rate limiter unavailable" 503)
|
||||
(p/let [result (.limit limiter #js {:key (str (aget claims "sub") ":"
|
||||
(:operation-id operation) ":" graph-id)})]
|
||||
(if (false? (aget result "success"))
|
||||
(rate-limit-response)
|
||||
(forward-semantic-request request env operation url)))))))))))))
|
||||
|
||||
(defn- request-user-id
|
||||
[request]
|
||||
(let [token (auth/token-from-request request)
|
||||
claims (when (string? token)
|
||||
(auth/unsafe-jwt-claims token))
|
||||
user-id (some-> claims (aget "sub"))]
|
||||
(when (string? user-id)
|
||||
user-id)))
|
||||
|
||||
(defn- <safe-touch-activity!
|
||||
[request ^js env graph-id]
|
||||
(let [db (aget env "DB")
|
||||
user-id (request-user-id request)]
|
||||
(if (and db (string? graph-id))
|
||||
(try
|
||||
(-> (p/let [_ (index/<graph-activity-touch! db graph-id)
|
||||
_ (when (string? user-id)
|
||||
(index/<user-activity-touch! db user-id))]
|
||||
nil)
|
||||
(p/catch (fn [error]
|
||||
(log/warn :db-sync/activity-touch-failed
|
||||
{:graph-id graph-id
|
||||
:user-id user-id
|
||||
:error error})
|
||||
nil)))
|
||||
(catch :default error
|
||||
(log/warn :db-sync/activity-touch-failed
|
||||
{:graph-id graph-id
|
||||
:user-id user-id
|
||||
:error error})
|
||||
(p/resolved nil)))
|
||||
(p/resolved nil))))
|
||||
|
||||
(defn handle-worker-fetch [request ^js env]
|
||||
(->
|
||||
(p/do
|
||||
(let [url (platform/request-url request)
|
||||
path (.-pathname url)
|
||||
method (.-method request)]
|
||||
(cond
|
||||
(= path "/health")
|
||||
(http/json-response :worker/health {:ok true})
|
||||
|
||||
(= path "/openapi.json")
|
||||
(http/json-response nil (semantic-routes/openapi-document (or (aget env "COGNITO_ISSUER") "")))
|
||||
|
||||
(semantic-routes/match-public method path)
|
||||
(handle-semantic-request request env url (semantic-routes/match-public method path))
|
||||
|
||||
(or (= path "/graphs")
|
||||
(string/starts-with? path "/graphs/"))
|
||||
(index-handler/handle-fetch #js {:env env :d1 (aget env "DB")} request)
|
||||
|
||||
(string/starts-with? path "/admin/graphs/")
|
||||
(index-handler/handle-fetch #js {:env env :d1 (aget env "DB")} request)
|
||||
|
||||
(string/starts-with? path "/e2ee")
|
||||
(index-handler/handle-fetch #js {:env env :d1 (aget env "DB")} request)
|
||||
|
||||
(string/starts-with? path "/assets/")
|
||||
(if (= method "OPTIONS")
|
||||
(assets-handler/handle request env)
|
||||
(if-let [{:keys [graph-id]} (assets-handler/parse-asset-path path)]
|
||||
(if (admin-token-valid? request env)
|
||||
(assets-handler/handle request env)
|
||||
(p/let [signed? (if (= method "GET") (asset-link/<valid-request? request env) false)]
|
||||
(if signed?
|
||||
(assets-handler/handle request env)
|
||||
(p/let [access-resp (index-handler/graph-access-response request env graph-id)]
|
||||
(if (.-ok access-resp)
|
||||
(assets-handler/handle request env)
|
||||
access-resp)))))
|
||||
(http/bad-request "invalid asset path")))
|
||||
|
||||
(= method "OPTIONS")
|
||||
(common/options-response)
|
||||
|
||||
(string/starts-with? path "/sync/")
|
||||
(let [prefix (count "/sync/")
|
||||
rest-path (subs path prefix)
|
||||
rest-path (if (string/starts-with? rest-path "/")
|
||||
(subs rest-path 1)
|
||||
rest-path)
|
||||
slash-idx (or (string/index-of rest-path "/") -1)
|
||||
graph-id (if (neg? slash-idx) rest-path (subs rest-path 0 slash-idx))
|
||||
tail (if (neg? slash-idx)
|
||||
"/"
|
||||
(subs rest-path slash-idx))
|
||||
new-url (js/URL. (str (.-origin url) tail (.-search url)))]
|
||||
(cond
|
||||
(not (seq graph-id))
|
||||
(http/bad-request "missing graph id")
|
||||
|
||||
(or (= tail "/semantic")
|
||||
(string/starts-with? tail "/semantic/"))
|
||||
(http/not-found)
|
||||
|
||||
:else
|
||||
(if (= method "OPTIONS")
|
||||
(common/options-response)
|
||||
(if (admin-token-valid? request env)
|
||||
(forward-sync-request request env graph-id new-url)
|
||||
(p/let [access-resp (index-handler/graph-access-response request env graph-id)]
|
||||
(if (.-ok access-resp)
|
||||
(p/let [response (forward-sync-request request env graph-id new-url)
|
||||
_ (when (< (.-status response) 400)
|
||||
(<safe-touch-activity! request env graph-id))]
|
||||
response)
|
||||
access-resp))))))
|
||||
|
||||
:else
|
||||
(http/not-found))))
|
||||
(p/catch (fn [error]
|
||||
(let [err-type (str (type error))
|
||||
message (try (.-message error) (catch :default _ nil))
|
||||
data (try (ex-data error) (catch :default _ nil))
|
||||
stack (try (.-stack error) (catch :default _ nil))
|
||||
json-str (try (js/JSON.stringify error) (catch :default _ nil))]
|
||||
(common/json-response
|
||||
{:error "dispatch error"
|
||||
:debug-type err-type
|
||||
:debug-message message
|
||||
:debug-data (when data (pr-str data))
|
||||
:debug-json json-str
|
||||
:debug-stack stack}
|
||||
500))))))
|
||||
247
deps/db-sync/src/logseq/db_sync/worker/handler/assets.cljs
vendored
Normal file
247
deps/db-sync/src/logseq/db_sync/worker/handler/assets.cljs
vendored
Normal file
@@ -0,0 +1,247 @@
|
||||
(ns logseq.db-sync.worker.handler.assets
|
||||
(:require [cljs-bean.core :as bean]
|
||||
[clojure.string :as string]
|
||||
[logseq.db-sync.common :as common :refer [cors-headers]]
|
||||
[logseq.db-sync.index :as index]
|
||||
[logseq.db-sync.worker.http :as http]
|
||||
[promesa.core :as p]))
|
||||
|
||||
(def max-asset-size (* 100 1024 1024))
|
||||
(def max-encrypted-asset-size (* 200 1024 1024))
|
||||
|
||||
(def ^:private asset-type->content-type
|
||||
{"png" "image/png"
|
||||
"jpg" "image/jpeg"
|
||||
"jpeg" "image/jpeg"
|
||||
"gif" "image/gif"
|
||||
"webp" "image/webp"
|
||||
"bmp" "image/bmp"
|
||||
"svg" "image/svg+xml"
|
||||
"ico" "image/x-icon"
|
||||
"pdf" "application/pdf"})
|
||||
|
||||
(defn- response-content-type [stored-content-type asset-type]
|
||||
(or (get asset-type->content-type (string/lower-case asset-type))
|
||||
stored-content-type
|
||||
"application/octet-stream"))
|
||||
|
||||
(defn- parse-size
|
||||
[size]
|
||||
(cond
|
||||
(number? size) size
|
||||
(string? size) (let [n (js/parseInt size 10)]
|
||||
(when-not (js/isNaN n)
|
||||
n))
|
||||
:else nil))
|
||||
|
||||
(defn- fixed-length-body
|
||||
[body size]
|
||||
(when (and (number? size)
|
||||
(exists? js/FixedLengthStream)
|
||||
(some? body)
|
||||
(fn? (.-pipeTo body)))
|
||||
(let [^js fixed (js/FixedLengthStream. size)]
|
||||
{:body (.-readable fixed)
|
||||
:pipe-promise (.pipeTo body (.-writable fixed))})))
|
||||
|
||||
(defn- decoded-base64-chunk
|
||||
[value]
|
||||
(when-not (re-matches #"(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?" value)
|
||||
(throw (js/Error. "invalid base64 asset body")))
|
||||
(let [decoded (js/atob value)
|
||||
result (js/Uint8Array. (.-length decoded))]
|
||||
(dotimes [index (.-length decoded)]
|
||||
(aset result index (.charCodeAt decoded index)))
|
||||
result))
|
||||
|
||||
(defn- base64-decoded-body
|
||||
[body]
|
||||
(let [remainder (atom "")
|
||||
decoder (js/TextDecoder.)
|
||||
transform (js/TransformStream.
|
||||
#js {:transform
|
||||
(fn [chunk controller]
|
||||
(let [text (string/replace
|
||||
(str @remainder (.decode decoder chunk #js {:stream true}))
|
||||
#"\s" "")
|
||||
complete-length (* 4 (max 0 (dec (quot (count text) 4))))
|
||||
complete (subs text 0 complete-length)]
|
||||
(reset! remainder (subs text complete-length))
|
||||
(when (seq complete)
|
||||
(.enqueue controller (decoded-base64-chunk complete)))))
|
||||
:flush
|
||||
(fn [controller]
|
||||
(let [tail (string/replace (str @remainder (.decode decoder)) #"\s" "")]
|
||||
(when (seq tail)
|
||||
(.enqueue controller (decoded-base64-chunk tail)))))})]
|
||||
{:body (.-readable transform)
|
||||
:pipe-promise (.pipeTo body (.-writable transform))}))
|
||||
|
||||
(defn- response-fixed-length-body
|
||||
[body size]
|
||||
(let [{stream-body :body pipe-promise :pipe-promise} (fixed-length-body body size)]
|
||||
(when pipe-promise
|
||||
;; The response consumes the paired readable stream after this handler returns.
|
||||
(p/catch pipe-promise (fn [_] nil)))
|
||||
stream-body))
|
||||
|
||||
(defn <put-stream!
|
||||
"Streams `body` with a declared byte length to the R2 `bucket` without buffering it in Worker memory.
|
||||
|
||||
Options:
|
||||
|
||||
| key | description |
|
||||
|-----------------|-------------|
|
||||
| `:size` | Exact payload size in bytes |
|
||||
| `:content-type` | HTTP content type stored in R2 metadata |
|
||||
| `:checksum` | Client-computed SHA-256 checksum |
|
||||
| `:asset-type` | File extension stored in custom metadata |
|
||||
| `:encoding` | Optional `base64` streaming transfer encoding |"
|
||||
[^js bucket key body {:keys [size content-type checksum asset-type encoding]}]
|
||||
(cond
|
||||
(or (not (number? size)) (neg? size))
|
||||
(p/resolved (http/error-response "invalid asset size" 400))
|
||||
|
||||
(> size max-asset-size)
|
||||
(p/resolved (http/error-response "asset too large" 413))
|
||||
|
||||
(nil? body)
|
||||
(p/resolved (http/error-response "missing asset body" 400))
|
||||
|
||||
:else
|
||||
(let [{decoded-body :body decode-promise :pipe-promise}
|
||||
(when (= "base64" encoding) (base64-decoded-body body))
|
||||
source-body (or decoded-body body)
|
||||
{stream-body :body fixed-length-promise :pipe-promise} (fixed-length-body source-body size)
|
||||
put-promise (.put bucket key (or stream-body source-body)
|
||||
#js {:httpMetadata #js {:contentType (or content-type "application/octet-stream")}
|
||||
:customMetadata #js {:checksum checksum :type asset-type}})]
|
||||
(-> (p/all (cond-> [put-promise]
|
||||
decode-promise (conj decode-promise)
|
||||
fixed-length-promise (conj fixed-length-promise)))
|
||||
(p/then (fn [_]
|
||||
(http/json-response :assets/put {:ok true} 200)))
|
||||
(p/catch (fn [error]
|
||||
(if (= "invalid base64 asset body" (.-message error))
|
||||
(http/error-response (.-message error) 400)
|
||||
(throw error))))))))
|
||||
|
||||
(defn- <body-with-known-length
|
||||
[body size]
|
||||
(cond
|
||||
(nil? body)
|
||||
(p/resolved nil)
|
||||
|
||||
(and (number? size)
|
||||
(exists? js/FixedLengthStream)
|
||||
(fn? (some-> body .-pipeTo)))
|
||||
(p/resolved (response-fixed-length-body body size))
|
||||
|
||||
;; Some runtimes drop content-length for streamed bodies without a fixed-length wrapper.
|
||||
;; Buffer as a fallback so clients still receive the header.
|
||||
(and (number? size)
|
||||
(fn? (some-> body .-getReader)))
|
||||
(p/let [resp (js/Response. body)
|
||||
buf (.arrayBuffer resp)]
|
||||
buf)
|
||||
|
||||
:else
|
||||
(p/resolved body)))
|
||||
|
||||
(defn- handle-get-asset
|
||||
[^js bucket key asset-type]
|
||||
(.then (.get bucket key)
|
||||
(fn [^js obj]
|
||||
(if (nil? obj)
|
||||
(http/error-response "not found" 404)
|
||||
(let [metadata (.-httpMetadata obj)
|
||||
content-type (response-content-type (.-contentType metadata) asset-type)
|
||||
content-encoding (.-contentEncoding metadata)
|
||||
cache-control (.-cacheControl metadata)
|
||||
size (parse-size (or (.-size obj)
|
||||
(some-> (.-body obj) .-byteLength)))
|
||||
content-length (cond
|
||||
(number? size) (str size)
|
||||
(string? size) size
|
||||
:else nil)]
|
||||
(p/let [body (<body-with-known-length (.-body obj) size)
|
||||
headers (cond-> {"content-type" content-type
|
||||
"content-disposition" (str "inline; filename=\""
|
||||
(last (string/split key #"/")) "\"")
|
||||
"x-asset-type" asset-type}
|
||||
(and (string? content-length)
|
||||
(pos? (.-length content-length)))
|
||||
(assoc "content-length" content-length)
|
||||
(and (string? content-length)
|
||||
(pos? (.-length content-length)))
|
||||
(assoc "x-asset-size" content-length)
|
||||
(and (string? content-encoding)
|
||||
(not= content-encoding "null")
|
||||
(pos? (.-length content-encoding)))
|
||||
(assoc "content-encoding" content-encoding)
|
||||
(and (string? cache-control)
|
||||
(pos? (.-length cache-control)))
|
||||
(assoc "cache-control" cache-control)
|
||||
true
|
||||
(bean/->js))]
|
||||
(js/Response. body
|
||||
#js {:status 200
|
||||
:headers (js/Object.assign
|
||||
headers
|
||||
(cors-headers))})))))))
|
||||
|
||||
(defn parse-asset-path [path]
|
||||
(let [prefix "/assets/"]
|
||||
(when (string/starts-with? path prefix)
|
||||
(let [rest-path (subs path (count prefix))
|
||||
slash-idx (string/index-of rest-path "/")
|
||||
graph-id (when (and slash-idx (pos? slash-idx)) (subs rest-path 0 slash-idx))
|
||||
file (when (and slash-idx (pos? slash-idx)) (subs rest-path (inc slash-idx)))
|
||||
dot-idx (when file (string/last-index-of file "."))
|
||||
asset-uuid (when (and dot-idx (pos? dot-idx)) (subs file 0 dot-idx))
|
||||
asset-type (when (and dot-idx (pos? dot-idx)) (subs file (inc dot-idx)))]
|
||||
(when (and (seq graph-id) (seq asset-uuid) (seq asset-type))
|
||||
{:graph-id graph-id
|
||||
:asset-uuid asset-uuid
|
||||
:asset-type asset-type
|
||||
:key (str graph-id "/" asset-uuid "." asset-type)})))))
|
||||
|
||||
(defn handle [request ^js env]
|
||||
(let [url (js/URL. (.-url request))
|
||||
path (.-pathname url)
|
||||
method (.-method request)]
|
||||
(cond
|
||||
(= method "OPTIONS")
|
||||
(js/Response. nil #js {:status 204 :headers (cors-headers)})
|
||||
|
||||
:else
|
||||
(if-let [{:keys [graph-id key asset-type]} (parse-asset-path path)]
|
||||
(let [^js bucket (.-LOGSEQ_SYNC_ASSETS env)]
|
||||
(if-not bucket
|
||||
(http/error-response "missing assets bucket" 500)
|
||||
(case method
|
||||
"GET"
|
||||
(handle-get-asset bucket key asset-type)
|
||||
|
||||
"PUT"
|
||||
(p/let [e2ee? (index/<graph-e2ee? (.-DB env) graph-id)
|
||||
buf (.arrayBuffer request)
|
||||
size-limit (if e2ee? max-encrypted-asset-size max-asset-size)]
|
||||
(if (> (.-byteLength buf) size-limit)
|
||||
(http/error-response "asset too large" 413)
|
||||
(p/let [_ (.put bucket
|
||||
key
|
||||
buf
|
||||
#js {:httpMetadata #js {:contentType (or (.get (.-headers request) "content-type")
|
||||
"application/octet-stream")}
|
||||
:customMetadata #js {:checksum (.get (.-headers request) "x-amz-meta-checksum")
|
||||
:type asset-type}})]
|
||||
(http/json-response :assets/put {:ok true} 200))))
|
||||
|
||||
"DELETE"
|
||||
(.then (.delete bucket key)
|
||||
(fn [_]
|
||||
(http/json-response :assets/delete {:ok true} 200)))
|
||||
|
||||
(http/error-response "method not allowed" 405))))
|
||||
(http/error-response "invalid asset path" 400)))))
|
||||
539
deps/db-sync/src/logseq/db_sync/worker/handler/index.cljs
vendored
Normal file
539
deps/db-sync/src/logseq/db_sync/worker/handler/index.cljs
vendored
Normal file
@@ -0,0 +1,539 @@
|
||||
(ns logseq.db-sync.worker.handler.index
|
||||
(:require [lambdaisland.glogi :as log]
|
||||
[logseq.db-sync.common :as common]
|
||||
[logseq.db-sync.index :as index]
|
||||
[logseq.db-sync.worker.auth :as auth]
|
||||
[logseq.db-sync.worker.http :as http]
|
||||
[logseq.db-sync.worker.routes.index :as routes]
|
||||
[promesa.core :as p]))
|
||||
|
||||
(defn- index-db [^js self]
|
||||
(let [db (.-d1 self)]
|
||||
(when-not db
|
||||
(log/error :db-sync/index-db-missing {:binding "DB"}))
|
||||
db))
|
||||
|
||||
(defn- admin-token-valid?
|
||||
[request ^js env]
|
||||
(let [expected (aget env "DB_SYNC_ADMIN_TOKEN")
|
||||
actual (.get (.-headers request) "x-db-sync-admin-token")]
|
||||
(and (string? expected)
|
||||
(seq expected)
|
||||
(= expected actual))))
|
||||
|
||||
(defn- <delete-graph-do! [^js env ^js url graph-id]
|
||||
(let [^js namespace (.-LOGSEQ_SYNC_DO env)
|
||||
_ (when-not namespace
|
||||
(throw (ex-info "missing LOGSEQ_SYNC_DO binding"
|
||||
{:graph-id graph-id
|
||||
:binding "LOGSEQ_SYNC_DO"})))
|
||||
do-id (.idFromName namespace graph-id)
|
||||
stub (.get namespace do-id)
|
||||
reset-url (str (.-origin url) "/admin/reset")]
|
||||
(p/let [resp (.fetch stub (js/Request. reset-url #js {:method "DELETE"}))]
|
||||
(when-not (.-ok resp)
|
||||
(throw (ex-info "graph DO delete failed"
|
||||
{:graph-id graph-id
|
||||
:status (.-status resp)})))
|
||||
resp)))
|
||||
|
||||
(defn- <delete-graph-storage!
|
||||
[^js env ^js url graph-id]
|
||||
(let [delete-graph-fn (aget env "DB_SYNC_DELETE_GRAPH")]
|
||||
(if (fn? delete-graph-fn)
|
||||
(delete-graph-fn graph-id)
|
||||
(<delete-graph-do! env url graph-id))))
|
||||
|
||||
(defn- <delete-graph! [db ^js env ^js url graph-id]
|
||||
(p/do!
|
||||
(index/<graph-delete-metadata! db graph-id)
|
||||
(<delete-graph-storage! env url graph-id)
|
||||
(index/<graph-delete-index-entry! db graph-id)))
|
||||
|
||||
(defn- <safe-user-activity-touch!
|
||||
[db user-id]
|
||||
(if (string? user-id)
|
||||
(try
|
||||
(-> (index/<user-activity-touch! db user-id)
|
||||
(p/catch (fn [error]
|
||||
(log/warn :db-sync/activity-touch-user-failed
|
||||
{:user-id user-id
|
||||
:error error})
|
||||
nil)))
|
||||
(catch :default error
|
||||
(log/warn :db-sync/activity-touch-user-failed
|
||||
{:user-id user-id
|
||||
:error error})
|
||||
(p/resolved nil)))
|
||||
(p/resolved nil)))
|
||||
|
||||
(defn- <safe-graph-activity-touch!
|
||||
[db graph-id]
|
||||
(if (string? graph-id)
|
||||
(try
|
||||
(-> (index/<graph-activity-touch! db graph-id)
|
||||
(p/catch (fn [error]
|
||||
(log/warn :db-sync/activity-touch-graph-failed
|
||||
{:graph-id graph-id
|
||||
:error error})
|
||||
nil)))
|
||||
(catch :default error
|
||||
(log/warn :db-sync/activity-touch-graph-failed
|
||||
{:graph-id graph-id
|
||||
:error error})
|
||||
(p/resolved nil)))
|
||||
(p/resolved nil)))
|
||||
|
||||
(defn ^:large-vars/cleanup-todo handle [{:keys [db ^js env request url claims route]}]
|
||||
(let [path-params (:path-params route)
|
||||
graph-id (:graph-id path-params)
|
||||
member-id (:member-id path-params)
|
||||
user-id (some-> claims (aget "sub"))]
|
||||
(case (:handler route)
|
||||
:graphs/list
|
||||
(if (string? user-id)
|
||||
(p/let [graphs (index/<index-list db user-id)
|
||||
user-rsa-key-pair (index/<user-rsa-key-pair db user-id)
|
||||
user-rsa-keys-exists?
|
||||
(and (string? (:public-key user-rsa-key-pair))
|
||||
(string? (:encrypted-private-key user-rsa-key-pair)))]
|
||||
(http/json-response :graphs/list
|
||||
{:graphs graphs
|
||||
:user-rsa-keys-exists? user-rsa-keys-exists?}))
|
||||
(http/unauthorized))
|
||||
|
||||
:graphs/create
|
||||
(.then (common/read-json request)
|
||||
(fn [result]
|
||||
(if (nil? result)
|
||||
(http/bad-request "missing body")
|
||||
(let [body (js->clj result :keywordize-keys true)
|
||||
body (http/coerce-http-request :graphs/create body)
|
||||
graph-id (str (random-uuid))]
|
||||
(cond
|
||||
(not (string? user-id))
|
||||
(http/unauthorized)
|
||||
|
||||
(nil? body)
|
||||
(http/bad-request "invalid body")
|
||||
|
||||
:else
|
||||
(p/let [{:keys [graph-name schema-version graph-e2ee? graph-ready-for-use?]} body
|
||||
graph-e2ee? (if (nil? graph-e2ee?) true (true? graph-e2ee?))
|
||||
graph-ready-for-use? (if (nil? graph-ready-for-use?) true (true? graph-ready-for-use?))
|
||||
name-exists? (index/<graph-name-exists? db graph-name user-id)
|
||||
user-rsa-key-pair (index/<user-rsa-key-pair db user-id)
|
||||
has-user-rsa-key-pair?
|
||||
(and (string? (:public-key user-rsa-key-pair))
|
||||
(string? (:encrypted-private-key user-rsa-key-pair)))]
|
||||
(if name-exists?
|
||||
(http/bad-request "duplicate graph name")
|
||||
(if (and graph-e2ee? (not has-user-rsa-key-pair?))
|
||||
(http/bad-request "missing user rsa key pair")
|
||||
(p/let [_ (index/<index-upsert! db graph-id graph-name user-id schema-version graph-e2ee? graph-ready-for-use?)
|
||||
_ (index/<graph-member-upsert! db graph-id user-id "manager" user-id)]
|
||||
(http/json-response :graphs/create {:graph-id graph-id
|
||||
:graph-e2ee? graph-e2ee?
|
||||
:graph-ready-for-use? graph-ready-for-use?}))))))))))
|
||||
|
||||
:graphs/access
|
||||
(cond
|
||||
(not (string? user-id))
|
||||
(http/unauthorized)
|
||||
|
||||
:else
|
||||
(p/let [owns? (index/<user-has-access-to-graph? db graph-id user-id)]
|
||||
(if owns?
|
||||
(http/json-response :graphs/access {:ok true})
|
||||
(http/forbidden))))
|
||||
|
||||
:graph-members/list
|
||||
(cond
|
||||
(not (string? user-id))
|
||||
(http/unauthorized)
|
||||
|
||||
:else
|
||||
(p/let [can-access? (index/<user-has-access-to-graph? db graph-id user-id)]
|
||||
(if (not can-access?)
|
||||
(http/forbidden)
|
||||
(p/let [members (index/<graph-members-list db graph-id)]
|
||||
(http/json-response :graph-members/list {:members members})))))
|
||||
|
||||
:graph-members/create
|
||||
(cond
|
||||
(not (string? user-id))
|
||||
(http/unauthorized)
|
||||
|
||||
:else
|
||||
(.then (common/read-json request)
|
||||
(fn [result]
|
||||
(if (nil? result)
|
||||
(http/bad-request "missing body")
|
||||
(let [body (js->clj result :keywordize-keys true)
|
||||
body (http/coerce-http-request :graph-members/create body)
|
||||
member-id (:user-id body)
|
||||
email (:email body)
|
||||
role (or (:role body) "member")]
|
||||
(cond
|
||||
(nil? body)
|
||||
(http/bad-request "invalid body")
|
||||
|
||||
(and (not (string? member-id))
|
||||
(not (string? email)))
|
||||
(http/bad-request "invalid user")
|
||||
|
||||
:else
|
||||
(p/let [manager? (index/<user-is-manager? db graph-id user-id)
|
||||
resolved-id (if (string? member-id)
|
||||
(p/resolved member-id)
|
||||
(index/<user-id-by-email db email))]
|
||||
(if (not manager?)
|
||||
(http/forbidden)
|
||||
(if-not (string? resolved-id)
|
||||
(http/bad-request "user not found")
|
||||
(p/let [_ (index/<graph-member-upsert! db graph-id resolved-id role user-id)]
|
||||
(http/json-response :graph-members/create {:ok true})))))))))))
|
||||
|
||||
:graph-members/update
|
||||
(cond
|
||||
(not (string? user-id))
|
||||
(http/unauthorized)
|
||||
|
||||
(not (string? member-id))
|
||||
(http/bad-request "invalid user id")
|
||||
|
||||
:else
|
||||
(.then (common/read-json request)
|
||||
(fn [result]
|
||||
(if (nil? result)
|
||||
(http/bad-request "missing body")
|
||||
(let [body (js->clj result :keywordize-keys true)
|
||||
body (http/coerce-http-request :graph-members/update body)
|
||||
role (:role body)]
|
||||
(cond
|
||||
(nil? body)
|
||||
(http/bad-request "invalid body")
|
||||
|
||||
:else
|
||||
(p/let [manager? (index/<user-is-manager? db graph-id user-id)]
|
||||
(if (not manager?)
|
||||
(http/forbidden)
|
||||
(p/let [_ (index/<graph-member-update-role! db graph-id member-id role)]
|
||||
(http/json-response :graph-members/update {:ok true}))))))))))
|
||||
|
||||
:graph-members/delete
|
||||
(cond
|
||||
(not (string? user-id))
|
||||
(http/unauthorized)
|
||||
|
||||
(not (string? member-id))
|
||||
(http/bad-request "invalid user id")
|
||||
|
||||
:else
|
||||
(p/let [manager? (index/<user-is-manager? db graph-id user-id)
|
||||
target-role (index/<graph-member-role db graph-id member-id)
|
||||
self-leave? (and (= user-id member-id)
|
||||
(= "member" target-role))]
|
||||
(cond
|
||||
(and manager? (not= "manager" target-role))
|
||||
(p/let [_ (index/<graph-member-delete! db graph-id member-id)]
|
||||
(http/json-response :graph-members/delete {:ok true}))
|
||||
|
||||
self-leave?
|
||||
(p/let [_ (index/<graph-member-delete! db graph-id member-id)]
|
||||
(http/json-response :graph-members/delete {:ok true}))
|
||||
|
||||
:else
|
||||
(http/forbidden))))
|
||||
|
||||
:graphs/delete
|
||||
(cond
|
||||
(not (seq graph-id))
|
||||
(http/bad-request "missing graph id")
|
||||
|
||||
(not (string? user-id))
|
||||
(http/unauthorized)
|
||||
|
||||
:else
|
||||
(p/let [owns? (index/<user-has-access-to-graph? db graph-id user-id)]
|
||||
(if (not owns?)
|
||||
(http/forbidden)
|
||||
(p/let [_ (<delete-graph! db env url graph-id)]
|
||||
(http/json-response :graphs/delete {:graph-id graph-id :deleted true})))))
|
||||
|
||||
:admin-graphs/delete
|
||||
(if (seq graph-id)
|
||||
(p/let [_ (<delete-graph! db env url graph-id)]
|
||||
(http/json-response :graphs/delete {:graph-id graph-id :deleted true}))
|
||||
(http/bad-request "missing graph id"))
|
||||
|
||||
:e2ee/user-keys-get
|
||||
(if (string? user-id)
|
||||
(p/let [pair (index/<user-rsa-key-pair db user-id)]
|
||||
(http/json-response :e2ee/user-keys (or pair {})))
|
||||
(http/unauthorized))
|
||||
|
||||
:e2ee/user-keys-post
|
||||
(.then (common/read-json request)
|
||||
(fn [result]
|
||||
(if (nil? result)
|
||||
(http/bad-request "missing body")
|
||||
(let [body (js->clj result :keywordize-keys true)
|
||||
body (http/coerce-http-request :e2ee/user-keys body)]
|
||||
(cond
|
||||
(not (string? user-id))
|
||||
(http/unauthorized)
|
||||
|
||||
(nil? body)
|
||||
(http/bad-request "invalid body")
|
||||
|
||||
:else
|
||||
(let [{:keys [public-key encrypted-private-key]} body]
|
||||
(p/let [_ (index/<user-rsa-key-pair-upsert! db user-id public-key encrypted-private-key)]
|
||||
(http/json-response :e2ee/user-keys {:public-key public-key
|
||||
:encrypted-private-key encrypted-private-key}))))))))
|
||||
|
||||
:e2ee/user-public-key-get
|
||||
(let [email (.get (.-searchParams url) "email")]
|
||||
(p/let [public-key (index/<user-rsa-public-key-by-email db email)]
|
||||
(http/json-response :e2ee/user-public-key
|
||||
(cond-> {}
|
||||
(some? public-key)
|
||||
(assoc :public-key public-key)))))
|
||||
|
||||
:e2ee/graph-aes-key-get
|
||||
(cond
|
||||
(not (string? user-id))
|
||||
(http/unauthorized)
|
||||
|
||||
:else
|
||||
(p/let [access? (index/<user-has-access-to-graph? db graph-id user-id)]
|
||||
(if (not access?)
|
||||
(http/forbidden)
|
||||
(p/let [encrypted-aes-key (index/<graph-encrypted-aes-key db graph-id user-id)]
|
||||
(http/json-response :e2ee/graph-aes-key {:encrypted-aes-key encrypted-aes-key})))))
|
||||
|
||||
:e2ee/graph-aes-key-post
|
||||
(cond
|
||||
(not (string? user-id))
|
||||
(http/unauthorized)
|
||||
|
||||
:else
|
||||
(.then (common/read-json request)
|
||||
(fn [result]
|
||||
(if (nil? result)
|
||||
(http/bad-request "missing body")
|
||||
(let [body (js->clj result :keywordize-keys true)
|
||||
body (http/coerce-http-request :e2ee/graph-aes-key body)]
|
||||
(if (nil? body)
|
||||
(http/bad-request "invalid body")
|
||||
(p/let [access? (index/<user-has-access-to-graph? db graph-id user-id)]
|
||||
(if (not access?)
|
||||
(http/forbidden)
|
||||
(let [{:keys [encrypted-aes-key]} body]
|
||||
(p/let [_ (index/<graph-encrypted-aes-key-upsert! db graph-id user-id encrypted-aes-key)]
|
||||
(http/json-response :e2ee/graph-aes-key {:encrypted-aes-key encrypted-aes-key})))))))))))
|
||||
|
||||
:e2ee/grant-access
|
||||
(if (not (string? user-id))
|
||||
(http/unauthorized)
|
||||
(.then (common/read-json request)
|
||||
(fn [result]
|
||||
(if (nil? result)
|
||||
(http/bad-request "missing body")
|
||||
(let [body (js->clj result :keywordize-keys true)
|
||||
body (http/coerce-http-request :e2ee/grant-access body)]
|
||||
(if (nil? body)
|
||||
(http/bad-request "invalid body")
|
||||
(p/let [manager? (index/<user-is-manager? db graph-id user-id)]
|
||||
(if (not manager?)
|
||||
(http/forbidden)
|
||||
(let [entries (:target-user-email+encrypted-aes-key-coll body)
|
||||
missing (atom [])]
|
||||
(p/let [_ (p/all
|
||||
(map (fn [entry]
|
||||
(let [email (:email entry)
|
||||
encrypted-aes-key (:encrypted-aes-key entry)]
|
||||
(p/let [target-user-id (index/<user-id-by-email db email)
|
||||
access? (and target-user-id
|
||||
(index/<user-has-access-to-graph? db graph-id target-user-id))]
|
||||
(if (and target-user-id access?)
|
||||
(index/<graph-encrypted-aes-key-upsert! db graph-id target-user-id encrypted-aes-key)
|
||||
(swap! missing conj email)))))
|
||||
entries))]
|
||||
(http/json-response :e2ee/grant-access
|
||||
(cond-> {:ok true}
|
||||
(seq @missing)
|
||||
(assoc :missing-users @missing)))))))))))))
|
||||
|
||||
(http/not-found))))
|
||||
|
||||
(defn handle-fetch [^js self request]
|
||||
(let [db (index-db self)
|
||||
env (.-env self)
|
||||
url (js/URL. (.-url request))
|
||||
path (.-pathname url)
|
||||
method (.-method request)]
|
||||
(try
|
||||
(cond
|
||||
(contains? #{"OPTIONS" "HEAD"} method)
|
||||
(common/options-response)
|
||||
|
||||
(nil? db)
|
||||
(http/error-response "server error" 500)
|
||||
|
||||
:else
|
||||
(let [route (routes/match-route method path)]
|
||||
(cond
|
||||
(nil? route)
|
||||
(http/not-found)
|
||||
|
||||
(= :admin-graphs/delete (:handler route))
|
||||
(if (admin-token-valid? request env)
|
||||
(handle {:db db
|
||||
:env env
|
||||
:request request
|
||||
:url url
|
||||
:claims nil
|
||||
:route route})
|
||||
(http/unauthorized))
|
||||
|
||||
:else
|
||||
(p/let [claims (auth/auth-claims request env)
|
||||
_ (when claims
|
||||
(index/<user-upsert! db claims))]
|
||||
(if (nil? claims)
|
||||
(http/unauthorized)
|
||||
(p/let [user-id (aget claims "sub")
|
||||
_ (<safe-user-activity-touch! db user-id)
|
||||
response (handle {:db db
|
||||
:env env
|
||||
:request request
|
||||
:url url
|
||||
:claims claims
|
||||
:route route})
|
||||
graph-id (some-> route :path-params :graph-id)
|
||||
_ (when (and (string? user-id)
|
||||
(string? graph-id)
|
||||
(< (.-status response) 400))
|
||||
(<safe-graph-activity-touch! db graph-id))]
|
||||
response))))))
|
||||
(catch :default error
|
||||
(js/console.error "DEBUG handle-fetch error:" error)
|
||||
(log/error :db-sync/index-error error)
|
||||
(http/error-response (str "server error: " error) 500)))))
|
||||
|
||||
(def ^:private graph-access-cache-ttl-ms 5000)
|
||||
(def ^:private graph-access-cache-capacity 256)
|
||||
(defonce ^:private *graph-access-cache (atom {}))
|
||||
|
||||
(defn- now-ms []
|
||||
(.now js/Date))
|
||||
|
||||
(defn- unauthorized-timing [jwt-verify-ms]
|
||||
{:access-ok? false
|
||||
:cache-hit? false
|
||||
:jwt-verify-ms jwt-verify-ms
|
||||
:access-query-ms 0
|
||||
:access-check-ms jwt-verify-ms})
|
||||
|
||||
(defn- fresh-cache?
|
||||
[cached-at current-ms]
|
||||
(and (number? cached-at)
|
||||
(< (- current-ms cached-at) graph-access-cache-ttl-ms)))
|
||||
|
||||
(defn- lookup-graph-access-cache
|
||||
[graph-id token current-ms]
|
||||
(let [cache-key [graph-id token]]
|
||||
(when-let [{:keys [allowed? cached-at]} (get @*graph-access-cache cache-key)]
|
||||
(if (fresh-cache? cached-at current-ms)
|
||||
{:allowed? allowed?}
|
||||
(do
|
||||
(swap! *graph-access-cache dissoc cache-key)
|
||||
nil)))))
|
||||
|
||||
(defn- prune-graph-access-cache
|
||||
[cache current-ms]
|
||||
(let [fresh (into {}
|
||||
(filter (fn [[_ {:keys [cached-at]}]]
|
||||
(fresh-cache? cached-at current-ms)))
|
||||
cache)]
|
||||
(if (<= (count fresh) graph-access-cache-capacity)
|
||||
fresh
|
||||
(let [drop-count (- (count fresh) graph-access-cache-capacity)]
|
||||
(->> fresh
|
||||
(sort-by (comp :cached-at val))
|
||||
(drop drop-count)
|
||||
(into {}))))))
|
||||
|
||||
(defn- cache-graph-access!
|
||||
[graph-id token allowed? current-ms]
|
||||
(let [cache-key [graph-id token]]
|
||||
(swap! *graph-access-cache
|
||||
(fn [cache]
|
||||
(-> cache
|
||||
(assoc cache-key {:allowed? allowed? :cached-at current-ms})
|
||||
(prune-graph-access-cache current-ms))))))
|
||||
|
||||
(defn graph-access-response-with-timing
|
||||
[request env graph-id]
|
||||
(let [token (auth/token-from-request request)
|
||||
db (aget env "DB")]
|
||||
(cond
|
||||
(or (not (string? token))
|
||||
(not (seq token)))
|
||||
(p/resolved {:response (http/unauthorized)
|
||||
:timing (unauthorized-timing 0)})
|
||||
|
||||
(nil? db)
|
||||
(p/resolved {:response (http/error-response "server error" 500)
|
||||
:timing {:access-ok? false
|
||||
:cache-hit? false}})
|
||||
|
||||
:else
|
||||
(let [current-ms (now-ms)]
|
||||
(if-let [{:keys [allowed?]} (lookup-graph-access-cache graph-id token current-ms)]
|
||||
(p/resolved {:response (if allowed?
|
||||
(http/json-response :graphs/access {:ok true})
|
||||
(http/forbidden))
|
||||
:timing {:access-ok? allowed?
|
||||
:cache-hit? true
|
||||
:jwt-verify-ms 0
|
||||
:access-query-ms 0
|
||||
:access-check-ms 0}})
|
||||
(let [jwt-start-ms (now-ms)]
|
||||
(->
|
||||
(p/let [claims (auth/auth-claims request env)
|
||||
jwt-end-ms (now-ms)
|
||||
jwt-verify-ms (- jwt-end-ms jwt-start-ms)]
|
||||
(if (nil? claims)
|
||||
{:response (http/unauthorized)
|
||||
:timing (unauthorized-timing jwt-verify-ms)}
|
||||
(let [user-id (aget claims "sub")]
|
||||
(if-not (string? user-id)
|
||||
{:response (http/unauthorized)
|
||||
:timing (unauthorized-timing jwt-verify-ms)}
|
||||
(p/let [query-start-ms (now-ms)
|
||||
access? (index/<user-has-access-to-graph? db graph-id user-id)
|
||||
query-end-ms (now-ms)
|
||||
access-query-ms (- query-end-ms query-start-ms)
|
||||
access-check-ms (+ jwt-verify-ms access-query-ms)
|
||||
_ (cache-graph-access! graph-id token (true? access?) query-end-ms)
|
||||
response (if access?
|
||||
(http/json-response :graphs/access {:ok true})
|
||||
(http/forbidden))]
|
||||
{:response response
|
||||
:timing {:access-ok? (true? access?)
|
||||
:cache-hit? false
|
||||
:jwt-verify-ms jwt-verify-ms
|
||||
:access-query-ms access-query-ms
|
||||
:access-check-ms access-check-ms}})))))
|
||||
(p/catch (fn [error]
|
||||
(log/error :db-sync/index-error error)
|
||||
(p/resolved {:response (http/error-response (str "server error: " error) 500)
|
||||
:timing {:access-ok? false
|
||||
:cache-hit? false}}))))))))))
|
||||
|
||||
(defn graph-access-response [request env graph-id]
|
||||
(p/let [{:keys [response]} (graph-access-response-with-timing request env graph-id)]
|
||||
response))
|
||||
996
deps/db-sync/src/logseq/db_sync/worker/handler/semantic.cljs
vendored
Normal file
996
deps/db-sync/src/logseq/db_sync/worker/handler/semantic.cljs
vendored
Normal file
@@ -0,0 +1,996 @@
|
||||
(ns logseq.db-sync.worker.handler.semantic
|
||||
(:require [clojure.string :as string]
|
||||
[datascript.core :as d]
|
||||
[logseq.common.util :as common-util]
|
||||
[logseq.common.util.date-time :as date-time-util]
|
||||
[logseq.common.util.page-ref :as page-ref]
|
||||
[logseq.db :as ldb]
|
||||
[logseq.db-sync.common :as common]
|
||||
[logseq.db-sync.storage :as storage]
|
||||
[logseq.db-sync.worker.asset-link :as asset-link]
|
||||
[logseq.db-sync.worker.handler.assets :as assets-handler]
|
||||
[logseq.db-sync.worker.http :as http]
|
||||
[logseq.db-sync.worker.ws :as ws]
|
||||
[logseq.db.frontend.asset :as db-asset]
|
||||
[logseq.db.frontend.content :as db-content]
|
||||
[logseq.db.frontend.db :as db-db]
|
||||
[logseq.db.frontend.entity-util :as entity-util]
|
||||
[logseq.db.frontend.property.type :as db-property-type]
|
||||
[logseq.outliner.core :as outliner-core]
|
||||
[logseq.outliner.page :as outliner-page]
|
||||
[logseq.outliner.property :as outliner-property]
|
||||
[logseq.outliner.tree :as otree]
|
||||
[promesa.core :as p]))
|
||||
|
||||
(def ^:private default-limit 50)
|
||||
(def ^:private max-limit 200)
|
||||
|
||||
(defn- ensure-conn! [^js self]
|
||||
(when-not (true? (.-schema-ready self))
|
||||
(storage/init-schema! (.-sql self))
|
||||
(set! (.-schema-ready self) true))
|
||||
(when-not (.-conn self)
|
||||
(set! (.-conn self) (storage/open-conn (.-sql self))))
|
||||
(.-conn self))
|
||||
|
||||
(defn- uuid-string [value]
|
||||
(when value (str value)))
|
||||
|
||||
(defn- page? [entity]
|
||||
(or (ldb/internal-page? entity) (ldb/journal? entity)))
|
||||
|
||||
(defn- asset? [entity]
|
||||
(string? (:logseq.property.asset/type entity)))
|
||||
|
||||
(defn- entity-ident [entity]
|
||||
(when-let [ident (:db/ident entity)]
|
||||
(str (namespace ident) "/" (name ident))))
|
||||
|
||||
(defn- tag? [entity]
|
||||
(ldb/class? entity))
|
||||
|
||||
(defn- property? [entity]
|
||||
(ldb/property? entity))
|
||||
|
||||
(defn- block-kind [entity]
|
||||
(cond
|
||||
(asset? entity) "asset"
|
||||
(property? entity) "property"
|
||||
(tag? entity) "tag"
|
||||
(page? entity) "page"
|
||||
:else "block"))
|
||||
|
||||
(defn- block-response [block]
|
||||
(cond-> {:uuid (uuid-string (:block/uuid block))
|
||||
:kind (block-kind block)
|
||||
:title (:block/title block)
|
||||
:order (:block/order block)}
|
||||
(:block/parent block) (assoc :parent-id (uuid-string (:block/uuid (:block/parent block))))
|
||||
(:block/page block) (assoc :page-id (uuid-string (:block/uuid (:block/page block))))
|
||||
(seq (:block/children block)) (assoc :children (mapv block-response (:block/children block)))))
|
||||
|
||||
(defn- tag-response [entity]
|
||||
{:uuid (uuid-string (:block/uuid entity))
|
||||
:ident (entity-ident entity)
|
||||
:title (:block/title entity)})
|
||||
|
||||
(defn- property-choice-response [entity]
|
||||
{:uuid (uuid-string (:block/uuid entity))
|
||||
:ident (entity-ident entity)
|
||||
:title (:block/title entity)
|
||||
:value (:logseq.property/value entity)})
|
||||
|
||||
(defn- property-response [entity]
|
||||
(cond-> {:uuid (uuid-string (:block/uuid entity))
|
||||
:ident (entity-ident entity)
|
||||
:title (:block/title entity)
|
||||
:type (some-> (:logseq.property/type entity) name)
|
||||
:cardinality (some-> (:db/cardinality entity) name)}
|
||||
(seq (:block/_closed-value-property entity))
|
||||
(assoc :choices (mapv property-choice-response (:block/_closed-value-property entity)))))
|
||||
|
||||
(defn- asset-response [entity]
|
||||
{:uuid (uuid-string (:block/uuid entity))
|
||||
:title (:block/title entity)
|
||||
:type (:logseq.property.asset/type entity)
|
||||
:size (:logseq.property.asset/size entity)
|
||||
:checksum (:logseq.property.asset/checksum entity)})
|
||||
|
||||
(defn- entities-by-avet [db attribute & [value]]
|
||||
(->> (if (some? value)
|
||||
(d/datoms db :avet attribute value)
|
||||
(d/datoms db :avet attribute))
|
||||
(map #(d/entity db (:e %)))
|
||||
(remove entity-util/hidden?)))
|
||||
|
||||
(defn- parse-limit [url]
|
||||
(let [raw (.get (.-searchParams url) "limit")
|
||||
parsed (when raw (js/parseInt raw 10))]
|
||||
(cond
|
||||
(nil? raw) default-limit
|
||||
(or (js/isNaN parsed) (not (pos? parsed))) nil
|
||||
:else (min parsed max-limit))))
|
||||
|
||||
(def ^:private text-encoder (js/TextEncoder.))
|
||||
(def ^:private text-decoder (js/TextDecoder.))
|
||||
|
||||
(defn- encode-cursor [key]
|
||||
(let [payload (.encode text-encoder (js/JSON.stringify (clj->js key)))
|
||||
binary (.join (.map (js/Array.from payload) #(js/String.fromCharCode %)) "")]
|
||||
(js/btoa binary)))
|
||||
|
||||
(defn- decode-cursor [value]
|
||||
(when value
|
||||
(try
|
||||
(let [binary (js/atob value)
|
||||
payload (js/Uint8Array.from (js/Array.from binary #(.charCodeAt % 0)))
|
||||
decoded (js->clj (js/JSON.parse (.decode text-decoder payload)))]
|
||||
(when (and (vector? decoded) (= 2 (count decoded)) (every? string? decoded)) decoded))
|
||||
(catch :default _ nil))))
|
||||
|
||||
(defn- sort-key [item]
|
||||
[(or (:order item) (:block/order item)
|
||||
(string/lower-case (or (:title item) (:block/title item) "")))
|
||||
(str (or (:uuid item) (:block/uuid item) ""))])
|
||||
|
||||
(defn- paginate [items limit cursor]
|
||||
(let [sorted (sort-by sort-key items)
|
||||
remaining (if cursor (drop-while #(not (pos? (compare (sort-key %) cursor))) sorted) sorted)
|
||||
selected (vec (take (inc limit) remaining))
|
||||
more? (> (count selected) limit)
|
||||
page (vec (take limit selected))]
|
||||
(cond-> {:items page}
|
||||
more? (assoc :next-cursor (encode-cursor (sort-key (last page)))))))
|
||||
|
||||
(defn- paginate-bounded [items limit cursor]
|
||||
(let [selected (reduce (fn [selected item]
|
||||
(if (or (nil? cursor)
|
||||
(pos? (compare (sort-key item) cursor)))
|
||||
(->> (conj selected item)
|
||||
(sort-by sort-key)
|
||||
(take (inc limit))
|
||||
vec)
|
||||
selected))
|
||||
[]
|
||||
items)
|
||||
more? (> (count selected) limit)
|
||||
page (vec (take limit selected))]
|
||||
(cond-> {:items page}
|
||||
more? (assoc :next-cursor (encode-cursor (sort-key (last page)))))))
|
||||
|
||||
(defn- pagination-options [url]
|
||||
(let [raw-cursor (.get (.-searchParams url) "cursor")]
|
||||
{:limit (parse-limit url)
|
||||
:raw-cursor raw-cursor
|
||||
:cursor (decode-cursor raw-cursor)}))
|
||||
|
||||
(defn- parse-timestamp [raw]
|
||||
(cond
|
||||
(nil? raw) nil
|
||||
(not (re-matches #"\d+" raw)) ::invalid
|
||||
:else (js/parseInt raw 10)))
|
||||
|
||||
(defn- time-filter-options [url]
|
||||
(let [created-after (parse-timestamp (.get (.-searchParams url) "created-after"))
|
||||
updated-after (parse-timestamp (.get (.-searchParams url) "updated-after"))]
|
||||
{:created-after created-after
|
||||
:updated-after updated-after
|
||||
:invalid? (or (= ::invalid created-after) (= ::invalid updated-after))}))
|
||||
|
||||
(defn- after-time-filters? [entity {:keys [created-after updated-after]}]
|
||||
(and (or (nil? created-after)
|
||||
(and (number? (:block/created-at entity))
|
||||
(> (:block/created-at entity) created-after)))
|
||||
(or (nil? updated-after)
|
||||
(and (number? (:block/updated-at entity))
|
||||
(> (:block/updated-at entity) updated-after)))))
|
||||
|
||||
(defn- paginated-response
|
||||
([url response-key items]
|
||||
(paginated-response url response-key items identity))
|
||||
([url response-key items response-fn]
|
||||
(paginated-response url response-key items response-fn paginate))
|
||||
([url response-key items response-fn paginate-fn]
|
||||
(let [{:keys [limit raw-cursor cursor]} (pagination-options url)
|
||||
time-filters (time-filter-options url)]
|
||||
(cond
|
||||
(nil? limit) (http/bad-request "invalid limit")
|
||||
(and raw-cursor (nil? cursor)) (http/bad-request "invalid cursor")
|
||||
(:invalid? time-filters) (http/bad-request "invalid time filter")
|
||||
:else (let [items (filter #(after-time-filters? % time-filters) items)
|
||||
{:keys [items next-cursor]} (paginate-fn items limit cursor)]
|
||||
(http/json-response nil (cond-> {response-key (mapv response-fn items)}
|
||||
next-cursor (assoc :next-cursor next-cursor))))))))
|
||||
|
||||
(defn- indexed-reference-response
|
||||
([db url response-key attribute target-id]
|
||||
(indexed-reference-response db url response-key attribute target-id {}))
|
||||
([db url response-key attribute target-id {:keys [entity-filter response-fn]
|
||||
:or {entity-filter (constantly true)
|
||||
response-fn block-response}}]
|
||||
(let [{:keys [limit raw-cursor cursor]} (pagination-options url)
|
||||
time-filters (time-filter-options url)
|
||||
cursor-e (when (= "entity-id" (first cursor))
|
||||
(js/parseInt (second cursor) 10))]
|
||||
(cond
|
||||
(nil? limit) (http/bad-request "invalid limit")
|
||||
(and raw-cursor (or (nil? cursor) (js/isNaN cursor-e))) (http/bad-request "invalid cursor")
|
||||
(:invalid? time-filters) (http/bad-request "invalid time filter")
|
||||
:else
|
||||
(let [datoms (cond->> (d/datoms db :avet attribute target-id)
|
||||
cursor-e (drop-while #(<= (:e %) cursor-e))
|
||||
true (remove #(entity-util/hidden? (d/entity db (:e %))))
|
||||
true (filter #(after-time-filters? (d/entity db (:e %)) time-filters))
|
||||
true (filter #(entity-filter (d/entity db (:e %)))))
|
||||
selected (vec (take (inc limit) datoms))
|
||||
more? (> (count selected) limit)
|
||||
page (take limit selected)]
|
||||
(http/json-response
|
||||
nil
|
||||
(cond-> {response-key (mapv #(response-fn (d/entity db (:e %))) page)}
|
||||
more? (assoc :next-cursor
|
||||
(encode-cursor ["entity-id" (str (:e (last page)))])))))))))
|
||||
|
||||
(defn- find-entity [db id]
|
||||
(when-let [parsed (try (uuid id) (catch :default _ nil))]
|
||||
(d/entity db [:block/uuid parsed])))
|
||||
|
||||
(defn- find-visible-entity [db id]
|
||||
(when-let [entity (find-entity db id)]
|
||||
(when-not (entity-util/hidden? entity) entity)))
|
||||
|
||||
(defn- page-root-blocks [db page]
|
||||
(let [page-id (:db/id page)]
|
||||
(->> (d/datoms db :avet :block/page page-id)
|
||||
(filter #(seq (d/datoms db :eavt (:e %) :block/parent page-id)))
|
||||
(map #(:e %))
|
||||
(map #(d/pull db [:db/id :block/uuid :block/title :block/order
|
||||
:block/created-at :block/updated-at
|
||||
:logseq.property/hide? :logseq.property/deleted-at
|
||||
{:block/parent [:db/id :block/uuid]}
|
||||
{:block/page [:db/id :block/uuid]}]
|
||||
%))
|
||||
(remove entity-util/hidden?))))
|
||||
|
||||
(def ^:private page-block-pull-keys
|
||||
[:db/id :block/uuid :block/title :block/order
|
||||
:logseq.property/hide? :logseq.property/deleted-at
|
||||
{:block/parent [:db/id :block/uuid]}
|
||||
{:block/page [:db/id :block/uuid]}])
|
||||
|
||||
(defn- page-block-tree [db root]
|
||||
(let [blocks (->> (ldb/get-block-and-children db (:block/uuid root))
|
||||
(remove #(entity-util/hidden? (d/entity db (:db/id %)))))]
|
||||
(some->> blocks
|
||||
(map #(d/pull db page-block-pull-keys (:db/id %)))
|
||||
(otree/non-consecutive-blocks->vec-tree)
|
||||
first
|
||||
block-response)))
|
||||
|
||||
(defn- paginated-page-blocks-response [db page url]
|
||||
(let [{:keys [limit raw-cursor cursor]} (pagination-options url)
|
||||
time-filters (time-filter-options url)]
|
||||
(cond
|
||||
(nil? limit) (http/bad-request "invalid limit")
|
||||
(and raw-cursor (nil? cursor)) (http/bad-request "invalid cursor")
|
||||
(:invalid? time-filters) (http/bad-request "invalid time filter")
|
||||
:else
|
||||
(let [{roots :items next-cursor :next-cursor}
|
||||
(paginate (filter #(after-time-filters? % time-filters)
|
||||
(page-root-blocks db page))
|
||||
limit
|
||||
cursor)]
|
||||
(http/json-response
|
||||
nil
|
||||
(cond-> {:blocks (mapv #(page-block-tree db %) roots)}
|
||||
next-cursor (assoc :next-cursor next-cursor)))))))
|
||||
|
||||
(defn- body-clj [request]
|
||||
(p/let [body (common/read-json request)]
|
||||
(when body (js->clj body :keywordize-keys true))))
|
||||
|
||||
(defn- broadcast-change! [^js self]
|
||||
(ws/broadcast! self nil {:type "changed" :t (storage/get-t (.-sql self))}))
|
||||
|
||||
(defn- list-pages [db]
|
||||
(->> [:logseq.class/Page :logseq.class/Journal]
|
||||
(keep #(d/entid db %))
|
||||
(mapcat #(entities-by-avet db :block/tags %))))
|
||||
|
||||
(defn- page-ref-values [title]
|
||||
(when (string? title)
|
||||
(->> (re-seq page-ref/page-ref-re title)
|
||||
(map (comp string/trim second))
|
||||
(remove string/blank?)
|
||||
distinct)))
|
||||
|
||||
(defn- integer-string? [value]
|
||||
(boolean (re-matches #"-?\d+" value)))
|
||||
|
||||
(defn- resolve-page-ref! [conn value]
|
||||
(cond
|
||||
(common-util/uuid-string? value)
|
||||
(when-let [entity (d/entity @conn [:block/uuid (uuid value)])]
|
||||
{:block/uuid (:block/uuid entity)
|
||||
:block/title value})
|
||||
|
||||
(integer-string? value)
|
||||
(when-let [entity (d/entity @conn (js/parseInt value 10))]
|
||||
{:block/uuid (:block/uuid entity)
|
||||
:block/title value})
|
||||
|
||||
:else
|
||||
(let [page (or (ldb/get-page @conn value)
|
||||
(let [[_ page-id] (outliner-page/create! conn value {})]
|
||||
(d/entity @conn [:block/uuid page-id])))]
|
||||
{:block/uuid (:block/uuid page)
|
||||
:block/title (:block/title page)})))
|
||||
|
||||
(defn- prepare-block-title! [conn title]
|
||||
(let [refs (->> (page-ref-values title)
|
||||
(keep #(resolve-page-ref! conn %))
|
||||
(reduce (fn [result ref]
|
||||
(assoc result (:block/uuid ref) ref)) {})
|
||||
vals
|
||||
vec)]
|
||||
{:block/title (db-content/title-ref->id-ref title refs :replace-tag? false)
|
||||
:block/refs (mapv #(vector :block/uuid (:block/uuid %)) refs)}))
|
||||
|
||||
(defn- tree-block [conn node]
|
||||
(assoc (prepare-block-title! conn (:title node)) :block/uuid (random-uuid)))
|
||||
|
||||
(defn- insert-tree! [conn target nodes position]
|
||||
(let [nodes (vec nodes)
|
||||
blocks (mapv #(tree-block conn %) nodes)]
|
||||
(when (seq blocks)
|
||||
(outliner-core/insert-blocks! conn blocks target
|
||||
{:sibling? false :top? (= position "prepend")
|
||||
:bottom? (not= position "prepend") :keep-uuid? true})
|
||||
(mapv (fn [node block]
|
||||
(let [children (when (seq (:children node))
|
||||
(insert-tree! conn (d/entity @conn [:block/uuid (:block/uuid block)])
|
||||
(:children node) "append"))]
|
||||
(cond-> {:uuid (str (:block/uuid block)) :title (:title node)}
|
||||
(seq children) (assoc :children children))))
|
||||
nodes blocks))))
|
||||
|
||||
(defn- today-journal-day []
|
||||
(let [now (js/Date.)]
|
||||
(+ (* (.getFullYear now) 10000) (* (inc (.getMonth now)) 100) (.getDate now))))
|
||||
|
||||
(defn- ensure-today-page! [conn]
|
||||
(let [day (today-journal-day)]
|
||||
(or (ldb/get-journal-page-by-day @conn day)
|
||||
(let [formatter (:logseq.property.journal/title-format (d/entity @conn :logseq.class/Journal))
|
||||
title (date-time-util/int->journal-title day formatter)
|
||||
[_ page-id] (outliner-page/create! conn title {:journal? true :today-journal? true})]
|
||||
(d/entity @conn [:block/uuid page-id])))))
|
||||
|
||||
(defn- resolve-property-ident [db value]
|
||||
(or (when-let [entity (find-entity db value)] (:db/ident entity))
|
||||
(when (string? value)
|
||||
(let [ident (keyword value)]
|
||||
(or (when (d/entity db ident) ident)
|
||||
(some->> (d/datoms db :avet :block/title value)
|
||||
(map #(d/entity db (:e %)))
|
||||
(filter property?)
|
||||
first
|
||||
:db/ident))))))
|
||||
|
||||
(defn- resolve-property [db value]
|
||||
(when-let [ident (resolve-property-ident db value)]
|
||||
(let [property (d/entity db ident)]
|
||||
(when-not (entity-util/hidden? property)
|
||||
property))))
|
||||
|
||||
(def ^:private allowed-property-types
|
||||
(into (set db-property-type/user-built-in-property-types)
|
||||
db-property-type/user-allowed-internal-property-types))
|
||||
|
||||
(def ^:private allowed-cardinalities
|
||||
#{:db.cardinality/one :db.cardinality/many})
|
||||
|
||||
(defn- valid-property-schema? [{:keys [type cardinality]}]
|
||||
(and (or (nil? type)
|
||||
(and (string? type) (contains? allowed-property-types (keyword type))))
|
||||
(or (nil? cardinality)
|
||||
(and (string? cardinality) (contains? allowed-cardinalities (keyword cardinality))))))
|
||||
|
||||
(def ^:private task-status-idents
|
||||
{"todo" :logseq.property/status.todo
|
||||
"doing" :logseq.property/status.doing
|
||||
"in-review" :logseq.property/status.in-review
|
||||
"done" :logseq.property/status.done
|
||||
"canceled" :logseq.property/status.canceled
|
||||
"backlog" :logseq.property/status.backlog})
|
||||
|
||||
(def ^:private task-priority-idents
|
||||
{"low" :logseq.property/priority.low
|
||||
"medium" :logseq.property/priority.medium
|
||||
"high" :logseq.property/priority.high
|
||||
"urgent" :logseq.property/priority.urgent})
|
||||
|
||||
(def ^:private property-choice-aliases
|
||||
{:logseq.property/status task-status-idents
|
||||
:logseq.property/priority task-priority-idents})
|
||||
|
||||
(defn- resolve-ref-entity [db property-ident value]
|
||||
(let [value (if (string? value)
|
||||
(get-in property-choice-aliases
|
||||
[property-ident (string/lower-case value)] value)
|
||||
value)
|
||||
entity (cond
|
||||
(number? value) (d/entity db value)
|
||||
(qualified-keyword? value) (d/entity db value)
|
||||
(and (string? value) (common-util/uuid-string? value))
|
||||
(d/entity db [:block/uuid (uuid value)])
|
||||
(string? value)
|
||||
(or (d/entity db (keyword value))
|
||||
(some->> (d/datoms db :avet :block/title value)
|
||||
(map #(d/entity db (:e %)))
|
||||
(filter #(if (= property-ident :block/tags)
|
||||
(entity-util/class? %)
|
||||
true))
|
||||
first))
|
||||
:else nil)]
|
||||
(when (and entity
|
||||
(or (not= property-ident :block/tags) (entity-util/class? entity)))
|
||||
(:db/id entity))))
|
||||
|
||||
(defn- prepare-property-value [db property-ident value]
|
||||
(let [property (d/entity db property-ident)]
|
||||
(if-not (= :db.type/ref (:db/valueType property))
|
||||
{:value value}
|
||||
(let [many-input? (or (sequential? value) (set? value))
|
||||
values (if many-input? value [value])
|
||||
resolved (mapv #(resolve-ref-entity db property-ident %) values)]
|
||||
(if (every? some? resolved)
|
||||
{:value (if many-input? resolved (first resolved))}
|
||||
{:error (if (= property-ident :block/tags)
|
||||
"property value must resolve to an existing class by UUID, ident, or title"
|
||||
"property value must resolve to an existing entity by UUID, ident, or title")})))))
|
||||
|
||||
(defn- set-prepared-property! [conn block-id property-ident value reset-existing-values?]
|
||||
(if (coll? value)
|
||||
(do
|
||||
(when reset-existing-values?
|
||||
(outliner-property/remove-block-property! conn block-id property-ident))
|
||||
(doseq [item value]
|
||||
(outliner-property/batch-set-property! conn [block-id] property-ident item {:entity-id? true})))
|
||||
(outliner-property/set-block-property! conn block-id property-ident value)))
|
||||
|
||||
(defn- task-response [task]
|
||||
(cond-> (block-response task)
|
||||
(:logseq.property/status task)
|
||||
(assoc :status (property-choice-response (:logseq.property/status task)))
|
||||
(:logseq.property/priority task)
|
||||
(assoc :priority (property-choice-response (:logseq.property/priority task)))
|
||||
(:logseq.property/scheduled task) (assoc :scheduled (:logseq.property/scheduled task))
|
||||
(:logseq.property/deadline task) (assoc :deadline (:logseq.property/deadline task))))
|
||||
|
||||
(defn- resolve-property-choice [db property-ident aliases selector]
|
||||
(let [selector (get aliases selector selector)
|
||||
choice-id (resolve-ref-entity db property-ident selector)
|
||||
choice (when choice-id (d/entity db choice-id))
|
||||
property-id (d/entid db property-ident)]
|
||||
(when (and choice-id
|
||||
(some #(= property-id (:v %))
|
||||
(d/datoms db :eavt choice-id :block/closed-value-property)))
|
||||
choice)))
|
||||
|
||||
(defn- search-result-response [entity]
|
||||
(let [kind (block-kind entity)
|
||||
type (case kind "tag" "tags" "property" "properties" "asset" "assets" "blocks")
|
||||
response (case kind
|
||||
"tag" (tag-response entity)
|
||||
"property" (property-response entity)
|
||||
"asset" (asset-response entity)
|
||||
(block-response entity))]
|
||||
(assoc response :resource type)))
|
||||
|
||||
(defn- search-results [db query types]
|
||||
(let [needle (string/lower-case query)
|
||||
enabled (if (seq types) (set (string/split types #",")) #{"blocks" "tags" "properties" "assets"})
|
||||
title-matches (keep (fn [datom]
|
||||
(try
|
||||
(when (string/includes? (string/lower-case (:v datom)) needle)
|
||||
(:e datom))
|
||||
(catch :default _
|
||||
nil)))
|
||||
(d/datoms db :avet :block/title))]
|
||||
(->> title-matches
|
||||
(map #(d/entity db %))
|
||||
(remove entity-util/hidden?)
|
||||
(filter (fn [entity]
|
||||
(let [kind (block-kind entity)
|
||||
type (case kind "tag" "tags" "property" "properties" "asset" "assets" "blocks")]
|
||||
(and (contains? enabled type)
|
||||
(string/includes? (string/lower-case (or (:block/title entity) "")) needle))))))))
|
||||
|
||||
(defn- handle-pages [{:keys [^js self request ^js url conn db handler path-params]}]
|
||||
(case handler
|
||||
:semantic/pages-list
|
||||
(paginated-response url :blocks (list-pages db) block-response)
|
||||
|
||||
:semantic/pages-create
|
||||
(p/let [body (body-clj request)]
|
||||
(if-not (seq (:title body))
|
||||
(http/bad-request "missing title")
|
||||
(let [[title page-id] (outliner-page/create! conn (:title body) {})]
|
||||
(broadcast-change! self)
|
||||
(http/json-response nil {:uuid (str page-id) :kind "page" :title title} 201))))
|
||||
|
||||
:semantic/pages-blocks
|
||||
(if-let [page (find-visible-entity db (:page-id path-params))]
|
||||
(if-not (page? page)
|
||||
(http/bad-request "block is not a page")
|
||||
(paginated-page-blocks-response db page url))
|
||||
(http/not-found))
|
||||
|
||||
:semantic/pages-references
|
||||
(if-let [page (find-visible-entity db (:page-id path-params))]
|
||||
(if (page? page)
|
||||
(indexed-reference-response db url :references :block/refs (:db/id page))
|
||||
(http/not-found))
|
||||
(http/not-found))
|
||||
|
||||
:semantic/pages-get
|
||||
(if-let [page (find-visible-entity db (:page-id path-params))]
|
||||
(if (page? page) (http/json-response nil (block-response page)) (http/not-found))
|
||||
(http/not-found))
|
||||
|
||||
:semantic/pages-update
|
||||
(p/let [body (body-clj request)
|
||||
page (find-entity db (:page-id path-params))]
|
||||
(if (or (not (page? page)) (not (seq (:title body))))
|
||||
(http/bad-request "invalid page-id or title")
|
||||
(do
|
||||
(outliner-core/save-block! conn {:block/uuid (:block/uuid page) :block/title (:title body)})
|
||||
(broadcast-change! self)
|
||||
(http/json-response nil (assoc (block-response page) :title (:title body))))))
|
||||
|
||||
:semantic/pages-delete
|
||||
(if-let [page (find-entity db (:page-id path-params))]
|
||||
(if-not (page? page)
|
||||
(http/bad-request "block is not a page")
|
||||
(do
|
||||
(outliner-page/delete! conn (:block/uuid page))
|
||||
(broadcast-change! self)
|
||||
(js/Response. nil #js {:status 204})))
|
||||
(http/not-found))))
|
||||
|
||||
(defn- handle-blocks [{:keys [^js self request conn db handler path-params]}]
|
||||
(case handler
|
||||
|
||||
:semantic/blocks-get
|
||||
(if-let [block (find-visible-entity db (:block-id path-params))]
|
||||
(http/json-response nil (block-response block))
|
||||
(http/not-found))
|
||||
|
||||
:semantic/blocks-update
|
||||
(p/let [body (body-clj request)
|
||||
block (find-entity db (:block-id path-params))]
|
||||
(if (or (nil? block) (not (seq (:title body))))
|
||||
(http/bad-request "invalid block-id or title")
|
||||
(do
|
||||
(outliner-core/save-block! conn
|
||||
(assoc (prepare-block-title! conn (:title body))
|
||||
:block/uuid (:block/uuid block)))
|
||||
(broadcast-change! self)
|
||||
(http/json-response nil {:uuid (str (:block/uuid block)) :kind (block-kind block) :title (:title body)}))))
|
||||
|
||||
:semantic/blocks-delete
|
||||
(if-let [block (find-entity db (:block-id path-params))]
|
||||
(do
|
||||
(if (page? block)
|
||||
(outliner-page/delete! conn (:block/uuid block))
|
||||
(outliner-core/delete-blocks! conn [block] {}))
|
||||
(broadcast-change! self)
|
||||
(js/Response. nil #js {:status 204}))
|
||||
(http/not-found))
|
||||
|
||||
:semantic/blocks-move
|
||||
(p/let [body (body-clj request)
|
||||
block-ids (:block-ids body)
|
||||
blocks (mapv #(find-entity db %) block-ids)
|
||||
target (find-entity db (:target-id body))]
|
||||
(if (or (not (seq block-ids))
|
||||
(some nil? blocks) (nil? target)
|
||||
(not= (count blocks) (count (distinct (map :db/id blocks))))
|
||||
(some #(= (:db/id target) (:db/id %)) blocks)
|
||||
(not (contains? #{"before" "after" "first-child" "last-child"} (:position body))))
|
||||
(http/bad-request "invalid blocks, target, or position")
|
||||
(let [position (:position body)
|
||||
opts (case position
|
||||
"before" {:sibling? true :top? true}
|
||||
"after" {:sibling? true :bottom? true}
|
||||
"first-child" {:sibling? false :top? true}
|
||||
"last-child" {:sibling? false :bottom? true})]
|
||||
(outliner-core/move-blocks! conn blocks target opts)
|
||||
(broadcast-change! self)
|
||||
(http/json-response nil {:uuids (mapv #(str (:block/uuid %)) blocks) :moved true}))))
|
||||
|
||||
:semantic/blocks-insert-children
|
||||
(p/let [body (body-clj request)
|
||||
target (find-entity db (:block-id path-params))]
|
||||
(if (or (nil? target) (not (contains? #{"append" "prepend"} (:position body)))
|
||||
(not (seq (:blocks body))))
|
||||
(http/bad-request "invalid target, position, or blocks")
|
||||
(let [inserted (insert-tree! conn target (:blocks body) (:position body))]
|
||||
(broadcast-change! self)
|
||||
(http/json-response nil {:blocks inserted} 201))))
|
||||
|
||||
:semantic/blocks-insert-tree
|
||||
(p/let [body (body-clj request)
|
||||
target (find-entity db (:target-id body))]
|
||||
(if (or (nil? target) (not (seq (:blocks body))))
|
||||
(http/bad-request "invalid target or blocks")
|
||||
(let [inserted (insert-tree! conn target (:blocks body) (or (:position body) "append"))]
|
||||
(broadcast-change! self)
|
||||
(http/json-response nil {:blocks inserted} 201))))))
|
||||
|
||||
(defn- handle-block-properties [{:keys [^js self request conn db handler path-params]}]
|
||||
(case handler
|
||||
|
||||
:semantic/blocks-set-property
|
||||
(p/let [body (body-clj request)
|
||||
block (find-entity db (:block-id path-params))
|
||||
property-ident (resolve-property-ident db (:property-id path-params))
|
||||
prepared (when property-ident (prepare-property-value db property-ident (:value body)))]
|
||||
(cond
|
||||
(nil? block) (http/bad-request "block not found")
|
||||
(nil? property-ident) (http/bad-request "property not found")
|
||||
(not (contains? body :value)) (http/bad-request "missing property value")
|
||||
(:error prepared) (http/bad-request (:error prepared))
|
||||
:else
|
||||
(do (set-prepared-property! conn (:db/id block) property-ident (:value prepared) true)
|
||||
(broadcast-change! self)
|
||||
(http/json-response nil {:updated true}))))
|
||||
|
||||
:semantic/blocks-delete-property
|
||||
(let [block (find-entity db (:block-id path-params))
|
||||
property-ident (resolve-property-ident db (:property-id path-params))]
|
||||
(if (or (nil? block) (nil? property-ident))
|
||||
(http/bad-request "invalid block or property")
|
||||
(do
|
||||
(outliner-property/remove-block-property! conn (:db/id block) property-ident)
|
||||
(broadcast-change! self)
|
||||
(js/Response. nil #js {:status 204}))))
|
||||
|
||||
:semantic/blocks-batch-set-property
|
||||
(p/let [body (body-clj request)
|
||||
entries (:entries body)
|
||||
resolved (mapv (fn [{:keys [block-id property-id value]}]
|
||||
(let [property-ident (resolve-property-ident db property-id)]
|
||||
{:block (find-entity db block-id)
|
||||
:property-ident property-ident
|
||||
:prepared (when property-ident
|
||||
(prepare-property-value db property-ident value))})) entries)]
|
||||
(if (or (not (seq entries))
|
||||
(some #(or (nil? (:block %)) (nil? (:property-ident %))
|
||||
(:error (:prepared %))) resolved))
|
||||
(http/bad-request "invalid batch property entry")
|
||||
(do (ldb/batch-transact-with-temp-conn!
|
||||
conn {:outliner-op :batch-set-property}
|
||||
(fn [temp-conn]
|
||||
(doseq [{:keys [block property-ident prepared]} resolved]
|
||||
(set-prepared-property! temp-conn (:db/id block) property-ident (:value prepared)
|
||||
(true? (:isResetExistingValues body))))))
|
||||
(broadcast-change! self)
|
||||
(http/json-response nil {:updated (count resolved)}))))
|
||||
|
||||
:semantic/blocks-batch-delete-property
|
||||
(p/let [body (body-clj request)
|
||||
entries (:entries body)
|
||||
resolved (mapv (fn [{:keys [block-id property-id]}]
|
||||
{:block (find-entity db block-id)
|
||||
:property-ident (resolve-property-ident db property-id)}) entries)]
|
||||
(if (or (not (seq entries)) (some #(or (nil? (:block %)) (nil? (:property-ident %))) resolved))
|
||||
(http/bad-request "invalid batch property entry")
|
||||
(do
|
||||
(ldb/batch-transact-with-temp-conn!
|
||||
conn {:outliner-op :batch-remove-property}
|
||||
(fn [temp-conn]
|
||||
(doseq [{:keys [block property-ident]} resolved]
|
||||
(outliner-property/remove-block-property! temp-conn (:db/id block) property-ident))))
|
||||
(broadcast-change! self)
|
||||
(http/json-response nil {:deleted (count resolved)}))))))
|
||||
|
||||
(defn- handle-capture-and-tags [{:keys [^js self request ^js url conn db handler path-params]}]
|
||||
(case handler
|
||||
|
||||
:semantic/capture
|
||||
(p/let [body (body-clj request)]
|
||||
(if-not (seq (:blocks body))
|
||||
(http/bad-request "missing blocks")
|
||||
(let [today (ensure-today-page! conn)
|
||||
inserted (insert-tree! conn today (:blocks body) "append")]
|
||||
(broadcast-change! self)
|
||||
(http/json-response nil {:page-id (str (:block/uuid today)) :blocks inserted} 201))))
|
||||
|
||||
:semantic/tags-list
|
||||
(paginated-response url :tags (entities-by-avet db :block/tags :logseq.class/Tag) tag-response)
|
||||
|
||||
:semantic/tags-create
|
||||
(p/let [body (body-clj request)]
|
||||
(if-not (seq (:title body))
|
||||
(http/bad-request "missing title")
|
||||
(let [[title tag-id] (outliner-page/create! conn (:title body) {:class? true})]
|
||||
(broadcast-change! self)
|
||||
(http/json-response nil {:uuid (str tag-id) :title title} 201))))
|
||||
|
||||
:semantic/tags-get
|
||||
(if-let [tag (find-visible-entity db (:tag-id path-params))]
|
||||
(if (tag? tag) (http/json-response nil (tag-response tag)) (http/not-found))
|
||||
(http/not-found))
|
||||
|
||||
:semantic/tags-objects
|
||||
(if-let [tag (find-visible-entity db (:tag-id path-params))]
|
||||
(if (tag? tag)
|
||||
(indexed-reference-response db url :objects :block/tags (:db/id tag))
|
||||
(http/not-found))
|
||||
(http/not-found))
|
||||
|
||||
:semantic/tags-update
|
||||
(p/let [body (body-clj request)
|
||||
tag (find-entity db (:tag-id path-params))]
|
||||
(if (or (not (tag? tag)) (not (seq (:title body))))
|
||||
(http/bad-request "invalid tag-id or title")
|
||||
(do
|
||||
(outliner-core/save-block! conn {:block/uuid (:block/uuid tag) :block/title (:title body)})
|
||||
(broadcast-change! self)
|
||||
(http/json-response nil (assoc (tag-response tag) :title (:title body))))))
|
||||
|
||||
:semantic/tags-delete
|
||||
(if-let [tag (find-entity db (:tag-id path-params))]
|
||||
(if-not (tag? tag)
|
||||
(http/not-found)
|
||||
(do
|
||||
(outliner-page/delete! conn (:block/uuid tag))
|
||||
(broadcast-change! self)
|
||||
(js/Response. nil #js {:status 204})))
|
||||
(http/not-found))))
|
||||
|
||||
(defn- handle-tasks [{:keys [^js self request ^js url conn db handler]}]
|
||||
(case handler
|
||||
:semantic/tasks-list
|
||||
(let [status (.get (.-searchParams url) "status")
|
||||
priority (.get (.-searchParams url) "priority")
|
||||
content (some-> (.get (.-searchParams url) "content") string/lower-case)
|
||||
status-choice (when status (resolve-property-choice db :logseq.property/status
|
||||
task-status-idents status))
|
||||
priority-choice (when priority (resolve-property-choice db :logseq.property/priority
|
||||
task-priority-idents priority))]
|
||||
(cond
|
||||
(and status (nil? status-choice)) (http/bad-request "invalid task status")
|
||||
(and priority (nil? priority-choice)) (http/bad-request "invalid task priority")
|
||||
:else
|
||||
(indexed-reference-response
|
||||
db url :tasks :block/tags :logseq.class/Task
|
||||
{:entity-filter
|
||||
(fn [entity]
|
||||
(and (or (nil? status-choice)
|
||||
(= (:db/id status-choice) (:db/id (:logseq.property/status entity))))
|
||||
(or (nil? priority-choice)
|
||||
(= (:db/id priority-choice) (:db/id (:logseq.property/priority entity))))
|
||||
(or (nil? content)
|
||||
(string/includes? (string/lower-case (or (:block/title entity) "")) content))))
|
||||
:response-fn task-response})))
|
||||
|
||||
:semantic/tasks-create
|
||||
(p/let [body (body-clj request)]
|
||||
(let [status (or (:status body) "todo")
|
||||
priority (:priority body)
|
||||
status-choice (resolve-property-choice db :logseq.property/status task-status-idents status)
|
||||
priority-choice (when priority
|
||||
(resolve-property-choice db :logseq.property/priority
|
||||
task-priority-idents priority))
|
||||
target (when-let [page-id (:page-id body)]
|
||||
(find-entity db page-id))]
|
||||
(cond
|
||||
(not (seq (:title body))) (http/bad-request "missing task title")
|
||||
(nil? status-choice) (http/bad-request "invalid task status")
|
||||
(and priority (nil? priority-choice)) (http/bad-request "invalid task priority")
|
||||
(and (:page-id body) (not (page? target))) (http/bad-request "invalid task page-id")
|
||||
:else
|
||||
(let [task-id (atom nil)]
|
||||
(ldb/batch-transact-with-temp-conn!
|
||||
conn {:outliner-op :create-task}
|
||||
(fn [temp-conn]
|
||||
(let [target (if target
|
||||
(d/entity @temp-conn [:block/uuid (:block/uuid target)])
|
||||
(ensure-today-page! temp-conn))
|
||||
inserted (first (insert-tree! temp-conn target [{:title (:title body)}] "append"))
|
||||
task-uuid (uuid (:uuid inserted))
|
||||
task (d/entity @temp-conn [:block/uuid task-uuid])]
|
||||
(reset! task-id task-uuid)
|
||||
(outliner-property/set-block-property! temp-conn (:db/id task) :block/tags
|
||||
(:db/id (d/entity @temp-conn :logseq.class/Task)))
|
||||
(outliner-property/set-block-property! temp-conn (:db/id task) :logseq.property/status
|
||||
(:db/id status-choice))
|
||||
(when priority-choice
|
||||
(outliner-property/set-block-property! temp-conn (:db/id task) :logseq.property/priority
|
||||
(:db/id priority-choice)))
|
||||
(doseq [property-ident [:logseq.property/scheduled :logseq.property/deadline]
|
||||
:let [value (get body (keyword (name property-ident)))]
|
||||
:when (some? value)]
|
||||
(outliner-property/set-block-property! temp-conn (:db/id task) property-ident value)))))
|
||||
(broadcast-change! self)
|
||||
(http/json-response nil (task-response (d/entity @conn [:block/uuid @task-id])) 201)))))))
|
||||
|
||||
(defn- handle-assets
|
||||
[{:keys [^js self request ^js url conn db handler path-params]}]
|
||||
(case handler
|
||||
:semantic/assets-list
|
||||
(paginated-response url :assets
|
||||
(entities-by-avet db :logseq.property.asset/type)
|
||||
asset-response)
|
||||
|
||||
:semantic/assets-create
|
||||
(let [file-name (.get (.-searchParams url) "file-name")
|
||||
raw-size (.get (.-searchParams url) "size")
|
||||
size (when (and raw-size (re-matches #"\d+" raw-size))
|
||||
(js/parseInt raw-size 10))
|
||||
checksum (some-> (.get (.-searchParams url) "checksum") string/lower-case)
|
||||
encoding (.get (.-searchParams url) "encoding")
|
||||
asset-type (when (string? file-name) (db-asset/asset-path->type file-name))
|
||||
page-id (.get (.-searchParams url) "page-id")
|
||||
target (when page-id (find-entity db page-id))
|
||||
^js bucket (some-> (.-env self) (aget "LOGSEQ_SYNC_ASSETS"))]
|
||||
(cond
|
||||
(or (not (seq file-name)) (not (seq asset-type))) (http/bad-request "invalid asset file-name")
|
||||
(nil? size) (http/bad-request "invalid asset size")
|
||||
(> size assets-handler/max-asset-size) (http/error-response "asset too large" 413)
|
||||
(not (and (string? checksum) (re-matches #"[0-9a-f]{64}" checksum)))
|
||||
(http/bad-request "invalid asset checksum")
|
||||
(and encoding (not= "base64" encoding)) (http/bad-request "invalid asset encoding")
|
||||
(and page-id (not (page? target))) (http/bad-request "invalid asset page-id")
|
||||
(nil? bucket) (http/error-response "missing assets bucket" 500)
|
||||
(seq (d/datoms db :avet :logseq.property.asset/checksum checksum))
|
||||
(http/error-response "asset checksum already exists" 409)
|
||||
:else
|
||||
(let [asset-id (random-uuid)
|
||||
key (str (.get (.-searchParams url) "graph-id") "/" asset-id "." asset-type)
|
||||
title (or (.get (.-searchParams url) "title")
|
||||
(db-asset/asset-name->title file-name))]
|
||||
(-> (p/let [upload-response (assets-handler/<put-stream!
|
||||
bucket key (.-body request)
|
||||
{:size size
|
||||
:content-type (.get (.-headers request) "content-type")
|
||||
:checksum checksum
|
||||
:asset-type asset-type
|
||||
:encoding encoding})]
|
||||
(if-not (.-ok upload-response)
|
||||
upload-response
|
||||
(-> (p/let [target (or target (ensure-today-page! conn))
|
||||
block {:block/uuid asset-id
|
||||
:block/title title
|
||||
:block/tags #{:logseq.class/Asset}
|
||||
:logseq.property.asset/type asset-type
|
||||
:logseq.property.asset/size size
|
||||
:logseq.property.asset/checksum checksum
|
||||
:logseq.property.asset/remote-metadata {:checksum checksum
|
||||
:type asset-type}}
|
||||
_ (outliner-core/insert-blocks! conn [block] target
|
||||
{:sibling? false :bottom? true
|
||||
:keep-uuid? true})
|
||||
asset (d/entity @conn [:block/uuid asset-id])]
|
||||
(broadcast-change! self)
|
||||
(http/json-response nil (asset-response asset) 201))
|
||||
(p/catch (fn [error]
|
||||
(p/let [_ (.delete bucket key)]
|
||||
(throw error)))))))
|
||||
(p/catch (fn [error]
|
||||
(http/error-response (or (.-message error) "asset upload failed") 500)))))))
|
||||
|
||||
:semantic/assets-get
|
||||
(if-let [asset (find-visible-entity db (:asset-block-id path-params))]
|
||||
(if-not (asset? asset)
|
||||
(http/bad-request "block is not an asset")
|
||||
(p/let [link (asset-link/<temporary-url request (.-env self)
|
||||
(.get (.-searchParams url) "graph-id")
|
||||
(:block/uuid asset)
|
||||
(:logseq.property.asset/type asset))]
|
||||
(http/json-response nil (merge (asset-response asset) link))))
|
||||
(http/not-found))))
|
||||
|
||||
(defn- handle-properties-and-search
|
||||
[{:keys [^js self request ^js url conn db handler path-params]}]
|
||||
(case handler
|
||||
|
||||
:semantic/properties-list
|
||||
(paginated-response url :properties (->> (db-db/get-all-properties db)
|
||||
(remove entity-util/hidden?))
|
||||
property-response)
|
||||
|
||||
:semantic/properties-create
|
||||
(p/let [body (body-clj request)]
|
||||
(if (or (not (seq (:title body))) (not (valid-property-schema? body)))
|
||||
(http/bad-request "invalid property schema")
|
||||
(let [property (outliner-property/upsert-property!
|
||||
conn nil
|
||||
{:logseq.property/type (keyword (or (:type body) "default"))
|
||||
:db/cardinality (keyword (or (:cardinality body) "db.cardinality/one"))}
|
||||
{:property-name (:title body)})]
|
||||
(broadcast-change! self)
|
||||
(http/json-response nil (property-response property) 201))))
|
||||
|
||||
:semantic/properties-get
|
||||
(if-let [property (resolve-property db (:property-id path-params))]
|
||||
(if (property? property) (http/json-response nil (property-response property)) (http/not-found))
|
||||
(http/not-found))
|
||||
|
||||
:semantic/properties-update
|
||||
(p/let [body (body-clj request)
|
||||
property (resolve-property db (:property-id path-params))]
|
||||
(if (or (not (property? property)) (empty? body) (not (valid-property-schema? body)))
|
||||
(http/bad-request "invalid property-id or update")
|
||||
(let [schema (cond-> {}
|
||||
(:type body) (assoc :logseq.property/type (keyword (:type body)))
|
||||
(:cardinality body) (assoc :db/cardinality (keyword (:cardinality body))))
|
||||
updated (outliner-property/upsert-property!
|
||||
conn (:db/ident property) schema
|
||||
(cond-> {} (:title body) (assoc :property-name (:title body))))]
|
||||
(broadcast-change! self)
|
||||
(http/json-response nil (property-response updated)))))
|
||||
|
||||
:semantic/properties-delete
|
||||
(if-let [property (resolve-property db (:property-id path-params))]
|
||||
(if-not (property? property)
|
||||
(http/not-found)
|
||||
(if (outliner-page/delete! conn (:block/uuid property))
|
||||
(do
|
||||
(broadcast-change! self)
|
||||
(js/Response. nil #js {:status 204}))
|
||||
(http/bad-request "property cannot be deleted")))
|
||||
(http/not-found))
|
||||
|
||||
:semantic/search
|
||||
(let [query (.get (.-searchParams url) "q")]
|
||||
(if-not (seq query)
|
||||
(http/bad-request "missing q")
|
||||
(paginated-response url :results
|
||||
(search-results db query (.get (.-searchParams url) "types"))
|
||||
search-result-response
|
||||
paginate-bounded)))))
|
||||
|
||||
(def ^:private page-handlers
|
||||
#{:semantic/pages-list :semantic/pages-create :semantic/pages-blocks :semantic/pages-references :semantic/pages-get
|
||||
:semantic/pages-update :semantic/pages-delete})
|
||||
|
||||
(def ^:private block-handlers
|
||||
#{:semantic/blocks-get :semantic/blocks-update :semantic/blocks-delete :semantic/blocks-move
|
||||
:semantic/blocks-insert-children :semantic/blocks-insert-tree})
|
||||
|
||||
(def ^:private block-property-handlers
|
||||
#{:semantic/blocks-set-property :semantic/blocks-delete-property
|
||||
:semantic/blocks-batch-set-property :semantic/blocks-batch-delete-property})
|
||||
|
||||
(def ^:private capture-and-tag-handlers
|
||||
#{:semantic/capture :semantic/tags-list :semantic/tags-create :semantic/tags-get :semantic/tags-objects
|
||||
:semantic/tags-update :semantic/tags-delete})
|
||||
|
||||
(def ^:private task-handlers
|
||||
#{:semantic/tasks-list :semantic/tasks-create})
|
||||
|
||||
(def ^:private asset-handlers
|
||||
#{:semantic/assets-list :semantic/assets-create :semantic/assets-get})
|
||||
|
||||
(def ^:private property-and-search-handlers
|
||||
#{:semantic/properties-list :semantic/properties-create :semantic/properties-get
|
||||
:semantic/properties-update :semantic/properties-delete
|
||||
:semantic/search})
|
||||
|
||||
(defn handle [{:keys [^js self route] :as context}]
|
||||
(let [conn (ensure-conn! self)
|
||||
handler (:handler route)
|
||||
context (assoc context :conn conn :db @conn :handler handler :path-params (:path-params route))]
|
||||
(cond
|
||||
(contains? page-handlers handler) (handle-pages context)
|
||||
(contains? block-handlers handler) (handle-blocks context)
|
||||
(contains? block-property-handlers handler) (handle-block-properties context)
|
||||
(contains? capture-and-tag-handlers handler) (handle-capture-and-tags context)
|
||||
(contains? task-handlers handler) (handle-tasks context)
|
||||
(contains? asset-handlers handler) (handle-assets context)
|
||||
(contains? property-and-search-handlers handler) (handle-properties-and-search context)
|
||||
:else (http/not-found))))
|
||||
928
deps/db-sync/src/logseq/db_sync/worker/handler/sync.cljs
vendored
Normal file
928
deps/db-sync/src/logseq/db_sync/worker/handler/sync.cljs
vendored
Normal file
@@ -0,0 +1,928 @@
|
||||
(ns logseq.db-sync.worker.handler.sync
|
||||
(:require [clojure.string :as string]
|
||||
[datascript.core :as d]
|
||||
[lambdaisland.glogi :as log]
|
||||
[logseq.db :as ldb]
|
||||
[logseq.db-sync.batch :as batch]
|
||||
[logseq.db-sync.checksum :as sync-checksum]
|
||||
[logseq.db-sync.common :as common]
|
||||
[logseq.db-sync.index :as index]
|
||||
[logseq.db-sync.protocol :as protocol]
|
||||
[logseq.db-sync.snapshot :as snapshot]
|
||||
[logseq.db-sync.storage :as storage]
|
||||
[logseq.db-sync.tx-sanitize :as tx-sanitize]
|
||||
[logseq.db-sync.worker.http :as http]
|
||||
[logseq.db-sync.worker.handler.semantic :as semantic-handler]
|
||||
[logseq.db-sync.worker.routes.semantic :as semantic-routes]
|
||||
[logseq.db-sync.worker.routes.sync :as sync-routes]
|
||||
[logseq.db-sync.worker.ws :as ws]
|
||||
[promesa.core :as p]))
|
||||
|
||||
(def ^:private snapshot-download-batch-size 10000)
|
||||
;; (def ^:private snapshot-cache-control "private, max-age=300")
|
||||
(def ^:private snapshot-content-type "application/transit+json")
|
||||
(def ^:private snapshot-content-encoding "gzip")
|
||||
(def ^:private snapshot-uploading-meta-key :snapshot-uploading?)
|
||||
(def ^:private large-tx-min-items 500)
|
||||
(def ^:private large-tx-max-chunk-items 500)
|
||||
;; 10m
|
||||
;; (def ^:private snapshot-multipart-part-size (* 10 1024 1024))
|
||||
|
||||
(defn parse-int [value]
|
||||
(when (some? value)
|
||||
(let [n (js/parseInt value 10)]
|
||||
(when-not (js/isNaN n)
|
||||
n))))
|
||||
|
||||
(defn- ensure-schema! [^js self]
|
||||
(when-not (true? (.-schema-ready self))
|
||||
(try
|
||||
(storage/init-schema! (.-sql self))
|
||||
(catch :default e
|
||||
;; Schema may already exist. If DDL writes are rejected, probe
|
||||
;; existing tables before deciding this is a fatal error.
|
||||
(try
|
||||
(common/sql-exec (.-sql self) "select 1 from kvs limit 1")
|
||||
(common/sql-exec (.-sql self) "select 1 from tx_log limit 1")
|
||||
(common/sql-exec (.-sql self) "select 1 from sync_meta limit 1")
|
||||
(catch :default _
|
||||
(throw e)))))
|
||||
(set! (.-schema-ready self) true)))
|
||||
|
||||
(defn- ensure-conn! [^js self]
|
||||
(ensure-schema! self)
|
||||
(when-not (.-conn self)
|
||||
(set! (.-conn self)
|
||||
(storage/open-conn (.-sql self)))))
|
||||
|
||||
(defn t-now [^js self]
|
||||
(ensure-schema! self)
|
||||
(storage/get-t (.-sql self)))
|
||||
|
||||
(defn current-checksum [^js self]
|
||||
(ensure-conn! self)
|
||||
(storage/get-checksum (.-sql self)))
|
||||
|
||||
(defn snapshot-upload-finished? [^js self]
|
||||
(ensure-schema! self)
|
||||
(not= "true" (storage/get-meta (.-sql self) snapshot-uploading-meta-key)))
|
||||
|
||||
(defn <ready-for-sync?
|
||||
[^js self graph-id]
|
||||
(if-not (snapshot-upload-finished? self)
|
||||
(p/resolved false)
|
||||
(if-let [db (some-> self .-env (aget "DB"))]
|
||||
(p/let [graph-ready-for-use? (index/<graph-ready-for-use? db graph-id)]
|
||||
(not= false graph-ready-for-use?))
|
||||
(p/resolved true))))
|
||||
|
||||
(defn- <set-graph-ready-for-use!
|
||||
[^js self graph-id graph-ready-for-use?]
|
||||
(if-let [db (some-> self .-env (aget "DB"))]
|
||||
(p/let [result (index/<graph-ready-for-use-set! db graph-id graph-ready-for-use?)
|
||||
meta (some-> result (aget "meta"))
|
||||
rows-affected (or (some-> meta (aget "changes"))
|
||||
(some-> meta (aget "rows_written"))
|
||||
(some-> result (aget "changes"))
|
||||
(some-> result (aget "rows_written")))]
|
||||
(when (or (nil? result)
|
||||
(false? (some-> result (aget "success"))))
|
||||
(throw (ex-info "failed to persist graph_ready_for_use"
|
||||
{:type :db-sync/graph-ready-for-use-set-failed
|
||||
:graph-id graph-id
|
||||
:graph-ready-for-use? graph-ready-for-use?
|
||||
:result result})))
|
||||
(when (and (number? rows-affected)
|
||||
(<= rows-affected 0))
|
||||
(throw (ex-info "graph_ready_for_use update affected no rows"
|
||||
{:type :db-sync/graph-ready-for-use-set-no-rows
|
||||
:graph-id graph-id
|
||||
:graph-ready-for-use? graph-ready-for-use?
|
||||
:rows-affected rows-affected
|
||||
:result result})))
|
||||
result)
|
||||
(p/rejected (ex-info "missing DB binding for graph_ready_for_use update"
|
||||
{:type :db-sync/missing-db-binding
|
||||
:graph-id graph-id
|
||||
:graph-ready-for-use? graph-ready-for-use?}))))
|
||||
|
||||
(defn- import-snapshot-rows!
|
||||
[sql table rows]
|
||||
(when (seq rows)
|
||||
(doseq [batch (batch/rows->insert-batches table rows nil)]
|
||||
(let [sql-str (:sql batch)
|
||||
args (:args batch)]
|
||||
(apply common/sql-exec sql sql-str args)))))
|
||||
|
||||
(defn- reset-import!
|
||||
[sql]
|
||||
(common/sql-exec sql "delete from kvs")
|
||||
(common/sql-exec sql "delete from tx_log")
|
||||
(common/sql-exec sql "delete from sync_meta")
|
||||
(storage/set-t! sql 0))
|
||||
|
||||
(defn- graph-id-from-sync-path
|
||||
[^js url]
|
||||
(let [path (.-pathname url)
|
||||
prefix "/sync/"]
|
||||
(when (string/starts-with? path prefix)
|
||||
(let [rest-path (subs path (count prefix))
|
||||
slash-idx (or (string/index-of rest-path "/") -1)
|
||||
graph-id (if (neg? slash-idx)
|
||||
rest-path
|
||||
(subs rest-path 0 slash-idx))]
|
||||
(when (seq graph-id)
|
||||
graph-id)))))
|
||||
|
||||
(defn graph-id-from-request [request]
|
||||
(let [header-id (.get (.-headers request) "x-graph-id")
|
||||
url (js/URL. (.-url request))
|
||||
param-id (.get (.-searchParams url) "graph-id")
|
||||
graph-id (or header-id param-id (graph-id-from-sync-path url))]
|
||||
(when (seq graph-id)
|
||||
graph-id)))
|
||||
|
||||
;; (defn- snapshot-key [graph-id snapshot-id]
|
||||
;; (str graph-id "/" snapshot-id ".snapshot"))
|
||||
|
||||
;; (defn- snapshot-url [request graph-id snapshot-id]
|
||||
;; (let [url (js/URL. (.-url request))]
|
||||
;; (str (.-origin url) "/assets/" graph-id "/" snapshot-id ".snapshot")))
|
||||
|
||||
(defn- snapshot-stream-url [request graph-id]
|
||||
(let [url (js/URL. (.-url request))]
|
||||
(str (.-origin url) "/sync/" graph-id "/snapshot/stream")))
|
||||
|
||||
(defn- maybe-decompress-stream [stream encoding]
|
||||
(if (and (= encoding snapshot-content-encoding) (exists? js/DecompressionStream))
|
||||
(.pipeThrough stream (js/DecompressionStream. "gzip"))
|
||||
stream))
|
||||
|
||||
(defn- maybe-compress-stream [stream]
|
||||
(.pipeThrough stream (js/CompressionStream. snapshot-content-encoding)))
|
||||
|
||||
(defn- snapshot-stream-gzip-enabled?
|
||||
[^js self]
|
||||
(let [v (some-> self .-env (aget "DB_SYNC_SNAPSHOT_STREAM_GZIP"))]
|
||||
(cond
|
||||
(nil? v) true
|
||||
(false? v) false
|
||||
(string? v) (not (contains? #{"false" "0" "off" "no"}
|
||||
(string/lower-case v)))
|
||||
:else (boolean v))))
|
||||
|
||||
;; (defn- <buffer-stream
|
||||
;; [stream]
|
||||
;; (p/let [resp (js/Response. stream)
|
||||
;; buf (.arrayBuffer resp)]
|
||||
;; buf))
|
||||
|
||||
;; (defn- ->uint8 [data]
|
||||
;; (cond
|
||||
;; (instance? js/Uint8Array data) data
|
||||
;; (instance? js/ArrayBuffer data) (js/Uint8Array. data)
|
||||
;; :else (js/Uint8Array. data)))
|
||||
|
||||
;; (defn- concat-uint8 [^js a ^js b]
|
||||
;; (cond
|
||||
;; (nil? a) b
|
||||
;; (nil? b) a
|
||||
;; :else
|
||||
;; (let [out (js/Uint8Array. (+ (.-byteLength a) (.-byteLength b)))]
|
||||
;; (.set out a 0)
|
||||
;; (.set out b (.-byteLength a))
|
||||
;; out)))
|
||||
|
||||
(defn- frame-bytes
|
||||
[^js data]
|
||||
(let [len (.-byteLength data)
|
||||
out (js/Uint8Array. (+ 4 len))
|
||||
view (js/DataView. (.-buffer out))]
|
||||
(.setUint32 view 0 len false)
|
||||
(.set out data 4)
|
||||
out))
|
||||
|
||||
(defn- fetch-snapshot-kvs-rows
|
||||
[sql last-addr limit]
|
||||
(let [rows (common/get-sql-rows
|
||||
(common/sql-exec sql
|
||||
"select addr, content, addresses from kvs where addr > ? order by addr asc limit ?"
|
||||
last-addr
|
||||
limit))]
|
||||
(mapv (fn [row]
|
||||
[(aget row "addr")
|
||||
(aget row "content")
|
||||
(aget row "addresses")])
|
||||
rows)))
|
||||
|
||||
(defn- snapshot-row-count
|
||||
[sql]
|
||||
(if-let [row (first (common/get-sql-rows
|
||||
(common/sql-exec sql "select count(*) as row_count from kvs")))]
|
||||
(or (aget row "row_count") 0)
|
||||
0))
|
||||
|
||||
(defn- snapshot-export-stream [^js self]
|
||||
(ensure-schema! self)
|
||||
(let [sql (.-sql self)
|
||||
last-addr (volatile! -1)]
|
||||
(js/ReadableStream.
|
||||
(clj->js
|
||||
{:pull (fn [controller]
|
||||
(let [batch (fetch-snapshot-kvs-rows sql @last-addr snapshot-download-batch-size)]
|
||||
(if (empty? batch)
|
||||
(.close controller)
|
||||
(let [payload (snapshot/encode-rows batch)]
|
||||
(vreset! last-addr (first (peek batch)))
|
||||
(.enqueue controller (frame-bytes payload))))))}))))
|
||||
|
||||
;; (defn- upload-multipart!
|
||||
;; [^js bucket key stream opts]
|
||||
;; (p/let [^js upload (.createMultipartUpload bucket key opts)]
|
||||
;; (let [reader (.getReader stream)]
|
||||
;; (-> (p/loop [buffer nil
|
||||
;; part-number 1
|
||||
;; parts []]
|
||||
;; (p/let [chunk (.read reader)]
|
||||
;; (if (.-done chunk)
|
||||
;; (cond
|
||||
;; (and buffer (pos? (.-byteLength buffer)))
|
||||
;; (p/let [^js resp (.uploadPart upload part-number buffer)
|
||||
;; parts (conj parts {:partNumber part-number :etag (.-etag resp)})]
|
||||
;; (p/let [_ (.complete upload (clj->js parts))]
|
||||
;; {:ok true}))
|
||||
|
||||
;; (seq parts)
|
||||
;; (p/let [_ (.complete upload (clj->js parts))]
|
||||
;; {:ok true})
|
||||
|
||||
;; :else
|
||||
;; (p/let [_ (.abort upload)]
|
||||
;; (.put bucket key (js/Uint8Array. 0) opts)))
|
||||
;; (let [value (.-value chunk)
|
||||
;; buffer (concat-uint8 buffer (->uint8 value))]
|
||||
;; (if (>= (.-byteLength buffer) snapshot-multipart-part-size)
|
||||
;; (let [part (.slice buffer 0 snapshot-multipart-part-size)
|
||||
;; rest-parts (.slice buffer snapshot-multipart-part-size (.-byteLength buffer))]
|
||||
;; (p/let [^js resp (.uploadPart upload part-number part)
|
||||
;; parts (conj parts {:partNumber part-number :etag (.-etag resp)})]
|
||||
;; (p/recur rest-parts (inc part-number) parts)))
|
||||
;; (p/recur buffer part-number parts))))))
|
||||
;; (p/catch (fn [error]
|
||||
;; (.abort upload)
|
||||
;; (throw error)))))))
|
||||
|
||||
(declare import-snapshot!)
|
||||
(defn- import-snapshot-stream! [^js self stream reset?]
|
||||
(let [reader (.getReader stream)
|
||||
reset-pending? (volatile! reset?)
|
||||
total-count (volatile! 0)]
|
||||
(p/let [buffer nil]
|
||||
(p/catch
|
||||
(p/loop [buffer buffer]
|
||||
(p/let [chunk (.read reader)]
|
||||
(if (.-done chunk)
|
||||
(let [rows (snapshot/finalize-framed-buffer buffer)
|
||||
rows-count (count rows)
|
||||
reset? (and @reset-pending? true)]
|
||||
(when (or reset? (seq rows))
|
||||
(import-snapshot! self rows reset?)
|
||||
(vreset! reset-pending? false))
|
||||
(vswap! total-count + rows-count)
|
||||
@total-count)
|
||||
(let [value (.-value chunk)
|
||||
{:keys [rows buffer]} (snapshot/parse-framed-chunk buffer value)
|
||||
rows-count (count rows)
|
||||
reset? (boolean (and @reset-pending? (seq rows)))]
|
||||
(when (seq rows)
|
||||
(import-snapshot! self rows reset?)
|
||||
(vreset! reset-pending? false))
|
||||
(vswap! total-count + rows-count)
|
||||
(p/recur buffer)))))
|
||||
(fn [error]
|
||||
(throw error))))))
|
||||
|
||||
(defn pull-response [^js self since]
|
||||
(let [sql (.-sql self)
|
||||
txs (storage/fetch-tx-since sql since)
|
||||
checksum (current-checksum self)]
|
||||
(cond-> {:type "pull/ok"
|
||||
:t (t-now self)
|
||||
:txs txs}
|
||||
(string? checksum) (assoc :checksum checksum))))
|
||||
|
||||
(defn- block-uuid-lookup-ref
|
||||
[entity-id]
|
||||
(when (and (sequential? entity-id)
|
||||
(= :block/uuid (first entity-id))
|
||||
(uuid? (second entity-id)))
|
||||
(second entity-id)))
|
||||
|
||||
(defn- missing-block-uuids-from-error
|
||||
[error]
|
||||
(loop [e error
|
||||
result []]
|
||||
(if e
|
||||
(let [missing-uuid (block-uuid-lookup-ref (:entity-id (ex-data e)))]
|
||||
(recur (ex-cause e)
|
||||
(cond-> result
|
||||
missing-uuid (conj missing-uuid))))
|
||||
(-> result distinct vec))))
|
||||
|
||||
(def ^:private delete-outliner-ops
|
||||
#{:delete-blocks
|
||||
:delete-page})
|
||||
|
||||
(defn- request-context->tx-meta
|
||||
[{:keys [graph-id client-revision username]}]
|
||||
(cond-> {}
|
||||
graph-id (assoc :graph-id graph-id)
|
||||
client-revision (assoc :client-revision client-revision)
|
||||
username (assoc :username username)))
|
||||
|
||||
(defn- tempid?
|
||||
[value]
|
||||
(or (and (integer? value) (neg? value))
|
||||
(string? value)))
|
||||
|
||||
(defn- ref-attr?
|
||||
[db attr]
|
||||
(= :db.type/ref (:db/valueType (d/entity db attr))))
|
||||
|
||||
(defn- tx-item-tempids
|
||||
[db item]
|
||||
(cond
|
||||
(and (map? item) (tempid? (:db/id item)))
|
||||
#{(:db/id item)}
|
||||
|
||||
(and (vector? item)
|
||||
(contains? #{:db/add :db/retract :db/cas :db.fn/cas} (first item))
|
||||
(<= 4 (count item)))
|
||||
(let [[_op entity attr value] item]
|
||||
(cond-> #{}
|
||||
(tempid? entity)
|
||||
(conj entity)
|
||||
(and (ref-attr? db attr)
|
||||
(tempid? value))
|
||||
(conj value)))
|
||||
|
||||
(and (vector? item)
|
||||
(contains? #{:db/retractEntity :db.fn/retractEntity} (first item))
|
||||
(= 2 (count item))
|
||||
(tempid? (second item)))
|
||||
#{(second item)}
|
||||
|
||||
:else
|
||||
#{}))
|
||||
|
||||
(defn- merge-ranges
|
||||
[ranges]
|
||||
(loop [remaining (sort-by first ranges)
|
||||
merged []]
|
||||
(if-let [[start end] (first remaining)]
|
||||
(if-let [[prev-start prev-end] (peek merged)]
|
||||
(if (<= start prev-end)
|
||||
(recur (next remaining)
|
||||
(conj (pop merged) [prev-start (max prev-end end)]))
|
||||
(recur (next remaining)
|
||||
(conj merged [start end])))
|
||||
(recur (next remaining) [[start end]]))
|
||||
merged)))
|
||||
|
||||
(defn- tempid-ranges
|
||||
[db tx-data]
|
||||
(let [ranges-by-tempid
|
||||
(reduce-kv
|
||||
(fn [acc idx item]
|
||||
(reduce (fn [acc* tempid]
|
||||
(update acc* tempid
|
||||
(fn [[start end]]
|
||||
[(if (some? start) (min start idx) idx)
|
||||
(if (some? end) (max end idx) idx)])))
|
||||
acc
|
||||
(tx-item-tempids db item)))
|
||||
{}
|
||||
tx-data)]
|
||||
(merge-ranges (vals ranges-by-tempid))))
|
||||
|
||||
(defn- tempid-range-by-start
|
||||
[db tx-data]
|
||||
(let [ranges (tempid-ranges db tx-data)
|
||||
range-by-start (into {} (map (fn [[start end]] [start end]) ranges))]
|
||||
range-by-start))
|
||||
|
||||
(defn- next-ordered-tx-group
|
||||
[tx-data range-by-start idx]
|
||||
(if-let [end (get range-by-start idx)]
|
||||
[(inc end) (subvec tx-data idx (inc end))]
|
||||
[(inc idx) [(nth tx-data idx)]]))
|
||||
|
||||
(defn- add-group-to-chunk
|
||||
[items group]
|
||||
(into items group))
|
||||
|
||||
(defn- reduce-ordered-tx-chunks
|
||||
[db f init tx-data]
|
||||
(let [tx-data (vec tx-data)
|
||||
item-count (count tx-data)
|
||||
range-by-start (tempid-range-by-start db tx-data)]
|
||||
(loop [idx 0
|
||||
chunk []
|
||||
acc init]
|
||||
(if (< idx item-count)
|
||||
(let [[next-idx group] (next-ordered-tx-group tx-data range-by-start idx)
|
||||
next-count (+ (count chunk) (count group))]
|
||||
(if (and (seq chunk)
|
||||
(> next-count large-tx-max-chunk-items))
|
||||
(recur idx [] (f acc chunk))
|
||||
(recur next-idx (add-group-to-chunk chunk group) acc)))
|
||||
(cond-> acc
|
||||
(seq chunk) (f chunk))))))
|
||||
|
||||
(defn- large-tx?
|
||||
[tx-data]
|
||||
(>= (count tx-data) large-tx-min-items))
|
||||
|
||||
(defn- import-snapshot! [^js self rows reset?]
|
||||
(let [sql (.-sql self)]
|
||||
(ensure-schema! self)
|
||||
(when reset?
|
||||
(set! (.-conn self) nil)
|
||||
(reset-import! sql))
|
||||
(import-snapshot-rows! sql "kvs" rows)))
|
||||
|
||||
(defn- apply-client-tx-meta
|
||||
[request-context outliner-op]
|
||||
(cond-> (merge {:op :apply-client-tx}
|
||||
(request-context->tx-meta request-context))
|
||||
outliner-op
|
||||
(assoc :outliner-op outliner-op)
|
||||
(= outliner-op :db-migrate)
|
||||
(assoc :db-migrate? true
|
||||
:skip-validate-db? true)))
|
||||
|
||||
(defn- apply-large-tx-entry!
|
||||
[self conn tx-data {:keys [tx-id outliner-op]} request-context]
|
||||
(let [db-before @conn
|
||||
tx-meta (apply-client-tx-meta request-context outliner-op)
|
||||
sql (when self (.-sql ^js self))
|
||||
prev-checksum (when sql (storage/get-checksum sql))
|
||||
logical-tx-data (volatile! [])
|
||||
chunk-count (volatile! 0)]
|
||||
(log/info :db-sync/apply-large-tx-entry-start
|
||||
{:graph-id (:graph-id request-context)
|
||||
:tx-id tx-id
|
||||
:outliner-op outliner-op
|
||||
:tx-count (count tx-data)
|
||||
:max-chunk-items large-tx-max-chunk-items})
|
||||
(try
|
||||
(when sql
|
||||
(d/listen! conn ::large-logical-tx-checksum
|
||||
(fn [{:keys [tx-data]}]
|
||||
(vswap! logical-tx-data into tx-data))))
|
||||
((if sql
|
||||
#(storage/with-sql-transaction! sql %)
|
||||
(fn [f] (f)))
|
||||
(fn []
|
||||
(try
|
||||
(reduce-ordered-tx-chunks
|
||||
db-before
|
||||
(fn [_ chunk]
|
||||
(vswap! chunk-count inc)
|
||||
(ldb/transact! conn
|
||||
chunk
|
||||
(cond-> tx-meta
|
||||
sql
|
||||
(assoc :db-sync/skip-checksum-update? true)))
|
||||
nil)
|
||||
nil
|
||||
tx-data)
|
||||
(when sql
|
||||
(storage/set-checksum!
|
||||
sql
|
||||
(sync-checksum/update-checksum
|
||||
prev-checksum
|
||||
{:db-before db-before
|
||||
:db-after @conn
|
||||
:tx-data @logical-tx-data})))
|
||||
(finally
|
||||
(when sql
|
||||
(d/unlisten! conn ::large-logical-tx-checksum))))))
|
||||
(log/info :db-sync/apply-large-tx-entry-done
|
||||
{:graph-id (:graph-id request-context)
|
||||
:tx-id tx-id
|
||||
:outliner-op outliner-op
|
||||
:tx-count (count tx-data)
|
||||
:chunk-count @chunk-count})
|
||||
true
|
||||
(catch :default error
|
||||
(log/info :db-sync/apply-large-tx-entry-failed
|
||||
{:graph-id (:graph-id request-context)
|
||||
:tx-id tx-id
|
||||
:outliner-op outliner-op
|
||||
:tx-count (count tx-data)
|
||||
:chunk-count @chunk-count})
|
||||
(reset! conn db-before)
|
||||
(throw error)))))
|
||||
|
||||
(defn- sanitize-tx-entry
|
||||
[db {:keys [tx outliner-op] :as tx-entry}]
|
||||
(let [input-tx-data (protocol/transit->tx tx)
|
||||
tx-data (tx-sanitize/sanitize-tx db
|
||||
input-tx-data
|
||||
{:drop-missing-retract-ops? (or (= outliner-op :fix)
|
||||
(contains? delete-outliner-ops outliner-op))
|
||||
:drop-ops-targeting-retracted-entities? (contains? delete-outliner-ops
|
||||
outliner-op)
|
||||
:retract-touched-descendants? (contains? delete-outliner-ops outliner-op)})]
|
||||
{:input-tx-data input-tx-data
|
||||
:tx-data tx-data
|
||||
:tx-entry tx-entry}))
|
||||
|
||||
(defn- apply-tx-entry!
|
||||
([conn tx-entry]
|
||||
(apply-tx-entry! nil conn tx-entry nil))
|
||||
([self conn {:keys [outliner-op] :as tx-entry} request-context]
|
||||
(let [sanitized (sanitize-tx-entry @conn tx-entry)
|
||||
input-tx-data (:input-tx-data sanitized)
|
||||
tx-data (:tx-data sanitized)
|
||||
sanitized-entry (:tx-entry sanitized)]
|
||||
(if (seq tx-data)
|
||||
(try
|
||||
(if (and (not= outliner-op :db-migrate)
|
||||
(large-tx? tx-data))
|
||||
(apply-large-tx-entry! self conn tx-data sanitized-entry request-context)
|
||||
(do
|
||||
(ldb/transact! conn tx-data (apply-client-tx-meta request-context outliner-op))
|
||||
true))
|
||||
(catch :default e
|
||||
;; Rebase/fix txs are inferred from local history and can become stale
|
||||
;; when concurrent remote edits remove referenced entities before upload.
|
||||
;; Treat stale :entity-id/missing rebases/fixes as no-op so sync can continue.
|
||||
(if (and (contains? #{:rebase :fix} outliner-op)
|
||||
(= :entity-id/missing (:error (ex-data e))))
|
||||
(do
|
||||
(log/warn :db-sync/drop-stale-rebase-tx
|
||||
{:outliner-op outliner-op
|
||||
:tx-data tx-data
|
||||
:error (str e)})
|
||||
false)
|
||||
(throw e))))
|
||||
(if (and (contains? delete-outliner-ops outliner-op)
|
||||
(empty? input-tx-data))
|
||||
(throw (ex-info "delete tx input is empty"
|
||||
{:type :db-sync/empty-delete-tx
|
||||
:outliner-op outliner-op}))
|
||||
false)))))
|
||||
|
||||
(defn- apply-tx! [^js self tx-entries request-context]
|
||||
(let [sql (.-sql self)]
|
||||
(ensure-conn! self)
|
||||
(let [conn (.-conn self)]
|
||||
(loop [remaining tx-entries
|
||||
applied? false
|
||||
successful-tx-ids []]
|
||||
(if-let [tx-entry (first remaining)]
|
||||
(let [tx-id (:tx-id tx-entry)
|
||||
applied-entry? (try
|
||||
(boolean (apply-tx-entry! self conn tx-entry request-context))
|
||||
(catch :default e
|
||||
(log/error :db-sync/transact-failed e)
|
||||
(let [missing-block-uuids (missing-block-uuids-from-error e)]
|
||||
(throw (ex-info "tx entry apply failed"
|
||||
(cond-> {:type :db-sync/tx-entry-failed
|
||||
:successful-tx-ids successful-tx-ids}
|
||||
tx-id (assoc :failed-tx-id tx-id)
|
||||
(seq missing-block-uuids)
|
||||
(assoc :missing-block-uuids missing-block-uuids))
|
||||
e)))))
|
||||
next-successful-tx-ids (cond-> successful-tx-ids
|
||||
tx-id (conj tx-id))]
|
||||
(recur (next remaining)
|
||||
(or applied? applied-entry?)
|
||||
next-successful-tx-ids))
|
||||
(let [new-t (storage/get-t sql)]
|
||||
{:t new-t
|
||||
:applied? applied?
|
||||
:successful-tx-ids successful-tx-ids}))))))
|
||||
|
||||
(defn handle-tx-batch!
|
||||
([^js self sender txs t-before]
|
||||
(handle-tx-batch! self sender txs t-before nil))
|
||||
([^js self sender txs t-before request-context]
|
||||
(let [current-t (t-now self)]
|
||||
(cond
|
||||
(not (snapshot-upload-finished? self))
|
||||
{:type "tx/reject"
|
||||
:reason "snapshot upload in progress"
|
||||
:t current-t}
|
||||
|
||||
(or (not (number? t-before)) (neg? t-before))
|
||||
{:type "tx/reject"
|
||||
:reason "invalid t-before"}
|
||||
|
||||
(not= t-before current-t)
|
||||
{:type "tx/reject"
|
||||
:reason "stale"
|
||||
:t current-t}
|
||||
|
||||
:else
|
||||
(if (seq txs)
|
||||
(try
|
||||
(let [{:keys [t applied?]} (apply-tx! self txs request-context)
|
||||
checksum (current-checksum self)]
|
||||
(when applied?
|
||||
;; Broadcast once per processed batch after tx-log/checksum settle.
|
||||
(ws/broadcast! self sender {:type "changed" :t t}))
|
||||
(cond-> {:type "tx/batch/ok"
|
||||
:t t}
|
||||
(string? checksum) (assoc :checksum checksum)))
|
||||
(catch :default e
|
||||
(let [new-t (t-now self)
|
||||
checksum (current-checksum self)
|
||||
{:keys [successful-tx-ids failed-tx-id missing-block-uuids]}
|
||||
(ex-data e)]
|
||||
(log/error :db-sync/transact-failed e)
|
||||
(when (> new-t current-t)
|
||||
;; Broadcast once when partial batch writes advanced the graph.
|
||||
(ws/broadcast! self sender {:type "changed" :t new-t}))
|
||||
(cond-> {:type "tx/reject"
|
||||
:reason "db transact failed"
|
||||
:error-detail (str e)
|
||||
:t new-t}
|
||||
(seq successful-tx-ids) (assoc :success-tx-ids successful-tx-ids)
|
||||
failed-tx-id (assoc :failed-tx-id failed-tx-id)
|
||||
(seq missing-block-uuids) (assoc :missing-block-uuids missing-block-uuids)
|
||||
(string? checksum) (assoc :checksum checksum)))))
|
||||
{:type "tx/reject"
|
||||
:reason "empty tx data"})))))
|
||||
|
||||
(defn- handle-sync-pull
|
||||
[^js self ^js url]
|
||||
(let [raw-since (.get (.-searchParams url) "since")
|
||||
since (if (some? raw-since) (parse-int raw-since) 0)
|
||||
graph-id (.get (.-searchParams url) "graph-id")]
|
||||
(if (or (and (some? raw-since) (not (number? since))) (neg? since))
|
||||
(http/bad-request "invalid since")
|
||||
(p/let [ready-for-sync? (<ready-for-sync? self graph-id)]
|
||||
(if-not ready-for-sync?
|
||||
(http/error-response "graph not ready" 409)
|
||||
(http/json-response :sync/pull (pull-response self since)))))))
|
||||
|
||||
(defn- normalize-diagnostic-block
|
||||
[{:keys [block/uuid block/parent block/page block/order] :as block}]
|
||||
(cond-> block
|
||||
uuid (assoc :block/uuid (str uuid))
|
||||
parent (assoc :block/parent (str parent))
|
||||
page (assoc :block/page (str page))
|
||||
order (assoc :block/order order)))
|
||||
|
||||
(defn- checksum-diagnostics-response
|
||||
[^js self]
|
||||
(ensure-conn! self)
|
||||
(-> (sync-checksum/recompute-checksum-diagnostics @(.-conn self))
|
||||
(update :blocks (fn [blocks]
|
||||
(mapv normalize-diagnostic-block blocks)))))
|
||||
|
||||
(defn- handle-sync-checksum-diagnostics
|
||||
[^js self request]
|
||||
(let [graph-id (graph-id-from-request request)]
|
||||
(if (not (seq graph-id))
|
||||
(http/bad-request "missing graph id")
|
||||
(p/let [ready-for-sync? (<ready-for-sync? self graph-id)]
|
||||
(if-not ready-for-sync?
|
||||
(http/error-response "graph not ready" 409)
|
||||
(http/json-response :sync/checksum-diagnostics
|
||||
(checksum-diagnostics-response self)))))))
|
||||
|
||||
(defn- handle-sync-snapshot-stream
|
||||
[^js self request]
|
||||
(let [graph-id (graph-id-from-request request)]
|
||||
(if (not (seq graph-id))
|
||||
(http/bad-request "missing graph id")
|
||||
(let [gzip? (and (snapshot-stream-gzip-enabled? self)
|
||||
(exists? js/CompressionStream))
|
||||
stream (cond-> (snapshot-export-stream self)
|
||||
gzip?
|
||||
(maybe-compress-stream))
|
||||
row-count (snapshot-row-count (.-sql self))
|
||||
headers (cond-> {"content-type" snapshot-content-type}
|
||||
gzip?
|
||||
(assoc "content-encoding" snapshot-content-encoding))]
|
||||
(js/Response. stream
|
||||
#js {:status 200
|
||||
:headers (js/Object.assign
|
||||
(clj->js headers)
|
||||
#js {"x-snapshot-row-count" (str row-count)}
|
||||
(common/cors-headers))})))))
|
||||
|
||||
(defn- handle-sync-snapshot-download
|
||||
[^js self request]
|
||||
(let [graph-id (graph-id-from-request request)]
|
||||
(cond
|
||||
(not (seq graph-id))
|
||||
(http/bad-request "missing graph id")
|
||||
|
||||
:else
|
||||
(p/let [ready-for-sync? (<ready-for-sync? self graph-id)]
|
||||
(if-not ready-for-sync?
|
||||
(http/error-response "graph not ready" 409)
|
||||
(let [key (str "stream/" graph-id ".snapshot")
|
||||
url (snapshot-stream-url request graph-id)
|
||||
content-encoding (when (and (snapshot-stream-gzip-enabled? self)
|
||||
(exists? js/CompressionStream))
|
||||
snapshot-content-encoding)]
|
||||
(http/json-response :sync/snapshot-download
|
||||
(cond-> {:ok true
|
||||
:key key
|
||||
:url url}
|
||||
content-encoding
|
||||
(assoc :content-encoding content-encoding)))))))))
|
||||
|
||||
(defn- handle-sync-admin-reset
|
||||
[^js self]
|
||||
(let [^js state (.-state self)
|
||||
^js storage (.-storage state)
|
||||
delete-all (.-deleteAll storage)
|
||||
delete-alarm (.-deleteAlarm storage)]
|
||||
(doseq [^js ws (.getWebSockets state)]
|
||||
(.close ws 1000 "graph deleted"))
|
||||
(p/let [_ (when (fn? delete-alarm)
|
||||
(.deleteAlarm storage))]
|
||||
(if (fn? delete-all)
|
||||
(p/let [_ (.deleteAll storage)]
|
||||
(set! (.-schema-ready self) false)
|
||||
(set! (.-conn self) nil)
|
||||
(http/json-response :sync/admin-reset {:ok true}))
|
||||
(do
|
||||
(common/sql-exec (.-sql self) "drop table if exists kvs")
|
||||
(common/sql-exec (.-sql self) "drop table if exists tx_log")
|
||||
(common/sql-exec (.-sql self) "drop table if exists sync_meta")
|
||||
(storage/init-schema! (.-sql self))
|
||||
(set! (.-schema-ready self) true)
|
||||
(set! (.-conn self) nil)
|
||||
(http/json-response :sync/admin-reset {:ok true}))))))
|
||||
|
||||
(defn- handle-sync-tx-batch
|
||||
[^js self request]
|
||||
(.then (common/read-json request)
|
||||
(fn [result]
|
||||
(if (nil? result)
|
||||
(http/bad-request "missing body")
|
||||
(let [body (js->clj result :keywordize-keys true)
|
||||
body (http/coerce-http-request :sync/tx-batch body)
|
||||
graph-id (graph-id-from-request request)]
|
||||
(if (nil? body)
|
||||
(http/bad-request "invalid tx")
|
||||
(let [{:keys [client-revision txs t-before]} body
|
||||
t-before (parse-int t-before)]
|
||||
(if (sequential? txs)
|
||||
(p/let [ready-for-sync? (<ready-for-sync? self graph-id)]
|
||||
(if-not ready-for-sync?
|
||||
(http/error-response "graph not ready" 409)
|
||||
(http/json-response :sync/tx-batch
|
||||
(handle-tx-batch! self nil txs t-before
|
||||
{:graph-id graph-id
|
||||
:client-revision client-revision}))))
|
||||
(http/bad-request "invalid tx")))))))))
|
||||
|
||||
(defn- parse-reset-param
|
||||
[value]
|
||||
(if (nil? value)
|
||||
true
|
||||
(not (contains? #{"false" "0"} value))))
|
||||
|
||||
(defn- parse-finished-param
|
||||
[value]
|
||||
(contains? #{"true" "1"} value))
|
||||
|
||||
(defn- sqlite-too-big-error?
|
||||
[error]
|
||||
(let [message (-> (or (ex-message error)
|
||||
(some-> error .-message)
|
||||
(str error))
|
||||
string/lower-case)]
|
||||
(or (string/includes? message "sqlite_toobig")
|
||||
(string/includes? message "string or blob too big")
|
||||
(string/includes? message "statement too long"))))
|
||||
|
||||
(defn- handle-sync-snapshot-upload
|
||||
[^js self request url]
|
||||
(let [graph-id (graph-id-from-request request)
|
||||
reset-param (.get (.-searchParams url) "reset")
|
||||
reset? (parse-reset-param reset-param)
|
||||
finished-param (.get (.-searchParams url) "finished")
|
||||
finished? (parse-finished-param finished-param)
|
||||
checksum-param (.get (.-searchParams url) "checksum")
|
||||
req-encoding (.get (.-headers request) "content-encoding")]
|
||||
(cond
|
||||
(not (seq graph-id))
|
||||
(http/bad-request "missing graph id")
|
||||
|
||||
(nil? (.-body request))
|
||||
(http/bad-request "missing body")
|
||||
|
||||
:else
|
||||
(let [stream (.-body request)
|
||||
encoding (or req-encoding "")]
|
||||
(if (and (= encoding snapshot-content-encoding)
|
||||
(not (exists? js/DecompressionStream)))
|
||||
(http/error-response "gzip not supported" 500)
|
||||
(p/catch
|
||||
(p/let [_ (ensure-schema! self)
|
||||
_ (when reset?
|
||||
(storage/set-meta! (.-sql self) snapshot-uploading-meta-key true))
|
||||
_ (when reset?
|
||||
(<set-graph-ready-for-use! self graph-id false))
|
||||
stream (maybe-decompress-stream stream encoding)
|
||||
count (import-snapshot-stream! self stream reset?)
|
||||
_ (when finished?
|
||||
(storage/set-meta! (.-sql self) snapshot-uploading-meta-key false))
|
||||
_ (when finished?
|
||||
(when (seq checksum-param)
|
||||
(storage/set-initial-checksum! (.-sql self) checksum-param)))
|
||||
_ (when finished?
|
||||
(<set-graph-ready-for-use! self graph-id true))]
|
||||
(http/json-response :sync/snapshot-upload {:ok true
|
||||
:count count}))
|
||||
(fn [error]
|
||||
(if (sqlite-too-big-error? error)
|
||||
(http/error-response "snapshot row too large" 413)
|
||||
(throw error)))))))))
|
||||
|
||||
(defn handle [{:keys [^js self request url route]}]
|
||||
(case (:handler route)
|
||||
:sync/health
|
||||
(http/json-response :sync/health {:ok true})
|
||||
|
||||
:sync/pull
|
||||
(handle-sync-pull self url)
|
||||
|
||||
:sync/checksum-diagnostics
|
||||
(handle-sync-checksum-diagnostics self request)
|
||||
|
||||
:sync/snapshot-stream
|
||||
(handle-sync-snapshot-stream self request)
|
||||
|
||||
:sync/snapshot-download
|
||||
(handle-sync-snapshot-download self request)
|
||||
|
||||
:sync/admin-reset
|
||||
(handle-sync-admin-reset self)
|
||||
|
||||
:sync/tx-batch
|
||||
(handle-sync-tx-batch self request)
|
||||
|
||||
:sync/snapshot-upload
|
||||
(handle-sync-snapshot-upload self request url)
|
||||
|
||||
(http/not-found)))
|
||||
|
||||
(defn- strip-sync-prefix [path]
|
||||
(if (string/starts-with? path "/sync/")
|
||||
(let [rest-path (subs path (count "/sync/"))
|
||||
slash-idx (string/index-of rest-path "/")]
|
||||
(if (neg? slash-idx)
|
||||
"/"
|
||||
(subs rest-path slash-idx)))
|
||||
path))
|
||||
|
||||
(defn handle-http [^js self request]
|
||||
(letfn [(with-cors-error [resp]
|
||||
(if (instance? js/Promise resp)
|
||||
(.catch resp
|
||||
(fn [e]
|
||||
(log/error :db-sync/http-error {:error e})
|
||||
(common/json-response
|
||||
{:error "server error"
|
||||
:debug-message (str e)
|
||||
:debug-stack (when (instance? js/Error e) (.-stack e))}
|
||||
500)))
|
||||
resp))]
|
||||
(try
|
||||
(let [url (js/URL. (.-url request))
|
||||
raw-path (.-pathname url)
|
||||
path (strip-sync-prefix raw-path)
|
||||
method (.-method request)]
|
||||
(with-cors-error
|
||||
(cond
|
||||
(= method "OPTIONS")
|
||||
(common/options-response)
|
||||
|
||||
:else
|
||||
(if-let [route (or (sync-routes/match-route method path)
|
||||
(semantic-routes/match-internal method path))]
|
||||
(if (string/starts-with? path "/semantic/")
|
||||
(semantic-handler/handle {:self self :request request :url url :route route})
|
||||
(handle {:self self
|
||||
:request request
|
||||
:url url
|
||||
:route route}))
|
||||
(http/not-found)))))
|
||||
(catch :default e
|
||||
(log/error :db-sync/http-error {:error e})
|
||||
(common/json-response
|
||||
{:error "server error"
|
||||
:debug-message (str e)
|
||||
:debug-stack (when (instance? js/Error e) (.-stack e))}
|
||||
500)))))
|
||||
78
deps/db-sync/src/logseq/db_sync/worker/handler/ws.cljs
vendored
Normal file
78
deps/db-sync/src/logseq/db_sync/worker/handler/ws.cljs
vendored
Normal file
@@ -0,0 +1,78 @@
|
||||
(ns logseq.db-sync.worker.handler.ws
|
||||
(:require [logseq.db-sync.protocol :as protocol]
|
||||
[logseq.db-sync.worker.auth :as auth]
|
||||
[logseq.db-sync.worker.handler.sync :as sync-handler]
|
||||
[logseq.db-sync.worker.http :as http]
|
||||
[logseq.db-sync.worker.presence :as presence]
|
||||
[logseq.db-sync.worker.ws :as ws]
|
||||
[promesa.core :as p]))
|
||||
|
||||
(defn handle-ws-message! [^js self ^js ws raw]
|
||||
(let [message (-> raw protocol/parse-message ws/coerce-ws-client-message)]
|
||||
(if-not (map? message)
|
||||
(ws/send! ws {:type "error" :message "invalid request"})
|
||||
(case (:type message)
|
||||
"hello"
|
||||
(let [checksum (sync-handler/current-checksum self)]
|
||||
(ws/send! ws (cond-> {:type "hello"
|
||||
:t (sync-handler/t-now self)}
|
||||
(string? checksum) (assoc :checksum checksum))))
|
||||
|
||||
"ping"
|
||||
(ws/send! ws {:type "pong"})
|
||||
|
||||
"presence"
|
||||
(let [editing-block-uuid (:editing-block-uuid message)
|
||||
user (presence/get-user self ws)]
|
||||
(presence/update-presence! self ws {:editing-block-uuid editing-block-uuid})
|
||||
(ws/broadcast! self ws {:type "presence"
|
||||
:editing-block-uuid editing-block-uuid
|
||||
:user-id (:user-id user)}))
|
||||
|
||||
"pull"
|
||||
(let [raw-since (:since message)
|
||||
since (if (some? raw-since) (sync-handler/parse-int raw-since) 0)]
|
||||
(if (or (and (some? raw-since) (not (number? since))) (neg? since))
|
||||
(ws/send! ws {:type "error" :message "invalid since"})
|
||||
(ws/send! ws (sync-handler/pull-response self since))))
|
||||
|
||||
;; "snapshot"
|
||||
;; (send! ws (snapshot-response self))
|
||||
|
||||
"tx/batch"
|
||||
(let [txs (:txs message)
|
||||
user (presence/get-user self ws)
|
||||
t-before (sync-handler/parse-int (:t-before message))]
|
||||
(if (sequential? txs)
|
||||
(ws/send! ws (sync-handler/handle-tx-batch!
|
||||
self
|
||||
ws
|
||||
txs
|
||||
t-before
|
||||
(cond-> {:graph-id (aget self "graph-id")}
|
||||
(:client-revision message)
|
||||
(assoc :client-revision (:client-revision message))
|
||||
(:username user)
|
||||
(assoc :username (:username user)))))
|
||||
(ws/send! ws {:type "tx/reject" :reason "invalid tx"})))
|
||||
|
||||
(ws/send! ws {:type "error" :message "unknown type"})))))
|
||||
|
||||
(defn handle-ws [^js self request]
|
||||
(let [graph-id (sync-handler/graph-id-from-request request)]
|
||||
(p/let [ready-for-sync? (sync-handler/<ready-for-sync? self graph-id)]
|
||||
(if-not ready-for-sync?
|
||||
(http/error-response "graph not ready" 409)
|
||||
(let [pair (js/WebSocketPair.)
|
||||
client (aget pair 0)
|
||||
server (aget pair 1)
|
||||
state (.-state self)]
|
||||
(aset self "graph-id" graph-id)
|
||||
(.acceptWebSocket state server)
|
||||
(let [token (auth/token-from-request request)
|
||||
claims (auth/unsafe-jwt-claims token)
|
||||
user (presence/claims->user claims)]
|
||||
(when user
|
||||
(presence/add-presence! self server user))
|
||||
(presence/broadcast-online-users! self)
|
||||
(js/Response. nil #js {:status 101 :webSocket client})))))))
|
||||
42
deps/db-sync/src/logseq/db_sync/worker/http.cljs
vendored
Normal file
42
deps/db-sync/src/logseq/db_sync/worker/http.cljs
vendored
Normal file
@@ -0,0 +1,42 @@
|
||||
(ns logseq.db-sync.worker.http
|
||||
(:require [logseq.db-sync.common :as common]
|
||||
[logseq.db-sync.malli-schema :as db-sync-schema]
|
||||
[logseq.db-sync.worker.coerce :as coerce]))
|
||||
|
||||
(defn coerce-http-request [schema-key body]
|
||||
(if-let [coercer (get db-sync-schema/http-request-coercers schema-key)]
|
||||
(let [coerced (coerce/coerce coercer body {:schema schema-key :dir :request})]
|
||||
(when-not (= coerced coerce/invalid-coerce)
|
||||
coerced))
|
||||
body))
|
||||
|
||||
(defn json-response
|
||||
([schema-key data] (json-response schema-key data 200))
|
||||
([schema-key data status]
|
||||
(if-let [coercer (get db-sync-schema/http-response-coercers schema-key)]
|
||||
(let [coerced (coerce/coerce coercer data {:schema schema-key :dir :response})]
|
||||
(if (= coerced coerce/invalid-coerce)
|
||||
(do
|
||||
(js/console.error "DEBUG json-response coercion FAILED for" (pr-str schema-key) "data:" (pr-str data))
|
||||
(common/json-response
|
||||
{:error "server error"
|
||||
:debug-coercion-failed (pr-str schema-key)
|
||||
:debug-data (pr-str data)}
|
||||
500))
|
||||
(common/json-response coerced status)))
|
||||
(common/json-response data status))))
|
||||
|
||||
(defn error-response [message status]
|
||||
(json-response :error {:error message} status))
|
||||
|
||||
(defn bad-request [message]
|
||||
(error-response message 400))
|
||||
|
||||
(defn unauthorized []
|
||||
(error-response "unauthorized" 401))
|
||||
|
||||
(defn forbidden []
|
||||
(error-response "forbidden" 403))
|
||||
|
||||
(defn not-found []
|
||||
(error-response "not found" 404))
|
||||
68
deps/db-sync/src/logseq/db_sync/worker/presence.cljs
vendored
Normal file
68
deps/db-sync/src/logseq/db_sync/worker/presence.cljs
vendored
Normal file
@@ -0,0 +1,68 @@
|
||||
(ns logseq.db-sync.worker.presence
|
||||
(:require [cljs-bean.core :as bean]
|
||||
[clojure.string :as string]
|
||||
[logseq.db-sync.worker.ws :as ws]))
|
||||
|
||||
(defn claims->user
|
||||
[claims]
|
||||
(when claims
|
||||
(let [user-id (aget claims "sub")
|
||||
email (aget claims "email")
|
||||
username (or (aget claims "preferred_username")
|
||||
(aget claims "cognito:username")
|
||||
(aget claims "username"))
|
||||
name (aget claims "name")]
|
||||
(when (string? user-id)
|
||||
(cond-> {:user-id user-id}
|
||||
(string? email) (assoc :email email)
|
||||
(string? username) (assoc :username username)
|
||||
(string? name) (assoc :name name))))))
|
||||
|
||||
(defn attachment->user
|
||||
[attachment]
|
||||
(:presence/user (bean/->clj attachment)))
|
||||
|
||||
(defn- serialize-attachment!
|
||||
[^js ws user]
|
||||
(.serializeAttachment ws (bean/->js {:presence/user user})))
|
||||
|
||||
(defn presence*
|
||||
[^js self]
|
||||
(or (.-presence self)
|
||||
(set! (.-presence self) (atom {}))))
|
||||
|
||||
(defn online-users
|
||||
[^js self]
|
||||
(vec (distinct (vals @(presence* self)))))
|
||||
|
||||
(defn broadcast-online-users!
|
||||
[^js self]
|
||||
(ws/broadcast! self nil {:type "online-users" :online-users (online-users self)}))
|
||||
|
||||
(defn add-presence!
|
||||
[^js self ^js ws user]
|
||||
(swap! (presence* self) assoc ws user)
|
||||
(serialize-attachment! ws user))
|
||||
|
||||
(defn update-presence!
|
||||
[^js self ^js ws {:keys [editing-block-uuid] :as updates}]
|
||||
(swap! (presence* self)
|
||||
(fn [presence]
|
||||
(if-let [user (get presence ws)]
|
||||
(let [user' (if (contains? updates :editing-block-uuid)
|
||||
(if (and (string? editing-block-uuid)
|
||||
(not (string/blank? editing-block-uuid)))
|
||||
(assoc user :editing-block-uuid editing-block-uuid)
|
||||
(dissoc user :editing-block-uuid))
|
||||
user)]
|
||||
(serialize-attachment! ws user')
|
||||
(assoc presence ws user'))
|
||||
presence))))
|
||||
|
||||
(defn get-user
|
||||
[^js self ^js ws]
|
||||
(get @(presence* self) ws))
|
||||
|
||||
(defn remove-presence!
|
||||
[^js self ^js ws]
|
||||
(swap! (presence* self) dissoc ws))
|
||||
34
deps/db-sync/src/logseq/db_sync/worker/routes/index.cljs
vendored
Normal file
34
deps/db-sync/src/logseq/db_sync/worker/routes/index.cljs
vendored
Normal file
@@ -0,0 +1,34 @@
|
||||
(ns logseq.db-sync.worker.routes.index
|
||||
(:require [reitit.core :as r]))
|
||||
|
||||
(def ^:private route-data
|
||||
[["/admin"
|
||||
["/graphs/:graph-id" {:methods {"DELETE" :admin-graphs/delete}}]]
|
||||
|
||||
["/graphs"
|
||||
["" {:methods {"GET" :graphs/list
|
||||
"POST" :graphs/create}}]
|
||||
["/:graph-id"
|
||||
["/access" {:methods {"GET" :graphs/access}}]
|
||||
["/members" {:methods {"GET" :graph-members/list
|
||||
"POST" :graph-members/create}}]
|
||||
["/members/:member-id" {:methods {"PUT" :graph-members/update
|
||||
"DELETE" :graph-members/delete}}]
|
||||
["" {:methods {"DELETE" :graphs/delete}}]]]
|
||||
|
||||
["/e2ee"
|
||||
["/user-keys" {:methods {"GET" :e2ee/user-keys-get
|
||||
"POST" :e2ee/user-keys-post}}]
|
||||
["/user-public-key" {:methods {"GET" :e2ee/user-public-key-get}}]
|
||||
["/graphs/:graph-id"
|
||||
["/aes-key" {:methods {"GET" :e2ee/graph-aes-key-get
|
||||
"POST" :e2ee/graph-aes-key-post}}]
|
||||
["/grant-access" {:methods {"POST" :e2ee/grant-access}}]]]])
|
||||
|
||||
(def ^:private router
|
||||
(r/router route-data))
|
||||
|
||||
(defn match-route [method path]
|
||||
(when-let [match (r/match-by-path router path)]
|
||||
(when-let [handler (get-in match [:data :methods method])]
|
||||
(assoc match :handler handler))))
|
||||
410
deps/db-sync/src/logseq/db_sync/worker/routes/semantic.cljs
vendored
Normal file
410
deps/db-sync/src/logseq/db_sync/worker/routes/semantic.cljs
vendored
Normal file
@@ -0,0 +1,410 @@
|
||||
(ns logseq.db-sync.worker.routes.semantic
|
||||
(:require [clojure.string :as string]
|
||||
[flatland.ordered.map :refer [ordered-map]]
|
||||
[reitit.core :as r]))
|
||||
|
||||
(def operations
|
||||
[{:method "GET" :path "/api/v1/graphs"
|
||||
:handler :semantic/graphs-list :operation-id "listGraphs" :scope "logseq/read" :rate-class :read}
|
||||
{:method "GET" :path "/api/v1/graphs/:graph-id/pages" :internal-path "/semantic/pages"
|
||||
:handler :semantic/pages-list :operation-id "listPages" :scope "logseq/read" :rate-class :read}
|
||||
{:method "POST" :path "/api/v1/graphs/:graph-id/pages" :internal-path "/semantic/pages"
|
||||
:handler :semantic/pages-create :operation-id "createPage" :scope "logseq/write" :rate-class :write}
|
||||
{:method "GET" :path "/api/v1/graphs/:graph-id/pages/:page-id/blocks" :internal-path "/semantic/pages/:page-id/blocks"
|
||||
:handler :semantic/pages-blocks :operation-id "listPageBlocks" :scope "logseq/read" :rate-class :read}
|
||||
{:method "GET" :path "/api/v1/graphs/:graph-id/pages/:page-id/references" :internal-path "/semantic/pages/:page-id/references"
|
||||
:handler :semantic/pages-references :operation-id "listPageReferences" :scope "logseq/read" :rate-class :read}
|
||||
{:method "DELETE" :path "/api/v1/graphs/:graph-id/pages/:page-id" :internal-path "/semantic/pages/:page-id"
|
||||
:handler :semantic/pages-delete :operation-id "deletePage" :scope "logseq/write" :rate-class :write}
|
||||
{:method "GET" :path "/api/v1/graphs/:graph-id/pages/:page-id" :internal-path "/semantic/pages/:page-id"
|
||||
:handler :semantic/pages-get :operation-id "getPage" :scope "logseq/read" :rate-class :read}
|
||||
{:method "PATCH" :path "/api/v1/graphs/:graph-id/pages/:page-id" :internal-path "/semantic/pages/:page-id"
|
||||
:handler :semantic/pages-update :operation-id "updatePage" :scope "logseq/write" :rate-class :write}
|
||||
{:method "GET" :path "/api/v1/graphs/:graph-id/blocks/:block-id" :internal-path "/semantic/blocks/:block-id"
|
||||
:handler :semantic/blocks-get :operation-id "getBlock" :scope "logseq/read" :rate-class :read}
|
||||
{:method "PATCH" :path "/api/v1/graphs/:graph-id/blocks/:block-id" :internal-path "/semantic/blocks/:block-id"
|
||||
:handler :semantic/blocks-update :operation-id "updateBlock" :scope "logseq/write" :rate-class :write}
|
||||
{:method "DELETE" :path "/api/v1/graphs/:graph-id/blocks/:block-id" :internal-path "/semantic/blocks/:block-id"
|
||||
:handler :semantic/blocks-delete :operation-id "deleteBlock" :scope "logseq/write" :rate-class :write}
|
||||
{:method "POST" :path "/api/v1/graphs/:graph-id/block-moves" :internal-path "/semantic/block-moves"
|
||||
:handler :semantic/blocks-move :operation-id "moveBlocks" :scope "logseq/write" :rate-class :write}
|
||||
{:method "POST" :path "/api/v1/graphs/:graph-id/blocks/:block-id/children" :internal-path "/semantic/blocks/:block-id/children"
|
||||
:handler :semantic/blocks-insert-children :operation-id "insertBlockChildren" :scope "logseq/write" :rate-class :write}
|
||||
{:method "POST" :path "/api/v1/graphs/:graph-id/block-trees" :internal-path "/semantic/block-trees"
|
||||
:handler :semantic/blocks-insert-tree :operation-id "insertBlockTree" :scope "logseq/write" :rate-class :write}
|
||||
{:method "PUT" :path "/api/v1/graphs/:graph-id/blocks/:block-id/properties/:property-id" :internal-path "/semantic/blocks/:block-id/properties/:property-id"
|
||||
:handler :semantic/blocks-set-property :operation-id "setBlockProperty" :scope "logseq/write" :rate-class :write}
|
||||
{:method "DELETE" :path "/api/v1/graphs/:graph-id/blocks/:block-id/properties/:property-id" :internal-path "/semantic/blocks/:block-id/properties/:property-id"
|
||||
:handler :semantic/blocks-delete-property :operation-id "deleteBlockProperty" :scope "logseq/write" :rate-class :write}
|
||||
{:method "POST" :path "/api/v1/graphs/:graph-id/block-properties/batch-set" :internal-path "/semantic/block-properties/batch-set"
|
||||
:handler :semantic/blocks-batch-set-property :operation-id "batchSetBlockProperty" :scope "logseq/write" :rate-class :write}
|
||||
{:method "POST" :path "/api/v1/graphs/:graph-id/block-properties/batch-delete" :internal-path "/semantic/block-properties/batch-delete"
|
||||
:handler :semantic/blocks-batch-delete-property :operation-id "batchDeleteBlockProperty" :scope "logseq/write" :rate-class :write}
|
||||
{:method "POST" :path "/api/v1/graphs/:graph-id/capture" :internal-path "/semantic/capture"
|
||||
:handler :semantic/capture :operation-id "captureToToday" :scope "logseq/write" :rate-class :write}
|
||||
{:method "GET" :path "/api/v1/graphs/:graph-id/tasks" :internal-path "/semantic/tasks"
|
||||
:handler :semantic/tasks-list :operation-id "listTasks" :scope "logseq/read" :rate-class :read}
|
||||
{:method "POST" :path "/api/v1/graphs/:graph-id/tasks" :internal-path "/semantic/tasks"
|
||||
:handler :semantic/tasks-create :operation-id "createTask" :scope "logseq/write" :rate-class :write}
|
||||
{:method "GET" :path "/api/v1/graphs/:graph-id/tags" :internal-path "/semantic/tags"
|
||||
:handler :semantic/tags-list :operation-id "listTags" :scope "logseq/read" :rate-class :read}
|
||||
{:method "POST" :path "/api/v1/graphs/:graph-id/tags" :internal-path "/semantic/tags"
|
||||
:handler :semantic/tags-create :operation-id "createTag" :scope "logseq/write" :rate-class :write}
|
||||
{:method "GET" :path "/api/v1/graphs/:graph-id/tags/:tag-id" :internal-path "/semantic/tags/:tag-id"
|
||||
:handler :semantic/tags-get :operation-id "getTag" :scope "logseq/read" :rate-class :read}
|
||||
{:method "GET" :path "/api/v1/graphs/:graph-id/tags/:tag-id/objects" :internal-path "/semantic/tags/:tag-id/objects"
|
||||
:handler :semantic/tags-objects :operation-id "listTagObjects" :scope "logseq/read" :rate-class :read}
|
||||
{:method "PATCH" :path "/api/v1/graphs/:graph-id/tags/:tag-id" :internal-path "/semantic/tags/:tag-id"
|
||||
:handler :semantic/tags-update :operation-id "updateTag" :scope "logseq/write" :rate-class :write}
|
||||
{:method "DELETE" :path "/api/v1/graphs/:graph-id/tags/:tag-id" :internal-path "/semantic/tags/:tag-id"
|
||||
:handler :semantic/tags-delete :operation-id "deleteTag" :scope "logseq/write" :rate-class :write}
|
||||
{:method "GET" :path "/api/v1/graphs/:graph-id/properties" :internal-path "/semantic/properties"
|
||||
:handler :semantic/properties-list :operation-id "listProperties" :scope "logseq/read" :rate-class :read}
|
||||
{:method "POST" :path "/api/v1/graphs/:graph-id/properties" :internal-path "/semantic/properties"
|
||||
:handler :semantic/properties-create :operation-id "createProperty" :scope "logseq/write" :rate-class :write}
|
||||
{:method "GET" :path "/api/v1/graphs/:graph-id/properties/:property-id" :internal-path "/semantic/properties/:property-id"
|
||||
:handler :semantic/properties-get :operation-id "getProperty" :scope "logseq/read" :rate-class :read}
|
||||
{:method "PATCH" :path "/api/v1/graphs/:graph-id/properties/:property-id" :internal-path "/semantic/properties/:property-id"
|
||||
:handler :semantic/properties-update :operation-id "updateProperty" :scope "logseq/write" :rate-class :write}
|
||||
{:method "DELETE" :path "/api/v1/graphs/:graph-id/properties/:property-id" :internal-path "/semantic/properties/:property-id"
|
||||
:handler :semantic/properties-delete :operation-id "deleteProperty" :scope "logseq/write" :rate-class :write}
|
||||
{:method "GET" :path "/api/v1/graphs/:graph-id/assets" :internal-path "/semantic/assets"
|
||||
:handler :semantic/assets-list :operation-id "listAssets" :scope "logseq/read" :rate-class :read}
|
||||
{:method "POST" :path "/api/v1/graphs/:graph-id/assets" :internal-path "/semantic/assets"
|
||||
:handler :semantic/assets-create :operation-id "createAsset" :scope "logseq/write" :rate-class :write}
|
||||
{:method "GET" :path "/api/v1/graphs/:graph-id/assets/:asset-block-id" :internal-path "/semantic/assets/:asset-block-id"
|
||||
:handler :semantic/assets-get :operation-id "getAsset" :scope "logseq/read" :rate-class :read}
|
||||
{:method "GET" :path "/api/v1/graphs/:graph-id/search" :internal-path "/semantic/search"
|
||||
:handler :semantic/search :operation-id "searchGraph" :scope "logseq/read" :rate-class :read}])
|
||||
|
||||
(def ^:private operation-docs
|
||||
{"listGraphs" ["List available graphs" "Returns cursor-paginated non-E2EE graphs available to the authenticated user. Use the optional exact name filter to resolve a graph name to its UUID."]
|
||||
"listPages" ["List pages" "Returns a cursor-paginated list of page blocks in the graph."]
|
||||
"createPage" ["Create a page" "Creates a page block with the supplied title."]
|
||||
"listPageBlocks" ["List a page's blocks" "Returns a cursor-paginated list of the page's top-level blocks, including each selected block's descendant tree."]
|
||||
"listPageReferences" ["List references to a page" "Returns a cursor-paginated flat list of blocks and pages that reference the addressed page. Results are not recursively expanded."]
|
||||
"getPage" ["Get a page" "Returns one page block by UUID without loading its block tree."]
|
||||
"updatePage" ["Update a page" "Renames one page identified by its block UUID."]
|
||||
"deletePage" ["Delete a page" "Deletes one page through Logseq's page deletion rules, including its block tree."]
|
||||
"getBlock" ["Get a block" "Returns one block or page by UUID without loading an unbounded tree."]
|
||||
"updateBlock" ["Update a block" "Replaces the title of one block identified by UUID."]
|
||||
"deleteBlock" ["Delete a block" "Deletes one block and its descendants. Asset blocks are deleted through this same operation."]
|
||||
"moveBlocks" ["Move blocks" "Moves one or more blocks together before, after, into the first-child position, or into the last-child position of a target block. Logseq preserves the blocks' outliner order."]
|
||||
"insertBlockChildren" ["Insert child block trees" "Appends or prepends one or more recursive block trees as children of the addressed block."]
|
||||
"insertBlockTree" ["Insert block trees" "Inserts one or more recursive block trees relative to an explicit target block."]
|
||||
"setBlockProperty" ["Set a block property" "Idempotently adds or replaces one typed property value on one block in a DB graph. The property path value may be its UUID or ident. Never write legacy file-graph key:: value syntax into the block title."]
|
||||
"deleteBlockProperty" ["Delete a block property" "Idempotently removes one property from one block. The property definition itself is not deleted."]
|
||||
"batchSetBlockProperty" ["Set block properties in a batch" "Atomically adds or replaces multiple typed block-property values in a DB graph after validating every entry. Property IDs are UUIDs or idents; never use legacy file-graph key:: value syntax."]
|
||||
"batchDeleteBlockProperty" ["Delete block properties in a batch" "Atomically removes multiple block-property values after validating every entry."]
|
||||
"captureToToday" ["Capture to today's journal" "Creates today's journal page when necessary and appends the supplied recursive block trees to its end."]
|
||||
"listTasks" ["List DB tasks" "Lists cursor-paginated DB Task objects using the Task class index, with optional status, priority, and title-content filters. Use this operation instead of graph search to retrieve tasks."]
|
||||
"createTask" ["Create a DB task" "Creates a DB Task object with the Task class and typed status, priority, scheduled, and deadline properties. Never encode a task as Markdown TODO text."]
|
||||
"listTags" ["List tags" "Returns a cursor-paginated list of tag entities in the graph."]
|
||||
"createTag" ["Create a tag" "Creates a Logseq tag with the supplied title."]
|
||||
"getTag" ["Get a tag" "Returns one tag by UUID."]
|
||||
"listTagObjects" ["List objects with a tag" "Returns a cursor-paginated flat list of blocks and pages tagged with the addressed tag. Results are not recursively expanded."]
|
||||
"updateTag" ["Update a tag" "Renames one tag identified by UUID."]
|
||||
"deleteTag" ["Delete a tag" "Deletes one tag through Logseq's page deletion rules."]
|
||||
"listProperties" ["List property definitions" "Returns a cursor-paginated list of property definitions in the graph."]
|
||||
"createProperty" ["Create a property definition" "Creates a typed property definition in a DB graph with its name, value type, and cardinality. Use the returned UUID or ident to set values; never use legacy file-graph key:: value syntax."]
|
||||
"getProperty" ["Get a property definition" "Returns one property definition by UUID."]
|
||||
"updateProperty" ["Update a property definition" "Updates the name, value type, or cardinality of one typed property definition in a DB graph. Never use legacy file-graph key:: value syntax."]
|
||||
"deleteProperty" ["Delete a property definition" "Deletes one property definition through Logseq's property deletion rules."]
|
||||
"listAssets" ["List assets" "Returns a cursor-paginated list of asset blocks with optional creation and update time filters."]
|
||||
"createAsset" ["Upload an asset" "Streams a file of at most 100MB to R2 and creates its Asset-class block. Requires a client-computed SHA-256 checksum."]
|
||||
"getAsset" ["Get an asset download link" "Returns asset metadata and a five-minute signed URL for the asset block's R2 object. For an image asset, fetch or use the URL directly to display image content; it does not require an Authorization header."]
|
||||
"searchGraph" ["Search graph resources" "Searches blocks, tags, properties, and assets by title and returns cursor-paginated results."]})
|
||||
|
||||
(defn- operation-routes [path-key]
|
||||
(->> operations
|
||||
(filter path-key)
|
||||
(group-by path-key)
|
||||
(mapv (fn [[path path-operations]]
|
||||
[path {:methods (into {} (map (juxt :method identity)) path-operations)}]))))
|
||||
|
||||
(def ^:private public-router
|
||||
(r/router (operation-routes :path)))
|
||||
|
||||
(def ^:private internal-router
|
||||
(r/router (operation-routes :internal-path)))
|
||||
|
||||
(defn- match [router method path]
|
||||
(when-let [route (r/match-by-path router path)]
|
||||
(when-let [operation (get-in route [:data :methods method])]
|
||||
(merge operation (select-keys route [:path-params])))))
|
||||
|
||||
(defn match-public [method path]
|
||||
(match public-router method path))
|
||||
|
||||
(defn match-internal [method path]
|
||||
(match internal-router method path))
|
||||
|
||||
(defn- openapi-path [path]
|
||||
(string/replace path #":([^/]+)" "{$1}"))
|
||||
|
||||
(defn- path-parameters [path]
|
||||
(mapv (fn [[_ parameter]]
|
||||
(cond-> {:name parameter :in "path" :required true :schema {:type "string"}}
|
||||
(= "property-id" parameter)
|
||||
(assoc :description "Property UUID, qualified ident, or exact title.")))
|
||||
(re-seq #":([^/]+)" path)))
|
||||
|
||||
(def ^:private pagination-parameters
|
||||
[{:name "limit" :in "query" :schema {:type "integer" :minimum 1 :maximum 200 :default 50}}
|
||||
{:name "cursor" :in "query" :schema {:type "string"}}])
|
||||
|
||||
(def ^:private time-filter-parameters
|
||||
[{:name "created-after" :in "query"
|
||||
:description "Return entities created strictly after this Unix epoch timestamp in milliseconds."
|
||||
:schema {:type "integer" :minimum 0}}
|
||||
{:name "updated-after" :in "query"
|
||||
:description "Return entities updated strictly after this Unix epoch timestamp in milliseconds."
|
||||
:schema {:type "integer" :minimum 0}}])
|
||||
|
||||
(def ^:private asset-upload-parameters
|
||||
[{:name "file-name" :in "query" :required true :schema {:type "string"}}
|
||||
{:name "size" :in "query" :required true
|
||||
:description "Exact decoded file size in bytes. The trusted MCP host recalculates it for encoding=base64; direct API uploads must supply it. Uploads larger than 100MB are rejected before R2."
|
||||
:schema {:type "integer" :minimum 0 :maximum 104857600}}
|
||||
{:name "title" :in "query" :schema {:type "string"}}
|
||||
{:name "page-id" :in "query"
|
||||
:description "Destination page UUID. Omit to append to today's journal."
|
||||
:schema {:type "string"}}
|
||||
{:name "checksum" :in "query" :required true
|
||||
:description "Lowercase or uppercase SHA-256 hex digest of the file."
|
||||
:schema {:type "string" :pattern "^[0-9a-fA-F]{64}$"}}
|
||||
{:name "encoding" :in "query"
|
||||
:description "Use base64 when the caller cannot transfer raw binary, including ChatGPT Code Mode. The server decodes it as a stream before writing to R2; size and checksum describe the decoded file."
|
||||
:schema {:type "string" :enum ["base64"]}}])
|
||||
|
||||
(defn- request-schema [operation-id]
|
||||
(case operation-id
|
||||
"createPage" {:required ["title"] :properties {:title {:type "string"}}}
|
||||
"updatePage" {:required ["title"] :properties {:title {:type "string"}}}
|
||||
"updateBlock" {:required ["title"] :properties {:title {:type "string"}}}
|
||||
"moveBlocks" {:required ["block-ids" "target-id" "position"]
|
||||
:properties {:block-ids {:type "array" :minItems 1 :uniqueItems true
|
||||
:items {:type "string"}}
|
||||
:target-id {:type "string"}
|
||||
:position {:enum ["before" "after" "first-child" "last-child"]}}}
|
||||
"insertBlockChildren" {:required ["position" "blocks"]
|
||||
:properties {:position {:enum ["append" "prepend"]}
|
||||
:blocks {:type "array" :items {:$ref "#/components/schemas/BlockTree"}}}}
|
||||
"insertBlockTree" {:required ["target-id" "blocks"]
|
||||
:properties {:target-id {:type "string"}
|
||||
:position {:enum ["append" "prepend"] :default "append"}
|
||||
:blocks {:type "array" :items {:$ref "#/components/schemas/BlockTree"}}}}
|
||||
"setBlockProperty" {:required ["value"]
|
||||
:properties {:value {:$ref "#/components/schemas/PropertyValue"}}}
|
||||
"batchSetBlockProperty" {:required ["entries"]
|
||||
:properties {:isResetExistingValues
|
||||
{:type "boolean" :default false
|
||||
:description "For cardinality-many values, replace existing values when true; append when false or omitted."}
|
||||
:entries {:type "array"
|
||||
:items {:type "object" :required ["block-id" "property-id" "value"]
|
||||
:properties {:block-id {:type "string"}
|
||||
:property-id {:type "string"}
|
||||
:value {:$ref "#/components/schemas/PropertyValue"}}}}}}
|
||||
"batchDeleteBlockProperty" {:required ["entries"]
|
||||
:properties {:entries {:type "array"
|
||||
:items {:type "object" :required ["block-id" "property-id"]
|
||||
:properties {:block-id {:type "string"}
|
||||
:property-id {:type "string"}}}}}}
|
||||
"captureToToday" {:required ["blocks"]
|
||||
:properties {:blocks {:type "array" :items {:$ref "#/components/schemas/BlockTree"}}}}
|
||||
"createTask" {:required ["title"]
|
||||
:properties {:title {:type "string"}
|
||||
:page-id {:type "string"
|
||||
:description "Destination page UUID. Omit to append to today's journal."}
|
||||
:status {:$ref "#/components/schemas/TaskStatusSelector"}
|
||||
:priority {:$ref "#/components/schemas/TaskPrioritySelector"}
|
||||
:scheduled {:type "integer" :description "Scheduled date as YYYYMMDD."}
|
||||
:deadline {:type "integer" :description "Deadline date as YYYYMMDD."}}}
|
||||
"createTag" {:required ["title"] :properties {:title {:type "string"}}}
|
||||
"updateTag" {:required ["title"] :properties {:title {:type "string"}}}
|
||||
"createProperty" {:required ["title"]
|
||||
:properties {:title {:type "string"} :type {:$ref "#/components/schemas/PropertyType"}
|
||||
:cardinality {:enum ["db.cardinality/one" "db.cardinality/many"]}}}
|
||||
"updateProperty" {:properties {:title {:type "string"} :type {:$ref "#/components/schemas/PropertyType"}
|
||||
:cardinality {:enum ["db.cardinality/one" "db.cardinality/many"]}}}
|
||||
nil))
|
||||
|
||||
(defn- operation-parameters [{:keys [operation-id path]}]
|
||||
(cond-> (path-parameters path)
|
||||
(contains? #{"listGraphs" "listPages" "listPageBlocks" "listPageReferences" "listTasks" "listTags"
|
||||
"listTagObjects" "listProperties" "listAssets" "searchGraph"} operation-id)
|
||||
(into pagination-parameters)
|
||||
(contains? #{"listPages" "listTasks" "listTags" "listTagObjects" "listProperties" "listAssets" "searchGraph"}
|
||||
operation-id)
|
||||
(into time-filter-parameters)
|
||||
(= "createAsset" operation-id)
|
||||
(into asset-upload-parameters)
|
||||
(= "listGraphs" operation-id)
|
||||
(into [{:name "name" :in "query" :schema {:type "string"}
|
||||
:description "Exact graph name, matched case-insensitively."}])
|
||||
(= "searchGraph" operation-id)
|
||||
(into [{:name "q" :in "query" :required true :schema {:type "string"}}
|
||||
{:name "types" :in "query" :schema {:type "string"}}])
|
||||
(= "listTasks" operation-id)
|
||||
(into [{:name "status" :in "query" :schema {:$ref "#/components/schemas/TaskStatusSelector"}}
|
||||
{:name "priority" :in "query" :schema {:$ref "#/components/schemas/TaskPrioritySelector"}}
|
||||
{:name "content" :in "query" :schema {:type "string"}
|
||||
:description "Case-insensitive task title filter."}])))
|
||||
|
||||
(defn- request-body [operation-id]
|
||||
(if (= "createAsset" operation-id)
|
||||
{:required true
|
||||
:content {"application/octet-stream"
|
||||
{:schema {:type "string" :format "binary"}}}}
|
||||
(when-let [schema (request-schema operation-id)]
|
||||
{:required true
|
||||
:content {"application/json" {:schema (assoc schema :type "object")}}})))
|
||||
|
||||
(defn- list-response [response-key schema description]
|
||||
{:description description
|
||||
:content {"application/json"
|
||||
{:schema {:type "object"
|
||||
:required [(name response-key)]
|
||||
:properties {response-key {:type "array"
|
||||
:items {:$ref (str "#/components/schemas/" schema)}}
|
||||
:next-cursor {:type "string"}}}}}})
|
||||
|
||||
(defn- entity-response [schema description]
|
||||
{:description description
|
||||
:content {"application/json" {:schema {:$ref (str "#/components/schemas/" schema)}}}})
|
||||
|
||||
(defn- operation-responses [operation-id]
|
||||
(merge {"200" (case operation-id
|
||||
"listPages" (list-response :pages "PageResponse" "A cursor page of pages")
|
||||
"listPageBlocks"
|
||||
(list-response :blocks "BlockResponse" "A cursor page of top-level block trees")
|
||||
"listPageReferences" (list-response :references "BlockResponse" "A cursor page of page references")
|
||||
"listTasks"
|
||||
(list-response :tasks "TaskResponse" "A cursor page of DB Task objects")
|
||||
"listTags" (list-response :tags "TagResponse" "A cursor page of tags")
|
||||
"listTagObjects" (list-response :objects "BlockResponse" "A cursor page of tagged objects")
|
||||
"listProperties" (list-response :properties "PropertyResponse" "A cursor page of property definitions")
|
||||
"listAssets"
|
||||
(list-response :assets "AssetResponse" "A cursor page of asset blocks")
|
||||
"searchGraph" (list-response :results "SearchResultResponse" "A cursor page of search results")
|
||||
"getPage" (entity-response "PageResponse" "A page")
|
||||
"updatePage" (entity-response "PageResponse" "The updated page")
|
||||
"getBlock" (entity-response "BlockResponse" "A block")
|
||||
"updateBlock" (entity-response "BlockResponse" "The updated block")
|
||||
"getTag" (entity-response "TagResponse" "A tag")
|
||||
"updateTag" (entity-response "TagResponse" "The updated tag")
|
||||
"getProperty"
|
||||
(entity-response "PropertyResponse" "A property definition and its user-extensible choices")
|
||||
"updateProperty" (entity-response "PropertyResponse" "The updated property definition")
|
||||
"getAsset" (entity-response "AssetResponse" "Asset metadata and a temporary URL")
|
||||
{:description "Success"})
|
||||
"400" {:description "Invalid request"}
|
||||
"403" {:description "Forbidden"}
|
||||
"409" {:description "Unavailable for E2EE graphs"}
|
||||
"429" {:description "Rate limit exceeded"}}
|
||||
(case operation-id
|
||||
"createPage" {"201" (entity-response "PageResponse" "The created page")}
|
||||
"insertBlockChildren" {"201" {:description "The inserted block trees"}}
|
||||
"insertBlockTree" {"201" {:description "The inserted block trees"}}
|
||||
"createTag" {"201" (entity-response "TagResponse" "The created tag")}
|
||||
"createProperty" {"201" (entity-response "PropertyResponse" "The created property definition")}
|
||||
"createTask"
|
||||
{"201" (entity-response "TaskResponse" "The created DB Task object")}
|
||||
"createAsset"
|
||||
{"201" (entity-response "AssetResponse" "The created asset block")}
|
||||
{})))
|
||||
|
||||
(defn openapi-document [issuer]
|
||||
{:openapi "3.1.0"
|
||||
:info {:title "Logseq Server API" :version "1.0.0"}
|
||||
:servers [{:url "/"}]
|
||||
:components
|
||||
{:schemas (ordered-map
|
||||
:BlockTree {:type "object" :required ["title"]
|
||||
:properties {:title {:type "string"}
|
||||
:children {:type "array" :items {:$ref "#/components/schemas/BlockTree"}}}}
|
||||
:BlockResponse {:type "object" :required ["uuid" "kind" "title"]
|
||||
:properties {:uuid {:type "string"}
|
||||
:kind {:type "string"}
|
||||
:title {:type "string"}
|
||||
:order {:type "string"}
|
||||
:parent-id {:type "string"}
|
||||
:page-id {:type "string"}
|
||||
:children {:type "array"
|
||||
:items {:$ref "#/components/schemas/BlockResponse"}}}}
|
||||
:PageResponse {:allOf [{:$ref "#/components/schemas/BlockResponse"}]}
|
||||
:TagResponse {:type "object" :required ["uuid" "title"]
|
||||
:properties {:uuid {:type "string"}
|
||||
:title {:type "string"}
|
||||
:ident {:type "string"}}}
|
||||
:SearchResultResponse {:type "object" :required ["uuid" "title" "resource"]
|
||||
:properties {:uuid {:type "string"}
|
||||
:title {:type "string"}
|
||||
:resource {:enum ["blocks" "tags" "properties" "assets"]}}}
|
||||
:EntitySelector {:type "string"
|
||||
:description "An existing entity UUID, qualified ident, or exact title."}
|
||||
:TaskStatusSelector
|
||||
{:description "A built-in status alias or any user-extensible Status property choice selected by UUID, ident, or exact title."
|
||||
:anyOf [{:type "string" :enum ["todo" "doing" "in-review" "done" "canceled" "backlog"]}
|
||||
{:$ref "#/components/schemas/EntitySelector"}]}
|
||||
:TaskPrioritySelector
|
||||
{:description "A built-in priority alias or any user-extensible Priority property choice selected by UUID, ident, or exact title."
|
||||
:anyOf [{:type "string" :enum ["low" "medium" "high" "urgent"]}
|
||||
{:$ref "#/components/schemas/EntitySelector"}]}
|
||||
:PropertyChoice {:type "object" :required ["uuid" "title"]
|
||||
:properties {:uuid {:type "string"}
|
||||
:ident {:type "string"}
|
||||
:title {:type "string"}
|
||||
:value {}}}
|
||||
:PropertyResponse {:type "object" :required ["uuid" "title" "type" "cardinality"]
|
||||
:properties {:uuid {:type "string"}
|
||||
:ident {:type "string"}
|
||||
:title {:type "string"}
|
||||
:type {:type "string"}
|
||||
:cardinality {:type "string"}
|
||||
:choices {:type "array"
|
||||
:description "The current user-extensible choices for this property."
|
||||
:items {:$ref "#/components/schemas/PropertyChoice"}}}}
|
||||
:PropertyType {:type "string"
|
||||
:enum ["default" "number" "date" "datetime" "checkbox" "url" "node" "asset"
|
||||
"map" "json" "string"]}
|
||||
:AssetResponse {:type "object" :required ["uuid" "title" "type" "size" "checksum"]
|
||||
:properties {:uuid {:type "string"}
|
||||
:title {:type "string"}
|
||||
:type {:type "string"}
|
||||
:size {:type "integer" :minimum 0 :maximum 104857600}
|
||||
:checksum {:type "string" :pattern "^[0-9a-f]{64}$"}
|
||||
:url {:type "string" :format "uri"}
|
||||
:expires-at {:type "integer"}}}
|
||||
:TaskResponse {:allOf [{:$ref "#/components/schemas/BlockResponse"}
|
||||
{:type "object"
|
||||
:required ["status"]
|
||||
:properties {:status {:$ref "#/components/schemas/PropertyChoice"}
|
||||
:priority {:$ref "#/components/schemas/PropertyChoice"}
|
||||
:scheduled {:type "integer"}
|
||||
:deadline {:type "integer"}}}]}
|
||||
:PropertyValue
|
||||
{:description "For ref properties, pass an existing entity by UUID, ident, or title. Status and Priority also accept case-insensitive built-in aliases such as TODO, DONE, and HIGH; user-defined choices use UUID, ident, or exact title. Class properties such as Tags accept a class UUID, ident, or title, or an array of those selectors. Scalar properties accept their JSON scalar value."
|
||||
:oneOf [{:type "string"} {:type "number"} {:type "boolean"}
|
||||
{:type "array" :items {:oneOf [{:type "string"} {:type "number"}]}}]})
|
||||
:securitySchemes
|
||||
{:oauth {:type "oauth2"
|
||||
:flows {:authorizationCode
|
||||
{:authorizationUrl (str issuer "/oauth2/authorize")
|
||||
:tokenUrl (str issuer "/oauth2/token")
|
||||
:scopes {:logseq/read "Read pages and blocks"
|
||||
:logseq/write "Create and edit pages and blocks"}}}}}}
|
||||
:paths
|
||||
(reduce (fn [paths {:keys [method path operation-id scope] :as operation}]
|
||||
(let [[summary description] (get operation-docs operation-id)]
|
||||
(assoc-in paths [(openapi-path path) (keyword (.toLowerCase method))]
|
||||
(cond-> {:operationId operation-id
|
||||
:summary summary
|
||||
:description description
|
||||
:parameters (operation-parameters operation)
|
||||
:security [{:oauth [scope]}]
|
||||
:responses (operation-responses operation-id)}
|
||||
(request-body operation-id)
|
||||
(assoc :requestBody (request-body operation-id))))))
|
||||
(ordered-map)
|
||||
operations)})
|
||||
20
deps/db-sync/src/logseq/db_sync/worker/routes/sync.cljs
vendored
Normal file
20
deps/db-sync/src/logseq/db_sync/worker/routes/sync.cljs
vendored
Normal file
@@ -0,0 +1,20 @@
|
||||
(ns logseq.db-sync.worker.routes.sync
|
||||
(:require [reitit.core :as r]))
|
||||
|
||||
(def ^:private route-data
|
||||
[["/health" {:methods {"GET" :sync/health}}]
|
||||
["/pull" {:methods {"GET" :sync/pull}}]
|
||||
["/checksum/diagnostics" {:methods {"GET" :sync/checksum-diagnostics}}]
|
||||
["/snapshot/download" {:methods {"GET" :sync/snapshot-download}}]
|
||||
["/snapshot/stream" {:methods {"GET" :sync/snapshot-stream}}]
|
||||
["/admin/reset" {:methods {"DELETE" :sync/admin-reset}}]
|
||||
["/tx/batch" {:methods {"POST" :sync/tx-batch}}]
|
||||
["/snapshot/upload" {:methods {"POST" :sync/snapshot-upload}}]])
|
||||
|
||||
(def ^:private router
|
||||
(r/router route-data))
|
||||
|
||||
(defn match-route [method path]
|
||||
(when-let [match (r/match-by-path router path)]
|
||||
(when-let [handler (get-in match [:data :methods method])]
|
||||
(assoc match :handler handler))))
|
||||
20
deps/db-sync/src/logseq/db_sync/worker/timing.cljs
vendored
Normal file
20
deps/db-sync/src/logseq/db_sync/worker/timing.cljs
vendored
Normal file
@@ -0,0 +1,20 @@
|
||||
(ns logseq.db-sync.worker.timing)
|
||||
|
||||
(defn summary
|
||||
"Returns duration summary from `start-ms` to `end-ms` with ordered `steps`.
|
||||
|
||||
`steps` is a vector of `[label ms]` pairs where each `ms` is an absolute
|
||||
timestamp in milliseconds."
|
||||
[start-ms steps end-ms]
|
||||
(let [steps (or steps [])
|
||||
result (reduce (fn [{:keys [prev entries]} [label ms]]
|
||||
{:prev ms
|
||||
:entries (conj entries {:label label
|
||||
:ms (- ms prev)})})
|
||||
{:prev start-ms :entries []}
|
||||
steps)
|
||||
last-ms (:prev result)
|
||||
tail-ms (- end-ms last-ms)]
|
||||
{:total-ms (- end-ms start-ms)
|
||||
:steps (:entries result)
|
||||
:tail-ms tail-ms}))
|
||||
36
deps/db-sync/src/logseq/db_sync/worker/ws.cljs
vendored
Normal file
36
deps/db-sync/src/logseq/db_sync/worker/ws.cljs
vendored
Normal file
@@ -0,0 +1,36 @@
|
||||
(ns logseq.db-sync.worker.ws
|
||||
(:require [lambdaisland.glogi :as log]
|
||||
[logseq.db-sync.malli-schema :as db-sync-schema]
|
||||
[logseq.db-sync.protocol :as protocol]
|
||||
[logseq.db-sync.worker.coerce :as coerce]))
|
||||
|
||||
(defn ws-open? [ws]
|
||||
(= 1 (.-readyState ws)))
|
||||
|
||||
(defn coerce-ws-client-message [message]
|
||||
(when message
|
||||
(let [coerced (coerce/coerce db-sync-schema/ws-client-message-coercer message {:schema :ws/client})]
|
||||
(when-not (= coerced coerce/invalid-coerce)
|
||||
coerced))))
|
||||
|
||||
(defn coerce-ws-server-message [message]
|
||||
(when message
|
||||
(let [coerced (coerce/coerce db-sync-schema/ws-server-message-coercer message {:schema :ws/server})]
|
||||
(when-not (= coerced coerce/invalid-coerce)
|
||||
coerced))))
|
||||
|
||||
(defn send! [ws msg]
|
||||
(when (ws-open? ws)
|
||||
(if-let [coerced (coerce-ws-server-message msg)]
|
||||
(.send ws (protocol/encode-message coerced))
|
||||
(do
|
||||
(log/error :db-sync/ws-response-invalid {:message msg})
|
||||
(.send ws (protocol/encode-message {:type "error" :message "server error"}))))))
|
||||
|
||||
(defn broadcast! [^js self sender msg]
|
||||
(when-let [state (some-> self .-state)]
|
||||
(when (fn? (.-getWebSockets state))
|
||||
(let [clients (.getWebSockets state)]
|
||||
(doseq [ws clients]
|
||||
(when (and (not= ws sender) (ws-open? ws))
|
||||
(send! ws msg)))))))
|
||||
17
deps/db-sync/start.sh
vendored
Executable file
17
deps/db-sync/start.sh
vendored
Executable file
@@ -0,0 +1,17 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
: "${DB_SYNC_PORT:=8787}"
|
||||
|
||||
# Defaults match the local `pnpm watch` app auth config.
|
||||
# Override these env vars for production pool values if needed.
|
||||
: "${COGNITO_ISSUER:=https://cognito-idp.us-east-1.amazonaws.com/us-east-1_dtagLnju8}"
|
||||
: "${COGNITO_CLIENT_ID:=69cs1lgme7p8kbgld8n5kseii6}"
|
||||
: "${COGNITO_JWKS_URL:=https://cognito-idp.us-east-1.amazonaws.com/us-east-1_dtagLnju8/.well-known/jwks.json}"
|
||||
|
||||
export DB_SYNC_PORT
|
||||
export COGNITO_ISSUER
|
||||
export COGNITO_CLIENT_ID
|
||||
export COGNITO_JWKS_URL
|
||||
|
||||
node worker/dist/node-adapter.js
|
||||
17
deps/db-sync/test/logseq/db_sync/batch_test.cljs
vendored
Normal file
17
deps/db-sync/test/logseq/db_sync/batch_test.cljs
vendored
Normal file
@@ -0,0 +1,17 @@
|
||||
(ns logseq.db-sync.batch-test
|
||||
(:require [cljs.test :refer [deftest is testing]]
|
||||
[logseq.db-sync.batch :as batch]))
|
||||
|
||||
(deftest rows->insert-batches-test
|
||||
(testing "splits rows into batches based on max params"
|
||||
(let [rows [[1 "a" nil]
|
||||
[2 "b" nil]
|
||||
[3 "c" nil]]
|
||||
batches (batch/rows->insert-batches "kvs_import" rows {:max-params 6})]
|
||||
(is (= 2 (count batches)))
|
||||
(is (= [1 "a" nil 2 "b" nil] (:args (first batches))))
|
||||
(is (= [3 "c" nil] (:args (second batches))))
|
||||
(is (string? (:sql (first batches))))
|
||||
(is (string? (:sql (second batches))))))
|
||||
(testing "empty rows returns no batches"
|
||||
(is (= [] (batch/rows->insert-batches "kvs_import" [] {:max-params 6})))))
|
||||
547
deps/db-sync/test/logseq/db_sync/checksum_test.cljs
vendored
Normal file
547
deps/db-sync/test/logseq/db_sync/checksum_test.cljs
vendored
Normal file
@@ -0,0 +1,547 @@
|
||||
(ns logseq.db-sync.checksum-test
|
||||
(:require ["fs" :as fs]
|
||||
[cljs.reader :as reader]
|
||||
[cljs.test :refer [deftest is testing]]
|
||||
[datascript.core :as d]
|
||||
[logseq.db :as ldb]
|
||||
[logseq.db-sync.checksum :as checksum]
|
||||
[logseq.db.frontend.schema :as db-schema]))
|
||||
|
||||
(defn- sample-db
|
||||
[]
|
||||
(let [page-a-uuid (random-uuid)
|
||||
page-b-uuid (random-uuid)
|
||||
parent-uuid (random-uuid)
|
||||
child-uuid (random-uuid)]
|
||||
(-> (d/empty-db db-schema/schema)
|
||||
(d/db-with [{:db/id 1
|
||||
:block/uuid page-a-uuid
|
||||
:block/name "page-a"
|
||||
:block/title "Page A"}
|
||||
{:db/id 2
|
||||
:block/uuid page-b-uuid
|
||||
:block/name "page-b"
|
||||
:block/title "Page B"}
|
||||
{:db/id 3
|
||||
:block/uuid parent-uuid
|
||||
:block/title "Parent"
|
||||
:block/parent 1
|
||||
:block/page 1}
|
||||
{:db/id 4
|
||||
:block/uuid child-uuid
|
||||
:block/title "Child"
|
||||
:block/parent 3
|
||||
:block/page 1}]))))
|
||||
|
||||
(defn- assert-incremental=full!
|
||||
[db-before checksum-before tx-data]
|
||||
(let [tx-report (d/with db-before tx-data)
|
||||
full (checksum/recompute-checksum (:db-after tx-report))
|
||||
incremental (checksum/update-checksum checksum-before tx-report)]
|
||||
(is (= full incremental)
|
||||
(str "Expected checksum parity for tx-data: " (pr-str tx-data)))
|
||||
{:db (:db-after tx-report)
|
||||
:checksum incremental}))
|
||||
|
||||
(defn- load-rebased-retract-checksum-fixture
|
||||
[]
|
||||
(let [payload (-> (.readFileSync fs "test/logseq/db_sync/fixtures/rebased_retract_checksum_payload.edn" "utf8")
|
||||
reader/read-string)]
|
||||
{:db-before (ldb/read-transit-str (:db-before payload))
|
||||
:db-after (ldb/read-transit-str (:db-after payload))
|
||||
:tx-data (ldb/read-transit-str (:tx-data payload))}))
|
||||
|
||||
(defn- load-parent-order-rebase-checksum-fixture
|
||||
[]
|
||||
(let [payload (-> (.readFileSync fs "test/logseq/db_sync/fixtures/parent_order_rebase_checksum_payload.edn" "utf8")
|
||||
reader/read-string)]
|
||||
{:db-before (ldb/read-transit-str (:db-before payload))
|
||||
:db-after (ldb/read-transit-str (:db-after payload))
|
||||
:tx-data (ldb/read-transit-str (:tx-data payload))}))
|
||||
|
||||
(deftest checksum-ignores-unrelated-datoms-test
|
||||
(testing "recompute and incremental checksums ignore unrelated datoms"
|
||||
(let [db-before (sample-db)
|
||||
checksum-before (checksum/recompute-checksum db-before)
|
||||
tx-data [[:db/add 4 :block/updated-at 1773661308002]
|
||||
[:db/add 4 :logseq.property/created-by-ref 99]]
|
||||
tx-report (d/with db-before tx-data)]
|
||||
(is (= checksum-before
|
||||
(checksum/recompute-checksum (:db-after tx-report))))
|
||||
(is (= checksum-before
|
||||
(checksum/update-checksum checksum-before tx-report))))))
|
||||
|
||||
(deftest incremental-checksum-matches-recompute-on-rebased-retract-entity-log-repro-test
|
||||
(testing "incremental checksum should equal full recompute on rebased retract-entity replay payload"
|
||||
(let [{:keys [db-before db-after tx-data]} (load-rebased-retract-checksum-fixture)
|
||||
checksum-before (checksum/recompute-checksum db-before)
|
||||
tx-report {:db-before db-before
|
||||
:db-after db-after
|
||||
:tx-data tx-data}
|
||||
full (checksum/recompute-checksum db-after)
|
||||
incremental (checksum/update-checksum checksum-before tx-report)]
|
||||
(is (not= checksum-before full))
|
||||
(is (= full incremental)))))
|
||||
|
||||
(deftest incremental-checksum-matches-recompute-on-parent-order-rebase-log-repro-test
|
||||
(testing "incremental checksum should equal full recompute on parent/order rebase payload"
|
||||
(let [{:keys [db-before db-after tx-data]} (load-parent-order-rebase-checksum-fixture)
|
||||
checksum-before (checksum/recompute-checksum db-before)
|
||||
tx-report {:db-before db-before
|
||||
:db-after db-after
|
||||
:tx-data tx-data}
|
||||
full (checksum/recompute-checksum db-after)
|
||||
incremental (checksum/update-checksum checksum-before tx-report)]
|
||||
(is (= full incremental)))))
|
||||
|
||||
(deftest incremental-checksum-matches-recompute-on-replace-datom-test
|
||||
(testing "incremental checksum matches full recompute when replacing existing values"
|
||||
(let [db-before (sample-db)
|
||||
tx-report (d/with db-before [[:db/add 4 :block/title "Child updated"]
|
||||
[:db/add 1 :block/name "page-a-updated"]])]
|
||||
(is (= (checksum/recompute-checksum (:db-after tx-report))
|
||||
(checksum/update-checksum (checksum/recompute-checksum db-before) tx-report))))))
|
||||
|
||||
(deftest incremental-checksum-matches-recompute-across-mixed-mutations-test
|
||||
(testing "incremental checksum stays equal to full recompute across typical tx sequences"
|
||||
(let [db0 (sample-db)
|
||||
new-block-uuid (random-uuid)
|
||||
{:keys [db checksum]} (reduce
|
||||
(fn [{:keys [db checksum]} {:keys [tx-data]}]
|
||||
(assert-incremental=full! db checksum tx-data))
|
||||
{:db db0
|
||||
:checksum (checksum/recompute-checksum db0)}
|
||||
[{:tx-data [[:db/add 4 :block/title "Child edited"]]}
|
||||
{:tx-data [[:db/add 1 :block/name "page-a-renamed"]
|
||||
[:db/add 1 :block/title "Page A Renamed"]]}
|
||||
{:tx-data [[:db/add 4 :block/parent 2]
|
||||
[:db/add 4 :block/page 2]]}
|
||||
{:tx-data [[:db/add -1 :block/uuid new-block-uuid]
|
||||
[:db/add -1 :block/title "New block"]
|
||||
[:db/add -1 :block/parent 2]
|
||||
[:db/add -1 :block/page 2]]}
|
||||
{:tx-data [[:db/retract 3 :block/title "Parent"]]}
|
||||
{:tx-data [[:db/retractEntity [:block/uuid new-block-uuid]]]}
|
||||
{:tx-data [[:db/add 4 :block/updated-at 1773661308002]]}])]
|
||||
(is (= checksum (checksum/recompute-checksum db))))))
|
||||
|
||||
(deftest incremental-checksum-uses-recompute-when-initial-checksum-missing-test
|
||||
(testing "nil initial checksum uses db-before recompute as baseline"
|
||||
(let [db-before (sample-db)
|
||||
tx-report (d/with db-before [[:db/add 4 :block/title "Child updated"]])]
|
||||
(is (= (checksum/recompute-checksum (:db-after tx-report))
|
||||
(checksum/update-checksum nil tx-report))))))
|
||||
|
||||
(deftest checksum-e2ee-ignores-title-and-name-test
|
||||
(testing "with E2EE enabled, checksum ignores title/name changes for both modes"
|
||||
(let [db-before (-> (sample-db)
|
||||
(d/db-with [{:db/ident :logseq.kv/graph-rtc-e2ee?
|
||||
:kv/value true}]))
|
||||
checksum-before (checksum/recompute-checksum db-before)
|
||||
tx-report (d/with db-before [[:db/add 4 :block/title "Encrypted title update"]
|
||||
[:db/add 1 :block/name "encrypted-name-update"]])]
|
||||
(is (= checksum-before
|
||||
(checksum/recompute-checksum (:db-after tx-report))))
|
||||
(is (= checksum-before
|
||||
(checksum/update-checksum checksum-before tx-report))))))
|
||||
|
||||
(deftest incremental-checksum-recomputes-when-e2ee-mode-toggles-test
|
||||
(testing "incremental checksum falls back to full recompute when E2EE mode changes"
|
||||
(let [db-before (sample-db)
|
||||
tx-report (d/with db-before [{:db/ident :logseq.kv/graph-rtc-e2ee?
|
||||
:kv/value true}])]
|
||||
(is (= (checksum/recompute-checksum (:db-after tx-report))
|
||||
(checksum/update-checksum (checksum/recompute-checksum db-before) tx-report))))))
|
||||
|
||||
(deftest incremental-checksum-matches-recompute-when-referenced-entity-disappears-test
|
||||
(testing "incremental checksum tracks blocks whose parent/page UUID becomes unresolved after retracting referenced entities"
|
||||
(let [db-before (sample-db)
|
||||
before-checksum (checksum/recompute-checksum db-before)
|
||||
tx-report (d/with db-before [[:db/retractEntity 3]
|
||||
[:db/retractEntity 1]])
|
||||
db-after (:db-after tx-report)
|
||||
full (checksum/recompute-checksum db-after)
|
||||
incremental (checksum/update-checksum before-checksum tx-report)]
|
||||
(is (not= before-checksum full))
|
||||
(is (= full incremental)))))
|
||||
|
||||
(deftest incremental-checksum-matches-recompute-when-parent-uuid-changes-test
|
||||
(testing "incremental checksum tracks children whose normalized parent UUID changes via parent :block/uuid update"
|
||||
(let [db-before (sample-db)
|
||||
before-checksum (checksum/recompute-checksum db-before)
|
||||
tx-report (d/with db-before [[:db/add 3 :block/uuid (random-uuid)]])
|
||||
db-after (:db-after tx-report)
|
||||
full (checksum/recompute-checksum db-after)
|
||||
incremental (checksum/update-checksum before-checksum tx-report)]
|
||||
(is (not= before-checksum full))
|
||||
(is (= full incremental)))))
|
||||
|
||||
(deftest incremental-checksum-matches-recompute-when-page-uuid-changes-test
|
||||
(testing "incremental checksum tracks blocks whose normalized page UUID changes via page :block/uuid update"
|
||||
(let [db-before (sample-db)
|
||||
before-checksum (checksum/recompute-checksum db-before)
|
||||
tx-report (d/with db-before [[:db/add 1 :block/uuid (random-uuid)]])
|
||||
db-after (:db-after tx-report)
|
||||
full (checksum/recompute-checksum db-after)
|
||||
incremental (checksum/update-checksum before-checksum tx-report)]
|
||||
(is (not= before-checksum full))
|
||||
(is (= full incremental)))))
|
||||
|
||||
(deftest incremental-checksum-matches-recompute-when-parent-uuid-changes-on-ineligible-target-test
|
||||
(testing "incremental checksum tracks children when parent UUID changes on a non-checksum-eligible target entity"
|
||||
(let [db-before (-> (d/empty-db db-schema/schema)
|
||||
(d/db-with [{:db/id 1
|
||||
:block/uuid (random-uuid)
|
||||
:block/name "page-a"
|
||||
:block/title "Page A"}
|
||||
{:db/id 2
|
||||
:block/uuid (random-uuid)
|
||||
:block/title "Orphan parent"}
|
||||
{:db/id 3
|
||||
:block/uuid (random-uuid)
|
||||
:block/title "Child"
|
||||
:block/parent 2
|
||||
:block/page 1}]))
|
||||
before-checksum (checksum/recompute-checksum db-before)
|
||||
tx-report (d/with db-before [[:db/add 2 :block/uuid (random-uuid)]])
|
||||
db-after (:db-after tx-report)
|
||||
full (checksum/recompute-checksum db-after)
|
||||
incremental (checksum/update-checksum before-checksum tx-report)]
|
||||
(is (not= before-checksum full))
|
||||
(is (= full incremental)))))
|
||||
|
||||
(deftest incremental-checksum-matches-recompute-when-ineligible-referenced-entity-is-retracted-test
|
||||
(testing "incremental checksum tracks children when normalized tx data retracts a referenced non-checksum-eligible entity"
|
||||
(let [parent-uuid (random-uuid)
|
||||
child-uuid (random-uuid)
|
||||
page-uuid (random-uuid)
|
||||
db-before (-> (d/empty-db db-schema/schema)
|
||||
(d/db-with [{:db/id 1
|
||||
:block/uuid page-uuid
|
||||
:block/name "page-a"
|
||||
:block/title "Page A"}
|
||||
{:db/id 2
|
||||
:block/uuid parent-uuid
|
||||
:block/title "Orphan parent"}
|
||||
{:db/id 3
|
||||
:block/uuid child-uuid
|
||||
:block/title "Child"
|
||||
:block/parent 2
|
||||
:block/page 1}]))
|
||||
before-checksum (checksum/recompute-checksum db-before)
|
||||
tx-report* (d/with db-before [[:db/retractEntity [:block/uuid parent-uuid]]])
|
||||
tx-report (assoc tx-report*
|
||||
:tx-data [[:db/retractEntity [:block/uuid parent-uuid]]])
|
||||
full (checksum/recompute-checksum (:db-after tx-report))
|
||||
incremental (checksum/update-checksum before-checksum tx-report)]
|
||||
(is (not= before-checksum full))
|
||||
(is (= full incremental)))))
|
||||
|
||||
(deftest incremental-checksum-matches-recompute-when-block-is-readded-test
|
||||
(testing "incremental checksum remains equal to recompute when a block is deleted and re-added with the same UUID"
|
||||
(let [db0 (sample-db)
|
||||
checksum0 (checksum/recompute-checksum db0)
|
||||
child-uuid (:block/uuid (d/entity db0 4))
|
||||
parent-uuid (:block/uuid (d/entity db0 3))
|
||||
page-uuid (:block/uuid (d/entity db0 1))
|
||||
{:keys [db checksum]} (assert-incremental=full! db0 checksum0 [[:db/retractEntity [:block/uuid child-uuid]]])]
|
||||
(assert-incremental=full! db checksum [{:db/id -1
|
||||
:block/uuid child-uuid
|
||||
:block/title "Child"
|
||||
:block/parent [:block/uuid parent-uuid]
|
||||
:block/page [:block/uuid page-uuid]}]))))
|
||||
|
||||
(deftest incremental-checksum-matches-recompute-when-block-is-readded-with-uuid-tempid-test
|
||||
(testing "server rebase txs can delete and recreate the same block using a UUID string tempid"
|
||||
(let [db0 (sample-db)
|
||||
checksum0 (checksum/recompute-checksum db0)
|
||||
block-uuid (random-uuid)
|
||||
block-tempid (str block-uuid)
|
||||
page-uuid (:block/uuid (d/entity db0 1))
|
||||
parent-uuid (:block/uuid (d/entity db0 3))
|
||||
tx-data [[:db/add block-tempid :block/uuid block-uuid 536871536]
|
||||
[:db/add block-tempid :block/title "undo redo block" 536871536]
|
||||
[:db/add block-tempid :block/parent [:block/uuid parent-uuid] 536871536]
|
||||
[:db/add block-tempid :block/order "a0" 536871536]
|
||||
[:db/add block-tempid :block/page [:block/uuid page-uuid] 536871536]]
|
||||
{:keys [db checksum]} (assert-incremental=full! db0 checksum0 tx-data)
|
||||
{:keys [db checksum]} (assert-incremental=full! db checksum [[:db/retractEntity [:block/uuid block-uuid]]])]
|
||||
(assert-incremental=full!
|
||||
db
|
||||
checksum
|
||||
[[:db/add block-tempid :block/uuid block-uuid 536871538]
|
||||
[:db/add block-tempid :block/title "undo redo block" 536871538]
|
||||
[:db/add block-tempid :block/parent [:block/uuid parent-uuid] 536871538]
|
||||
[:db/add block-tempid :block/order "a0" 536871538]
|
||||
[:db/add block-tempid :block/page [:block/uuid page-uuid] 536871538]]))))
|
||||
|
||||
(deftest incremental-checksum-matches-recompute-with-normalized-vector-tx-data-test
|
||||
(testing "incremental checksum tracks normalized tx vectors persisted in server tx_log"
|
||||
(let [db-before (sample-db)
|
||||
checksum-before (checksum/recompute-checksum db-before)
|
||||
block-uuid (random-uuid)
|
||||
block-tempid (str block-uuid)
|
||||
parent-uuid (:block/uuid (d/entity db-before 3))
|
||||
page-uuid (:block/uuid (d/entity db-before 1))
|
||||
normalized-tx-data [[:db/add block-tempid :block/uuid block-uuid 536871538]
|
||||
[:db/add block-tempid :block/title "normalized block" 536871538]
|
||||
[:db/add block-tempid :block/parent [:block/uuid parent-uuid] 536871538]
|
||||
[:db/add block-tempid :block/order "a0" 536871538]
|
||||
[:db/add block-tempid :block/page [:block/uuid page-uuid] 536871538]]
|
||||
tx-report (d/with db-before normalized-tx-data)
|
||||
full (checksum/recompute-checksum (:db-after tx-report))
|
||||
incremental (checksum/update-checksum
|
||||
checksum-before
|
||||
(assoc tx-report :tx-data normalized-tx-data))]
|
||||
(is (not= checksum-before full))
|
||||
(is (= full incremental)))))
|
||||
|
||||
(deftest incremental-checksum-matches-recompute-when-delete-tree-undo-and-delete-again-test
|
||||
(testing "incremental checksum matches recompute across delete-tree, undo-all, then delete-tree-again"
|
||||
(let [db0 (sample-db)
|
||||
parent-uuid (:block/uuid (d/entity db0 3))
|
||||
child-uuid (:block/uuid (d/entity db0 4))
|
||||
page-uuid (:block/uuid (d/entity db0 1))
|
||||
tx-seq [{:tx-data [[:db/retractEntity [:block/uuid child-uuid]]
|
||||
[:db/retractEntity [:block/uuid parent-uuid]]]}
|
||||
{:tx-data [{:db/id -1
|
||||
:block/uuid parent-uuid
|
||||
:block/title "Parent"
|
||||
:block/parent [:block/uuid page-uuid]
|
||||
:block/page [:block/uuid page-uuid]}
|
||||
{:db/id -2
|
||||
:block/uuid child-uuid
|
||||
:block/title "Child"
|
||||
:block/parent [:block/uuid parent-uuid]
|
||||
:block/page [:block/uuid page-uuid]}]}
|
||||
{:tx-data [[:db/retractEntity [:block/uuid child-uuid]]
|
||||
[:db/retractEntity [:block/uuid parent-uuid]]]}]
|
||||
{:keys [db checksum]} (reduce
|
||||
(fn [{:keys [db checksum]} {:keys [tx-data]}]
|
||||
(assert-incremental=full! db checksum tx-data))
|
||||
{:db db0
|
||||
:checksum (checksum/recompute-checksum db0)}
|
||||
tx-seq)]
|
||||
(is (= checksum (checksum/recompute-checksum db))))))
|
||||
|
||||
(deftest recompute-checksum-diagnostics-includes-relevant-attrs-test
|
||||
(testing "diagnostics includes checksum attrs and block values used for checksum export"
|
||||
(let [db (sample-db)
|
||||
{:keys [checksum attrs blocks e2ee?]} (checksum/recompute-checksum-diagnostics db)
|
||||
child-uuid (:block/uuid (d/entity db 4))
|
||||
child-parent-uuid (:block/uuid (:block/parent (d/entity db 4)))
|
||||
child-page-uuid (:block/uuid (:block/page (d/entity db 4)))
|
||||
child (some #(when (= child-uuid (:block/uuid %)) %) blocks)]
|
||||
(is (false? e2ee?))
|
||||
(is (= (checksum/recompute-checksum db) checksum))
|
||||
(is (= #{:block/uuid :block/title :block/name :block/parent :block/page :block/order}
|
||||
(set attrs)))
|
||||
(is (= 4 (count blocks)))
|
||||
(is (= child-parent-uuid (:block/parent child)))
|
||||
(is (= child-page-uuid (:block/page child)))
|
||||
(is (string? (:block/title child))))))
|
||||
|
||||
(deftest recompute-checksum-diagnostics-omits-title-and-name-in-e2ee-test
|
||||
(testing "diagnostics for E2EE graphs omits title/name from checksum attrs and export blocks"
|
||||
(let [db (-> (sample-db)
|
||||
(d/db-with [{:db/ident :logseq.kv/graph-rtc-e2ee?
|
||||
:kv/value true}]))
|
||||
{:keys [checksum attrs blocks e2ee?]} (checksum/recompute-checksum-diagnostics db)]
|
||||
(is e2ee?)
|
||||
(is (= (checksum/recompute-checksum db) checksum))
|
||||
(is (= #{:block/uuid :block/parent :block/page :block/order}
|
||||
(set attrs)))
|
||||
(is (every? #(not (contains? % :block/title)) blocks))
|
||||
(is (every? #(not (contains? % :block/name)) blocks)))))
|
||||
|
||||
(deftest incremental-checksum-is-invariant-across-tx-partitioning-test
|
||||
(testing "incremental checksum converges to the same value regardless of tx partitioning"
|
||||
(let [db0 (sample-db)
|
||||
tx-a [[:db/add 4 :block/order "aBL"]
|
||||
[:db/add 4 :block/title "Child v2"]]
|
||||
tx-b [[:db/add 3 :block/order "aBK"]
|
||||
[:db/add 4 :block/parent 2]
|
||||
[:db/add 4 :block/page 2]]
|
||||
one-shot-report (d/with db0 (into tx-a tx-b))
|
||||
one-shot-checksum (checksum/update-checksum (checksum/recompute-checksum db0)
|
||||
one-shot-report)
|
||||
checksum0 (checksum/recompute-checksum db0)
|
||||
report-a (d/with db0 tx-a)
|
||||
checksum-a (checksum/update-checksum checksum0 report-a)
|
||||
db-a (:db-after report-a)
|
||||
report-b (d/with db-a tx-b)
|
||||
checksum-b (checksum/update-checksum checksum-a report-b)
|
||||
db-final (:db-after report-b)
|
||||
full-final (checksum/recompute-checksum db-final)]
|
||||
(is (= full-final one-shot-checksum))
|
||||
(is (= full-final checksum-b))
|
||||
(is (= one-shot-checksum checksum-b)))))
|
||||
|
||||
(deftest incremental-checksum-is-invariant-across-commuting-batch-order-test
|
||||
(testing "incremental checksum converges when commuting tx batches are applied in different order"
|
||||
(let [db0 (sample-db)
|
||||
checksum0 (checksum/recompute-checksum db0)
|
||||
tx-a [[:db/add 3 :block/title "Parent v2"]
|
||||
[:db/add 4 :block/order "a9"]]
|
||||
tx-b [[:db/add 1 :block/name "page-a-v2"]
|
||||
[:db/add 2 :block/title "Page B v2"]]
|
||||
report-a (d/with db0 tx-a)
|
||||
checksum-a (checksum/update-checksum checksum0 report-a)
|
||||
db-a (:db-after report-a)
|
||||
report-b-after-a (d/with db-a tx-b)
|
||||
checksum-ab (checksum/update-checksum checksum-a report-b-after-a)
|
||||
db-ab (:db-after report-b-after-a)
|
||||
full-ab (checksum/recompute-checksum db-ab)
|
||||
|
||||
report-b (d/with db0 tx-b)
|
||||
checksum-b (checksum/update-checksum checksum0 report-b)
|
||||
db-b (:db-after report-b)
|
||||
report-a-after-b (d/with db-b tx-a)
|
||||
checksum-ba (checksum/update-checksum checksum-b report-a-after-b)
|
||||
db-ba (:db-after report-a-after-b)
|
||||
full-ba (checksum/recompute-checksum db-ba)]
|
||||
(is (= full-ab full-ba))
|
||||
(is (= full-ab checksum-ab))
|
||||
(is (= full-ba checksum-ba))
|
||||
(is (= checksum-ab checksum-ba)))))
|
||||
|
||||
(deftest incremental-checksum-is-invariant-across-intra-batch-datom-order-test
|
||||
(testing "incremental checksum converges when same tx-data uses different datom order"
|
||||
(let [db0 (sample-db)
|
||||
checksum0 (checksum/recompute-checksum db0)
|
||||
uuid-a (random-uuid)
|
||||
uuid-b (random-uuid)
|
||||
tx-order-a [{:db/id -1
|
||||
:block/uuid uuid-a
|
||||
:block/title "Inserted A"
|
||||
:block/order "a5"
|
||||
:block/parent 3
|
||||
:block/page 1}
|
||||
{:db/id -2
|
||||
:block/uuid uuid-b
|
||||
:block/title "Inserted B"
|
||||
:block/order "a6"
|
||||
:block/parent 3
|
||||
:block/page 1}]
|
||||
tx-order-b (vec (reverse tx-order-a))
|
||||
report-a (d/with db0 tx-order-a)
|
||||
db-a (:db-after report-a)
|
||||
full-a (checksum/recompute-checksum db-a)
|
||||
incremental-a (checksum/update-checksum checksum0 report-a)
|
||||
report-b (d/with db0 tx-order-b)
|
||||
db-b (:db-after report-b)
|
||||
full-b (checksum/recompute-checksum db-b)
|
||||
incremental-b (checksum/update-checksum checksum0 report-b)]
|
||||
(is (= full-a full-b))
|
||||
(is (= full-a incremental-a))
|
||||
(is (= full-b incremental-b))
|
||||
(is (= incremental-a incremental-b)))))
|
||||
|
||||
(deftest incremental-checksum-handles-rebase-like-toggle-churn-test
|
||||
(testing "incremental checksum uses net tuple delta when batch contains add/retract/add churn"
|
||||
(let [db0 (sample-db)
|
||||
checksum0 (checksum/recompute-checksum db0)
|
||||
;; Simulate rebase churn on a tuple absent in db0: add -> retract -> add
|
||||
report-1 (d/with db0 [[:db/add 4 :block/order "aBL"]])
|
||||
db1 (:db-after report-1)
|
||||
report-2 (d/with db1 [[:db/retract 4 :block/order "aBL"]])
|
||||
db2 (:db-after report-2)
|
||||
report-3 (d/with db2 [[:db/add 4 :block/order "aBL"]])
|
||||
db3 (:db-after report-3)
|
||||
batch-report {:db-before db0
|
||||
:db-after db3
|
||||
:tx-data (vec (concat (:tx-data report-1)
|
||||
(:tx-data report-2)
|
||||
(:tx-data report-3)))}
|
||||
full-final (checksum/recompute-checksum db3)
|
||||
incremental (checksum/update-checksum checksum0 batch-report)]
|
||||
(is (not= checksum0 full-final))
|
||||
(is (= full-final incremental)))))
|
||||
|
||||
(deftest incremental-checksum-handles-transient-entity-churn-in-one-batch-test
|
||||
(testing "incremental checksum remains stable when a newly created block is retracted in the same batch"
|
||||
(let [db0 (sample-db)
|
||||
checksum0 (checksum/recompute-checksum db0)
|
||||
page-id 1
|
||||
parent-id 3
|
||||
transient-uuid (random-uuid)
|
||||
report-1 (d/with db0 [{:db/id -1
|
||||
:block/uuid transient-uuid
|
||||
:block/title "Transient"
|
||||
:block/parent parent-id
|
||||
:block/page page-id}])
|
||||
db1 (:db-after report-1)
|
||||
report-2 (d/with db1 [[:db/retractEntity [:block/uuid transient-uuid]]])
|
||||
db2 (:db-after report-2)
|
||||
batch-report {:db-before db0
|
||||
:db-after db2
|
||||
:tx-data (vec (concat (:tx-data report-1)
|
||||
(:tx-data report-2)))}
|
||||
full-final (checksum/recompute-checksum db2)
|
||||
incremental (checksum/update-checksum checksum0 batch-report)]
|
||||
(is (= checksum0 full-final))
|
||||
(is (= full-final incremental)))))
|
||||
|
||||
(deftest incremental-checksum-handles-new-entity-attr-replacement-in-one-batch-test
|
||||
(testing "incremental checksum cancels replaced attrs on entities created inside the same batch"
|
||||
(let [db0 (sample-db)
|
||||
checksum0 (checksum/recompute-checksum db0)
|
||||
page-id 1
|
||||
parent-id 3
|
||||
transient-uuid (random-uuid)
|
||||
report-1 (d/with db0 [{:db/id -1
|
||||
:block/uuid transient-uuid
|
||||
:block/title "Transient"
|
||||
:block/order "a1"
|
||||
:block/parent parent-id
|
||||
:block/page page-id}])
|
||||
db1 (:db-after report-1)
|
||||
report-2 (d/with db1 [[:db/add [:block/uuid transient-uuid] :block/order "a2"]])
|
||||
db2 (:db-after report-2)
|
||||
batch-report {:db-before db0
|
||||
:db-after db2
|
||||
:tx-data (vec (concat (:tx-data report-1)
|
||||
(:tx-data report-2)))}
|
||||
full-final (checksum/recompute-checksum db2)
|
||||
incremental (checksum/update-checksum checksum0 batch-report)]
|
||||
(is (= full-final incremental)))))
|
||||
|
||||
(deftest incremental-checksum-matches-recompute-with-duplicate-block-uuid-tx-churn-test
|
||||
(testing "incremental checksum stays equal to recompute when tx-data retracts+adds the same :block/uuid in one tx"
|
||||
(let [db0 (sample-db)
|
||||
shared-uuid (random-uuid)
|
||||
db1 (:db-after (d/with db0 [{:db/id 5
|
||||
:block/uuid shared-uuid
|
||||
:block/title "Peer"
|
||||
:block/parent 1
|
||||
:block/page 1}]))
|
||||
original-parent-uuid (:block/uuid (d/entity db1 3))
|
||||
tx-data [(d/datom 3 :block/uuid original-parent-uuid 200 false)
|
||||
(d/datom 5 :block/uuid shared-uuid 200 false)
|
||||
(d/datom 3 :block/uuid shared-uuid 200 true)
|
||||
(d/datom 3 :block/order "a9" 200 true)]
|
||||
tx-report (d/with db1 tx-data)
|
||||
db-before (:db-before tx-report)
|
||||
checksum-before (checksum/recompute-checksum db-before)
|
||||
full (checksum/recompute-checksum (:db-after tx-report))
|
||||
incremental (checksum/update-checksum checksum-before tx-report)]
|
||||
(is (= full incremental)))))
|
||||
|
||||
(deftest checksum-ignores-non-page-non-block-entities-test
|
||||
(testing "entities with uuid but without page semantics do not affect checksum"
|
||||
(let [db0 (sample-db)
|
||||
checksum0 (checksum/recompute-checksum db0)
|
||||
internal-uuid (random-uuid)
|
||||
tx-report (d/with db0 [{:db/id -1
|
||||
:block/uuid internal-uuid
|
||||
:block/order "zz"}])
|
||||
db1 (:db-after tx-report)
|
||||
full1 (checksum/recompute-checksum db1)
|
||||
incremental1 (checksum/update-checksum checksum0 tx-report)
|
||||
tx-report-2 (d/with db1 [[:db/add [:block/uuid internal-uuid] :block/order "aa"]])
|
||||
full2 (checksum/recompute-checksum (:db-after tx-report-2))
|
||||
incremental2 (checksum/update-checksum full1 tx-report-2)]
|
||||
(is (= checksum0 full1))
|
||||
(is (= checksum0 incremental1))
|
||||
(is (= checksum0 full2))
|
||||
(is (= full2 incremental2)))))
|
||||
91
deps/db-sync/test/logseq/db_sync/common_test.cljs
vendored
Normal file
91
deps/db-sync/test/logseq/db_sync/common_test.cljs
vendored
Normal file
@@ -0,0 +1,91 @@
|
||||
(ns logseq.db-sync.common-test
|
||||
(:require [cljs.test :refer [async deftest is]]
|
||||
[logseq.db-sync.common :as common]
|
||||
[promesa.core :as p]))
|
||||
|
||||
(defn- make-stmt [calls result]
|
||||
(let [stmt (js-obj)]
|
||||
(aset stmt
|
||||
"bind"
|
||||
(fn [& args]
|
||||
(swap! calls assoc :bind-args (vec args))
|
||||
stmt))
|
||||
(aset stmt
|
||||
"all"
|
||||
(fn []
|
||||
(swap! calls update :all-count (fnil inc 0))
|
||||
result))
|
||||
stmt))
|
||||
|
||||
(deftest d1-all-falls-back-when-with-session-missing-test
|
||||
(async done
|
||||
(let [calls (atom {})
|
||||
result #js {:results #js [#js {"id" "u1"}]}
|
||||
stmt (make-stmt calls result)
|
||||
db #js {:prepare (fn [sql]
|
||||
(swap! calls assoc :prepare-sql sql)
|
||||
stmt)}]
|
||||
(-> (p/let [query-result (common/<d1-all db
|
||||
"select id from users where id = ?"
|
||||
"u1")]
|
||||
(is (= result query-result))
|
||||
(is (= "select id from users where id = ?"
|
||||
(:prepare-sql @calls)))
|
||||
(is (= ["u1"] (:bind-args @calls)))
|
||||
(is (= 1 (:all-count @calls))))
|
||||
(p/then (fn [] (done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
|
||||
(deftest d1-all-uses-with-session-mode-when-provided-test
|
||||
(async done
|
||||
(let [calls (atom {})
|
||||
result #js {:results #js [#js {"id" "u1"}]}
|
||||
stmt (make-stmt calls result)
|
||||
session #js {:prepare (fn [sql]
|
||||
(swap! calls assoc :prepare-sql sql)
|
||||
stmt)}
|
||||
db #js {:prepare (fn [_]
|
||||
(throw (js/Error. "db.prepare should not be called")))
|
||||
:withSession (fn [& args]
|
||||
(swap! calls assoc :with-session-args (vec args))
|
||||
session)}]
|
||||
(-> (p/let [query-result (common/<d1-all db
|
||||
{:session "first-primary"}
|
||||
"select id from users where id = ?"
|
||||
"u1")]
|
||||
(is (= result query-result))
|
||||
(is (= ["first-primary"] (:with-session-args @calls)))
|
||||
(is (= "select id from users where id = ?"
|
||||
(:prepare-sql @calls)))
|
||||
(is (= ["u1"] (:bind-args @calls)))
|
||||
(is (= 1 (:all-count @calls))))
|
||||
(p/then (fn [] (done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
|
||||
(deftest d1-all-uses-default-with-session-when-no-mode-test
|
||||
(async done
|
||||
(let [calls (atom {})
|
||||
result #js {:results #js []}
|
||||
stmt (make-stmt calls result)
|
||||
session #js {:prepare (fn [sql]
|
||||
(swap! calls assoc :prepare-sql sql)
|
||||
stmt)}
|
||||
db #js {:prepare (fn [_]
|
||||
(throw (js/Error. "db.prepare should not be called")))
|
||||
:withSession (fn [& args]
|
||||
(swap! calls assoc :with-session-args (vec args))
|
||||
session)}]
|
||||
(-> (p/let [query-result (common/<d1-all db "select 1")]
|
||||
(is (= result query-result))
|
||||
(is (= [] (:with-session-args @calls)))
|
||||
(is (= "select 1" (:prepare-sql @calls)))
|
||||
(is (nil? (:bind-args @calls)))
|
||||
(is (= 1 (:all-count @calls))))
|
||||
(p/then (fn [] (done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
229
deps/db-sync/test/logseq/db_sync/cycle_test.cljs
vendored
Normal file
229
deps/db-sync/test/logseq/db_sync/cycle_test.cljs
vendored
Normal file
@@ -0,0 +1,229 @@
|
||||
(ns logseq.db-sync.cycle-test
|
||||
(:require [cljs.test :refer [deftest is testing]]
|
||||
[datascript.core :as d]
|
||||
[logseq.db-sync.cycle :as cycle]
|
||||
[logseq.db.test.helper :as db-test]))
|
||||
|
||||
(defn- new-conn []
|
||||
(db-test/create-conn))
|
||||
|
||||
(defn- fix-cycle!
|
||||
[temp-conn remote-tx-report rebase-tx-report]
|
||||
(let [tx-metas (atom [])]
|
||||
(cycle/fix-cycle! temp-conn remote-tx-report rebase-tx-report
|
||||
{:transact! (fn [conn tx-data tx-meta]
|
||||
(swap! tx-metas conj tx-meta)
|
||||
(d/transact! conn tx-data tx-meta))})
|
||||
@tx-metas))
|
||||
|
||||
(defn- create-page!
|
||||
[conn title]
|
||||
(let [page-uuid (random-uuid)]
|
||||
(d/transact! conn [{:block/uuid page-uuid
|
||||
:block/name title
|
||||
:block/title title}])
|
||||
page-uuid))
|
||||
|
||||
(defn- block-eid
|
||||
[db uuid]
|
||||
(d/entid db [:block/uuid uuid]))
|
||||
|
||||
(deftest block-parent-2-node-cycle-test
|
||||
(let [conn (new-conn)
|
||||
page-uuid (create-page! conn "page")
|
||||
a (random-uuid)
|
||||
b (random-uuid)]
|
||||
(d/transact! conn [{:block/uuid a
|
||||
:block/page [:block/uuid page-uuid]
|
||||
:block/parent [:block/uuid page-uuid]}
|
||||
{:block/uuid b
|
||||
:block/page [:block/uuid page-uuid]
|
||||
:block/parent [:block/uuid a]}])
|
||||
(testing "breaks a 2-node :block/parent cycle and reparents to the page"
|
||||
(let [remote-report (d/transact! conn [{:block/uuid a :block/parent [:block/uuid b]}])
|
||||
tx-metas (fix-cycle! conn remote-report nil)
|
||||
a' (d/entity @conn [:block/uuid a])
|
||||
b' (d/entity @conn [:block/uuid b])]
|
||||
(is (= (block-eid @conn page-uuid) (:db/id (:block/parent a'))))
|
||||
(is (= (block-eid @conn a) (:db/id (:block/parent b'))))
|
||||
(is (some #(= :fix-cycle (:outliner-op %)) tx-metas))
|
||||
(is (every? #(false? (:gen-undo-ops? %)) tx-metas))
|
||||
(is (every? #(false? (:persist-op? %)) tx-metas))))))
|
||||
|
||||
(deftest block-parent-3-node-cycle-test
|
||||
(let [conn (new-conn)
|
||||
page-uuid (create-page! conn "page")
|
||||
a (random-uuid)
|
||||
b (random-uuid)
|
||||
c (random-uuid)]
|
||||
(d/transact! conn [{:block/uuid a
|
||||
:block/page [:block/uuid page-uuid]
|
||||
:block/parent [:block/uuid page-uuid]}
|
||||
{:block/uuid b
|
||||
:block/page [:block/uuid page-uuid]
|
||||
:block/parent [:block/uuid a]}
|
||||
{:block/uuid c
|
||||
:block/page [:block/uuid page-uuid]
|
||||
:block/parent [:block/uuid b]}])
|
||||
(testing "breaks a 3-node :block/parent cycle and reparents to the page"
|
||||
(let [remote-report (d/transact! conn [{:block/uuid a :block/parent [:block/uuid c]}])
|
||||
tx-metas (fix-cycle! conn remote-report nil)]
|
||||
(let [a' (d/entity @conn [:block/uuid a])
|
||||
b' (d/entity @conn [:block/uuid b])
|
||||
c' (d/entity @conn [:block/uuid c])]
|
||||
(is (= (block-eid @conn page-uuid) (:db/id (:block/parent a'))))
|
||||
(is (= (block-eid @conn a) (:db/id (:block/parent b'))))
|
||||
(is (= (block-eid @conn b) (:db/id (:block/parent c')))))
|
||||
(is (some #(= :fix-cycle (:outliner-op %)) tx-metas))
|
||||
(is (every? #(false? (:gen-undo-ops? %)) tx-metas))
|
||||
(is (every? #(false? (:persist-op? %)) tx-metas))))))
|
||||
|
||||
(deftest block-parent-cycle-prefers-remote-parent-test
|
||||
(let [conn (new-conn)
|
||||
page-uuid (create-page! conn "page")
|
||||
a (random-uuid)
|
||||
b (random-uuid)
|
||||
c (random-uuid)]
|
||||
(d/transact! conn [{:block/uuid a
|
||||
:block/page [:block/uuid page-uuid]
|
||||
:block/parent [:block/uuid page-uuid]}
|
||||
{:block/uuid b
|
||||
:block/page [:block/uuid page-uuid]
|
||||
:block/parent [:block/uuid page-uuid]}
|
||||
{:block/uuid c
|
||||
:block/page [:block/uuid page-uuid]
|
||||
:block/parent [:block/uuid page-uuid]}])
|
||||
(testing "prefers remote parent when breaking local cycle"
|
||||
(let [remote-report (d/transact! conn [{:block/uuid b :block/parent [:block/uuid c]}
|
||||
{:block/uuid a :block/parent [:block/uuid b]}])
|
||||
rebase-report (d/transact! conn [{:block/uuid b :block/parent [:block/uuid a]}
|
||||
{:block/uuid c :block/parent [:block/uuid b]}])
|
||||
tx-metas (fix-cycle! conn remote-report rebase-report)
|
||||
a' (d/entity @conn [:block/uuid a])
|
||||
b' (d/entity @conn [:block/uuid b])
|
||||
c' (d/entity @conn [:block/uuid c])]
|
||||
(is (= (block-eid @conn b) (:db/id (:block/parent a'))))
|
||||
(is (= (block-eid @conn c) (:db/id (:block/parent b'))))
|
||||
(is (= (block-eid @conn page-uuid) (:db/id (:block/parent c'))))
|
||||
(is (some #(= :fix-cycle (:outliner-op %)) tx-metas))
|
||||
(is (every? #(false? (:gen-undo-ops? %)) tx-metas))
|
||||
(is (every? #(false? (:persist-op? %)) tx-metas))))))
|
||||
|
||||
(deftest block-parent-cycle-avoids-descendant-remote-parent-test
|
||||
(let [conn (new-conn)
|
||||
page-uuid (create-page! conn "page")
|
||||
a (random-uuid)
|
||||
b (random-uuid)
|
||||
c (random-uuid)]
|
||||
(d/transact! conn [{:block/uuid a
|
||||
:block/page [:block/uuid page-uuid]
|
||||
:block/parent [:block/uuid page-uuid]}
|
||||
{:block/uuid b
|
||||
:block/page [:block/uuid page-uuid]
|
||||
:block/parent [:block/uuid a]}
|
||||
{:block/uuid c
|
||||
:block/page [:block/uuid page-uuid]
|
||||
:block/parent [:block/uuid b]}])
|
||||
(testing "falls back to page when remote parent is a descendant"
|
||||
(let [remote-report (d/transact! conn [{:block/uuid b :block/parent [:block/uuid c]}])
|
||||
rebase-report (d/transact! conn [{:block/uuid a :block/parent [:block/uuid b]}])
|
||||
tx-metas (fix-cycle! conn remote-report rebase-report)
|
||||
a' (d/entity @conn [:block/uuid a])
|
||||
b' (d/entity @conn [:block/uuid b])
|
||||
c' (d/entity @conn [:block/uuid c])]
|
||||
(is (= (block-eid @conn page-uuid) (:db/id (:block/parent b'))))
|
||||
(is (= (block-eid @conn b) (:db/id (:block/parent a'))))
|
||||
(is (= (block-eid @conn b) (:db/id (:block/parent c'))))
|
||||
(is (some #(= :fix-cycle (:outliner-op %)) tx-metas))
|
||||
(is (every? #(false? (:gen-undo-ops? %)) tx-metas))
|
||||
(is (every? #(false? (:persist-op? %)) tx-metas))))))
|
||||
|
||||
(deftest block-parent-cycle-preserves-safe-remote-edges-test
|
||||
(let [conn (new-conn)
|
||||
page-uuid (create-page! conn "page")
|
||||
a (random-uuid)
|
||||
b (random-uuid)
|
||||
c (random-uuid)]
|
||||
(d/transact! conn [{:block/uuid a
|
||||
:block/page [:block/uuid page-uuid]
|
||||
:block/parent [:block/uuid page-uuid]}
|
||||
{:block/uuid b
|
||||
:block/page [:block/uuid page-uuid]
|
||||
:block/parent [:block/uuid a]}
|
||||
{:block/uuid c
|
||||
:block/page [:block/uuid page-uuid]
|
||||
:block/parent [:block/uuid b]}])
|
||||
(testing "preserves remote parent when it doesn't reintroduce a cycle"
|
||||
(let [remote-report (d/transact! conn [{:block/uuid b :block/parent [:block/uuid c]}
|
||||
{:block/uuid a :block/parent [:block/uuid b]}])
|
||||
rebase-report (d/transact! conn [{:block/uuid b :block/parent [:block/uuid a]}
|
||||
{:block/uuid c :block/parent [:block/uuid b]}])
|
||||
tx-metas (fix-cycle! conn remote-report rebase-report)
|
||||
a' (d/entity @conn [:block/uuid a])
|
||||
b' (d/entity @conn [:block/uuid b])
|
||||
c' (d/entity @conn [:block/uuid c])]
|
||||
(is (= (block-eid @conn b) (:db/id (:block/parent a'))))
|
||||
(is (= (block-eid @conn page-uuid) (:db/id (:block/parent b'))))
|
||||
(is (= (block-eid @conn b) (:db/id (:block/parent c'))))
|
||||
(is (some #(= :fix-cycle (:outliner-op %)) tx-metas))
|
||||
(is (every? #(false? (:gen-undo-ops? %)) tx-metas))
|
||||
(is (every? #(false? (:persist-op? %)) tx-metas))))))
|
||||
|
||||
(deftest class-extends-2-node-cycle-test
|
||||
(let [conn (new-conn)]
|
||||
(d/transact! conn [{:db/ident :logseq.class/Root}
|
||||
{:db/ident :user.class/B}
|
||||
{:db/ident :user.class/A :logseq.property.class/extends :user.class/B}])
|
||||
(testing "breaks a 2-node :logseq.property.class/extends cycle"
|
||||
(let [remote-report (d/transact! conn [{:db/ident :user.class/B
|
||||
:logseq.property.class/extends :user.class/A}])
|
||||
tx-metas (fix-cycle! conn remote-report nil)]
|
||||
(let [b (d/entity @conn :user.class/B)
|
||||
extends (set (map :db/ident (:logseq.property.class/extends b)))]
|
||||
(is (not (contains? extends :user.class/A)))
|
||||
(is (contains? extends :logseq.class/Root)))
|
||||
(is (some #(= :fix-cycle (:outliner-op %)) tx-metas))
|
||||
(is (every? #(false? (:gen-undo-ops? %)) tx-metas))
|
||||
(is (every? #(false? (:persist-op? %)) tx-metas))))))
|
||||
|
||||
(deftest class-extends-3-node-cycle-with-multiple-values-test
|
||||
(let [conn (new-conn)]
|
||||
(d/transact! conn [{:db/ident :logseq.class/Root}
|
||||
{:db/ident :user.class/C}
|
||||
{:db/ident :user.class/D}
|
||||
{:db/ident :user.class/B :logseq.property.class/extends :user.class/C}
|
||||
{:db/ident :user.class/A :logseq.property.class/extends :user.class/B}])
|
||||
(testing "breaks a 3-node :logseq.property.class/extends cycle while preserving other extends"
|
||||
(let [remote-report (d/transact! conn [{:db/ident :user.class/C
|
||||
:logseq.property.class/extends #{:user.class/A :user.class/D}}])
|
||||
tx-metas (fix-cycle! conn remote-report nil)]
|
||||
(let [c (d/entity @conn :user.class/C)
|
||||
extends (set (map :db/ident (:logseq.property.class/extends c)))]
|
||||
(is (not (contains? extends :user.class/A)))
|
||||
(is (contains? extends :user.class/D))
|
||||
(is (contains? extends :logseq.class/Root)))
|
||||
(is (some #(= :fix-cycle (:outliner-op %)) tx-metas))
|
||||
(is (every? #(false? (:gen-undo-ops? %)) tx-metas))
|
||||
(is (every? #(false? (:persist-op? %)) tx-metas))))))
|
||||
|
||||
(deftest class-extends-cycle-prefers-remote-set-test
|
||||
(let [conn (new-conn)]
|
||||
(d/transact! conn [{:db/ident :logseq.class/Root}
|
||||
{:db/ident :user.class/B}
|
||||
{:db/ident :user.class/A :logseq.property.class/extends :logseq.class/Root}])
|
||||
(testing "prefers remote extends set when breaking cycles"
|
||||
(let [remote-report (d/transact! conn [{:db/ident :user.class/B
|
||||
:logseq.property.class/extends :user.class/A}])
|
||||
rebase-report (d/transact! conn [{:db/ident :user.class/A
|
||||
:logseq.property.class/extends :user.class/B}])
|
||||
tx-metas (fix-cycle! conn remote-report rebase-report)]
|
||||
(let [a (d/entity @conn :user.class/A)
|
||||
b (d/entity @conn :user.class/B)
|
||||
extends-a (set (map :db/ident (:logseq.property.class/extends a)))
|
||||
extends-b (set (map :db/ident (:logseq.property.class/extends b)))]
|
||||
(is (not (contains? extends-a :user.class/B)))
|
||||
(is (contains? extends-a :logseq.class/Root))
|
||||
(is (contains? extends-b :user.class/A)))
|
||||
(is (some #(= :fix-cycle (:outliner-op %)) tx-metas))
|
||||
(is (every? #(false? (:gen-undo-ops? %)) tx-metas))
|
||||
(is (every? #(false? (:persist-op? %)) tx-metas))))))
|
||||
1
deps/db-sync/test/logseq/db_sync/fixtures/parent_order_rebase_checksum_payload.edn
vendored
Normal file
1
deps/db-sync/test/logseq/db_sync/fixtures/parent_order_rebase_checksum_payload.edn
vendored
Normal file
File diff suppressed because one or more lines are too long
1
deps/db-sync/test/logseq/db_sync/fixtures/rebased_retract_checksum_payload.edn
vendored
Normal file
1
deps/db-sync/test/logseq/db_sync/fixtures/rebased_retract_checksum_payload.edn
vendored
Normal file
File diff suppressed because one or more lines are too long
197
deps/db-sync/test/logseq/db_sync/index_test.cljs
vendored
Normal file
197
deps/db-sync/test/logseq/db_sync/index_test.cljs
vendored
Normal file
@@ -0,0 +1,197 @@
|
||||
(ns logseq.db-sync.index-test
|
||||
(:require [cljs.test :refer [async deftest is]]
|
||||
[clojure.string :as string]
|
||||
[logseq.db-sync.common :as common]
|
||||
[logseq.db-sync.index :as index]
|
||||
[promesa.core :as p]))
|
||||
|
||||
(def ^:private graph-e2ee-migration-sql
|
||||
"alter table graphs add column graph_e2ee integer default 1")
|
||||
(def ^:private graph-ready-for-use-migration-sql
|
||||
"alter table graphs add column graph_ready_for_use integer default 1")
|
||||
(def ^:private graph-members-graph-id-created-at-index-sql
|
||||
"create index if not exists idx_graph_members_graph_id_created_at on graph_members (graph_id, created_at)")
|
||||
(def ^:private graphs-user-id-updated-at-index-sql
|
||||
"create index if not exists idx_graphs_user_id_updated_at on graphs (user_id, updated_at desc)")
|
||||
(def ^:private users-email-index-sql
|
||||
"create index if not exists idx_users_email on users (email)")
|
||||
|
||||
(deftest index-list-includes-graph-e2ee-flag-test
|
||||
(async done
|
||||
(let [rows #js [#js {"graph_id" "graph-1"
|
||||
"graph_name" "Graph 1"
|
||||
"schema_version" "65"
|
||||
"role" "manager"
|
||||
"invited_by" nil
|
||||
"created_at" 10
|
||||
"updated_at" 20
|
||||
"graph_e2ee" 0}
|
||||
#js {"graph_id" "graph-2"
|
||||
"graph_name" "Graph 2"
|
||||
"schema_version" "65"
|
||||
"role" "member"
|
||||
"invited_by" "u1"
|
||||
"created_at" 11
|
||||
"updated_at" 21
|
||||
"graph_e2ee" 1
|
||||
"graph_ready_for_use" 0}]]
|
||||
(-> (p/with-redefs [common/<d1-all (fn [& _]
|
||||
(p/resolved #js {:results rows}))
|
||||
common/get-sql-rows (fn [result]
|
||||
(aget result "results"))]
|
||||
(index/<index-list :db "user-1"))
|
||||
(p/then (fn [graphs]
|
||||
(is (= 2 (count graphs)))
|
||||
(is (= false (:graph-e2ee? (first graphs))))
|
||||
(is (= true (:graph-e2ee? (second graphs))))
|
||||
(is (= true (:graph-ready-for-use? (first graphs))))
|
||||
(is (= false (:graph-ready-for-use? (second graphs))))
|
||||
(done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
|
||||
(deftest semantic-graphs-list-is-bounded-and-excludes-e2ee-test
|
||||
(async done
|
||||
(let [call (atom nil)
|
||||
rows #js [#js {"graph_id" "graph-1" "graph_name" "test-mcp"
|
||||
"updated_at" 20 "created_at" 10 "schema_version" "65"
|
||||
"graph_ready_for_use" 1 "role" "manager" "invited_by" nil}
|
||||
#js {"graph_id" "graph-2" "graph_name" "test-mcp-2"
|
||||
"updated_at" 19 "created_at" 11 "schema_version" "65"
|
||||
"graph_ready_for_use" 1 "role" "member" "invited_by" "u2"}]]
|
||||
(-> (p/with-redefs [common/<d1-all
|
||||
(fn [_db sql & args]
|
||||
(reset! call {:sql sql :args args})
|
||||
(p/resolved #js {:results rows}))
|
||||
common/get-sql-rows (fn [result] (aget result "results"))]
|
||||
(index/<semantic-graphs-list :db "user-1" {:name "test-mcp" :limit 1 :cursor nil}))
|
||||
(p/then (fn [{:keys [graphs next-cursor]}]
|
||||
(is (= 1 (count graphs)))
|
||||
(is (string? next-cursor))
|
||||
(is (string/includes? (:sql @call) "g.graph_e2ee = 0"))
|
||||
(is (string/includes? (:sql @call) "limit ?"))
|
||||
(is (= 2 (last (:args @call))))
|
||||
(done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
|
||||
(deftest index-upsert-persists-graph-e2ee-flag-test
|
||||
(async done
|
||||
(let [called (atom nil)]
|
||||
(-> (p/with-redefs [common/now-ms (fn [] 1234)
|
||||
common/<d1-run (fn [_db sql & args]
|
||||
(reset! called {:sql sql
|
||||
:args args})
|
||||
(p/resolved {:ok true}))]
|
||||
(index/<index-upsert! :db "graph-1" "Graph 1" "user-1" "65" false))
|
||||
(p/then (fn [_]
|
||||
(is (string/includes? (:sql @called) "graph_e2ee"))
|
||||
(is (= ["graph-1" "Graph 1" "user-1" "65" 0 1 1234 1234]
|
||||
(:args @called)))
|
||||
(done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
|
||||
(deftest index-upsert-persists-graph-ready-for-use-flag-test
|
||||
(async done
|
||||
(let [called (atom nil)]
|
||||
(-> (p/with-redefs [common/now-ms (fn [] 1234)
|
||||
common/<d1-run (fn [_db sql & args]
|
||||
(reset! called {:sql sql
|
||||
:args args})
|
||||
(p/resolved {:ok true}))]
|
||||
(index/<index-upsert! :db "graph-1" "Graph 1" "user-1" "65" false false))
|
||||
(p/then (fn [_]
|
||||
(is (string/includes? (:sql @called) "graph_ready_for_use"))
|
||||
(is (= ["graph-1" "Graph 1" "user-1" "65" 0 0 1234 1234]
|
||||
(:args @called)))
|
||||
(done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
|
||||
(deftest index-init-runs-graph-e2ee-migration-test
|
||||
(async done
|
||||
(let [sql-calls (atom [])]
|
||||
(-> (p/with-redefs [common/<d1-all (fn [& _]
|
||||
(p/resolved #js {:results #js []}))
|
||||
common/get-sql-rows (fn [result]
|
||||
(aget result "results"))
|
||||
common/<d1-run (fn [_db sql & _args]
|
||||
(swap! sql-calls conj (string/lower-case sql))
|
||||
(p/resolved {:ok true}))]
|
||||
(index/<index-init! :db))
|
||||
(p/then (fn [_]
|
||||
(is (some #(string/includes? % graph-e2ee-migration-sql)
|
||||
@sql-calls))
|
||||
(done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
|
||||
(deftest index-init-runs-graph-ready-for-use-migration-test
|
||||
(async done
|
||||
(let [sql-calls (atom [])]
|
||||
(-> (p/with-redefs [common/<d1-all (fn [& _]
|
||||
(p/resolved #js {:results #js []}))
|
||||
common/get-sql-rows (fn [result]
|
||||
(aget result "results"))
|
||||
common/<d1-run (fn [_db sql & _args]
|
||||
(swap! sql-calls conj (string/lower-case sql))
|
||||
(p/resolved {:ok true}))]
|
||||
(index/<index-init! :db))
|
||||
(p/then (fn [_]
|
||||
(is (some #(string/includes? % graph-ready-for-use-migration-sql)
|
||||
@sql-calls))
|
||||
(done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
|
||||
(deftest index-init-ignores-duplicate-graph-e2ee-column-error-test
|
||||
(async done
|
||||
(let [sql-calls (atom [])]
|
||||
(-> (p/with-redefs [common/<d1-all (fn [& _]
|
||||
(p/resolved #js {:results #js []}))
|
||||
common/get-sql-rows (fn [result]
|
||||
(aget result "results"))
|
||||
common/<d1-run (fn [_db sql & _args]
|
||||
(let [sql' (string/lower-case sql)]
|
||||
(swap! sql-calls conj sql')
|
||||
(if (string/includes? sql' graph-e2ee-migration-sql)
|
||||
(p/rejected (ex-info "duplicate column name: graph_e2ee" {}))
|
||||
(p/resolved {:ok true}))))]
|
||||
(index/<index-init! :db))
|
||||
(p/then (fn [_]
|
||||
(is (some #(string/includes? % graph-e2ee-migration-sql)
|
||||
@sql-calls))
|
||||
(done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
|
||||
(deftest index-init-creates-indexes-test
|
||||
(async done
|
||||
(let [sql-calls (atom [])]
|
||||
(-> (p/with-redefs [common/<d1-all (fn [& _]
|
||||
(p/resolved #js {:results #js []}))
|
||||
common/get-sql-rows (fn [result]
|
||||
(aget result "results"))
|
||||
common/<d1-run (fn [_db sql & _args]
|
||||
(swap! sql-calls conj (string/lower-case sql))
|
||||
(p/resolved {:ok true}))]
|
||||
(index/<index-init! :db))
|
||||
(p/then (fn [_]
|
||||
(is (some #(string/includes? % graph-members-graph-id-created-at-index-sql)
|
||||
@sql-calls))
|
||||
(is (some #(string/includes? % graphs-user-id-updated-at-index-sql)
|
||||
@sql-calls))
|
||||
(is (some #(string/includes? % users-email-index-sql)
|
||||
@sql-calls))
|
||||
(done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
42
deps/db-sync/test/logseq/db_sync/malli_schema_test.cljs
vendored
Normal file
42
deps/db-sync/test/logseq/db_sync/malli_schema_test.cljs
vendored
Normal file
@@ -0,0 +1,42 @@
|
||||
(ns logseq.db-sync.malli-schema-test
|
||||
(:require [cljs.test :refer [deftest is testing]]
|
||||
[logseq.db-sync.malli-schema :as db-sync-schema]))
|
||||
|
||||
(def ^:private request-samples
|
||||
{:graphs/create {:graph-name "Demo"}
|
||||
:graph-members/create {:email "user@example.com"}
|
||||
:graph-members/update {:role "member"}
|
||||
:sync/tx-batch {:t-before 0 :txs []}
|
||||
:e2ee/user-keys {:public-key "public"
|
||||
:encrypted-private-key "private"}
|
||||
:e2ee/graph-aes-key {:encrypted-aes-key "aes"}
|
||||
:e2ee/grant-access {:target-user-email+encrypted-aes-key-coll
|
||||
[{:email "user@example.com"
|
||||
:encrypted-aes-key "aes"}]}})
|
||||
|
||||
(defn- coerce-request
|
||||
[schema-key body]
|
||||
((get db-sync-schema/http-request-coercers schema-key) body))
|
||||
|
||||
(deftest http-request-client-revision-is-optional-test
|
||||
(doseq [[schema-key body] request-samples]
|
||||
(testing schema-key
|
||||
(is (= body (coerce-request schema-key body))))))
|
||||
|
||||
(deftest tx-batch-request-client-revision-accepts-string-test
|
||||
(let [body' (assoc (:sync/tx-batch request-samples)
|
||||
:client-revision "test-revision")]
|
||||
(is (= body' (coerce-request :sync/tx-batch body')))))
|
||||
|
||||
(deftest ws-tx-batch-client-revision-accepts-string-test
|
||||
(let [body {:type "tx/batch"
|
||||
:t-before 0
|
||||
:txs []
|
||||
:client-revision "test-revision"}]
|
||||
(is (= body (db-sync-schema/ws-client-message-coercer body)))))
|
||||
|
||||
(deftest tx-batch-request-client-revision-rejects-non-string-test
|
||||
(is (thrown? js/Error
|
||||
(coerce-request :sync/tx-batch
|
||||
(assoc (:sync/tx-batch request-samples)
|
||||
:client-revision 42)))))
|
||||
108
deps/db-sync/test/logseq/db_sync/node_adapter_test.cljs
vendored
Normal file
108
deps/db-sync/test/logseq/db_sync/node_adapter_test.cljs
vendored
Normal file
@@ -0,0 +1,108 @@
|
||||
(ns logseq.db-sync.node-adapter-test
|
||||
;; Linters disabled because commented out FIXME code causes false positives
|
||||
{:clj-kondo/config {:ignore true}}
|
||||
(:require [clojure.string :as string]
|
||||
[logseq.db-sync.node.server :as node-server]
|
||||
[logseq.db-sync.protocol :as protocol]
|
||||
[promesa.core :as p]))
|
||||
|
||||
(def test-token "test-token")
|
||||
|
||||
(defn- auth-headers []
|
||||
#js {"authorization" (str "Bearer " test-token)
|
||||
"content-type" "application/json"})
|
||||
|
||||
(defn- post-json [url body]
|
||||
(js/fetch url #js {:method "POST"
|
||||
:headers (auth-headers)
|
||||
:body (js/JSON.stringify (clj->js body))}))
|
||||
|
||||
(defn- get-json [url]
|
||||
(js/fetch url #js {:method "GET" :headers (auth-headers)}))
|
||||
|
||||
(defn- parse-json [resp]
|
||||
(.json resp))
|
||||
|
||||
(defn- start-test-server []
|
||||
(let [suffix (str (random-uuid))
|
||||
dir (str "tmp/db-sync-node-test/" suffix)]
|
||||
(node-server/start! {:port 0
|
||||
:data-dir dir})))
|
||||
;; FIXME: Tests are disabled until they stop hanging
|
||||
#_(deftest node-adapter-http-roundtrip-test
|
||||
(async done
|
||||
(p/let [{:keys [base-url stop!]} (start-test-server)
|
||||
health-resp (js/fetch (str base-url "/health"))
|
||||
health-body (parse-json health-resp)]
|
||||
(testing "health"
|
||||
(is (.-ok health-resp))
|
||||
(is (= true (aget health-body "ok"))))
|
||||
;; FIXME: Test hangs here due to an exception
|
||||
(p/let [create-resp (post-json (str base-url "/graphs") {:graph-name "Test Graph"})
|
||||
create-body (parse-json create-resp)
|
||||
graph-id (aget create-body "graph-id")
|
||||
access-resp (get-json (str base-url "/graphs/" graph-id "/access"))
|
||||
access-body (parse-json access-resp)
|
||||
sync-health (get-json (str base-url "/sync/" graph-id "/health"))
|
||||
sync-health-body (parse-json sync-health)]
|
||||
(testing "graph access"
|
||||
(is (.-ok create-resp))
|
||||
(is (string? graph-id))
|
||||
(is (.-ok access-resp))
|
||||
(is (= true (aget access-body "ok"))))
|
||||
(testing "sync health"
|
||||
(is (.-ok sync-health))
|
||||
(is (= true (aget sync-health-body "ok"))))
|
||||
(p/let [tx-data [{:block/uuid (random-uuid)
|
||||
:block/title "hello"}]
|
||||
tx-entry {:tx (protocol/tx->transit tx-data)
|
||||
:outliner-op :save-block}
|
||||
tx-resp (post-json (str base-url "/sync/" graph-id "/tx/batch")
|
||||
{:t-before 0
|
||||
:txs [tx-entry]})
|
||||
tx-body (parse-json tx-resp)
|
||||
pull-resp (get-json (str base-url "/sync/" graph-id "/pull?since=0"))
|
||||
pull-body (parse-json pull-resp)]
|
||||
(testing "tx batch"
|
||||
(is (.-ok tx-resp))
|
||||
(is (= "tx/batch/ok" (aget tx-body "type"))))
|
||||
(testing "pull"
|
||||
(is (.-ok pull-resp))
|
||||
(is (= "pull/ok" (aget pull-body "type")))
|
||||
(is (pos? (count (aget pull-body "txs"))))
|
||||
(is (= "save-block"
|
||||
(aget (aget pull-body "txs" 0) "outliner-op"))))
|
||||
(p/then (stop!) (fn [] (done))))))))
|
||||
|
||||
#_(deftest node-adapter-websocket-test
|
||||
(async done
|
||||
(p/let [{:keys [base-url stop!]} (start-test-server)
|
||||
create-resp (post-json (str base-url "/graphs") {:graph-name "WS Graph"})
|
||||
create-body (parse-json create-resp)
|
||||
graph-id (aget create-body "graph-id")]
|
||||
(testing "websocket hello and changed"
|
||||
(let [ws-url (str (string/replace base-url "http" "ws") "/sync/" graph-id)
|
||||
ws-module (js/require "ws")
|
||||
WebSocket (or (.-WebSocket ws-module) ws-module)
|
||||
client (new WebSocket ws-url #js {:headers (auth-headers)})
|
||||
messages (atom [])
|
||||
push-message (fn [data]
|
||||
(let [text (if (string? data) data (.toString data))]
|
||||
(swap! messages conj (js/JSON.parse text))))]
|
||||
(.on client "message" push-message)
|
||||
(.on client "open"
|
||||
(fn []
|
||||
(.send client (protocol/encode-message {:type "hello" :client "test"}))))
|
||||
(p/let [_ (p/delay 50)
|
||||
tx-data [{:block/uuid (random-uuid)
|
||||
:block/title "ws"}]
|
||||
txs (protocol/tx->transit tx-data)
|
||||
_ (post-json (str base-url "/sync/" graph-id "/tx/batch")
|
||||
{:t-before 0
|
||||
:txs txs})
|
||||
_ (p/delay 100)]
|
||||
(let [types (set (map #(aget % "type") @messages))]
|
||||
(is (contains? types "hello"))
|
||||
(is (contains? types "changed")))
|
||||
(.close client)
|
||||
(p/then (stop!) (fn [] (done)))))))))
|
||||
31
deps/db-sync/test/logseq/db_sync/node_config_test.cljs
vendored
Normal file
31
deps/db-sync/test/logseq/db_sync/node_config_test.cljs
vendored
Normal file
@@ -0,0 +1,31 @@
|
||||
(ns logseq.db-sync.node-config-test
|
||||
(:require [cljs.test :refer [deftest is testing]]
|
||||
[logseq.db-sync.node.config :as config]))
|
||||
|
||||
(defn- throws? [f]
|
||||
(try
|
||||
(f)
|
||||
false
|
||||
(catch :default _ true)))
|
||||
|
||||
(deftest normalize-config-drops-unknown-keys-test
|
||||
(let [cfg (config/normalize-config {:port 7777
|
||||
:unknown-key "value"
|
||||
:legacy-auth-key "value"})]
|
||||
(is (= 7777 (:port cfg)))
|
||||
(is (nil? (:unknown-key cfg)))
|
||||
(is (nil? (:legacy-auth-key cfg)))))
|
||||
|
||||
(deftest normalize-config-storage-driver-test
|
||||
(testing "sqlite storage driver accepted"
|
||||
(let [cfg (config/normalize-config {:storage-driver "sqlite"})]
|
||||
(is (= "sqlite" (:storage-driver cfg)))))
|
||||
(testing "unsupported storage driver throws"
|
||||
(is (throws? #(config/normalize-config {:storage-driver "other"})))))
|
||||
|
||||
(deftest normalize-config-assets-driver-test
|
||||
(testing "filesystem assets driver accepted"
|
||||
(let [cfg (config/normalize-config {:assets-driver "filesystem"})]
|
||||
(is (= "filesystem" (:assets-driver cfg)))))
|
||||
(testing "unsupported assets driver throws"
|
||||
(is (throws? #(config/normalize-config {:assets-driver "s3"})))))
|
||||
163
deps/db-sync/test/logseq/db_sync/node_server_test.cljs
vendored
Normal file
163
deps/db-sync/test/logseq/db_sync/node_server_test.cljs
vendored
Normal file
@@ -0,0 +1,163 @@
|
||||
(ns logseq.db-sync.node-server-test
|
||||
(:require [cljs.test :refer [async deftest is]]
|
||||
[logseq.db-sync.node.server :as node-server]
|
||||
[logseq.db-sync.platform.node :as platform-node]
|
||||
[logseq.db-sync.worker.auth :as auth]
|
||||
[promesa.core :as p]))
|
||||
|
||||
(defn- fetch-with-timeout [url timeout-ms]
|
||||
(let [timeout-sentinel ::timeout]
|
||||
{:sentinel timeout-sentinel
|
||||
:promise (js/Promise.
|
||||
(fn [resolve reject]
|
||||
(let [controller (js/AbortController.)
|
||||
timeout-id (js/setTimeout
|
||||
(fn []
|
||||
(.abort controller))
|
||||
timeout-ms)]
|
||||
(-> (js/fetch url #js {:method "GET"
|
||||
:headers #js {"authorization" "Bearer test.token.sig"}
|
||||
:signal (.-signal controller)})
|
||||
(.then (fn [response]
|
||||
(js/clearTimeout timeout-id)
|
||||
(resolve response)))
|
||||
(.catch (fn [error]
|
||||
(js/clearTimeout timeout-id)
|
||||
(if (= "AbortError" (.-name error))
|
||||
(resolve timeout-sentinel)
|
||||
(reject error))))))))}))
|
||||
|
||||
(deftest node-server-returns-500-when-auth-claims-rejects-test
|
||||
(async done
|
||||
(let [stop-server! (atom nil)
|
||||
test-url (atom nil)]
|
||||
(-> (p/with-redefs [auth/auth-claims
|
||||
(fn [_request _env]
|
||||
(p/rejected (ex-info "jwks" {})))]
|
||||
(p/let [{:keys [base-url stop!]} (node-server/start! {:port 0
|
||||
:data-dir (str "tmp/db-sync-node-server-test/" (random-uuid))})
|
||||
_ (reset! stop-server! stop!)
|
||||
_ (reset! test-url (str base-url "/graphs"))
|
||||
{:keys [promise sentinel]} (fetch-with-timeout @test-url 1200)
|
||||
response promise]
|
||||
(if (identical? response sentinel)
|
||||
(is false "request timed out")
|
||||
(p/let [body (.json response)]
|
||||
(is (= 500 (.-status response)))
|
||||
(is (= "server error" (aget body "error")))))))
|
||||
(p/then
|
||||
(fn []
|
||||
(if-let [stop! @stop-server!]
|
||||
(-> (stop!)
|
||||
(p/then (fn [] (done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done))))
|
||||
(done))))
|
||||
(p/catch
|
||||
(fn [error]
|
||||
(if-let [stop! @stop-server!]
|
||||
(-> (stop!)
|
||||
(p/then (fn []
|
||||
(is false (str error))
|
||||
(done)))
|
||||
(p/catch (fn [stop-error]
|
||||
(is false (str error))
|
||||
(is false (str stop-error))
|
||||
(done))))
|
||||
(do
|
||||
(is false (str error))
|
||||
(done)))))))))
|
||||
|
||||
(deftest node-server-request-origin-uses-configured-base-url-host-test
|
||||
(async done
|
||||
(let [stop-server! (atom nil)
|
||||
request-opts (atom [])
|
||||
original-request-from-node platform-node/request-from-node]
|
||||
(-> (p/with-redefs [platform-node/request-from-node
|
||||
(fn [req opts]
|
||||
(swap! request-opts conj opts)
|
||||
(original-request-from-node req opts))]
|
||||
(p/let [{:keys [port stop!]} (node-server/start! {:port 0
|
||||
:base-url "https://sync.example.test:9443"
|
||||
:data-dir (str "tmp/db-sync-node-server-base-url-test/" (random-uuid))})
|
||||
_ (reset! stop-server! stop!)
|
||||
response (js/fetch (str "http://localhost:" port "/health"))]
|
||||
(is (= 200 (.-status response)))
|
||||
(is (some #(= {:scheme "https"
|
||||
:host "sync.example.test:9443"}
|
||||
%)
|
||||
@request-opts))))
|
||||
(p/then
|
||||
(fn []
|
||||
(if-let [stop! @stop-server!]
|
||||
(-> (stop!)
|
||||
(p/then (fn [] (done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done))))
|
||||
(done))))
|
||||
(p/catch
|
||||
(fn [error]
|
||||
(if-let [stop! @stop-server!]
|
||||
(-> (stop!)
|
||||
(p/then (fn []
|
||||
(is false (str error))
|
||||
(done)))
|
||||
(p/catch (fn [stop-error]
|
||||
(is false (str error))
|
||||
(is false (str stop-error))
|
||||
(done))))
|
||||
(do
|
||||
(is false (str error))
|
||||
(done)))))))))
|
||||
|
||||
(deftest node-server-logs-request-failed-marker-when-auth-claims-rejects-test
|
||||
(async done
|
||||
(let [stop-server! (atom nil)
|
||||
original-console-error (.-error js/console)
|
||||
logged-errors (atom [])]
|
||||
(aset js/console
|
||||
"error"
|
||||
(fn [& args]
|
||||
(swap! logged-errors conj args)))
|
||||
(-> (p/with-redefs [auth/auth-claims
|
||||
(fn [_request _env]
|
||||
(p/rejected (ex-info "jwks" {})))]
|
||||
(p/let [{:keys [base-url stop!]} (node-server/start! {:port 0
|
||||
:data-dir (str "tmp/db-sync-node-server-log-test/" (random-uuid))})
|
||||
_ (reset! stop-server! stop!)
|
||||
_ (reset! logged-errors [])
|
||||
{:keys [promise sentinel]} (fetch-with-timeout (str base-url "/graphs") 1200)
|
||||
response promise]
|
||||
(if (identical? response sentinel)
|
||||
(is false "request timed out")
|
||||
(do
|
||||
(is (= 500 (.-status response)))
|
||||
(is (some #(= ":db-sync/node-request-failed" (first %))
|
||||
@logged-errors))))))
|
||||
(p/then
|
||||
(fn []
|
||||
(aset js/console "error" original-console-error)
|
||||
(if-let [stop! @stop-server!]
|
||||
(-> (stop!)
|
||||
(p/then (fn [] (done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done))))
|
||||
(done))))
|
||||
(p/catch
|
||||
(fn [error]
|
||||
(aset js/console "error" original-console-error)
|
||||
(if-let [stop! @stop-server!]
|
||||
(-> (stop!)
|
||||
(p/then (fn []
|
||||
(is false (str error))
|
||||
(done)))
|
||||
(p/catch (fn [stop-error]
|
||||
(is false (str error))
|
||||
(is false (str stop-error))
|
||||
(done))))
|
||||
(do
|
||||
(is false (str error))
|
||||
(done)))))))))
|
||||
382
deps/db-sync/test/logseq/db_sync/normalize_test.cljs
vendored
Normal file
382
deps/db-sync/test/logseq/db_sync/normalize_test.cljs
vendored
Normal file
@@ -0,0 +1,382 @@
|
||||
(ns logseq.db-sync.normalize-test
|
||||
(:require [cljs.test :refer [deftest is testing]]
|
||||
[datascript.core :as d]
|
||||
[logseq.db :as ldb]
|
||||
[logseq.db.common.normalize :as db-normalize]
|
||||
[logseq.db.test.helper :as db-test]))
|
||||
|
||||
(defn- new-conn []
|
||||
(db-test/create-conn))
|
||||
|
||||
(defn- create-page!
|
||||
[conn title]
|
||||
(let [page-uuid (random-uuid)]
|
||||
(d/transact! conn [{:block/uuid page-uuid
|
||||
:block/name title
|
||||
:block/title title}])
|
||||
page-uuid))
|
||||
|
||||
(defn- op-e-a-v
|
||||
[datom]
|
||||
(subvec (vec datom) 0 4))
|
||||
|
||||
(defn- tx-touches-uuid?
|
||||
[tx-data block-uuid]
|
||||
(let [lookup [:block/uuid block-uuid]
|
||||
block-uuid-str (str block-uuid)]
|
||||
(boolean
|
||||
(some (fn [item]
|
||||
(let [item (vec item)]
|
||||
(case (count item)
|
||||
5 (or (= lookup (nth item 1))
|
||||
(= block-uuid-str (nth item 1))
|
||||
(= block-uuid (nth item 3)))
|
||||
2 (or (= lookup (second item))
|
||||
(= block-uuid-str (second item)))
|
||||
false)))
|
||||
tx-data))))
|
||||
|
||||
(defn- seeded-rng
|
||||
[seed0]
|
||||
(let [state (atom (bit-or (long seed0) 0))]
|
||||
(fn []
|
||||
(let [s (swap! state
|
||||
(fn [x]
|
||||
(let [x (bit-xor x (bit-shift-left x 13))
|
||||
x (bit-xor x (bit-shift-right x 17))
|
||||
x (bit-xor x (bit-shift-left x 5))]
|
||||
(bit-or x 0))))]
|
||||
(/ (double (unsigned-bit-shift-right s 0)) 4294967296.0)))))
|
||||
|
||||
(defn- rand-int*
|
||||
[rng n]
|
||||
(js/Math.floor (* (rng) n)))
|
||||
|
||||
(defn- pick-rand
|
||||
[rng coll]
|
||||
(when (seq coll)
|
||||
(nth coll (rand-int* rng (count coll)))))
|
||||
|
||||
(defn- normal-block-uuids
|
||||
[db]
|
||||
(->> (d/datoms db :avet :block/uuid)
|
||||
(map :e)
|
||||
distinct
|
||||
(keep (fn [eid]
|
||||
(let [ent (d/entity db eid)]
|
||||
(when (and (uuid? (:block/uuid ent))
|
||||
(not (ldb/built-in? ent))
|
||||
(nil? (:block/name ent))
|
||||
(some? (:block/page ent)))
|
||||
(:block/uuid ent)))))
|
||||
vec))
|
||||
|
||||
(defn- block-state
|
||||
[db]
|
||||
(->> (d/datoms db :avet :block/uuid)
|
||||
(map :e)
|
||||
distinct
|
||||
(keep (fn [eid]
|
||||
(let [ent (d/entity db eid)]
|
||||
(when (and (uuid? (:block/uuid ent))
|
||||
(not (ldb/built-in? ent))
|
||||
(or (string? (:block/name ent))
|
||||
(some? (:block/page ent))))
|
||||
[(:block/uuid ent)
|
||||
{:block/uuid (:block/uuid ent)
|
||||
:block/name (:block/name ent)
|
||||
:block/title (:block/title ent)
|
||||
:block/page (some-> ent :block/page :block/uuid)
|
||||
:block/parent (some-> ent :block/parent :block/uuid)
|
||||
:block/order (:block/order ent)}]))))
|
||||
(into (sorted-map))))
|
||||
|
||||
(deftest normalize-tx-data-keeps-title-retract-without-replacement-test
|
||||
(let [conn (new-conn)
|
||||
page-uuid (create-page! conn "Page")
|
||||
tx-report (d/transact! conn [[:db/retract [:block/uuid page-uuid] :block/title "Page"]])
|
||||
normalized (db-normalize/normalize-tx-data (:db-after tx-report)
|
||||
(:db-before tx-report)
|
||||
(:tx-data tx-report))
|
||||
tx-data (mapv op-e-a-v normalized)]
|
||||
(testing "keeps :block/title retract when no replacement title exists in same tx"
|
||||
(is (= [[:db/retract [:block/uuid page-uuid] :block/title "Page"]]
|
||||
tx-data)))))
|
||||
|
||||
(deftest normalize-tx-data-drops-title-retract-when-replaced-test
|
||||
(let [conn (new-conn)
|
||||
page-uuid (create-page! conn "Page")
|
||||
tx-report (d/transact! conn [{:block/uuid page-uuid
|
||||
:block/title "Page 2"}])
|
||||
normalized (db-normalize/normalize-tx-data (:db-after tx-report)
|
||||
(:db-before tx-report)
|
||||
(:tx-data tx-report))
|
||||
tx-data (mapv op-e-a-v normalized)]
|
||||
(testing "keeps old :block/title retract and new add during title update"
|
||||
(is (some #(= [:db/add [:block/uuid page-uuid] :block/title "Page 2"] %) tx-data))
|
||||
(is (some #(= [:db/retract [:block/uuid page-uuid] :block/title "Page"] %) tx-data)))))
|
||||
|
||||
(deftest normalize-tx-data-keeps-recreated-normal-blocks-test
|
||||
(testing "retract + recreate for normal blocks should not drop recreated entity datoms"
|
||||
(let [conn (new-conn)
|
||||
page-uuid (create-page! conn "Page")
|
||||
target-uuid (random-uuid)
|
||||
sibling-uuid (random-uuid)
|
||||
child-uuid (random-uuid)
|
||||
_ (d/transact! conn [{:block/uuid target-uuid
|
||||
:block/title "old-target"
|
||||
:block/page [:block/uuid page-uuid]
|
||||
:block/parent [:block/uuid page-uuid]
|
||||
:block/order "a0"}
|
||||
{:block/uuid sibling-uuid
|
||||
:block/title "sibling"
|
||||
:block/page [:block/uuid page-uuid]
|
||||
:block/parent [:block/uuid page-uuid]
|
||||
:block/order "a1"}
|
||||
{:block/uuid child-uuid
|
||||
:block/title "child"
|
||||
:block/page [:block/uuid page-uuid]
|
||||
:block/parent [:block/uuid target-uuid]
|
||||
:block/order "a0"}])
|
||||
db-before @conn
|
||||
|
||||
;; Shape A: retract then recreate with tempid + explicit :block/uuid add.
|
||||
tx-report-a (d/with db-before
|
||||
[[:db/retractEntity [:block/uuid target-uuid]]
|
||||
[:db/add -1 :block/uuid target-uuid]
|
||||
[:db/add -1 :block/title "new-target-a"]
|
||||
[:db/add -1 :block/page [:block/uuid page-uuid]]
|
||||
[:db/add -1 :block/parent [:block/uuid page-uuid]]
|
||||
[:db/add -1 :block/order "a0"]
|
||||
[:db/add [:block/uuid sibling-uuid] :block/title "sibling-2"]]
|
||||
{})
|
||||
normalized-a (db-normalize/normalize-tx-data (:db-after tx-report-a)
|
||||
(:db-before tx-report-a)
|
||||
(:tx-data tx-report-a))
|
||||
|
||||
;; Shape B: same recreate, plus child reparent to recreated target.
|
||||
tx-report-b (d/with db-before
|
||||
[[:db/retractEntity [:block/uuid target-uuid]]
|
||||
[:db/add -1 :block/uuid target-uuid]
|
||||
[:db/add -1 :block/title "new-target-b"]
|
||||
[:db/add -1 :block/page [:block/uuid page-uuid]]
|
||||
[:db/add -1 :block/parent [:block/uuid page-uuid]]
|
||||
[:db/add -1 :block/order "a2"]
|
||||
[:db/add [:block/uuid child-uuid] :block/parent [:block/uuid target-uuid]]
|
||||
[:db/add [:block/uuid sibling-uuid] :block/title "sibling-3"]]
|
||||
{})
|
||||
normalized-b (db-normalize/normalize-tx-data (:db-after tx-report-b)
|
||||
(:db-before tx-report-b)
|
||||
(:tx-data tx-report-b))]
|
||||
(is (some? (d/entity (:db-after tx-report-a) [:block/uuid target-uuid])))
|
||||
(is (some? (d/entity (:db-after tx-report-b) [:block/uuid target-uuid])))
|
||||
(is (tx-touches-uuid? normalized-a target-uuid)
|
||||
(str "shape A unexpectedly dropped recreated block from normalized tx: " (pr-str normalized-a)))
|
||||
(is (tx-touches-uuid? normalized-b target-uuid)
|
||||
(str "shape B unexpectedly dropped recreated block from normalized tx: " (pr-str normalized-b))))))
|
||||
|
||||
(deftest apply-client-delete-blocks-removes-current-server-subtree-test
|
||||
(testing "a delete-blocks tx deletes children that exist on the server but not in the client tx"
|
||||
(let [conn (new-conn)
|
||||
page-uuid (create-page! conn "Page")
|
||||
parent-uuid (random-uuid)
|
||||
child-uuid (random-uuid)
|
||||
now (js/Date.now)
|
||||
_ (d/transact! conn [{:block/uuid parent-uuid
|
||||
:block/title "parent"
|
||||
:block/created-at now
|
||||
:block/updated-at now
|
||||
:block/page [:block/uuid page-uuid]
|
||||
:block/parent [:block/uuid page-uuid]
|
||||
:block/order "a0"}
|
||||
{:block/uuid child-uuid
|
||||
:block/title "child"
|
||||
:block/created-at now
|
||||
:block/updated-at now
|
||||
:block/page [:block/uuid page-uuid]
|
||||
:block/parent [:block/uuid parent-uuid]
|
||||
:block/order "a0"}])
|
||||
result (try
|
||||
(ldb/transact! conn
|
||||
[[:db/retractEntity [:block/uuid parent-uuid]]]
|
||||
{:op :apply-client-tx
|
||||
:outliner-op :delete-blocks})
|
||||
:ok
|
||||
(catch :default e
|
||||
{:message (.-message e)
|
||||
:data (ex-data e)}))]
|
||||
(is (= :ok result))
|
||||
(is (nil? (d/entity @conn [:block/uuid parent-uuid])))
|
||||
(is (nil? (d/entity @conn [:block/uuid child-uuid]))))))
|
||||
|
||||
(deftest normalize-delete-blocks-keeps-property-history-retract-test
|
||||
(testing "a normalized delete-blocks tx still removes property history rows for the deleted block"
|
||||
(let [conn (new-conn)
|
||||
page-uuid (create-page! conn "Page")
|
||||
block-uuid (random-uuid)
|
||||
history-uuid (random-uuid)
|
||||
property-uuid (random-uuid)
|
||||
value-uuid (random-uuid)
|
||||
now (js/Date.now)]
|
||||
(d/transact! conn [{:block/uuid property-uuid
|
||||
:block/name "p"
|
||||
:block/title "P"
|
||||
:db/ident :user.property/p
|
||||
:db/valueType :db.type/ref
|
||||
:db/cardinality :db.cardinality/one
|
||||
:logseq.property/type :default}
|
||||
{:block/uuid value-uuid
|
||||
:block/title "value"
|
||||
:block/page [:block/uuid page-uuid]
|
||||
:block/parent [:block/uuid page-uuid]
|
||||
:block/order "a0"}
|
||||
{:block/uuid block-uuid
|
||||
:block/title "target"
|
||||
:block/page [:block/uuid page-uuid]
|
||||
:block/parent [:block/uuid page-uuid]
|
||||
:block/order "a1"
|
||||
:block/created-at now
|
||||
:block/updated-at now}
|
||||
{:block/uuid history-uuid
|
||||
:block/created-at now
|
||||
:block/updated-at now
|
||||
:logseq.property.history/block [:block/uuid block-uuid]
|
||||
:logseq.property.history/property :user.property/p
|
||||
:logseq.property.history/ref-value [:block/uuid value-uuid]}])
|
||||
(let [client-conn (d/conn-from-db @conn)
|
||||
tx-report (ldb/transact! conn
|
||||
[[:db/retractEntity [:block/uuid block-uuid]]]
|
||||
{:outliner-op :delete-blocks})
|
||||
normalized (db-normalize/normalize-tx-data (:db-after tx-report)
|
||||
(:db-before tx-report)
|
||||
(:tx-data tx-report))]
|
||||
(is (some #(= [:db/retractEntity [:block/uuid history-uuid]] %) normalized))
|
||||
(ldb/transact! client-conn normalized {:transact-remote? true
|
||||
:outliner-op :delete-blocks})
|
||||
(is (nil? (d/entity @client-conn [:block/uuid block-uuid])))
|
||||
(is (nil? (d/entity @client-conn [:block/uuid history-uuid])))))))
|
||||
|
||||
(deftest remote-delete-blocks-history-ref-retract-deletes-history-entity-test
|
||||
(testing "remote delete-blocks tx fully removes property history rows when their history block ref is retracted"
|
||||
(let [conn (new-conn)
|
||||
page-uuid (create-page! conn "Page")
|
||||
block-uuid (random-uuid)
|
||||
history-uuid (random-uuid)
|
||||
property-uuid (random-uuid)
|
||||
value-uuid (random-uuid)
|
||||
now (js/Date.now)]
|
||||
(d/transact! conn [{:block/uuid property-uuid
|
||||
:block/name "p"
|
||||
:block/title "P"
|
||||
:db/ident :user.property/p
|
||||
:db/valueType :db.type/ref
|
||||
:db/cardinality :db.cardinality/one
|
||||
:logseq.property/type :default}
|
||||
{:block/uuid value-uuid
|
||||
:block/title "value"
|
||||
:block/page [:block/uuid page-uuid]
|
||||
:block/parent [:block/uuid page-uuid]
|
||||
:block/order "a0"}
|
||||
{:block/uuid block-uuid
|
||||
:block/title "target"
|
||||
:block/page [:block/uuid page-uuid]
|
||||
:block/parent [:block/uuid page-uuid]
|
||||
:block/order "a1"
|
||||
:block/created-at now
|
||||
:block/updated-at now}
|
||||
{:block/uuid history-uuid
|
||||
:block/created-at now
|
||||
:block/updated-at now
|
||||
:logseq.property.history/block [:block/uuid block-uuid]
|
||||
:logseq.property.history/property :user.property/p
|
||||
:logseq.property.history/ref-value [:block/uuid value-uuid]}])
|
||||
(ldb/transact! conn
|
||||
[[:db/retract
|
||||
[:block/uuid history-uuid]
|
||||
:logseq.property.history/block
|
||||
[:block/uuid block-uuid]]]
|
||||
{:transact-remote? true
|
||||
:outliner-op :delete-blocks})
|
||||
(is (nil? (d/entity @conn [:block/uuid history-uuid]))))))
|
||||
|
||||
(deftest normalize-tx-data-replay-equivalence-for-retract-recreate-fuzz-test
|
||||
(testing "normalized tx-data should replay to same db-after for normal-block retract/recreate patterns"
|
||||
(let [conn (new-conn)
|
||||
page-uuid (create-page! conn "Page")]
|
||||
(d/transact! conn (mapv (fn [idx]
|
||||
{:block/uuid (random-uuid)
|
||||
:block/title (str "seed-block-" idx)
|
||||
:block/page [:block/uuid page-uuid]
|
||||
:block/parent [:block/uuid page-uuid]
|
||||
:block/order (str "a" idx)})
|
||||
(range 5)))
|
||||
(dotimes [seed 300]
|
||||
(let [rng (seeded-rng (+ 777 seed))
|
||||
db-before @conn
|
||||
blocks (normal-block-uuids db-before)
|
||||
target (pick-rand rng blocks)
|
||||
sibling (pick-rand rng (remove #(= % target) blocks))
|
||||
move-candidate (pick-rand rng (remove #(= % target) blocks))
|
||||
case-id (mod seed 6)
|
||||
base [[:db/retractEntity [:block/uuid target]]
|
||||
[:db/add -1 :block/uuid target]
|
||||
[:db/add -1 :block/title (str "fuzz-target-" seed)]
|
||||
[:db/add -1 :block/page [:block/uuid page-uuid]]
|
||||
[:db/add -1 :block/parent [:block/uuid page-uuid]]
|
||||
[:db/add -1 :block/order (str "z" (mod seed 5))]]
|
||||
tx-data (case case-id
|
||||
0
|
||||
(cond-> base
|
||||
sibling (conj [:db/add [:block/uuid sibling] :block/title (str "fuzz-sibling-title-" seed)]))
|
||||
|
||||
1
|
||||
(cond-> base
|
||||
sibling (conj [:db/retractEntity [:block/uuid sibling]]))
|
||||
|
||||
2
|
||||
(if sibling
|
||||
(into base
|
||||
[[:db/retractEntity [:block/uuid sibling]]
|
||||
[:db/add -2 :block/uuid sibling]
|
||||
[:db/add -2 :block/title (str "fuzz-sibling-recreated-" seed)]
|
||||
[:db/add -2 :block/page [:block/uuid page-uuid]]
|
||||
[:db/add -2 :block/parent [:block/uuid page-uuid]]
|
||||
[:db/add -2 :block/order (str "y" (mod seed 5))]])
|
||||
base)
|
||||
|
||||
3
|
||||
(into base
|
||||
[[:db/add -2 :block/uuid (random-uuid)]
|
||||
[:db/add -2 :block/title (str "fuzz-fresh-" seed)]
|
||||
[:db/add -2 :block/page [:block/uuid page-uuid]]
|
||||
[:db/add -2 :block/parent [:block/uuid page-uuid]]
|
||||
[:db/add -2 :block/order (str "x" (mod seed 7))]])
|
||||
|
||||
4
|
||||
(if sibling
|
||||
(into base
|
||||
[[:db/retractEntity [:block/uuid sibling]]
|
||||
[:db/add -2 :block/uuid sibling]
|
||||
[:db/add -2 :block/title (str "fuzz-sibling-reparented-" seed)]
|
||||
[:db/add -2 :block/page [:block/uuid page-uuid]]
|
||||
[:db/add -2 :block/parent [:block/uuid target]]
|
||||
[:db/add -2 :block/order "a0"]])
|
||||
base)
|
||||
|
||||
;; 5
|
||||
(cond-> base
|
||||
move-candidate
|
||||
(conj [:db/add [:block/uuid move-candidate] :block/parent [:block/uuid target]])))
|
||||
tx-report (d/with db-before tx-data {})
|
||||
normalized (db-normalize/normalize-tx-data (:db-after tx-report)
|
||||
(:db-before tx-report)
|
||||
(:tx-data tx-report))
|
||||
replay-report (d/with db-before normalized {})
|
||||
expected (block-state (:db-after tx-report))
|
||||
actual (block-state (:db-after replay-report))]
|
||||
(is (= expected actual)
|
||||
(str "seed=" seed
|
||||
"\noriginal=" (pr-str tx-data)
|
||||
"\nnormalized=" (pr-str normalized)
|
||||
"\nexpected-count=" (count expected)
|
||||
" actual-count=" (count actual))))))))
|
||||
272
deps/db-sync/test/logseq/db_sync/storage_test.cljs
vendored
Normal file
272
deps/db-sync/test/logseq/db_sync/storage_test.cljs
vendored
Normal file
@@ -0,0 +1,272 @@
|
||||
(ns logseq.db-sync.storage-test
|
||||
(:require ["better-sqlite3" :as sqlite3]
|
||||
[clojure.string :as string]
|
||||
[cljs.test :refer [deftest is testing]]
|
||||
[datascript.core :as d]
|
||||
[logseq.db :as ldb]
|
||||
[logseq.db-sync.common :as common]
|
||||
[logseq.db-sync.storage :as storage]
|
||||
[logseq.db.common.normalize :as db-normalize]
|
||||
[logseq.db-sync.test-sql :as test-sql]))
|
||||
|
||||
(def sqlite (if (find-ns 'nbb.core) (aget sqlite3 "default") sqlite3))
|
||||
|
||||
(defn- select-sql?
|
||||
[sql]
|
||||
(string/starts-with? (-> sql string/trim string/lower-case) "select"))
|
||||
|
||||
(defn- run-sql
|
||||
[^js stmt args]
|
||||
(.apply (.-run stmt) stmt (to-array args)))
|
||||
|
||||
(defn- all-sql
|
||||
[^js stmt args]
|
||||
(.apply (.-all stmt) stmt (to-array args)))
|
||||
|
||||
(defn- with-memory-sql
|
||||
[f]
|
||||
(let [db (new sqlite ":memory:" nil)
|
||||
sql #js {:exec (fn [sql-str & args]
|
||||
(let [stmt (.prepare db sql-str)]
|
||||
(if (select-sql? sql-str)
|
||||
(all-sql stmt args)
|
||||
(do
|
||||
(run-sql stmt args)
|
||||
nil))))
|
||||
:close (fn []
|
||||
(.close db))}]
|
||||
(try
|
||||
(f sql)
|
||||
(finally
|
||||
(.close sql)))))
|
||||
|
||||
(defn- seeded-rng
|
||||
[seed0]
|
||||
(let [state (atom (bit-or (long seed0) 0))]
|
||||
(fn []
|
||||
(let [s (swap! state
|
||||
(fn [x]
|
||||
(let [x (bit-xor x (bit-shift-left x 13))
|
||||
x (bit-xor x (bit-shift-right x 17))
|
||||
x (bit-xor x (bit-shift-left x 5))]
|
||||
(bit-or x 0))))]
|
||||
(/ (double (unsigned-bit-shift-right s 0)) 4294967296.0)))))
|
||||
|
||||
(defn- rand-int*
|
||||
[rng n]
|
||||
(js/Math.floor (* (rng) n)))
|
||||
|
||||
(defn- pick-rand
|
||||
[rng coll]
|
||||
(when (seq coll)
|
||||
(nth coll (rand-int* rng (count coll)))))
|
||||
|
||||
(defn- normal-block-uuids
|
||||
[db]
|
||||
(->> (d/datoms db :avet :block/uuid)
|
||||
(map :e)
|
||||
distinct
|
||||
(keep (fn [eid]
|
||||
(let [ent (d/entity db eid)]
|
||||
(when (and (uuid? (:block/uuid ent))
|
||||
(not (ldb/built-in? ent))
|
||||
(nil? (:block/name ent))
|
||||
(some? (:block/page ent)))
|
||||
(:block/uuid ent)))))
|
||||
vec))
|
||||
|
||||
(deftest t-meta-test
|
||||
(let [sql (test-sql/make-sql)]
|
||||
(storage/init-schema! sql)
|
||||
(is (= 0 (storage/get-t sql)))
|
||||
(storage/set-t! sql 1)
|
||||
(is (= 1 (storage/get-t sql)))
|
||||
(storage/set-t! sql 2)
|
||||
(is (= 2 (storage/get-t sql)))))
|
||||
|
||||
(deftest tx-log-test
|
||||
(let [sql (test-sql/make-sql)]
|
||||
(storage/init-schema! sql)
|
||||
(storage/append-tx! sql 1 "tx-1" 100 :save-block)
|
||||
(storage/append-tx! sql 2 "tx-2" 200 :move-blocks)
|
||||
(storage/append-tx! sql 3 "tx-3" 300 nil)
|
||||
(let [result (storage/fetch-tx-since sql 1)]
|
||||
(is (= [{:t 2 :tx "tx-2" :outliner-op :move-blocks}
|
||||
{:t 3 :tx "tx-3" :outliner-op nil}]
|
||||
result)))))
|
||||
|
||||
(deftest stale-checksum-no-op-transact-does-not-throw-test
|
||||
(testing "a no-op tx should not throw and should keep incremental checksum state"
|
||||
(with-memory-sql
|
||||
(fn [sql]
|
||||
(let [stale-checksum "f4b78e83776d45fb"]
|
||||
(storage/init-schema! sql)
|
||||
(storage/set-checksum! sql stale-checksum)
|
||||
(let [conn (storage/open-conn sql)
|
||||
result (try
|
||||
(d/transact! conn
|
||||
[]
|
||||
{:outliner-op :rebase})
|
||||
:ok
|
||||
(catch :default e
|
||||
e))]
|
||||
(is (= :ok result))
|
||||
(is (= stale-checksum
|
||||
(storage/get-checksum sql)))))))))
|
||||
|
||||
(deftest initial-checksum-does-not-overwrite-existing-checksum-test
|
||||
(testing "snapshot checksum initialization must not replace an existing incremental checksum"
|
||||
(let [sql (test-sql/make-sql)
|
||||
existing-checksum "aaaaaaaaaaaaaaaa"
|
||||
snapshot-checksum "bbbbbbbbbbbbbbbb"]
|
||||
(storage/init-schema! sql)
|
||||
(storage/set-initial-checksum! sql existing-checksum)
|
||||
(let [result (try
|
||||
(storage/set-initial-checksum! sql snapshot-checksum)
|
||||
:ok
|
||||
(catch :default error
|
||||
error))]
|
||||
(is (not= :ok result))
|
||||
(is (= existing-checksum (storage/get-checksum sql)))))))
|
||||
|
||||
(deftest initial-checksum-rejects-non-empty-tx-log-test
|
||||
(testing "snapshot checksum initialization must not run after tx history already advanced"
|
||||
(let [sql (test-sql/make-sql)
|
||||
snapshot-checksum "bbbbbbbbbbbbbbbb"]
|
||||
(storage/init-schema! sql)
|
||||
(storage/append-tx! sql 1 "tx-1" 100 :save-block)
|
||||
(storage/set-t! sql 1)
|
||||
(let [result (try
|
||||
(storage/set-initial-checksum! sql snapshot-checksum)
|
||||
:ok
|
||||
(catch :default error
|
||||
error))]
|
||||
(is (not= :ok result))
|
||||
(is (nil? (storage/get-checksum sql)))))))
|
||||
|
||||
(deftest stale-checksum-transact-keeps-kvs-and-tx-log-consistent-test
|
||||
(testing "stale checksum should not fail transact; kvs and tx_log/t should advance together"
|
||||
(with-memory-sql
|
||||
(fn [sql]
|
||||
(storage/init-schema! sql)
|
||||
(let [conn (storage/open-conn sql)
|
||||
stale-checksum "ffffffffffffffff"
|
||||
page-uuid (random-uuid)]
|
||||
;; Use a stale checksum and ensure append path remains consistent.
|
||||
(storage/set-checksum! sql stale-checksum)
|
||||
(let [result (try
|
||||
(d/transact! conn [{:block/uuid page-uuid
|
||||
:block/name "repro-kvs-ahead-page"}])
|
||||
:ok
|
||||
(catch :default e
|
||||
e))]
|
||||
(is (= :ok result))
|
||||
(is (= 1 (storage/get-t sql)))
|
||||
(is (= 1 (count (storage/fetch-tx-since sql 0))))
|
||||
(is (not= stale-checksum (storage/get-checksum sql)))
|
||||
(let [restored-conn (storage/open-conn sql)]
|
||||
(is (= page-uuid
|
||||
(:block/uuid (d/entity @restored-conn [:block/uuid page-uuid])))))))))))
|
||||
|
||||
(deftest tx-log-append-failure-does-not-advance-checksum-test
|
||||
(testing "server checksum and t should not advance when the tx log append fails"
|
||||
(with-memory-sql
|
||||
(fn [sql]
|
||||
(storage/init-schema! sql)
|
||||
(let [conn (storage/open-conn sql)
|
||||
page-uuid (random-uuid)
|
||||
failed-block-uuid (random-uuid)]
|
||||
(d/transact! conn [{:block/uuid page-uuid
|
||||
:block/name "tx-log-failure-repro"}])
|
||||
(let [checksum-before (storage/get-checksum sql)
|
||||
t-before (storage/get-t sql)
|
||||
original-sql-exec common/sql-exec
|
||||
result (with-redefs [common/sql-exec
|
||||
(fn [sql sql-str & args]
|
||||
(if (string/includes? sql-str "insert into tx_log")
|
||||
(throw (js/Error. "tx log append failed"))
|
||||
(apply original-sql-exec sql sql-str args)))]
|
||||
(try
|
||||
(d/transact! conn [{:block/uuid failed-block-uuid
|
||||
:block/title "failed append"
|
||||
:block/page [:block/uuid page-uuid]
|
||||
:block/parent [:block/uuid page-uuid]}])
|
||||
:ok
|
||||
(catch :default error
|
||||
error)))]
|
||||
(is (not= :ok result))
|
||||
(is (= t-before (storage/get-t sql)))
|
||||
(is (= checksum-before (storage/get-checksum sql)))))))))
|
||||
|
||||
(deftest normalize-drop-can-hide-kvs-mutation-from-tx-log-test
|
||||
(testing "if normalize drops tx payload, tx_log can miss persisted kvs state changes"
|
||||
(with-memory-sql
|
||||
(fn [sql]
|
||||
(storage/init-schema! sql)
|
||||
(let [conn (storage/open-conn sql)
|
||||
page-uuid (random-uuid)]
|
||||
(with-redefs [db-normalize/normalize-tx-data (fn [_db-after _db-before _tx-data]
|
||||
[])]
|
||||
(d/transact! conn [{:block/uuid page-uuid
|
||||
:block/name "normalize-drop-repro"}]))
|
||||
(is (= 1 (storage/get-t sql)))
|
||||
(let [entries (storage/fetch-tx-since sql 0)]
|
||||
(is (= 1 (count entries)))
|
||||
(is (= []
|
||||
(common/read-transit (:tx (first entries)))))
|
||||
(let [restored-conn (storage/open-conn sql)]
|
||||
(is (= page-uuid
|
||||
(:block/uuid (d/entity @restored-conn [:block/uuid page-uuid])))))))))))
|
||||
|
||||
(deftest randomized-normal-block-retract-recreate-does-not-throw-checksum-mismatch-test
|
||||
(testing "normal block retract/recreate patterns should not naturally trigger server checksum mismatch"
|
||||
(with-memory-sql
|
||||
(fn [sql]
|
||||
(storage/init-schema! sql)
|
||||
(let [conn (storage/open-conn sql)
|
||||
rng (seeded-rng 424242)
|
||||
page-uuid (random-uuid)
|
||||
_ (d/transact! conn [{:block/uuid page-uuid
|
||||
:block/name "repro-page"
|
||||
:block/title "repro-page"}])
|
||||
_ (d/transact! conn (mapv (fn [idx]
|
||||
{:block/uuid (random-uuid)
|
||||
:block/title (str "seed-" idx)
|
||||
:block/page [:block/uuid page-uuid]
|
||||
:block/parent [:block/uuid page-uuid]
|
||||
:block/order (str "a" idx)})
|
||||
(range 5)))
|
||||
*mismatch (atom nil)]
|
||||
(dotimes [step 500]
|
||||
(when-not @*mismatch
|
||||
(let [db @conn
|
||||
blocks (normal-block-uuids db)
|
||||
target (pick-rand rng blocks)
|
||||
sibling (pick-rand rng (remove #(= % target) blocks))
|
||||
add-fresh? (< (rng) 0.3)
|
||||
tx-data (cond-> [[:db/retractEntity [:block/uuid target]]
|
||||
[:db/add -1 :block/uuid target]
|
||||
[:db/add -1 :block/title (str "rr-" step)]
|
||||
[:db/add -1 :block/page [:block/uuid page-uuid]]
|
||||
[:db/add -1 :block/parent [:block/uuid page-uuid]]
|
||||
[:db/add -1 :block/order (str "z" (mod step 7))]]
|
||||
sibling
|
||||
(conj [:db/add [:block/uuid sibling] :block/title (str "sib-" step)])
|
||||
add-fresh?
|
||||
(into [[:db/add -2 :block/uuid (random-uuid)]
|
||||
[:db/add -2 :block/title (str "fresh-" step)]
|
||||
[:db/add -2 :block/page [:block/uuid page-uuid]]
|
||||
[:db/add -2 :block/parent [:block/uuid page-uuid]]
|
||||
[:db/add -2 :block/order (str "x" (mod step 9))]]))]
|
||||
(try
|
||||
(d/transact! conn tx-data)
|
||||
(catch :default e
|
||||
(let [message (or (ex-message e) (some-> e .-message) (str e))]
|
||||
(if (string/includes? message "server checksum doesn't match")
|
||||
(reset! *mismatch {:step step
|
||||
:tx-data tx-data
|
||||
:error message})
|
||||
;; tx can be invalid due random -2 fresh fields; ignore non-checksum failures
|
||||
nil)))))))
|
||||
(is (nil? @*mismatch)
|
||||
(str "found checksum mismatch repro: " (pr-str @*mismatch))))))))
|
||||
36
deps/db-sync/test/logseq/db_sync/test_runner.cljs
vendored
Normal file
36
deps/db-sync/test/logseq/db_sync/test_runner.cljs
vendored
Normal file
@@ -0,0 +1,36 @@
|
||||
(ns logseq.db-sync.test-runner
|
||||
(:require [cljs.test :as ct]
|
||||
[logseq.db-sync.checksum-test]
|
||||
[logseq.db-sync.common-test]
|
||||
[logseq.db-sync.index-test]
|
||||
[logseq.db-sync.malli-schema-test]
|
||||
[logseq.db-sync.node-adapter-test]
|
||||
[logseq.db-sync.node-config-test]
|
||||
[logseq.db-sync.node-server-test]
|
||||
[logseq.db-sync.normalize-test]
|
||||
[logseq.db-sync.storage-test]
|
||||
[logseq.db-sync.tx-sanitize-test]
|
||||
[logseq.db-sync.worker-auth-test]
|
||||
[logseq.db-sync.worker-dispatch-test]
|
||||
[logseq.db-sync.worker-handler-assets-test]
|
||||
[logseq.db-sync.worker-handler-index-test]
|
||||
[logseq.db-sync.worker-handler-sync-test]
|
||||
[logseq.db-sync.worker-handler-ws-test]
|
||||
[shadow.test :as st]
|
||||
[shadow.test.env :as env]))
|
||||
|
||||
(derive ::node ::ct/default)
|
||||
|
||||
(defmethod ct/report [::node :end-run-tests] [m]
|
||||
(if (ct/successful? m)
|
||||
(js/process.exit 0)
|
||||
(js/process.exit 1)))
|
||||
|
||||
(defn ^:dev/after-load reset-test-data! []
|
||||
(-> (env/get-test-data)
|
||||
(env/reset-test-data!)))
|
||||
|
||||
(defn main [& _args]
|
||||
(reset-test-data!)
|
||||
(let [test-env (ct/empty-env ::node)]
|
||||
(st/run-all-tests test-env nil)))
|
||||
56
deps/db-sync/test/logseq/db_sync/test_sql.cljs
vendored
Normal file
56
deps/db-sync/test/logseq/db_sync/test_sql.cljs
vendored
Normal file
@@ -0,0 +1,56 @@
|
||||
(ns logseq.db-sync.test-sql
|
||||
(:require [clojure.string :as string]))
|
||||
|
||||
(defn- js-key [k]
|
||||
(cond
|
||||
(keyword? k) (string/replace (name k) "-" "_")
|
||||
(string? k) k
|
||||
:else (str k)))
|
||||
|
||||
(defn- js-row [m]
|
||||
(let [o (js-obj)]
|
||||
(doseq [[k v] m]
|
||||
(aset o (js-key k) v))
|
||||
o))
|
||||
|
||||
(defn- js-rows [rows]
|
||||
(into-array (map js-row rows)))
|
||||
|
||||
(defn make-sql []
|
||||
(let [state (atom {:tx-log {}
|
||||
:meta {}})]
|
||||
#js {:exec (fn [sql & args]
|
||||
(cond
|
||||
(string/includes? sql "insert into tx_log")
|
||||
(let [[t tx created-at outliner-op] args]
|
||||
(swap! state update :tx-log assoc t {:t t
|
||||
:tx tx
|
||||
:created-at created-at
|
||||
:outliner-op outliner-op})
|
||||
nil)
|
||||
|
||||
(string/includes? sql "select t, tx, outliner_op from tx_log")
|
||||
(let [since (first args)
|
||||
rows (->> (:tx-log @state)
|
||||
vals
|
||||
(filter (fn [row] (> (:t row) since)))
|
||||
(sort-by :t)
|
||||
(map (fn [row] {:t (:t row)
|
||||
:tx (:tx row)
|
||||
:outliner-op (:outliner-op row)})))]
|
||||
(js-rows rows))
|
||||
|
||||
(string/includes? sql "insert into sync_meta")
|
||||
(let [[k v] args]
|
||||
(swap! state update :meta assoc k v)
|
||||
nil)
|
||||
|
||||
(string/includes? sql "select value from sync_meta")
|
||||
(let [k (first args)
|
||||
value (get-in @state [:meta k])]
|
||||
(if (some? value)
|
||||
(js-rows [{:value value}])
|
||||
(js-rows [])))
|
||||
|
||||
:else
|
||||
nil))}))
|
||||
104
deps/db-sync/test/logseq/db_sync/tx_sanitize_test.cljs
vendored
Normal file
104
deps/db-sync/test/logseq/db_sync/tx_sanitize_test.cljs
vendored
Normal file
@@ -0,0 +1,104 @@
|
||||
(ns logseq.db-sync.tx-sanitize-test
|
||||
(:require [cljs.test :refer [deftest is testing]]
|
||||
[clojure.set :as set]
|
||||
[datascript.core :as d]
|
||||
[logseq.db-sync.tx-sanitize :as tx-sanitize]
|
||||
[logseq.db.test.helper :as db-test]))
|
||||
|
||||
(def ^:private migration-deleted-attrs
|
||||
#{:block/path-refs
|
||||
:block/pre-block?
|
||||
:logseq.property.embedding/hnsw-label
|
||||
:logseq.property.embedding/hnsw-label-updated-at})
|
||||
|
||||
(def ^:private graph-backup-folder-ops
|
||||
#{[:db/retractEntity :logseq.kv/graph-backup-folder]
|
||||
[:db/add :logseq.kv/graph-backup-folder :logseq.kv/value "/tmp/backup"]})
|
||||
|
||||
(defn- tx-item-attrs
|
||||
[item]
|
||||
(cond
|
||||
(map? item)
|
||||
(set (keys item))
|
||||
|
||||
(and (vector? item) (<= 4 (count item)))
|
||||
#{(nth item 2)}
|
||||
|
||||
:else
|
||||
#{}))
|
||||
|
||||
(deftest sanitize-tx-drops-migration-deleted-attrs-test
|
||||
(testing "remote txs from older clients should not reintroduce attrs deleted by client migrations"
|
||||
(let [conn (db-test/create-conn)
|
||||
block-uuid #uuid "11111111-1111-1111-1111-111111111111"
|
||||
tx-data [[:db/add [:block/uuid block-uuid] :block/title "remote title"]
|
||||
[:db/add [:block/uuid block-uuid] :block/path-refs #uuid "22222222-2222-2222-2222-222222222222"]
|
||||
[:db/retract [:block/uuid block-uuid] :block/pre-block? true]
|
||||
[:db/add [:block/uuid block-uuid] :logseq.property.embedding/hnsw-label "label"]
|
||||
[:db/retract [:block/uuid block-uuid] :logseq.property.embedding/hnsw-label-updated-at 123]]
|
||||
sanitized (tx-sanitize/sanitize-tx @conn tx-data)
|
||||
sanitized-attrs (set (mapcat tx-item-attrs sanitized))]
|
||||
(is (empty? (set/intersection migration-deleted-attrs sanitized-attrs)))
|
||||
(is (some #(= [:db/add [:block/uuid block-uuid] :block/title "remote title"] %) sanitized)))))
|
||||
|
||||
(deftest sanitize-tx-drops-ignored-kv-entity-ops-test
|
||||
(testing "remote txs should not apply KV entities that are excluded from sync"
|
||||
(let [block-uuid #uuid "11111111-1111-1111-1111-111111111111"
|
||||
conn (db-test/create-conn)
|
||||
tx-data (into [[:db/add [:block/uuid block-uuid] :block/title "remote title"]]
|
||||
graph-backup-folder-ops)
|
||||
sanitized (tx-sanitize/sanitize-tx @conn tx-data)]
|
||||
(is (empty? (set/intersection graph-backup-folder-ops (set sanitized))))
|
||||
(is (some #(= [:db/add [:block/uuid block-uuid] :block/title "remote title"] %) sanitized)))))
|
||||
|
||||
(deftest sanitize-tx-drops-same-tx-ignored-kv-tempid-ops-test
|
||||
(testing "remote txs should drop all ops for tempids identified as ignored KV entities"
|
||||
(let [block-uuid #uuid "11111111-1111-1111-1111-111111111111"
|
||||
conn (db-test/create-conn)
|
||||
ignored-ops #{[:db/add "kv-temp" :db/ident :logseq.kv/graph-backup-folder]
|
||||
[:db/add "kv-temp" :logseq.kv/value "/tmp/backup"]
|
||||
[:db/retractEntity "kv-temp"]}
|
||||
tx-data (into [[:db/add [:block/uuid block-uuid] :block/title "remote title"]]
|
||||
ignored-ops)
|
||||
sanitized (tx-sanitize/sanitize-tx @conn tx-data)]
|
||||
(is (empty? (set/intersection ignored-ops (set sanitized))))
|
||||
(is (some #(= [:db/add [:block/uuid block-uuid] :block/title "remote title"] %) sanitized)))))
|
||||
|
||||
(deftest sanitize-tx-drops-same-tx-ignored-kv-map-ops-test
|
||||
(testing "remote txs should drop map-form ignored KV entities and following tempid ops"
|
||||
(let [block-uuid #uuid "11111111-1111-1111-1111-111111111111"
|
||||
conn (db-test/create-conn)
|
||||
ignored-map {:db/id -1
|
||||
:db/ident :logseq.kv/graph-backup-folder
|
||||
:logseq.kv/value "/tmp/backup"}
|
||||
ignored-ops #{ignored-map
|
||||
[:db/add -1 :logseq.kv/value "/tmp/backup-2"]
|
||||
[:db/retractEntity -1]}
|
||||
tx-data (into [[:db/add [:block/uuid block-uuid] :block/title "remote title"]]
|
||||
ignored-ops)
|
||||
sanitized (tx-sanitize/sanitize-tx @conn tx-data)]
|
||||
(is (empty? (set/intersection ignored-ops (set sanitized))))
|
||||
(is (some #(= [:db/add [:block/uuid block-uuid] :block/title "remote title"] %) sanitized)))))
|
||||
|
||||
(deftest sanitize-delete-blocks-retracts-property-value-children-test
|
||||
(testing "delete-blocks adds retracts for generated property value children"
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:properties {:user.property/cli-http-prop {:logseq.property/type :default}}
|
||||
:pages-and-blocks
|
||||
[{:page {:block/title "Page"}
|
||||
:blocks [{:block/title "Parent"
|
||||
:build/properties
|
||||
{:user.property/cli-http-prop
|
||||
{:build/property-value :block
|
||||
:block/title "Property value"}}}]}]})
|
||||
parent (db-test/find-block-by-content @conn "Parent")
|
||||
property-value (db-test/find-block-by-content @conn "Property value")
|
||||
tx-data [[:db/retractEntity [:block/uuid (:block/uuid parent)]]]
|
||||
sanitized (tx-sanitize/sanitize-tx @conn
|
||||
tx-data
|
||||
{:drop-missing-retract-ops? true
|
||||
:drop-ops-targeting-retracted-entities? true
|
||||
:retract-touched-descendants? true})]
|
||||
(is (= (:db/id property-value)
|
||||
(:db/id (:user.property/cli-http-prop (d/entity @conn (:db/id parent))))))
|
||||
(is (some #(= [:db/retractEntity (:db/id property-value)] %) sanitized)))))
|
||||
107
deps/db-sync/test/logseq/db_sync/worker_auth_test.cljs
vendored
Normal file
107
deps/db-sync/test/logseq/db_sync/worker_auth_test.cljs
vendored
Normal file
@@ -0,0 +1,107 @@
|
||||
(ns logseq.db-sync.worker-auth-test
|
||||
(:require [cljs.test :refer [async deftest is]]
|
||||
[logseq.common.authorization :as authorization]
|
||||
[logseq.db-sync.worker.auth :as auth]
|
||||
[promesa.core :as p]))
|
||||
|
||||
(deftest cognito-client-id-allowlist-test
|
||||
(let [env #js {"COGNITO_CLIENT_ID" "web-client"
|
||||
"COGNITO_CLIENT_IDS" "chatgpt-client, admin-client"}]
|
||||
(is (authorization/client-id-allowed? env "web-client"))
|
||||
(is (authorization/client-id-allowed? env "chatgpt-client"))
|
||||
(is (authorization/client-id-allowed? env "admin-client"))
|
||||
(is (not (authorization/client-id-allowed? env "unknown-client")))))
|
||||
|
||||
(deftest cognito-client-id-allowlist-fails-closed-test
|
||||
(is (not (authorization/client-id-allowed? #js {} nil)))
|
||||
(is (not (authorization/client-id-allowed? #js {} "")))
|
||||
(is (not (authorization/client-id-allowed? #js {"COGNITO_CLIENT_ID" ""} "")))
|
||||
(is (not (authorization/client-id-allowed? #js {"COGNITO_CLIENT_IDS" " , "} "chatgpt-client")))
|
||||
(is (not (authorization/client-id-allowed? #js {"COGNITO_CLIENT_ID" "web-client"} nil))))
|
||||
|
||||
(deftest auth-claims-uses-jwt-verification-test
|
||||
(async done
|
||||
(let [request (js/Request. "http://localhost/graphs"
|
||||
#js {:headers #js {"authorization" "Bearer dev-token"}})]
|
||||
(-> (p/with-redefs [authorization/verify-jwt
|
||||
(fn [token _env]
|
||||
(js/Promise.resolve #js {"sub" (str "jwt:" token)}))]
|
||||
(p/let [claims (auth/auth-claims request #js {})]
|
||||
(is (= "jwt:dev-token" (aget claims "sub")))))
|
||||
(p/then (fn [] (done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
|
||||
(deftest auth-claims-without-token-returns-nil-test
|
||||
(async done
|
||||
(let [request (js/Request. "http://localhost/graphs")]
|
||||
(-> (p/let [claims (auth/auth-claims request #js {})]
|
||||
(is (nil? claims)))
|
||||
(p/then (fn [] (done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
|
||||
(deftest auth-claims-expired-token-returns-nil-test
|
||||
(async done
|
||||
(let [request (js/Request. "http://localhost/graphs"
|
||||
#js {:headers #js {"authorization" "Bearer expired-token"}})]
|
||||
(-> (p/with-redefs [authorization/verify-jwt
|
||||
(fn [_token _env]
|
||||
(p/rejected (ex-info "exp" {})))]
|
||||
(p/let [claims (auth/auth-claims request #js {})]
|
||||
(is (nil? claims))))
|
||||
(p/then (fn [] (done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
|
||||
(deftest auth-claims-jwks-error-propagates-test
|
||||
(async done
|
||||
(let [request (js/Request. "http://localhost/graphs"
|
||||
#js {:headers #js {"authorization" "Bearer broken-token"}})]
|
||||
(-> (p/with-redefs [authorization/verify-jwt
|
||||
(fn [_token _env]
|
||||
(p/rejected (ex-info "jwks" {})))]
|
||||
(auth/auth-claims request #js {}))
|
||||
(p/then (fn [_]
|
||||
(is false "expected rejection when jwks fetch fails")
|
||||
(done)))
|
||||
(p/catch (fn [error]
|
||||
(is (= "jwks" (ex-message error)))
|
||||
(done)))))))
|
||||
|
||||
(deftest auth-claims-jwks-error-falls-back-to-unsafe-claims-when-enabled-test
|
||||
(async done
|
||||
(let [token "eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJ1MSJ9.signature"
|
||||
request (js/Request. "http://localhost/graphs"
|
||||
#js {:headers #js {"authorization" (str "Bearer " token)}})
|
||||
env #js {"DB_SYNC_ALLOW_UNVERIFIED_JWT_CLAIMS" "true"}]
|
||||
(-> (p/with-redefs [authorization/verify-jwt
|
||||
(fn [_token _env]
|
||||
(p/rejected (ex-info "jwks" {})))]
|
||||
(p/let [claims (auth/auth-claims request env)]
|
||||
(is (= "u1" (aget claims "sub")))))
|
||||
(p/then (fn [] (done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
|
||||
(deftest auth-claims-expired-jwt-short-circuits-verification-test
|
||||
(async done
|
||||
(let [expired-token "eyJhbGciOiJSUzI1NiJ9.eyJleHAiOjEsInN1YiI6InUxIn0.signature"
|
||||
request (js/Request. "http://localhost/graphs"
|
||||
#js {:headers #js {"authorization" (str "Bearer " expired-token)}})
|
||||
verify-called? (atom false)]
|
||||
(-> (p/with-redefs [authorization/verify-jwt
|
||||
(fn [_token _env]
|
||||
(reset! verify-called? true)
|
||||
(p/rejected (ex-info "should-not-be-called" {})))]
|
||||
(p/let [claims (auth/auth-claims request #js {})]
|
||||
(is (nil? claims))
|
||||
(is (false? @verify-called?))))
|
||||
(p/then (fn [] (done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
392
deps/db-sync/test/logseq/db_sync/worker_dispatch_test.cljs
vendored
Normal file
392
deps/db-sync/test/logseq/db_sync/worker_dispatch_test.cljs
vendored
Normal file
@@ -0,0 +1,392 @@
|
||||
(ns logseq.db-sync.worker-dispatch-test
|
||||
(:require [cljs.test :refer [async deftest is]]
|
||||
[clojure.string :as string]
|
||||
[logseq.db-sync.common :as common]
|
||||
[logseq.db-sync.index :as index]
|
||||
[logseq.db-sync.worker.auth :as auth]
|
||||
[logseq.db-sync.worker.dispatch :as dispatch]
|
||||
[logseq.db-sync.worker.handler.assets :as assets-handler]
|
||||
[logseq.db-sync.worker.handler.index :as index-handler]
|
||||
[logseq.db-sync.worker.http :as http]
|
||||
[promesa.core :as p]))
|
||||
|
||||
(defn- ok-json-response []
|
||||
(js/Response. (js/JSON.stringify #js {:ok true})
|
||||
#js {:status 200
|
||||
:headers #js {"content-type" "application/json"}}))
|
||||
|
||||
(defn- make-do-namespace []
|
||||
#js {:idFromName (fn [_graph-id] "do-id")
|
||||
:get (fn [_do-id]
|
||||
#js {:fetch (fn [_request]
|
||||
(js/Promise.resolve (ok-json-response)))})})
|
||||
|
||||
(declare capturing-do-namespace json-body)
|
||||
|
||||
(deftest admin-token-bypasses-graph-access-check-for-sync-route-test
|
||||
(async done
|
||||
(let [access-check-calls (atom 0)
|
||||
request (js/Request. "http://localhost/sync/graph-1/snapshot/download"
|
||||
#js {:method "GET"
|
||||
:headers #js {"x-db-sync-admin-token" "test-admin-token"}})
|
||||
env #js {"DB_SYNC_ADMIN_TOKEN" "test-admin-token"
|
||||
"LOGSEQ_SYNC_DO" (make-do-namespace)}]
|
||||
(-> (p/with-redefs [index-handler/graph-access-response (fn [_request _env _graph-id]
|
||||
(swap! access-check-calls inc)
|
||||
(p/resolved (http/unauthorized)))]
|
||||
(p/let [resp (dispatch/handle-worker-fetch request env)
|
||||
text (.text resp)
|
||||
body (js->clj (js/JSON.parse text) :keywordize-keys true)]
|
||||
(is (= 200 (.-status resp)))
|
||||
(is (= true (:ok body)))
|
||||
(is (= 0 @access-check-calls))))
|
||||
(p/then (fn [] (done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
|
||||
(deftest public-sync-route-rejects-semantic-tail-without-touching-durable-object-test
|
||||
(async done
|
||||
(let [forwarded (atom [])
|
||||
request (js/Request. "http://localhost/sync/graph-1/semantic/pages"
|
||||
#js {:method "GET"})
|
||||
env #js {"LOGSEQ_SYNC_DO" (capturing-do-namespace forwarded)}]
|
||||
(-> (p/with-redefs [index-handler/graph-access-response
|
||||
(fn [_request _env _graph-id]
|
||||
(p/resolved (ok-json-response)))]
|
||||
(p/let [response (dispatch/handle-worker-fetch request env)
|
||||
body (json-body response)]
|
||||
(is (= 404 (.-status response)))
|
||||
(is (= "not found" (:error body)))
|
||||
(is (empty? @forwarded))))
|
||||
(p/then (fn [] (done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
|
||||
(deftest public-sync-route-still-forwards-legacy-snapshot-request-test
|
||||
(async done
|
||||
(let [forwarded (atom [])
|
||||
request (js/Request. "http://localhost/sync/graph-1/snapshot/download"
|
||||
#js {:method "GET"})
|
||||
env #js {"LOGSEQ_SYNC_DO" (capturing-do-namespace forwarded)}]
|
||||
(-> (p/with-redefs [index-handler/graph-access-response
|
||||
(fn [_request _env _graph-id]
|
||||
(p/resolved (ok-json-response)))]
|
||||
(p/let [response (dispatch/handle-worker-fetch request env)]
|
||||
(is (= 200 (.-status response)))
|
||||
(is (= 1 (count @forwarded)))
|
||||
(is (= "/snapshot/download"
|
||||
(.-pathname (js/URL. (.-url (first @forwarded))))))))
|
||||
(p/then (fn [] (done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
|
||||
(defn- json-body [response]
|
||||
(p/let [text (.text response)]
|
||||
(js->clj (js/JSON.parse text) :keywordize-keys true)))
|
||||
|
||||
(defn- semantic-request
|
||||
([path scope]
|
||||
(semantic-request path scope "GET"))
|
||||
([path scope method]
|
||||
{:request (js/Request. (str "http://localhost" path)
|
||||
#js {:method method
|
||||
:headers #js {"authorization" "Bearer semantic-token"}})
|
||||
:claims #js {"sub" "user-1" "scope" scope}}))
|
||||
|
||||
(defn- graph-row [e2ee?]
|
||||
#js {:graph_e2ee (if e2ee? 1 0)})
|
||||
|
||||
(defn- rate-limiter [success? calls]
|
||||
#js {:limit (fn [opts]
|
||||
(swap! calls conj (js->clj opts :keywordize-keys true))
|
||||
(p/resolved #js {:success success?}))})
|
||||
|
||||
(defn- capturing-do-namespace [requests]
|
||||
#js {:idFromName (fn [graph-id] graph-id)
|
||||
:get (fn [_]
|
||||
#js {:fetch (fn [request]
|
||||
(swap! requests conj request)
|
||||
(p/resolved (ok-json-response)))})})
|
||||
|
||||
(deftest openapi-document-describes-semantic-api-and-oauth-scopes-test
|
||||
(async done
|
||||
(-> (p/let [response (dispatch/handle-worker-fetch
|
||||
(js/Request. "http://localhost/openapi.json")
|
||||
#js {})
|
||||
body (json-body response)]
|
||||
(is (= 200 (.-status response)))
|
||||
(is (= "3.1.0" (:openapi body)))
|
||||
(is (= ["logseq/read"]
|
||||
(get-in body [:paths (keyword "/api/v1/graphs") :get :security 0 :oauth])))
|
||||
(is (= ["logseq/read"]
|
||||
(get-in body [:paths (keyword "/api/v1/graphs/{graph-id}/pages") :get :security 0 :oauth])))
|
||||
(is (= ["logseq/write"]
|
||||
(get-in body [:paths (keyword "/api/v1/graphs/{graph-id}/blocks/{block-id}") :patch :security 0 :oauth])))
|
||||
(is (= ["logseq/write"]
|
||||
(get-in body [:paths (keyword "/api/v1/graphs/{graph-id}/pages/{page-id}") :delete :security 0 :oauth])))
|
||||
(doseq [path ["/api/v1/graphs/{graph-id}/pages/{page-id}"
|
||||
"/api/v1/graphs/{graph-id}/tags/{tag-id}"
|
||||
"/api/v1/graphs/{graph-id}/properties/{property-id}"]]
|
||||
(is (= #{:get :patch :delete} (set (keys (get-in body [:paths (keyword path)]))))))
|
||||
(is (= ["logseq/write"]
|
||||
(get-in body [:paths (keyword "/api/v1/graphs/{graph-id}/blocks/{block-id}/properties/{property-id}")
|
||||
:delete :security 0 :oauth])))
|
||||
(let [description (->> (get-in body [:paths (keyword "/api/v1/graphs/{graph-id}/blocks/{block-id}/properties/{property-id}")
|
||||
:put :parameters])
|
||||
(filter #(= "property-id" (:name %))) first :description)]
|
||||
(is (and (string? description)
|
||||
(string/includes? description "exact title"))))
|
||||
(is (= ["logseq/write"]
|
||||
(get-in body [:paths (keyword "/api/v1/graphs/{graph-id}/block-properties/batch-delete")
|
||||
:post :security 0 :oauth])))
|
||||
(is (= ["block-ids" "target-id" "position"]
|
||||
(get-in body [:paths (keyword "/api/v1/graphs/{graph-id}/block-moves")
|
||||
:post :requestBody :content :application/json :schema :required])))
|
||||
(is (= "array"
|
||||
(get-in body [:paths (keyword "/api/v1/graphs/{graph-id}/block-moves")
|
||||
:post :requestBody :content :application/json :schema
|
||||
:properties :block-ids :type])))
|
||||
(is (= "#/components/schemas/BlockResponse"
|
||||
(get-in body [:paths (keyword "/api/v1/graphs/{graph-id}/pages/{page-id}/blocks")
|
||||
:get :responses :200 :content :application/json :schema
|
||||
:properties :blocks :items :$ref])))
|
||||
(is (= "#/components/schemas/BlockResponse"
|
||||
(get-in body [:components :schemas :BlockResponse :properties :children :items :$ref])))
|
||||
(is (= "listTasks"
|
||||
(get-in body [:paths (keyword "/api/v1/graphs/{graph-id}/tasks") :get :operationId])))
|
||||
(is (= "createTask"
|
||||
(get-in body [:paths (keyword "/api/v1/graphs/{graph-id}/tasks") :post :operationId])))
|
||||
(is (= ["todo" "doing" "in-review" "done" "canceled" "backlog"]
|
||||
(get-in body [:components :schemas :TaskStatusSelector :anyOf 0 :enum])))
|
||||
(is (= "#/components/schemas/EntitySelector"
|
||||
(get-in body [:components :schemas :TaskStatusSelector :anyOf 1 :$ref])))
|
||||
(is (= "#/components/schemas/PropertyChoice"
|
||||
(get-in body [:components :schemas :TaskResponse :allOf 1 :properties :status :$ref])))
|
||||
(is (= "#/components/schemas/TaskResponse"
|
||||
(get-in body [:paths (keyword "/api/v1/graphs/{graph-id}/tasks")
|
||||
:get :responses :200 :content :application/json :schema
|
||||
:properties :tasks :items :$ref])))
|
||||
(is (= "#/components/schemas/TaskResponse"
|
||||
(get-in body [:paths (keyword "/api/v1/graphs/{graph-id}/tasks")
|
||||
:post :responses :201 :content :application/json :schema :$ref])))
|
||||
(is (= "#/components/schemas/PropertyResponse"
|
||||
(get-in body [:paths (keyword "/api/v1/graphs/{graph-id}/properties/{property-id}")
|
||||
:get :responses :200 :content :application/json :schema :$ref])))
|
||||
(is (= ["default" "number" "date" "datetime" "checkbox" "url" "node" "asset"
|
||||
"map" "json" "string"]
|
||||
(get-in body [:components :schemas :PropertyType :enum])))
|
||||
(is (= "#/components/schemas/PropertyType"
|
||||
(get-in body [:paths (keyword "/api/v1/graphs/{graph-id}/properties")
|
||||
:post :requestBody :content :application/json :schema
|
||||
:properties :type :$ref])))
|
||||
(doseq [[path response-key schema]
|
||||
[["/api/v1/graphs/{graph-id}/pages" :pages "PageResponse"]
|
||||
["/api/v1/graphs/{graph-id}/tags" :tags "TagResponse"]
|
||||
["/api/v1/graphs/{graph-id}/properties" :properties "PropertyResponse"]
|
||||
["/api/v1/graphs/{graph-id}/search" :results "SearchResultResponse"]]]
|
||||
(is (= (str "#/components/schemas/" schema)
|
||||
(get-in body [:paths (keyword path) :get :responses :200
|
||||
:content :application/json :schema :properties
|
||||
response-key :items :$ref]))))
|
||||
(is (= "#/components/schemas/PropertyChoice"
|
||||
(get-in body [:components :schemas :PropertyResponse :properties
|
||||
:choices :items :$ref])))
|
||||
(is (= "#/components/schemas/PropertyValue"
|
||||
(get-in body [:paths (keyword "/api/v1/graphs/{graph-id}/blocks/{block-id}/properties/{property-id}")
|
||||
:put :requestBody :content :application/json :schema
|
||||
:properties :value :$ref])))
|
||||
(is (= {:type "boolean" :default false}
|
||||
(select-keys
|
||||
(get-in body [:paths (keyword "/api/v1/graphs/{graph-id}/block-properties/batch-set")
|
||||
:post :requestBody :content :application/json :schema
|
||||
:properties :isResetExistingValues])
|
||||
[:type :default])))
|
||||
(is (string/includes?
|
||||
(get-in body [:components :schemas :PropertyValue :description])
|
||||
"UUID, ident, or title"))
|
||||
(is (string/includes?
|
||||
(get-in body [:components :schemas :PropertyValue :description])
|
||||
"TODO"))
|
||||
(doseq [path ["/api/v1/graphs/{graph-id}/pages"
|
||||
"/api/v1/graphs/{graph-id}/tasks"
|
||||
"/api/v1/graphs/{graph-id}/tags"
|
||||
"/api/v1/graphs/{graph-id}/tags/{tag-id}/objects"
|
||||
"/api/v1/graphs/{graph-id}/properties"
|
||||
"/api/v1/graphs/{graph-id}/assets"
|
||||
"/api/v1/graphs/{graph-id}/search"]]
|
||||
(is (= #{"created-after" "updated-after"}
|
||||
(->> (get-in body [:paths (keyword path) :get :parameters])
|
||||
(map :name)
|
||||
(filter #{"created-after" "updated-after"})
|
||||
set))
|
||||
path))
|
||||
(is (= "listAssets"
|
||||
(get-in body [:paths (keyword "/api/v1/graphs/{graph-id}/assets") :get :operationId])))
|
||||
(is (= "createAsset"
|
||||
(get-in body [:paths (keyword "/api/v1/graphs/{graph-id}/assets") :post :operationId])))
|
||||
(is (string/includes?
|
||||
(get-in body [:paths (keyword "/api/v1/graphs/{graph-id}/assets/{asset-block-id}")
|
||||
:get :description])
|
||||
"display image"))
|
||||
(is (= "binary"
|
||||
(get-in body [:paths (keyword "/api/v1/graphs/{graph-id}/assets") :post
|
||||
:requestBody :content :application/octet-stream :schema :format])))
|
||||
(is (= #{"file-name" "size" "title" "page-id" "checksum" "encoding"}
|
||||
(->> (get-in body [:paths (keyword "/api/v1/graphs/{graph-id}/assets")
|
||||
:post :parameters])
|
||||
(map :name)
|
||||
(filter #{"file-name" "size" "title" "page-id" "checksum" "encoding"})
|
||||
set)))
|
||||
(is (= "#/components/schemas/AssetResponse"
|
||||
(get-in body [:paths (keyword "/api/v1/graphs/{graph-id}/assets") :get
|
||||
:responses :200 :content :application/json :schema
|
||||
:properties :assets :items :$ref])))
|
||||
(is (= "#/components/schemas/AssetResponse"
|
||||
(get-in body [:paths (keyword "/api/v1/graphs/{graph-id}/assets") :post
|
||||
:responses :201 :content :application/json :schema :$ref])))
|
||||
(doseq [[path method] [["/api/v1/graphs/{graph-id}/properties" :post]
|
||||
["/api/v1/graphs/{graph-id}/properties/{property-id}" :patch]
|
||||
["/api/v1/graphs/{graph-id}/blocks/{block-id}/properties/{property-id}" :put]
|
||||
["/api/v1/graphs/{graph-id}/block-properties/batch-set" :post]]]
|
||||
(let [description (get-in body [:paths (keyword path) method :description])]
|
||||
(is (string/includes? description "DB graph"))
|
||||
(is (string/includes? description "typed"))
|
||||
(is (string/includes? description "key:: value"))))
|
||||
(doseq [[_ path-operations] (:paths body)
|
||||
[_ operation] path-operations]
|
||||
(is (seq (:summary operation)))
|
||||
(is (seq (:description operation)))))
|
||||
(p/then (fn [] (done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done))))))
|
||||
|
||||
(deftest semantic-api-lists-non-e2ee-graphs-with-pagination-test
|
||||
(async done
|
||||
(let [{:keys [request claims]} (semantic-request "/api/v1/graphs?name=test-mcp&limit=1" "logseq/read")
|
||||
calls (atom [])
|
||||
limit-calls (atom [])
|
||||
env #js {"DB" #js {}
|
||||
"SEMANTIC_READ_RATE_LIMITER" (rate-limiter true limit-calls)}]
|
||||
(-> (p/with-redefs [auth/auth-claims (fn [_ _] (p/resolved claims))
|
||||
index/<semantic-graphs-list
|
||||
(fn [_db user-id opts]
|
||||
(swap! calls conj [user-id opts])
|
||||
(p/resolved {:graphs [{:graph-id "graph-1" :graph-name "test-mcp"}]
|
||||
:next-cursor "cursor-1"}))]
|
||||
(p/let [response (dispatch/handle-worker-fetch request env)
|
||||
body (json-body response)]
|
||||
(is (= 200 (.-status response)))
|
||||
(is (= [{:graph-id "graph-1" :graph-name "test-mcp"}] (:graphs body)))
|
||||
(is (= "cursor-1" (:next-cursor body)))
|
||||
(is (= [["user-1" {:name "test-mcp" :limit 1 :cursor nil}]] @calls))
|
||||
(is (= 1 (count @limit-calls)))))
|
||||
(p/then (fn [] (done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
|
||||
(deftest semantic-api-rejects-missing-operation-scope-test
|
||||
(async done
|
||||
(let [{:keys [request claims]} (semantic-request "/api/v1/graphs/graph-1/pages" "logseq/write")]
|
||||
(-> (p/with-redefs [auth/auth-claims (fn [_ _] (p/resolved claims))]
|
||||
(p/let [response (dispatch/handle-worker-fetch request #js {})
|
||||
body (json-body response)]
|
||||
(is (= 403 (.-status response)))
|
||||
(is (= "insufficient scope" (:error body)))))
|
||||
(p/then (fn [] (done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
|
||||
(deftest semantic-api-rejects-e2ee-graphs-before-durable-object-test
|
||||
(async done
|
||||
(let [{:keys [request claims]} (semantic-request "/api/v1/graphs/graph-1/pages" "logseq/read")
|
||||
forwarded (atom [])
|
||||
env #js {"DB" #js {}
|
||||
"LOGSEQ_SYNC_DO" (capturing-do-namespace forwarded)
|
||||
"SEMANTIC_READ_RATE_LIMITER" (rate-limiter true (atom []))}]
|
||||
(-> (p/with-redefs [auth/auth-claims (fn [_ _] (p/resolved claims))
|
||||
index-handler/graph-access-response (fn [_ _ _] (p/resolved (ok-json-response)))
|
||||
common/<d1-all (fn [& _] (p/resolved #js {:results #js [(graph-row true)]}))]
|
||||
(p/let [response (dispatch/handle-worker-fetch request env)
|
||||
body (json-body response)]
|
||||
(is (= 409 (.-status response)))
|
||||
(is (= "semantic-api-unavailable-for-e2ee" (:error body)))
|
||||
(is (empty? @forwarded))))
|
||||
(p/then (fn [] (done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
|
||||
(deftest semantic-api-rate-limit-rejection-does-not-call-durable-object-test
|
||||
(async done
|
||||
(let [{:keys [request claims]} (semantic-request "/api/v1/graphs/graph-1/pages" "logseq/read")
|
||||
forwarded (atom [])
|
||||
limit-calls (atom [])
|
||||
env #js {"DB" #js {}
|
||||
"LOGSEQ_SYNC_DO" (capturing-do-namespace forwarded)
|
||||
"SEMANTIC_READ_RATE_LIMITER" (rate-limiter false limit-calls)}]
|
||||
(-> (p/with-redefs [auth/auth-claims (fn [_ _] (p/resolved claims))
|
||||
index-handler/graph-access-response (fn [_ _ _] (p/resolved (ok-json-response)))
|
||||
common/<d1-all (fn [& _] (p/resolved #js {:results #js [(graph-row false)]}))]
|
||||
(p/let [response (dispatch/handle-worker-fetch request env)]
|
||||
(is (= 429 (.-status response)))
|
||||
(is (= "60" (.get (.-headers response) "retry-after")))
|
||||
(is (= 1 (count @limit-calls)))
|
||||
(is (empty? @forwarded))))
|
||||
(p/then (fn [] (done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
|
||||
(deftest semantic-api-forwards-authorized-non-e2ee-request-test
|
||||
(async done
|
||||
(let [{:keys [request claims]} (semantic-request "/api/v1/graphs/graph-1/pages?limit=10" "logseq/read")
|
||||
forwarded (atom [])
|
||||
limit-calls (atom [])
|
||||
env #js {"DB" #js {}
|
||||
"LOGSEQ_SYNC_DO" (capturing-do-namespace forwarded)
|
||||
"SEMANTIC_READ_RATE_LIMITER" (rate-limiter true limit-calls)}]
|
||||
(-> (p/with-redefs [auth/auth-claims (fn [_ _] (p/resolved claims))
|
||||
index-handler/graph-access-response (fn [_ _ _] (p/resolved (ok-json-response)))
|
||||
common/<d1-all (fn [& _] (p/resolved #js {:results #js [(graph-row false)]}))]
|
||||
(p/let [response (dispatch/handle-worker-fetch request env)
|
||||
forwarded-request (first @forwarded)
|
||||
forwarded-url (some-> forwarded-request .-url js/URL.)]
|
||||
(is (= 200 (.-status response)))
|
||||
(is (= 1 (count @forwarded)))
|
||||
(when forwarded-url
|
||||
(is (= "/semantic/pages" (.-pathname forwarded-url)))
|
||||
(is (= "10" (.get (.-searchParams forwarded-url) "limit")))
|
||||
(is (= "graph-1" (.get (.-searchParams forwarded-url) "graph-id"))))))
|
||||
(p/then (fn [] (done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
|
||||
(deftest admin-token-bypasses-graph-access-check-for-assets-route-test
|
||||
(async done
|
||||
(let [access-check-calls (atom 0)
|
||||
request (js/Request. "http://localhost/assets/graph-1/snapshot-1.snapshot"
|
||||
#js {:method "GET"
|
||||
:headers #js {"x-db-sync-admin-token" "test-admin-token"}})
|
||||
env #js {"DB_SYNC_ADMIN_TOKEN" "test-admin-token"}]
|
||||
(-> (p/with-redefs [index-handler/graph-access-response (fn [_request _env _graph-id]
|
||||
(swap! access-check-calls inc)
|
||||
(p/resolved (http/unauthorized)))
|
||||
assets-handler/handle (fn [_request _env]
|
||||
(ok-json-response))]
|
||||
(p/let [resp (dispatch/handle-worker-fetch request env)
|
||||
text (.text resp)
|
||||
body (js->clj (js/JSON.parse text) :keywordize-keys true)]
|
||||
(is (= 200 (.-status resp)))
|
||||
(is (= true (:ok body)))
|
||||
(is (= 0 @access-check-calls))))
|
||||
(p/then (fn [] (done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
144
deps/db-sync/test/logseq/db_sync/worker_handler_assets_test.cljs
vendored
Normal file
144
deps/db-sync/test/logseq/db_sync/worker_handler_assets_test.cljs
vendored
Normal file
@@ -0,0 +1,144 @@
|
||||
(ns logseq.db-sync.worker-handler-assets-test
|
||||
(:require [cljs.test :refer [async deftest is]]
|
||||
[logseq.db-sync.index :as index]
|
||||
[logseq.db-sync.worker.handler.assets :as assets]
|
||||
[promesa.core :as p]))
|
||||
|
||||
(defn- bytes->stream
|
||||
[^js payload]
|
||||
(js/ReadableStream.
|
||||
#js {:start (fn [controller]
|
||||
(.enqueue controller payload)
|
||||
(.close controller))}))
|
||||
|
||||
(defn- put-request-with-size
|
||||
[size]
|
||||
#js {:url "http://localhost/assets/graph-1/asset-1.bin"
|
||||
:method "PUT"
|
||||
:headers #js {:get (constantly nil)}
|
||||
:arrayBuffer (fn [] (p/resolved #js {:byteLength size}))})
|
||||
|
||||
(deftest encrypted-assets-allow-upload-at-larger-limit-test
|
||||
(async done
|
||||
(let [request (put-request-with-size (* 200 1024 1024))
|
||||
put-calls (atom 0)
|
||||
env #js {:DB #js {}
|
||||
:LOGSEQ_SYNC_ASSETS
|
||||
#js {:put (fn [& _]
|
||||
(swap! put-calls inc)
|
||||
(p/resolved nil))}}]
|
||||
(-> (p/with-redefs [index/<graph-e2ee? (fn [_ _] (p/resolved true))]
|
||||
(p/let [resp (assets/handle request env)]
|
||||
(is (= 200 (.-status resp)))
|
||||
(is (= 1 @put-calls))))
|
||||
(p/then (fn [] (done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
|
||||
(deftest unencrypted-assets-keep-existing-upload-limit-test
|
||||
(async done
|
||||
(let [request (put-request-with-size (* 150 1024 1024))
|
||||
put-calls (atom 0)
|
||||
env #js {:DB #js {}
|
||||
:LOGSEQ_SYNC_ASSETS
|
||||
#js {:put (fn [& _]
|
||||
(swap! put-calls inc)
|
||||
(p/resolved nil))}}]
|
||||
(-> (p/with-redefs [index/<graph-e2ee? (fn [_ _] (p/resolved false))]
|
||||
(p/let [resp (assets/handle request env)]
|
||||
(is (= 413 (.-status resp)))
|
||||
(is (zero? @put-calls))))
|
||||
(p/then (fn [] (done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
|
||||
(deftest encrypted-assets-still-reject-uploads-above-larger-limit-test
|
||||
(async done
|
||||
(let [request (put-request-with-size (inc (* 200 1024 1024)))
|
||||
put-calls (atom 0)
|
||||
env #js {:DB #js {}
|
||||
:LOGSEQ_SYNC_ASSETS
|
||||
#js {:put (fn [& _]
|
||||
(swap! put-calls inc)
|
||||
(p/resolved nil))}}]
|
||||
(-> (p/with-redefs [index/<graph-e2ee? (fn [_ _] (p/resolved true))]
|
||||
(p/let [resp (assets/handle request env)]
|
||||
(is (= 413 (.-status resp)))
|
||||
(is (zero? @put-calls))))
|
||||
(p/then (fn [] (done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
|
||||
(deftest assets-get-includes-content-length-header-test
|
||||
(async done
|
||||
(let [payload (js/Uint8Array. #js [1 2 3 4])
|
||||
request (js/Request. "http://localhost/assets/graph-1/snapshot-1.snapshot"
|
||||
#js {:method "GET"})
|
||||
env #js {:LOGSEQ_SYNC_ASSETS
|
||||
#js {:get (fn [_key]
|
||||
(js/Promise.resolve
|
||||
#js {:body payload
|
||||
:size 4
|
||||
:httpMetadata #js {:contentType "application/octet-stream"}}))}}]
|
||||
(-> (p/let [resp (assets/handle request env)]
|
||||
(is (= 200 (.-status resp)))
|
||||
(is (= "4" (.get (.-headers resp) "content-length")))
|
||||
(is (= "4" (.get (.-headers resp) "x-asset-size"))))
|
||||
(p/then (fn []
|
||||
(done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
|
||||
(deftest assets-get-includes-content-length-with-stream-body-without-fixed-length-stream-test
|
||||
(async done
|
||||
(let [payload (js/Uint8Array. #js [1 2 3 4])
|
||||
request (js/Request. "http://localhost/assets/graph-1/snapshot-1.snapshot"
|
||||
#js {:method "GET"})
|
||||
env #js {:LOGSEQ_SYNC_ASSETS
|
||||
#js {:get (fn [_key]
|
||||
(js/Promise.resolve
|
||||
#js {:body (bytes->stream payload)
|
||||
:size 4
|
||||
:httpMetadata #js {:contentType "application/octet-stream"}}))}}
|
||||
original-fixed-length-stream (.-FixedLengthStream js/globalThis)
|
||||
restore! #(aset js/globalThis "FixedLengthStream" original-fixed-length-stream)]
|
||||
(aset js/globalThis "FixedLengthStream" js/undefined)
|
||||
(-> (p/let [resp (assets/handle request env)
|
||||
buf (.arrayBuffer resp)]
|
||||
(is (= 200 (.-status resp)))
|
||||
(is (= "4" (.get (.-headers resp) "content-length")))
|
||||
(is (= "4" (.get (.-headers resp) "x-asset-size")))
|
||||
(is (= 4 (.-byteLength buf))))
|
||||
(p/then (fn []
|
||||
(restore!)
|
||||
(done)))
|
||||
(p/catch (fn [error]
|
||||
(restore!)
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
|
||||
(deftest assets-get-serves-image-inline-with-extension-content-type-test
|
||||
(async done
|
||||
(let [payload (js/Uint8Array. #js [1 2 3 4])
|
||||
request (js/Request. "http://localhost/assets/graph-1/image-id.png"
|
||||
#js {:method "GET"})
|
||||
env #js {:LOGSEQ_SYNC_ASSETS
|
||||
#js {:get (fn [_key]
|
||||
(p/resolved
|
||||
#js {:body payload
|
||||
:size 4
|
||||
;; Existing Code Mode uploads stored the transfer MIME type.
|
||||
:httpMetadata #js {:contentType "application/json"}}))}}]
|
||||
(-> (p/let [resp (assets/handle request env)]
|
||||
(is (= 200 (.-status resp)))
|
||||
(is (= "image/png" (.get (.-headers resp) "content-type")))
|
||||
(is (= "inline; filename=\"image-id.png\""
|
||||
(.get (.-headers resp) "content-disposition"))))
|
||||
(p/then (fn [] (done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
260
deps/db-sync/test/logseq/db_sync/worker_handler_index_test.cljs
vendored
Normal file
260
deps/db-sync/test/logseq/db_sync/worker_handler_index_test.cljs
vendored
Normal file
@@ -0,0 +1,260 @@
|
||||
(ns logseq.db-sync.worker-handler-index-test
|
||||
(:require [cljs.test :refer [async deftest is]]
|
||||
[clojure.string :as string]
|
||||
[logseq.db-sync.common :as common]
|
||||
[logseq.db-sync.index :as index]
|
||||
[logseq.db-sync.worker.auth :as auth]
|
||||
[logseq.db-sync.worker.handler.index :as index-handler]
|
||||
[promesa.core :as p]))
|
||||
|
||||
(deftest graph-access-response-with-timing-caches-result-test
|
||||
(async done
|
||||
(let [request (js/Request. "http://localhost/sync/graph-1"
|
||||
#js {:headers #js {"authorization" "Bearer token-cache-hit"}})
|
||||
env #js {"DB" #js {}}
|
||||
auth-count (atom 0)
|
||||
query-count (atom 0)]
|
||||
(-> (p/with-redefs [auth/auth-claims (fn [_request _env]
|
||||
(swap! auth-count inc)
|
||||
(p/resolved #js {"sub" "user-1"}))
|
||||
index/<user-has-access-to-graph? (fn [_db _graph-id _user-id]
|
||||
(swap! query-count inc)
|
||||
(p/resolved true))]
|
||||
(p/let [first-result (index-handler/graph-access-response-with-timing request env "graph-1")
|
||||
second-result (index-handler/graph-access-response-with-timing request env "graph-1")]
|
||||
(is (= 200 (.-status (:response first-result))))
|
||||
(is (= 200 (.-status (:response second-result))))
|
||||
(is (= 1 @auth-count))
|
||||
(is (= 1 @query-count))
|
||||
(is (false? (get-in first-result [:timing :cache-hit?])))
|
||||
(is (true? (get-in second-result [:timing :cache-hit?])))))
|
||||
(p/then (fn []
|
||||
(done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
|
||||
(deftest graph-access-response-with-timing-does-not-upsert-user-test
|
||||
(async done
|
||||
(let [request (js/Request. "http://localhost/sync/graph-2"
|
||||
#js {:headers #js {"authorization" "Bearer token-no-upsert"}})
|
||||
env #js {"DB" #js {}}]
|
||||
(-> (p/with-redefs [auth/auth-claims (fn [_request _env]
|
||||
(p/resolved #js {"sub" "user-2"}))
|
||||
index/<user-upsert! (fn [& _]
|
||||
(throw (ex-info "should-not-upsert" {})))
|
||||
index/<user-has-access-to-graph? (fn [_db _graph-id _user-id]
|
||||
(p/resolved true))]
|
||||
(p/let [result (index-handler/graph-access-response-with-timing request env "graph-2")]
|
||||
(is (= 200 (.-status (:response result))))
|
||||
(is (= true (get-in result [:timing :access-ok?])))))
|
||||
(p/then (fn []
|
||||
(done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
|
||||
(deftest graphs-create-defaults-ready-for-use-true-test
|
||||
(async done
|
||||
(let [request (js/Request. "http://localhost/graphs" #js {:method "POST"})
|
||||
url (js/URL. (.-url request))
|
||||
d1-runs (atom [])]
|
||||
(-> (p/with-redefs [common/read-json (fn [_]
|
||||
(p/resolved #js {"graph-name" "Graph 1"
|
||||
"schema-version" "65"}))
|
||||
index/<user-rsa-key-pair (fn [_db _user-id]
|
||||
(p/resolved {:public-key "pk"
|
||||
:encrypted-private-key "enc"}))
|
||||
common/<d1-all (fn [& _]
|
||||
(p/resolved #js {:results #js []}))
|
||||
common/get-sql-rows (fn [result]
|
||||
(aget result "results"))
|
||||
common/<d1-run (fn [_db sql & args]
|
||||
(swap! d1-runs conj {:sql sql
|
||||
:args args})
|
||||
(p/resolved {:ok true}))]
|
||||
(p/let [resp (index-handler/handle {:db :db
|
||||
:env #js {}
|
||||
:request request
|
||||
:url url
|
||||
:claims #js {"sub" "user-1"}
|
||||
:route {:handler :graphs/create
|
||||
:path-params {}}})
|
||||
text (.text resp)
|
||||
body (js->clj (js/JSON.parse text) :keywordize-keys true)
|
||||
graph-insert (first @d1-runs)]
|
||||
(is (= 200 (.-status resp)))
|
||||
(is (string/includes? (:sql graph-insert) "graph_ready_for_use"))
|
||||
(is (= 1 (nth (:args graph-insert) 5)))
|
||||
(is (= true (:graph-ready-for-use? body)))))
|
||||
(p/then (fn []
|
||||
(done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
|
||||
(deftest graphs-list-includes-user-rsa-keys-exists-flag-true-test
|
||||
(async done
|
||||
(let [request (js/Request. "http://localhost/graphs" #js {:method "GET"})
|
||||
url (js/URL. (.-url request))]
|
||||
(-> (p/with-redefs [index/<index-list (fn [_db _user-id]
|
||||
(p/resolved []))
|
||||
index/<user-rsa-key-pair (fn [_db _user-id]
|
||||
(p/resolved {:public-key "pk"
|
||||
:encrypted-private-key "enc"}))]
|
||||
(p/let [resp (index-handler/handle {:db :db
|
||||
:env #js {}
|
||||
:request request
|
||||
:url url
|
||||
:claims #js {"sub" "user-1"}
|
||||
:route {:handler :graphs/list
|
||||
:path-params {}}})
|
||||
text (.text resp)
|
||||
body (js->clj (js/JSON.parse text) :keywordize-keys true)]
|
||||
(is (= 200 (.-status resp)))
|
||||
(is (= [] (:graphs body)))
|
||||
(is (= true (:user-rsa-keys-exists? body)))))
|
||||
(p/then (fn []
|
||||
(done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
|
||||
(deftest graphs-list-includes-user-rsa-keys-exists-flag-false-test
|
||||
(async done
|
||||
(let [request (js/Request. "http://localhost/graphs" #js {:method "GET"})
|
||||
url (js/URL. (.-url request))]
|
||||
(-> (p/with-redefs [index/<index-list (fn [_db _user-id]
|
||||
(p/resolved []))
|
||||
index/<user-rsa-key-pair (fn [_db _user-id]
|
||||
(p/resolved nil))]
|
||||
(p/let [resp (index-handler/handle {:db :db
|
||||
:env #js {}
|
||||
:request request
|
||||
:url url
|
||||
:claims #js {"sub" "user-1"}
|
||||
:route {:handler :graphs/list
|
||||
:path-params {}}})
|
||||
text (.text resp)
|
||||
body (js->clj (js/JSON.parse text) :keywordize-keys true)]
|
||||
(is (= 200 (.-status resp)))
|
||||
(is (= [] (:graphs body)))
|
||||
(is (= false (:user-rsa-keys-exists? body)))))
|
||||
(p/then (fn []
|
||||
(done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
|
||||
(deftest graphs-create-e2ee-requires-user-rsa-key-pair-test
|
||||
(async done
|
||||
(let [request (js/Request. "http://localhost/graphs" #js {:method "POST"})
|
||||
url (js/URL. (.-url request))
|
||||
index-upsert-calls* (atom 0)]
|
||||
(-> (p/with-redefs [common/read-json (fn [_]
|
||||
(p/resolved #js {"graph-name" "Graph E2EE"
|
||||
"schema-version" "65"
|
||||
"graph-e2ee?" true}))
|
||||
index/<graph-name-exists? (fn [_db _graph-name _user-id]
|
||||
(p/resolved false))
|
||||
index/<user-rsa-key-pair (fn [_db _user-id]
|
||||
(p/resolved nil))
|
||||
index/<index-upsert! (fn
|
||||
([_db _graph-id _graph-name _user-id _schema-version _graph-e2ee?]
|
||||
(swap! index-upsert-calls* inc)
|
||||
(p/resolved nil))
|
||||
([_db _graph-id _graph-name _user-id _schema-version _graph-e2ee? _graph-ready-for-use?]
|
||||
(swap! index-upsert-calls* inc)
|
||||
(p/resolved nil)))
|
||||
index/<graph-member-upsert! (fn [& _]
|
||||
(p/resolved nil))]
|
||||
(p/let [resp (index-handler/handle {:db :db
|
||||
:env #js {}
|
||||
:request request
|
||||
:url url
|
||||
:claims #js {"sub" "user-1"}
|
||||
:route {:handler :graphs/create
|
||||
:path-params {}}})
|
||||
text (.text resp)
|
||||
body (js->clj (js/JSON.parse text) :keywordize-keys true)]
|
||||
(is (= 400 (.-status resp)))
|
||||
(is (= "missing user rsa key pair" (:error body)))
|
||||
(is (zero? @index-upsert-calls*))))
|
||||
(p/then (fn []
|
||||
(done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
|
||||
(deftest graphs-create-non-e2ee-skips-user-rsa-key-pair-test
|
||||
(async done
|
||||
(let [request (js/Request. "http://localhost/graphs" #js {:method "POST"})
|
||||
url (js/URL. (.-url request))
|
||||
index-upsert-calls* (atom 0)]
|
||||
(-> (p/with-redefs [common/read-json (fn [_]
|
||||
(p/resolved #js {"graph-name" "Graph Plain"
|
||||
"schema-version" "65"
|
||||
"graph-e2ee?" false}))
|
||||
index/<graph-name-exists? (fn [_db _graph-name _user-id]
|
||||
(p/resolved false))
|
||||
index/<user-rsa-key-pair (fn [_db _user-id]
|
||||
(p/resolved nil))
|
||||
index/<index-upsert! (fn
|
||||
([_db _graph-id _graph-name _user-id _schema-version _graph-e2ee?]
|
||||
(swap! index-upsert-calls* inc)
|
||||
(p/resolved nil))
|
||||
([_db _graph-id _graph-name _user-id _schema-version _graph-e2ee? _graph-ready-for-use?]
|
||||
(swap! index-upsert-calls* inc)
|
||||
(p/resolved nil)))
|
||||
index/<graph-member-upsert! (fn [& _]
|
||||
(p/resolved nil))]
|
||||
(p/let [resp (index-handler/handle {:db :db
|
||||
:env #js {}
|
||||
:request request
|
||||
:url url
|
||||
:claims #js {"sub" "user-1"}
|
||||
:route {:handler :graphs/create
|
||||
:path-params {}}})
|
||||
text (.text resp)
|
||||
body (js->clj (js/JSON.parse text) :keywordize-keys true)]
|
||||
(is (= 200 (.-status resp)))
|
||||
(is (string? (:graph-id body)))
|
||||
(is (= false (:graph-e2ee? body)))
|
||||
(is (= 1 @index-upsert-calls*))))
|
||||
(p/then (fn []
|
||||
(done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
|
||||
(deftest graphs-delete-supports-node-delete-hook-without-do-namespace-test
|
||||
(async done
|
||||
(let [request (js/Request. "http://localhost/graphs/graph-1" #js {:method "DELETE"})
|
||||
url (js/URL. (.-url request))
|
||||
hook-calls* (atom [])]
|
||||
(-> (p/with-redefs [index/<user-has-access-to-graph? (fn [_db _graph-id _user-id]
|
||||
(p/resolved true))
|
||||
index/<graph-delete-metadata! (fn [_db _graph-id]
|
||||
(p/resolved true))
|
||||
index/<graph-delete-index-entry! (fn [_db _graph-id]
|
||||
(p/resolved true))]
|
||||
(p/let [resp (index-handler/handle {:db :db
|
||||
:env #js {"DB_SYNC_DELETE_GRAPH"
|
||||
(fn [graph-id]
|
||||
(swap! hook-calls* conj graph-id)
|
||||
(p/resolved true))}
|
||||
:request request
|
||||
:url url
|
||||
:claims #js {"sub" "user-1"}
|
||||
:route {:handler :graphs/delete
|
||||
:path-params {:graph-id "graph-1"}}})
|
||||
text (.text resp)
|
||||
body (js->clj (js/JSON.parse text) :keywordize-keys true)]
|
||||
(is (= 200 (.-status resp)))
|
||||
(is (= {:graph-id "graph-1" :deleted true} body))
|
||||
(is (= ["graph-1"] @hook-calls*))))
|
||||
(p/then (fn []
|
||||
(done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
3041
deps/db-sync/test/logseq/db_sync/worker_handler_sync_test.cljs
vendored
Normal file
3041
deps/db-sync/test/logseq/db_sync/worker_handler_sync_test.cljs
vendored
Normal file
File diff suppressed because it is too large
Load Diff
190
deps/db-sync/test/logseq/db_sync/worker_handler_ws_test.cljs
vendored
Normal file
190
deps/db-sync/test/logseq/db_sync/worker_handler_ws_test.cljs
vendored
Normal file
@@ -0,0 +1,190 @@
|
||||
(ns logseq.db-sync.worker-handler-ws-test
|
||||
(:require [cljs-bean.core :as bean]
|
||||
[cljs.test :refer [async deftest is]]
|
||||
[datascript.core :as d]
|
||||
[logseq.db-sync.protocol :as protocol]
|
||||
[logseq.db-sync.storage :as storage]
|
||||
[logseq.db-sync.test-sql :as test-sql]
|
||||
[logseq.db-sync.worker.handler.sync :as sync-handler]
|
||||
[logseq.db-sync.worker.handler.ws :as ws-handler]
|
||||
[logseq.db-sync.worker.presence :as presence]
|
||||
[logseq.db-sync.worker.ws :as ws]
|
||||
[promesa.core :as p]))
|
||||
|
||||
(deftest presence-message-broadcast-excludes-source-client-test
|
||||
(let [source-ws #js {:readyState 1}
|
||||
peer-ws #js {:readyState 1}
|
||||
send-events (atom [])
|
||||
self #js {:state #js {:getWebSockets (fn [] #js [source-ws peer-ws])}}
|
||||
raw (protocol/encode-message {:type "presence"
|
||||
:editing-block-uuid "block-1"})]
|
||||
(with-redefs [presence/get-user (fn [_self _ws] {:user-id "user-1"})
|
||||
presence/update-presence! (fn [_self _ws _patch] nil)
|
||||
ws/send! (fn [target msg]
|
||||
(swap! send-events conj {:ws target
|
||||
:msg msg}))]
|
||||
(ws-handler/handle-ws-message! self source-ws raw))
|
||||
|
||||
(is (= [peer-ws]
|
||||
(mapv :ws @send-events)))
|
||||
(is (= [{:type "presence"
|
||||
:editing-block-uuid "block-1"
|
||||
:user-id "user-1"}]
|
||||
(mapv :msg @send-events)))))
|
||||
|
||||
(deftest hello-message-includes-checksum-test
|
||||
(let [sql (test-sql/make-sql)
|
||||
conn (storage/open-conn sql)
|
||||
ws #js {:readyState 1}
|
||||
sent (atom nil)
|
||||
self #js {:conn conn
|
||||
:schema-ready true
|
||||
:sql sql}
|
||||
raw (protocol/encode-message {:type "hello"
|
||||
:client "test"})]
|
||||
(d/transact! conn [{:block/uuid (random-uuid)
|
||||
:block/title "hello"}])
|
||||
(with-redefs [ws/send! (fn [_target msg]
|
||||
(reset! sent msg))]
|
||||
(ws-handler/handle-ws-message! self ws raw))
|
||||
(is (= "hello" (:type @sent)))
|
||||
(is (number? (:t @sent)))
|
||||
(is (string? (:checksum @sent)))))
|
||||
|
||||
(deftest hello-message-omits-nil-checksum-test
|
||||
(let [sql (test-sql/make-sql)
|
||||
conn (storage/open-conn sql)
|
||||
ws #js {:readyState 1}
|
||||
sent (atom nil)
|
||||
self #js {:conn conn
|
||||
:schema-ready true
|
||||
:sql sql}
|
||||
raw (protocol/encode-message {:type "hello"
|
||||
:client "test"})]
|
||||
(with-redefs [ws/send! (fn [_target msg]
|
||||
(reset! sent msg))]
|
||||
(ws-handler/handle-ws-message! self ws raw))
|
||||
(is (= "hello" (:type @sent)))
|
||||
(is (false? (contains? @sent :checksum)))))
|
||||
|
||||
(deftest tx-batch-message-adds-graph-and-user-context-to-transact-meta-test
|
||||
(let [sql (test-sql/make-sql)
|
||||
conn (storage/open-conn sql)
|
||||
ws #js {:readyState 1}
|
||||
sent (atom nil)
|
||||
tx-metas (atom [])
|
||||
self #js {:conn conn
|
||||
:graph-id "graph-ws"
|
||||
:schema-ready true
|
||||
:sql sql}
|
||||
raw (protocol/encode-message
|
||||
{:type "tx/batch"
|
||||
:client-revision "revision-ws"
|
||||
:t-before 0
|
||||
:txs [{:tx (protocol/tx->transit [[:db/add -1 :block/title "ws context"]])
|
||||
:outliner-op :save-block}]})]
|
||||
(with-redefs [presence/get-user (fn [_self _ws]
|
||||
{:user-id "user-1"
|
||||
:username "alice"})
|
||||
ws/broadcast! (fn [& _] nil)
|
||||
ws/send! (fn [_target msg]
|
||||
(reset! sent msg))]
|
||||
(d/listen! conn ::capture-ws-context-tx-meta
|
||||
(fn [tx-report]
|
||||
(swap! tx-metas conj (:tx-meta tx-report))))
|
||||
(try
|
||||
(ws-handler/handle-ws-message! self ws raw)
|
||||
(finally
|
||||
(d/unlisten! conn ::capture-ws-context-tx-meta))))
|
||||
(is (= "tx/batch/ok" (:type @sent)))
|
||||
(is (some #(= {:op :apply-client-tx
|
||||
:outliner-op :save-block
|
||||
:graph-id "graph-ws"
|
||||
:client-revision "revision-ws"
|
||||
:username "alice"}
|
||||
(select-keys % [:op :outliner-op :graph-id :client-revision :username]))
|
||||
@tx-metas))))
|
||||
|
||||
(deftest ws-send-serializes-tx-reject-uuids-as-strings-test
|
||||
(let [raw* (atom nil)
|
||||
ws #js {:readyState 1
|
||||
:send (fn [raw] (reset! raw* raw))}
|
||||
success-tx-id (random-uuid)
|
||||
failed-tx-id (random-uuid)]
|
||||
(ws/send! ws {:type "tx/reject"
|
||||
:reason "db transact failed"
|
||||
:t 3
|
||||
:success-tx-ids [success-tx-id]
|
||||
:failed-tx-id failed-tx-id})
|
||||
(let [message (-> @raw* js/JSON.parse (js->clj :keywordize-keys true))]
|
||||
(is (= "tx/reject" (:type message)))
|
||||
(is (= [(str success-tx-id)] (:success-tx-ids message)))
|
||||
(is (= (str failed-tx-id) (:failed-tx-id message))))))
|
||||
|
||||
(deftest online-users-broadcast-restored-attachment-user-test
|
||||
(let [attachment* (atom nil)
|
||||
ws #js {:readyState 1
|
||||
:serializeAttachment (fn [attachment]
|
||||
(reset! attachment* attachment))
|
||||
:deserializeAttachment (fn []
|
||||
@attachment*)}
|
||||
self #js {}
|
||||
restored-self #js {}
|
||||
user {:user-id "user-1"
|
||||
:email "user@example.com"
|
||||
:username "alice"}
|
||||
sent (atom nil)]
|
||||
(presence/add-presence! self ws user)
|
||||
(is (= {:presence/user user}
|
||||
(bean/->clj (.deserializeAttachment ws))))
|
||||
(swap! (presence/presence* restored-self)
|
||||
assoc
|
||||
ws
|
||||
(presence/attachment->user (.deserializeAttachment ws)))
|
||||
(with-redefs [ws/broadcast! (fn [_self _sender message]
|
||||
(reset! sent message))]
|
||||
(presence/broadcast-online-users! restored-self))
|
||||
(is (= {:type "online-users"
|
||||
:online-users [user]}
|
||||
@sent))))
|
||||
|
||||
(deftest websocket-connection-is-rejected-while-snapshot-upload-is-in-progress-test
|
||||
(async done
|
||||
(let [accepted (atom [])
|
||||
presence-events (atom [])
|
||||
self #js {:state #js {:acceptWebSocket (fn [socket]
|
||||
(swap! accepted conj socket))}}
|
||||
request (js/Request. "http://localhost/sync/graph-1/ws?graph-id=graph-1"
|
||||
#js {:method "GET"})]
|
||||
(-> (p/with-redefs [sync-handler/<ready-for-sync? (fn [_ _] (p/resolved false))
|
||||
presence/add-presence! (fn [& _]
|
||||
(swap! presence-events conj :add))
|
||||
presence/broadcast-online-users! (fn [& _]
|
||||
(swap! presence-events conj :broadcast))]
|
||||
(ws-handler/handle-ws self request))
|
||||
(p/then (fn [response]
|
||||
(is (= 409 (.-status response)))
|
||||
(is (empty? @accepted))
|
||||
(is (empty? @presence-events))
|
||||
(done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
|
||||
(deftest websocket-connection-uses-graph-id-from-sync-path-test
|
||||
(async done
|
||||
(let [seen-graph-id (atom ::unset)
|
||||
self #js {}
|
||||
request (js/Request. "http://localhost/sync/graph-from-path"
|
||||
#js {:method "GET"})]
|
||||
(-> (p/with-redefs [sync-handler/<ready-for-sync? (fn [_self graph-id]
|
||||
(reset! seen-graph-id graph-id)
|
||||
(p/resolved false))]
|
||||
(ws-handler/handle-ws self request))
|
||||
(p/then (fn [response]
|
||||
(is (= "graph-from-path" @seen-graph-id))
|
||||
(is (= 409 (.-status response)))
|
||||
(done)))
|
||||
(p/catch (fn [error]
|
||||
(is false (str error))
|
||||
(done)))))))
|
||||
216
deps/db-sync/test/logseq/db_sync/worker_large_op_memory_test.cljs
vendored
Normal file
216
deps/db-sync/test/logseq/db_sync/worker_large_op_memory_test.cljs
vendored
Normal file
@@ -0,0 +1,216 @@
|
||||
(ns logseq.db-sync.worker-large-op-memory-test
|
||||
(:require ["better-sqlite3" :as sqlite3]
|
||||
[clojure.string :as string]
|
||||
[datascript.core :as d]
|
||||
[logseq.db-sync.protocol :as protocol]
|
||||
[logseq.db-sync.storage :as storage]
|
||||
[logseq.db-sync.worker.handler.sync :as sync-handler]
|
||||
[logseq.db-sync.worker.ws :as ws]
|
||||
[logseq.db.frontend.validate :as db-validate]))
|
||||
|
||||
(def sqlite (if (find-ns 'nbb.core) (aget sqlite3 "default") sqlite3))
|
||||
|
||||
(defn- select-sql?
|
||||
[sql]
|
||||
(string/starts-with? (-> sql string/trim string/lower-case) "select"))
|
||||
|
||||
(defn- run-sql
|
||||
[^js stmt args]
|
||||
(.apply (.-run stmt) stmt (to-array args)))
|
||||
|
||||
(defn- all-sql
|
||||
[^js stmt args]
|
||||
(.apply (.-all stmt) stmt (to-array args)))
|
||||
|
||||
(defn- with-memory-sql
|
||||
[f]
|
||||
(let [db (new sqlite ":memory:" nil)
|
||||
sql #js {:_db db
|
||||
:exec (fn [sql-str & args]
|
||||
(let [stmt (.prepare db sql-str)]
|
||||
(if (select-sql? sql-str)
|
||||
(all-sql stmt args)
|
||||
(do
|
||||
(run-sql stmt args)
|
||||
nil))))
|
||||
:close (fn []
|
||||
(.close db))}]
|
||||
(try
|
||||
(f sql)
|
||||
(finally
|
||||
(.close sql)))))
|
||||
|
||||
(defn- large-block-insert-tx
|
||||
[page-uuid block-count]
|
||||
(let [block-uuids (mapv (fn [_idx] (random-uuid)) (range block-count))]
|
||||
{:block-uuids block-uuids
|
||||
:tx-data
|
||||
(vec
|
||||
(mapcat (fn [idx]
|
||||
(let [block-uuid (nth block-uuids idx)
|
||||
eid (str block-uuid)]
|
||||
[[:db/add eid :block/uuid block-uuid idx]
|
||||
[:db/add eid :block/title (str "large-memory-block-" idx) idx]
|
||||
[:db/add eid :block/page [:block/uuid page-uuid] idx]
|
||||
[:db/add eid :block/parent [:block/uuid page-uuid] idx]
|
||||
[:db/add eid :block/order "a0" idx]
|
||||
[:db/add eid :block/created-at idx idx]
|
||||
[:db/add eid :block/updated-at idx idx]]))
|
||||
(range block-count)))}))
|
||||
|
||||
(defn- sample-blocks-present?
|
||||
[db block-uuids]
|
||||
(every? (fn [idx]
|
||||
(some? (d/entity db [:block/uuid (nth block-uuids idx)])))
|
||||
[0 (quot (count block-uuids) 2) (dec (count block-uuids))]))
|
||||
|
||||
(defn- assert!
|
||||
[condition message]
|
||||
(when-not condition
|
||||
(throw (js/Error. message))))
|
||||
|
||||
(defn- run-large-op-memory-test!
|
||||
[{:keys [skip-final-store? skip-validation?]}]
|
||||
(with-memory-sql
|
||||
(fn [sql]
|
||||
(storage/init-schema! sql)
|
||||
(let [conn (storage/open-conn sql)
|
||||
page-uuid (random-uuid)
|
||||
_ (d/transact! conn [{:block/uuid page-uuid
|
||||
:block/name "large-memory-page"
|
||||
:block/title "large-memory-page"}])
|
||||
t-before (storage/get-t sql)
|
||||
{:keys [tx-data block-uuids]} (large-block-insert-tx page-uuid 2000)
|
||||
tx-entry {:tx (protocol/tx->transit tx-data)
|
||||
:tx-id (random-uuid)
|
||||
:outliner-op :insert-blocks}
|
||||
self #js {:sql sql
|
||||
:conn conn
|
||||
:schema-ready true}
|
||||
tx-report-count (atom 0)
|
||||
response (try
|
||||
(d/listen! conn ::large-op-memory-chunks
|
||||
(fn [_tx-report]
|
||||
(swap! tx-report-count inc)))
|
||||
(with-redefs [d/store (if skip-final-store?
|
||||
(fn [_db] nil)
|
||||
d/store)
|
||||
db-validate/validate-tx-report
|
||||
(if skip-validation?
|
||||
(fn [_tx-report _options] [true nil])
|
||||
db-validate/validate-tx-report)
|
||||
ws/broadcast! (fn [& _] nil)]
|
||||
(sync-handler/handle-tx-batch! self nil [tx-entry] t-before))
|
||||
(finally
|
||||
(d/unlisten! conn ::large-op-memory-chunks)))]
|
||||
(assert! (= "tx/batch/ok" (:type response))
|
||||
(str "expected tx/batch/ok, got " (pr-str response)))
|
||||
(assert! (> @tx-report-count 1)
|
||||
(str "expected more than one chunk, got " @tx-report-count))
|
||||
(assert! (= (+ t-before @tx-report-count) (:t response))
|
||||
(str "expected response t to advance by visible chunks, got " (:t response)))
|
||||
(assert! (= @tx-report-count (count (storage/fetch-tx-since sql t-before)))
|
||||
"expected large tx chunks to be persisted as visible tx log entries")
|
||||
(assert! (sample-blocks-present? @conn block-uuids)
|
||||
"expected sampled large tx blocks to be persisted")))))
|
||||
|
||||
(defn- run-client-split-large-op-memory-test!
|
||||
[{:keys [serial?]}]
|
||||
(with-memory-sql
|
||||
(fn [sql]
|
||||
(storage/init-schema! sql)
|
||||
(let [conn (storage/open-conn sql)
|
||||
page-uuid (random-uuid)
|
||||
_ (d/transact! conn [{:block/uuid page-uuid
|
||||
:block/name "large-memory-page"
|
||||
:block/title "large-memory-page"}])
|
||||
t-before (storage/get-t sql)
|
||||
{:keys [tx-data block-uuids]} (large-block-insert-tx page-uuid 2000)
|
||||
tx-id (random-uuid)
|
||||
tx-entries (mapv (fn [chunk]
|
||||
{:tx (protocol/tx->transit chunk)
|
||||
:tx-id tx-id
|
||||
:outliner-op :insert-blocks})
|
||||
(partition-all 392 tx-data))
|
||||
self #js {:sql sql
|
||||
:conn conn
|
||||
:schema-ready true}
|
||||
tx-report-count (atom 0)
|
||||
response (try
|
||||
(d/listen! conn ::large-op-memory-client-split
|
||||
(fn [_tx-report]
|
||||
(swap! tx-report-count inc)))
|
||||
(with-redefs [ws/broadcast! (fn [& _] nil)]
|
||||
(if serial?
|
||||
(loop [t-before* t-before
|
||||
[tx-entry & more] tx-entries
|
||||
response nil]
|
||||
(if tx-entry
|
||||
(let [response* (sync-handler/handle-tx-batch! self nil [tx-entry] t-before*)]
|
||||
(assert! (= "tx/batch/ok" (:type response*))
|
||||
(str "expected tx/batch/ok, got " (pr-str response*)))
|
||||
(recur (:t response*) more response*))
|
||||
response))
|
||||
(sync-handler/handle-tx-batch! self nil tx-entries t-before)))
|
||||
(finally
|
||||
(d/unlisten! conn ::large-op-memory-client-split)))]
|
||||
(assert! (= "tx/batch/ok" (:type response))
|
||||
(str "expected tx/batch/ok, got " (pr-str response)))
|
||||
(assert! (= (count tx-entries) @tx-report-count)
|
||||
(str "expected one tx report per client split entry, got " @tx-report-count))
|
||||
(assert! (= (+ t-before @tx-report-count) (:t response))
|
||||
(str "expected response t to advance by client split entries, got " (:t response)))
|
||||
(assert! (= @tx-report-count (count (storage/fetch-tx-since sql t-before)))
|
||||
"expected client split tx entries to be persisted as visible tx log entries")
|
||||
(assert! (sample-blocks-present? @conn block-uuids)
|
||||
"expected sampled client split blocks to be persisted")))))
|
||||
|
||||
(defn- run-open-sql-memory-test!
|
||||
[]
|
||||
(with-memory-sql
|
||||
(fn [sql]
|
||||
(storage/init-schema! sql)
|
||||
(let [conn (storage/open-conn sql)
|
||||
page-uuid (random-uuid)]
|
||||
(d/transact! conn [{:block/uuid page-uuid
|
||||
:block/name "large-memory-page"
|
||||
:block/title "large-memory-page"}])
|
||||
(assert! (some? (d/entity @conn [:block/uuid page-uuid]))
|
||||
"expected setup page to be persisted")))))
|
||||
|
||||
(defn- run-prepare-memory-test!
|
||||
[]
|
||||
(let [page-uuid (random-uuid)
|
||||
{:keys [tx-data]} (large-block-insert-tx page-uuid 2000)
|
||||
tx-entry {:tx (protocol/tx->transit tx-data)
|
||||
:tx-id (random-uuid)
|
||||
:outliner-op :insert-blocks}]
|
||||
(assert! (= 14000 (count tx-data))
|
||||
"expected generated tx data")
|
||||
(assert! (string? (:tx tx-entry))
|
||||
"expected transit tx payload")))
|
||||
|
||||
(defn- run-mode!
|
||||
[mode]
|
||||
(case mode
|
||||
"baseline" nil
|
||||
"open-sql" (run-open-sql-memory-test!)
|
||||
"prepare" (run-prepare-memory-test!)
|
||||
"apply" (run-large-op-memory-test! nil)
|
||||
"apply-client-split" (run-client-split-large-op-memory-test! nil)
|
||||
"apply-client-split-serial" (run-client-split-large-op-memory-test! {:serial? true})
|
||||
"apply-no-store" (run-large-op-memory-test! {:skip-final-store? true})
|
||||
"apply-no-validation" (run-large-op-memory-test! {:skip-validation? true})
|
||||
"apply-no-store-validation" (run-large-op-memory-test! {:skip-final-store? true
|
||||
:skip-validation? true})
|
||||
(run-large-op-memory-test! nil)))
|
||||
|
||||
(defn main [& args]
|
||||
(try
|
||||
(let [mode (or (first args) "apply")]
|
||||
(run-mode! mode)
|
||||
(js/console.log (str "large op memory test passed: " mode)))
|
||||
(js/process.exit 0)
|
||||
(catch :default error
|
||||
(js/console.error error)
|
||||
(js/process.exit 1))))
|
||||
89
deps/db-sync/test/logseq/db_sync/worker_routes_test.cljs
vendored
Normal file
89
deps/db-sync/test/logseq/db_sync/worker_routes_test.cljs
vendored
Normal file
@@ -0,0 +1,89 @@
|
||||
(ns logseq.db-sync.worker-routes-test
|
||||
(:require [cljs.test :refer [deftest is testing]]
|
||||
[logseq.db-sync.worker.routes.index :as routes]
|
||||
[logseq.db-sync.worker.routes.semantic :as semantic-routes]))
|
||||
|
||||
(deftest match-route-graphs-test
|
||||
(testing "graphs routes"
|
||||
(let [match (routes/match-route "GET" "/graphs")]
|
||||
(is (= :graphs/list (:handler match))))
|
||||
(let [match (routes/match-route "POST" "/graphs")]
|
||||
(is (= :graphs/create (:handler match))))
|
||||
(let [match (routes/match-route "GET" "/graphs/graph-1/access")]
|
||||
(is (= :graphs/access (:handler match)))
|
||||
(is (= "graph-1" (get-in match [:path-params :graph-id]))))
|
||||
(let [match (routes/match-route "GET" "/graphs/graph-2/members")]
|
||||
(is (= :graph-members/list (:handler match)))
|
||||
(is (= "graph-2" (get-in match [:path-params :graph-id]))))
|
||||
(let [match (routes/match-route "POST" "/graphs/graph-3/members")]
|
||||
(is (= :graph-members/create (:handler match))))
|
||||
(let [match (routes/match-route "PUT" "/graphs/graph-4/members/user-9")]
|
||||
(is (= :graph-members/update (:handler match)))
|
||||
(is (= "graph-4" (get-in match [:path-params :graph-id])))
|
||||
(is (= "user-9" (get-in match [:path-params :member-id]))))
|
||||
(let [match (routes/match-route "DELETE" "/graphs/graph-5")]
|
||||
(is (= :graphs/delete (:handler match)))
|
||||
(is (= "graph-5" (get-in match [:path-params :graph-id]))))))
|
||||
|
||||
(deftest match-route-e2ee-test
|
||||
(testing "e2ee routes"
|
||||
(let [match (routes/match-route "GET" "/e2ee/user-keys")]
|
||||
(is (= :e2ee/user-keys-get (:handler match))))
|
||||
(let [match (routes/match-route "POST" "/e2ee/user-keys")]
|
||||
(is (= :e2ee/user-keys-post (:handler match))))
|
||||
(let [match (routes/match-route "GET" "/e2ee/user-public-key")]
|
||||
(is (= :e2ee/user-public-key-get (:handler match))))
|
||||
(let [match (routes/match-route "GET" "/e2ee/graphs/graph-7/aes-key")]
|
||||
(is (= :e2ee/graph-aes-key-get (:handler match)))
|
||||
(is (= "graph-7" (get-in match [:path-params :graph-id]))))
|
||||
(let [match (routes/match-route "POST" "/e2ee/graphs/graph-8/aes-key")]
|
||||
(is (= :e2ee/graph-aes-key-post (:handler match)))
|
||||
(is (= "graph-8" (get-in match [:path-params :graph-id]))))
|
||||
(let [match (routes/match-route "POST" "/e2ee/graphs/graph-9/grant-access")]
|
||||
(is (= :e2ee/grant-access (:handler match)))
|
||||
(is (= "graph-9" (get-in match [:path-params :graph-id]))))))
|
||||
|
||||
(deftest match-route-method-mismatch-test
|
||||
(testing "method mismatch returns nil"
|
||||
(is (nil? (routes/match-route "GET" "/graphs/graph-1/members/user-9")))
|
||||
(is (nil? (routes/match-route "PUT" "/e2ee/user-keys")))))
|
||||
|
||||
(deftest semantic-internal-routes-ignore-public-only-operations-test
|
||||
(is (nil? (semantic-routes/match-internal "GET" "/api/v1/graphs")))
|
||||
(is (= :semantic/pages-list
|
||||
(:handler (semantic-routes/match-internal "GET" "/semantic/pages")))))
|
||||
|
||||
(deftest semantic-move-blocks-route-test
|
||||
(is (= :semantic/blocks-move
|
||||
(:handler (semantic-routes/match-public "POST" "/api/v1/graphs/graph-1/block-moves"))))
|
||||
(is (= :semantic/blocks-move
|
||||
(:handler (semantic-routes/match-internal "POST" "/semantic/block-moves"))))
|
||||
(is (nil? (semantic-routes/match-public "POST" "/api/v1/graphs/graph-1/blocks/block-1/move"))))
|
||||
|
||||
(deftest semantic-reverse-reference-routes-test
|
||||
(is (= :semantic/tags-objects
|
||||
(:handler (semantic-routes/match-public
|
||||
"GET" "/api/v1/graphs/graph-1/tags/tag-1/objects"))))
|
||||
(is (= :semantic/tags-objects
|
||||
(:handler (semantic-routes/match-internal "GET" "/semantic/tags/tag-1/objects"))))
|
||||
(is (= :semantic/pages-references
|
||||
(:handler (semantic-routes/match-public
|
||||
"GET" "/api/v1/graphs/graph-1/pages/page-1/references"))))
|
||||
(is (= :semantic/pages-references
|
||||
(:handler (semantic-routes/match-internal "GET" "/semantic/pages/page-1/references")))))
|
||||
|
||||
(deftest semantic-task-routes-test
|
||||
(doseq [[method handler] [["GET" :semantic/tasks-list]
|
||||
["POST" :semantic/tasks-create]]]
|
||||
(is (= handler
|
||||
(:handler (semantic-routes/match-public method "/api/v1/graphs/graph-1/tasks"))))
|
||||
(is (= handler
|
||||
(:handler (semantic-routes/match-internal method "/semantic/tasks"))))))
|
||||
|
||||
(deftest semantic-asset-collection-routes-test
|
||||
(doseq [[method handler] [["GET" :semantic/assets-list]
|
||||
["POST" :semantic/assets-create]]]
|
||||
(is (= handler
|
||||
(:handler (semantic-routes/match-public method "/api/v1/graphs/graph-1/assets"))))
|
||||
(is (= handler
|
||||
(:handler (semantic-routes/match-internal method "/semantic/assets"))))))
|
||||
27
deps/db-sync/test/logseq/db_sync/worker_sync_routes_test.cljs
vendored
Normal file
27
deps/db-sync/test/logseq/db_sync/worker_sync_routes_test.cljs
vendored
Normal file
@@ -0,0 +1,27 @@
|
||||
(ns logseq.db-sync.worker-sync-routes-test
|
||||
(:require [cljs.test :refer [deftest is testing]]
|
||||
[logseq.db-sync.worker.routes.sync :as sync-routes]))
|
||||
|
||||
(deftest match-route-sync-test
|
||||
(testing "sync routes"
|
||||
(let [match (sync-routes/match-route "GET" "/health")]
|
||||
(is (= :sync/health (:handler match))))
|
||||
(let [match (sync-routes/match-route "GET" "/pull")]
|
||||
(is (= :sync/pull (:handler match))))
|
||||
(is (nil? (sync-routes/match-route "GET" "/repair/blocks")))
|
||||
(let [match (sync-routes/match-route "GET" "/snapshot/download")]
|
||||
(is (= :sync/snapshot-download (:handler match))))
|
||||
(let [match (sync-routes/match-route "GET" "/snapshot/stream")]
|
||||
(is (= :sync/snapshot-stream (:handler match))))
|
||||
(let [match (sync-routes/match-route "DELETE" "/admin/reset")]
|
||||
(is (= :sync/admin-reset (:handler match))))
|
||||
(let [match (sync-routes/match-route "POST" "/tx/batch")]
|
||||
(is (= :sync/tx-batch (:handler match))))
|
||||
(let [match (sync-routes/match-route "POST" "/snapshot/upload")]
|
||||
(is (= :sync/snapshot-upload (:handler match))))))
|
||||
|
||||
(deftest match-route-sync-method-mismatch-test
|
||||
(testing "sync method mismatch returns nil"
|
||||
(is (nil? (sync-routes/match-route "POST" "/health")))
|
||||
(is (nil? (sync-routes/match-route "GET" "/admin/reset")))
|
||||
(is (nil? (sync-routes/match-route "PUT" "/tx/batch")))))
|
||||
30
deps/db-sync/test/logseq/db_sync/worker_test.cljs
vendored
Normal file
30
deps/db-sync/test/logseq/db_sync/worker_test.cljs
vendored
Normal file
@@ -0,0 +1,30 @@
|
||||
(ns logseq.db-sync.worker-test
|
||||
(:require [cljs.test :refer [deftest is]]
|
||||
[datascript.core :as d]
|
||||
[logseq.db-sync.order :as sync-order]
|
||||
[logseq.db.common.order :as db-order]
|
||||
[logseq.db.frontend.schema :as db-schema]))
|
||||
|
||||
(defn- new-conn []
|
||||
(d/create-conn db-schema/schema))
|
||||
|
||||
(deftest duplicate-order-fix-test
|
||||
(let [conn (new-conn)
|
||||
parent (random-uuid)
|
||||
block-a (random-uuid)
|
||||
block-b (random-uuid)
|
||||
order-a (db-order/gen-key nil nil)
|
||||
order-b (db-order/gen-key order-a nil)]
|
||||
(d/transact! conn [{:block/uuid parent}
|
||||
{:block/uuid block-a
|
||||
:block/parent [:block/uuid parent]
|
||||
:block/order order-a}
|
||||
{:block/uuid block-b
|
||||
:block/parent [:block/uuid parent]
|
||||
:block/order order-b}])
|
||||
(let [tx [[:db/add [:block/uuid block-b] :block/order order-a]]
|
||||
_ (sync-order/fix-duplicate-orders! conn tx {})
|
||||
order-a' (:block/order (d/entity @conn [:block/uuid block-a]))
|
||||
order-b' (:block/order (d/entity @conn [:block/uuid block-b]))]
|
||||
(is (= order-a order-a'))
|
||||
(is (not= order-a' order-b')))))
|
||||
9
deps/db-sync/worker/api_docs.mjs
vendored
Normal file
9
deps/db-sync/worker/api_docs.mjs
vendored
Normal file
@@ -0,0 +1,9 @@
|
||||
export function apiDocsResponse(path, html) {
|
||||
if (path !== "/api-docs" && path !== "/api-docs/") return null;
|
||||
return new Response(html, {
|
||||
headers: {
|
||||
"cache-control": "public, max-age=300",
|
||||
"content-type": "text/html; charset=utf-8",
|
||||
},
|
||||
});
|
||||
}
|
||||
19
deps/db-sync/worker/api_docs_test.mjs
vendored
Normal file
19
deps/db-sync/worker/api_docs_test.mjs
vendored
Normal file
@@ -0,0 +1,19 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
|
||||
import { apiDocsResponse } from "./api_docs.mjs";
|
||||
|
||||
test("API docs are served as standalone HTML from both docs paths", async () => {
|
||||
const html = "<!doctype html><title>Logseq Server API</title>";
|
||||
|
||||
for (const path of ["/api-docs", "/api-docs/"]) {
|
||||
const response = apiDocsResponse(path, html);
|
||||
assert.equal(response.status, 200);
|
||||
assert.equal(response.headers.get("content-type"), "text/html; charset=utf-8");
|
||||
assert.equal(await response.text(), html);
|
||||
}
|
||||
});
|
||||
|
||||
test("API docs helper ignores unrelated paths", () => {
|
||||
assert.equal(apiDocsResponse("/openapi.json", "docs"), null);
|
||||
});
|
||||
232
deps/db-sync/worker/asset_image.mjs
vendored
Normal file
232
deps/db-sync/worker/asset_image.mjs
vendored
Normal file
@@ -0,0 +1,232 @@
|
||||
function bytesToBase64(bytes) {
|
||||
const chunkSize = 32 * 1024;
|
||||
const chunks = [];
|
||||
for (let offset = 0; offset < bytes.length; offset += chunkSize) {
|
||||
chunks.push(String.fromCharCode(...bytes.subarray(offset, offset + chunkSize)));
|
||||
}
|
||||
return btoa(chunks.join(""));
|
||||
}
|
||||
|
||||
const MAX_DISPLAY_IMAGE_SIZE = 10 * 1024 * 1024;
|
||||
|
||||
export const ASSET_IMAGE_RESOURCE_URI = "ui://widget/logseq-asset-image.html";
|
||||
|
||||
const ASSET_IMAGE_COMPONENT_HTML = `
|
||||
<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<style>
|
||||
:root { color-scheme: light dark; }
|
||||
body {
|
||||
margin: 0;
|
||||
padding: 8px;
|
||||
font-family: ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
|
||||
}
|
||||
figure { margin: 0; }
|
||||
#image-button {
|
||||
display: block;
|
||||
width: 100%;
|
||||
padding: 0;
|
||||
border: 0;
|
||||
background: transparent;
|
||||
cursor: zoom-in;
|
||||
}
|
||||
img {
|
||||
display: block;
|
||||
width: 100%;
|
||||
height: auto;
|
||||
max-height: min(70vh, 720px);
|
||||
object-fit: contain;
|
||||
border-radius: 8px;
|
||||
}
|
||||
figcaption {
|
||||
margin-top: 6px;
|
||||
color: rgba(127, 127, 127, 0.95);
|
||||
font-size: 12px;
|
||||
line-height: 1.4;
|
||||
}
|
||||
.error {
|
||||
color: #b42318;
|
||||
font-size: 13px;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<figure>
|
||||
<button id="image-button" type="button" aria-label="Open image fullscreen">
|
||||
<img id="image" alt="">
|
||||
</button>
|
||||
<figcaption id="caption"></figcaption>
|
||||
</figure>
|
||||
<p id="error" class="error" hidden>Image data was not provided by the tool result.</p>
|
||||
<script>
|
||||
let objectUrl;
|
||||
|
||||
function currentToolResult() {
|
||||
const metadata = window.openai?.toolResponseMetadata;
|
||||
return metadata?.mcp_tool_result
|
||||
|| metadata?.call_tool_result
|
||||
|| { structuredContent: window.openai?.toolOutput };
|
||||
}
|
||||
|
||||
function imageFromResult(result) {
|
||||
const metadataImage = result?._meta?.image;
|
||||
if (metadataImage?.data && metadataImage?.mimeType) return metadataImage;
|
||||
|
||||
const contentImage = result?.content?.find((item) => item?.type === "image");
|
||||
if (!contentImage?.data || !contentImage?.mimeType) return null;
|
||||
|
||||
const details = result?.structuredContent || {};
|
||||
return {
|
||||
data: contentImage.data,
|
||||
mimeType: contentImage.mimeType,
|
||||
title: details.title,
|
||||
uuid: details.uuid,
|
||||
};
|
||||
}
|
||||
|
||||
function bytesFromBase64(base64) {
|
||||
const raw = atob(base64);
|
||||
const payload = new Uint8Array(raw.length);
|
||||
for (let i = 0; i < raw.length; i += 1) payload[i] = raw.charCodeAt(i);
|
||||
return payload;
|
||||
}
|
||||
|
||||
function notifyHeight() {
|
||||
const notify = window.openai?.notifyIntrinsicHeight;
|
||||
if (!notify) return;
|
||||
|
||||
const height = Math.ceil(document.documentElement.scrollHeight);
|
||||
try {
|
||||
notify({ height });
|
||||
} catch (_) {
|
||||
try {
|
||||
notify(height);
|
||||
} catch (_) {
|
||||
notify();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function scheduleHeightNotification() {
|
||||
requestAnimationFrame(() => notifyHeight());
|
||||
}
|
||||
|
||||
async function requestFullscreen() {
|
||||
await window.openai?.requestDisplayMode?.({ mode: "fullscreen" });
|
||||
scheduleHeightNotification();
|
||||
}
|
||||
|
||||
function render(result) {
|
||||
const image = imageFromResult(result);
|
||||
const buttonElement = document.getElementById("image-button");
|
||||
const imageElement = document.getElementById("image");
|
||||
const captionElement = document.getElementById("caption");
|
||||
const errorElement = document.getElementById("error");
|
||||
|
||||
if (!image) {
|
||||
buttonElement.hidden = true;
|
||||
imageElement.hidden = true;
|
||||
captionElement.textContent = "";
|
||||
errorElement.hidden = false;
|
||||
scheduleHeightNotification();
|
||||
return;
|
||||
}
|
||||
|
||||
if (objectUrl) URL.revokeObjectURL(objectUrl);
|
||||
objectUrl = URL.createObjectURL(new Blob([bytesFromBase64(image.data)], { type: image.mimeType }));
|
||||
imageElement.src = objectUrl;
|
||||
imageElement.alt = image.title || "Logseq image asset";
|
||||
buttonElement.hidden = false;
|
||||
imageElement.hidden = false;
|
||||
errorElement.hidden = true;
|
||||
captionElement.textContent = image.title
|
||||
? image.uuid ? image.title + " (" + image.uuid + ")" : image.title
|
||||
: image.uuid || "";
|
||||
scheduleHeightNotification();
|
||||
}
|
||||
|
||||
document.getElementById("image").addEventListener("load", notifyHeight);
|
||||
document.getElementById("image-button").addEventListener("click", () => {
|
||||
requestFullscreen().catch(() => {});
|
||||
});
|
||||
|
||||
render(currentToolResult());
|
||||
|
||||
window.addEventListener("message", (event) => {
|
||||
if (event.source !== window.parent) return;
|
||||
const message = event.data;
|
||||
if (!message || message.jsonrpc !== "2.0") return;
|
||||
if (message.method !== "ui/notifications/tool-result") return;
|
||||
render(message.params);
|
||||
}, { passive: true });
|
||||
|
||||
window.addEventListener("openai:set_globals", (event) => {
|
||||
const metadata = event.detail?.globals?.toolResponseMetadata;
|
||||
const output = event.detail?.globals?.toolOutput;
|
||||
render(metadata?.mcp_tool_result || metadata?.call_tool_result || { structuredContent: output });
|
||||
}, { passive: true });
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
`.trim();
|
||||
|
||||
export function assertDisplayableImageMetadata(metadata) {
|
||||
if (!Number.isSafeInteger(metadata.size) || metadata.size < 0) {
|
||||
throw new Error("asset has an invalid size");
|
||||
}
|
||||
if (metadata.size > MAX_DISPLAY_IMAGE_SIZE) {
|
||||
throw new Error("asset is too large to display directly (10MB maximum)");
|
||||
}
|
||||
}
|
||||
|
||||
export async function assetImageResult(response, metadata) {
|
||||
const mimeType = response.headers.get("content-type")?.split(";", 1)[0];
|
||||
if (!response.ok || !mimeType?.startsWith("image/")) {
|
||||
throw new Error(`asset response is not an image (${response.status} ${mimeType ?? "unknown"})`);
|
||||
}
|
||||
const payload = new Uint8Array(await response.arrayBuffer());
|
||||
const data = bytesToBase64(payload);
|
||||
const title = metadata.title ?? "Image asset";
|
||||
const uuid = metadata.uuid;
|
||||
return {
|
||||
structuredContent: {
|
||||
uuid,
|
||||
title,
|
||||
mimeType,
|
||||
size: payload.length,
|
||||
},
|
||||
content: [
|
||||
{ type: "image", data, mimeType },
|
||||
{ type: "text", text: `${title} (${uuid})` },
|
||||
],
|
||||
_meta: {
|
||||
ui: { resourceUri: ASSET_IMAGE_RESOURCE_URI },
|
||||
"openai/outputTemplate": ASSET_IMAGE_RESOURCE_URI,
|
||||
image: {
|
||||
data,
|
||||
mimeType,
|
||||
title,
|
||||
uuid,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function assetImageResource() {
|
||||
return {
|
||||
contents: [
|
||||
{
|
||||
uri: ASSET_IMAGE_RESOURCE_URI,
|
||||
mimeType: "text/html;profile=mcp-app",
|
||||
text: ASSET_IMAGE_COMPONENT_HTML,
|
||||
_meta: {
|
||||
ui: {
|
||||
prefersBorder: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
65
deps/db-sync/worker/asset_image_test.mjs
vendored
Normal file
65
deps/db-sync/worker/asset_image_test.mjs
vendored
Normal file
@@ -0,0 +1,65 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
|
||||
import * as assetImage from "./asset_image.mjs";
|
||||
|
||||
test("asset image result returns MCP image content", async () => {
|
||||
const response = new Response(new Uint8Array([1, 2, 3, 4]), {
|
||||
headers: { "content-type": "image/jpeg" },
|
||||
});
|
||||
|
||||
assert.deepEqual(await assetImage.assetImageResult(response, { uuid: "asset-1", title: "Photo" }), {
|
||||
structuredContent: {
|
||||
uuid: "asset-1",
|
||||
title: "Photo",
|
||||
mimeType: "image/jpeg",
|
||||
size: 4,
|
||||
},
|
||||
content: [
|
||||
{ type: "image", data: "AQIDBA==", mimeType: "image/jpeg" },
|
||||
{ type: "text", text: "Photo (asset-1)" },
|
||||
],
|
||||
_meta: {
|
||||
ui: { resourceUri: "ui://widget/logseq-asset-image.html" },
|
||||
"openai/outputTemplate": "ui://widget/logseq-asset-image.html",
|
||||
image: {
|
||||
data: "AQIDBA==",
|
||||
mimeType: "image/jpeg",
|
||||
title: "Photo",
|
||||
uuid: "asset-1",
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
test("asset image result rejects non-image responses", async () => {
|
||||
await assert.rejects(
|
||||
assetImage.assetImageResult(new Response("{}", { headers: { "content-type": "application/json" } }), {}),
|
||||
/not an image/,
|
||||
);
|
||||
});
|
||||
|
||||
test("asset image display rejects payloads that are unsafe to base64 in Worker memory", () => {
|
||||
assert.throws(
|
||||
() => assetImage.assertDisplayableImageMetadata({ size: 10 * 1024 * 1024 + 1 }),
|
||||
/too large to display/,
|
||||
);
|
||||
});
|
||||
|
||||
test("asset image resource is an MCP app component that renders tool results", async () => {
|
||||
assert.equal(typeof assetImage.assetImageResource, "function");
|
||||
const resource = await assetImage.assetImageResource();
|
||||
const content = resource.contents[0];
|
||||
|
||||
assert.equal(content.uri, assetImage.ASSET_IMAGE_RESOURCE_URI);
|
||||
assert.equal(assetImage.ASSET_IMAGE_RESOURCE_URI, "ui://widget/logseq-asset-image.html");
|
||||
assert.equal(content.mimeType, "text/html;profile=mcp-app");
|
||||
assert.match(content.text, /ui\/notifications\/tool-result/);
|
||||
assert.match(content.text, /toolResponseMetadata/);
|
||||
assert.match(content.text, /createObjectURL/);
|
||||
assert.match(content.text, /notifyIntrinsicHeight/);
|
||||
assert.match(content.text, /requestDisplayMode/);
|
||||
assert.match(content.text, /image-button/);
|
||||
assert.match(content.text, /zoom-in/);
|
||||
assert.doesNotMatch(content.text, /https?:\/\//);
|
||||
});
|
||||
160
deps/db-sync/worker/chatgpt_app.mjs
vendored
Normal file
160
deps/db-sync/worker/chatgpt_app.mjs
vendored
Normal file
@@ -0,0 +1,160 @@
|
||||
import { ASSET_IMAGE_RESOURCE_URI } from "./asset_image.mjs";
|
||||
|
||||
const writeScopes = ["logseq/read", "logseq/write"];
|
||||
|
||||
const imageOutputSchema = {
|
||||
type: "object",
|
||||
properties: {
|
||||
uuid: { type: "string", description: "Image asset block UUID" },
|
||||
title: { type: "string", description: "Image asset title" },
|
||||
mimeType: { type: "string", description: "Image MIME type" },
|
||||
size: { type: "integer", description: "Image size in bytes" },
|
||||
},
|
||||
required: ["uuid", "title", "mimeType", "size"],
|
||||
additionalProperties: false,
|
||||
};
|
||||
|
||||
const fileInputSchema = {
|
||||
type: "object",
|
||||
properties: {
|
||||
download_url: { type: "string", description: "Temporary HTTPS URL for the original ChatGPT file" },
|
||||
file_id: { type: "string", description: "ChatGPT file identifier" },
|
||||
mime_type: { type: "string", description: "File MIME type" },
|
||||
file_name: { type: "string", description: "Original file name including its extension" },
|
||||
},
|
||||
required: ["download_url", "file_id"],
|
||||
additionalProperties: false,
|
||||
};
|
||||
|
||||
const assetOutputSchema = {
|
||||
type: "object",
|
||||
properties: {
|
||||
uuid: { type: "string", description: "Asset block UUID" },
|
||||
title: { type: "string", description: "Asset block title" },
|
||||
type: { type: "string", description: "Asset file extension" },
|
||||
size: { type: "integer", description: "Original file size in bytes" },
|
||||
checksum: { type: "string", description: "SHA-256 checksum of the original file" },
|
||||
},
|
||||
required: ["uuid", "title", "type", "size", "checksum"],
|
||||
additionalProperties: false,
|
||||
};
|
||||
|
||||
function oauthSecurity(scopes) {
|
||||
return [{ type: "oauth2", scopes }];
|
||||
}
|
||||
|
||||
export function chatGptToolDescriptors() {
|
||||
const searchSecurity = oauthSecurity(writeScopes);
|
||||
const executeSecurity = oauthSecurity(writeScopes);
|
||||
const imageSecurity = oauthSecurity(["logseq/read"]);
|
||||
return [
|
||||
{
|
||||
name: "search",
|
||||
title: "Discover Logseq operations",
|
||||
description: "Search the Logseq DB graph OpenAPI document before making requests. For tasks, discover the dedicated /tasks operations: never use Markdown TODO syntax to create tasks and never use graph search to list tasks. Use upload_asset for ChatGPT attachments and never encode an attachment as base64 in Code Mode. This API uses DB-version typed properties; never use legacy file-graph key:: value syntax. Pass JavaScript as an async arrow function; codemode.spec() returns the OpenAPI document with references resolved inline.",
|
||||
inputSchema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
code: { type: "string", description: "JavaScript async arrow function that searches the Logseq OpenAPI document" },
|
||||
},
|
||||
required: ["code"],
|
||||
additionalProperties: false,
|
||||
},
|
||||
annotations: {
|
||||
readOnlyHint: true,
|
||||
openWorldHint: false,
|
||||
destructiveHint: false,
|
||||
idempotentHint: true,
|
||||
},
|
||||
securitySchemes: searchSecurity,
|
||||
_meta: {
|
||||
securitySchemes: searchSecurity,
|
||||
"openai/toolInvocation/invoking": "Discovering Logseq operations…",
|
||||
"openai/toolInvocation/invoked": "Logseq operations ready",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "execute",
|
||||
title: "Use Logseq",
|
||||
description: "Read or change the user's non-encrypted Logseq DB graph through the semantic API. Use the dedicated /tasks operations for DB tasks: never use Markdown TODO syntax to create tasks and never use graph search to list tasks. Use upload_asset for ChatGPT attachments and never encode an attachment as base64 in Code Mode. Use DB-version typed properties and property UUIDs or idents; never write legacy file-graph key:: value syntax into block titles. First use search to discover endpoints. Pass JavaScript as an async arrow function; codemode.request(options) calls only documented Logseq endpoints.",
|
||||
inputSchema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
code: { type: "string", description: "JavaScript async arrow function that calls the Logseq API" },
|
||||
},
|
||||
required: ["code"],
|
||||
additionalProperties: false,
|
||||
},
|
||||
annotations: {
|
||||
readOnlyHint: false,
|
||||
openWorldHint: false,
|
||||
destructiveHint: true,
|
||||
},
|
||||
securitySchemes: executeSecurity,
|
||||
_meta: {
|
||||
securitySchemes: executeSecurity,
|
||||
"openai/toolInvocation/invoking": "Updating Logseq…",
|
||||
"openai/toolInvocation/invoked": "Logseq request complete",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "upload_asset",
|
||||
title: "Upload Logseq asset",
|
||||
description: "Upload the original ChatGPT attachment, unchanged and without base64 re-encoding, to a non-encrypted Logseq DB graph and create its Asset block. Optionally append it to a page. The file must not exceed 100MB; its original name is preserved when available.",
|
||||
inputSchema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
graphId: { type: "string", description: "Graph UUID" },
|
||||
file: fileInputSchema,
|
||||
pageId: { type: "string", description: "Optional destination page UUID, ident, or title" },
|
||||
title: { type: "string", description: "Optional asset block title" },
|
||||
},
|
||||
required: ["graphId", "file"],
|
||||
additionalProperties: false,
|
||||
},
|
||||
outputSchema: assetOutputSchema,
|
||||
annotations: {
|
||||
readOnlyHint: false,
|
||||
openWorldHint: true,
|
||||
destructiveHint: false,
|
||||
idempotentHint: false,
|
||||
},
|
||||
securitySchemes: executeSecurity,
|
||||
_meta: {
|
||||
securitySchemes: executeSecurity,
|
||||
"openai/fileParams": ["file"],
|
||||
"openai/toolInvocation/invoking": "Uploading Logseq asset…",
|
||||
"openai/toolInvocation/invoked": "Logseq asset uploaded",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "get_asset_image",
|
||||
title: "Display Logseq image asset",
|
||||
description: "Fetch an image asset up to 10MB from a non-encrypted Logseq DB graph and return it as image content that ChatGPT can display directly. Use this tool instead of getAsset when the user asks to view or display an image. It is read-only and does not download to local storage or modify the graph.",
|
||||
inputSchema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
graphId: { type: "string", description: "Graph UUID" },
|
||||
assetBlockId: { type: "string", description: "Image asset block UUID" },
|
||||
},
|
||||
required: ["graphId", "assetBlockId"],
|
||||
additionalProperties: false,
|
||||
},
|
||||
outputSchema: imageOutputSchema,
|
||||
annotations: {
|
||||
readOnlyHint: true,
|
||||
openWorldHint: false,
|
||||
destructiveHint: false,
|
||||
idempotentHint: true,
|
||||
},
|
||||
securitySchemes: imageSecurity,
|
||||
_meta: {
|
||||
securitySchemes: imageSecurity,
|
||||
ui: { resourceUri: ASSET_IMAGE_RESOURCE_URI },
|
||||
"openai/outputTemplate": ASSET_IMAGE_RESOURCE_URI,
|
||||
"openai/toolInvocation/invoking": "Loading Logseq image…",
|
||||
"openai/toolInvocation/invoked": "Logseq image ready",
|
||||
},
|
||||
},
|
||||
];
|
||||
}
|
||||
128
deps/db-sync/worker/chatgpt_asset_upload.mjs
vendored
Normal file
128
deps/db-sync/worker/chatgpt_asset_upload.mjs
vendored
Normal file
@@ -0,0 +1,128 @@
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
const MAX_ASSET_SIZE = 100 * 1024 * 1024;
|
||||
const MIME_TYPE_EXTENSIONS = new Map([
|
||||
["image/png", "png"],
|
||||
["image/jpeg", "jpg"],
|
||||
["image/gif", "gif"],
|
||||
["image/webp", "webp"],
|
||||
["image/bmp", "bmp"],
|
||||
["image/svg+xml", "svg"],
|
||||
["image/x-icon", "ico"],
|
||||
["application/pdf", "pdf"],
|
||||
]);
|
||||
|
||||
function requiredFileLength(response) {
|
||||
const value = response.headers.get("content-length");
|
||||
if (!/^\d+$/.test(value ?? "")) {
|
||||
throw new Error("ChatGPT file download must include content-length");
|
||||
}
|
||||
const size = Number(value);
|
||||
if (!Number.isSafeInteger(size) || size > MAX_ASSET_SIZE) {
|
||||
throw new Error("ChatGPT file is larger than 100MB");
|
||||
}
|
||||
return size;
|
||||
}
|
||||
|
||||
function fixedLengthBody(body, size) {
|
||||
if (typeof FixedLengthStream !== "function") return { body, pipePromise: null };
|
||||
const fixed = new FixedLengthStream(size);
|
||||
return {
|
||||
body: fixed.readable,
|
||||
pipePromise: body.pipeTo(fixed.writable),
|
||||
};
|
||||
}
|
||||
|
||||
function validateInput({ graphId, file }) {
|
||||
if (!graphId) throw new Error("graphId is required");
|
||||
if (!file?.file_id) throw new Error("file.file_id is required");
|
||||
if (!file?.download_url) throw new Error("file.download_url is required");
|
||||
const downloadUrl = new URL(file.download_url);
|
||||
if (downloadUrl.protocol !== "https:") {
|
||||
throw new Error("file.download_url must use HTTPS");
|
||||
}
|
||||
return downloadUrl;
|
||||
}
|
||||
|
||||
function resolvedFileName(file, downloadUrl, contentType) {
|
||||
if (file.file_name) return file.file_name;
|
||||
const pathName = decodeURIComponent(downloadUrl.pathname.split("/").pop() ?? "");
|
||||
if (/\.[a-z0-9]+$/i.test(pathName)) return pathName;
|
||||
const extension = MIME_TYPE_EXTENSIONS.get(file.mime_type ?? contentType);
|
||||
if (!extension) throw new Error("file.file_name is required for this file type");
|
||||
return `${file.file_id}.${extension}`;
|
||||
}
|
||||
|
||||
export async function uploadChatGptAsset(input, dependencies) {
|
||||
const {
|
||||
origin,
|
||||
authorization,
|
||||
env,
|
||||
ctx,
|
||||
workerFetch,
|
||||
fetchFile = fetch,
|
||||
} = dependencies;
|
||||
const downloadUrl = validateInput(input);
|
||||
const bucket = env.LOGSEQ_SYNC_ASSETS;
|
||||
if (!bucket) throw new Error("missing assets bucket");
|
||||
|
||||
const fileResponse = await fetchFile(downloadUrl);
|
||||
if (!fileResponse.ok || !fileResponse.body) {
|
||||
throw new Error(`ChatGPT file download failed: ${fileResponse.status}`);
|
||||
}
|
||||
const size = requiredFileLength(fileResponse);
|
||||
const contentType = input.file.mime_type ?? fileResponse.headers.get("content-type");
|
||||
const fileName = resolvedFileName(input.file, downloadUrl, contentType);
|
||||
const hash = createHash("sha256");
|
||||
let received = 0;
|
||||
const hashingStream = new TransformStream({
|
||||
transform(chunk, controller) {
|
||||
const payload = chunk instanceof Uint8Array ? chunk : new Uint8Array(chunk);
|
||||
received += payload.byteLength;
|
||||
if (received > size) throw new Error("ChatGPT file exceeds declared content-length");
|
||||
hash.update(payload);
|
||||
controller.enqueue(payload);
|
||||
},
|
||||
});
|
||||
const hashedBody = fileResponse.body.pipeThrough(hashingStream);
|
||||
const { body, pipePromise } = fixedLengthBody(hashedBody, size);
|
||||
const tempKey = `chatgpt-uploads/${crypto.randomUUID()}`;
|
||||
|
||||
try {
|
||||
const uploadResults = await Promise.allSettled([
|
||||
bucket.put(tempKey, body, {
|
||||
httpMetadata: { contentType: contentType ?? "application/octet-stream" },
|
||||
}),
|
||||
pipePromise,
|
||||
].filter(Boolean));
|
||||
const uploadFailure = uploadResults.find((result) => result.status === "rejected");
|
||||
if (uploadFailure) throw uploadFailure.reason;
|
||||
if (received !== size) throw new Error("ChatGPT file is shorter than declared content-length");
|
||||
|
||||
const stored = await bucket.get(tempKey);
|
||||
if (!stored?.body) throw new Error("temporary ChatGPT upload is missing");
|
||||
const url = new URL(`/api/v1/graphs/${encodeURIComponent(input.graphId)}/assets`, origin);
|
||||
url.searchParams.set("file-name", fileName);
|
||||
url.searchParams.set("size", String(size));
|
||||
url.searchParams.set("checksum", hash.digest("hex"));
|
||||
if (input.pageId) url.searchParams.set("page-id", input.pageId);
|
||||
if (input.title) url.searchParams.set("title", input.title);
|
||||
|
||||
const headers = new Headers();
|
||||
if (authorization) headers.set("authorization", authorization);
|
||||
headers.set("content-type", contentType ?? "application/octet-stream");
|
||||
const response = await workerFetch(new Request(url, {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: stored.body,
|
||||
duplex: "half",
|
||||
}), env, ctx);
|
||||
const result = await response.json();
|
||||
if (!response.ok) {
|
||||
throw new Error(`Logseq API request failed: ${response.status} ${JSON.stringify(result)}`);
|
||||
}
|
||||
return result;
|
||||
} finally {
|
||||
await bucket.delete(tempKey);
|
||||
}
|
||||
}
|
||||
191
deps/db-sync/worker/entry.mjs
vendored
Normal file
191
deps/db-sync/worker/entry.mjs
vendored
Normal file
@@ -0,0 +1,191 @@
|
||||
import { DynamicWorkerExecutor } from "@cloudflare/codemode";
|
||||
import { openApiMcpServer } from "@cloudflare/codemode/mcp";
|
||||
import { WebStandardStreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/webStandardStreamableHttp.js";
|
||||
import { ListToolsRequestSchema } from "@modelcontextprotocol/sdk/types.js";
|
||||
import { z } from "zod";
|
||||
import { chatGptToolDescriptors } from "./chatgpt_app.mjs";
|
||||
import { uploadChatGptAsset } from "./chatgpt_asset_upload.mjs";
|
||||
import {
|
||||
ASSET_IMAGE_RESOURCE_URI,
|
||||
assertDisplayableImageMetadata,
|
||||
assetImageResource,
|
||||
assetImageResult,
|
||||
} from "./asset_image.mjs";
|
||||
import { apiDocsResponse } from "./api_docs.mjs";
|
||||
import { logseqIconResponse, setLogseqMcpServerInfoIcons } from "./logseq_icon.mjs";
|
||||
import { ensureMcpAcceptHeader } from "./mcp_request.mjs";
|
||||
import { openAiAppsChallengeResponse } from "./openai_app_challenge.mjs";
|
||||
import { semanticRequestBody, semanticRequestUrl } from "./semantic_request.mjs";
|
||||
import apiDocsHtml from "./dist/api-docs.generated.mjs";
|
||||
import worker, { SyncDO } from "./dist/worker/main.js";
|
||||
|
||||
function protectedResourceMetadata(request, env) {
|
||||
const url = new URL(request.url);
|
||||
return Response.json({
|
||||
resource: `${url.origin}/mcp`,
|
||||
authorization_servers: [env.COGNITO_ISSUER],
|
||||
scopes_supported: ["logseq/read", "logseq/write"],
|
||||
bearer_methods_supported: ["header"],
|
||||
});
|
||||
}
|
||||
|
||||
async function semanticSpec(request, env, ctx) {
|
||||
const url = new URL("/openapi.json", request.url);
|
||||
const response = await worker.fetch(new Request(url), env, ctx);
|
||||
if (!response.ok) {
|
||||
throw new Error(`OpenAPI request failed: ${response.status}`);
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async function handleMcp(request, env, ctx) {
|
||||
const authorization = request.headers.get("authorization");
|
||||
const server = openApiMcpServer({
|
||||
spec: await semanticSpec(request, env, ctx),
|
||||
executor: new DynamicWorkerExecutor({ loader: env.LOADER }),
|
||||
name: "logseq",
|
||||
version: "1.0.0",
|
||||
request: async (options) => {
|
||||
if (!options.path.startsWith("/api/v1/")) {
|
||||
throw new Error("Only Logseq server API paths are allowed");
|
||||
}
|
||||
const url = semanticRequestUrl(options, request.url);
|
||||
const headers = {};
|
||||
if (authorization) headers.authorization = authorization;
|
||||
if (options.contentType) headers["content-type"] = options.contentType;
|
||||
else if (options.body !== undefined) headers["content-type"] = "application/json";
|
||||
|
||||
const response = await worker.fetch(
|
||||
new Request(url, {
|
||||
method: options.method,
|
||||
headers,
|
||||
body: semanticRequestBody(options, url),
|
||||
}),
|
||||
env,
|
||||
ctx,
|
||||
);
|
||||
if (response.status === 204) return null;
|
||||
const result = (response.headers.get("content-type") ?? "").includes("application/json")
|
||||
? await response.json()
|
||||
: await response.text();
|
||||
if (!response.ok) {
|
||||
throw new Error(`Logseq API request failed: ${response.status} ${JSON.stringify(result)}`);
|
||||
}
|
||||
return result;
|
||||
},
|
||||
});
|
||||
setLogseqMcpServerInfoIcons(server, request);
|
||||
|
||||
server.registerResource("asset-image", ASSET_IMAGE_RESOURCE_URI, {
|
||||
title: "Logseq image asset",
|
||||
description: "Renders a Logseq image asset returned by the image asset tool.",
|
||||
mimeType: "text/html;profile=mcp-app",
|
||||
}, assetImageResource);
|
||||
|
||||
server.registerTool("upload_asset", {
|
||||
title: "Upload Logseq asset",
|
||||
description: "Upload an original ChatGPT attachment to a Logseq DB graph.",
|
||||
inputSchema: {
|
||||
graphId: z.string(),
|
||||
file: z.object({
|
||||
download_url: z.string(),
|
||||
file_id: z.string(),
|
||||
mime_type: z.string().optional(),
|
||||
file_name: z.string().optional(),
|
||||
}),
|
||||
pageId: z.string().optional(),
|
||||
title: z.string().optional(),
|
||||
},
|
||||
outputSchema: {
|
||||
uuid: z.string(),
|
||||
title: z.string(),
|
||||
type: z.string(),
|
||||
size: z.number().int().nonnegative(),
|
||||
checksum: z.string(),
|
||||
},
|
||||
annotations: {
|
||||
readOnlyHint: false,
|
||||
openWorldHint: true,
|
||||
destructiveHint: false,
|
||||
idempotentHint: false,
|
||||
},
|
||||
_meta: {
|
||||
"openai/fileParams": ["file"],
|
||||
},
|
||||
}, (input) => uploadChatGptAsset(input, {
|
||||
origin: new URL(request.url).origin,
|
||||
authorization,
|
||||
env,
|
||||
ctx,
|
||||
workerFetch: worker.fetch.bind(worker),
|
||||
}).then((result) => ({
|
||||
structuredContent: result,
|
||||
content: [{ type: "text", text: `Uploaded Logseq asset ${result.title} (${result.uuid}).` }],
|
||||
})));
|
||||
|
||||
server.registerTool("get_asset_image", {
|
||||
title: "Display Logseq image asset",
|
||||
description: "Return a Logseq image asset as MCP image content for direct display.",
|
||||
inputSchema: {
|
||||
graphId: z.string(),
|
||||
assetBlockId: z.string(),
|
||||
},
|
||||
outputSchema: {
|
||||
uuid: z.string(),
|
||||
title: z.string(),
|
||||
mimeType: z.string(),
|
||||
size: z.number().int().nonnegative(),
|
||||
},
|
||||
annotations: {
|
||||
readOnlyHint: true,
|
||||
openWorldHint: false,
|
||||
destructiveHint: false,
|
||||
idempotentHint: true,
|
||||
},
|
||||
_meta: {
|
||||
ui: { resourceUri: ASSET_IMAGE_RESOURCE_URI },
|
||||
"openai/outputTemplate": ASSET_IMAGE_RESOURCE_URI,
|
||||
},
|
||||
}, async ({ graphId, assetBlockId }) => {
|
||||
const metadataUrl = new URL(
|
||||
`/api/v1/graphs/${encodeURIComponent(graphId)}/assets/${encodeURIComponent(assetBlockId)}`,
|
||||
request.url,
|
||||
);
|
||||
const metadataResponse = await worker.fetch(new Request(metadataUrl, {
|
||||
headers: authorization ? { authorization } : {},
|
||||
}), env, ctx);
|
||||
const metadata = await metadataResponse.json();
|
||||
if (!metadataResponse.ok) {
|
||||
throw new Error(`Logseq API request failed: ${metadataResponse.status} ${JSON.stringify(metadata)}`);
|
||||
}
|
||||
assertDisplayableImageMetadata(metadata);
|
||||
const imageResponse = await worker.fetch(new Request(metadata.url), env, ctx);
|
||||
return assetImageResult(imageResponse, metadata);
|
||||
});
|
||||
|
||||
const transport = new WebStandardStreamableHTTPServerTransport();
|
||||
await server.connect(transport);
|
||||
server.server.setRequestHandler(ListToolsRequestSchema, () => ({
|
||||
tools: chatGptToolDescriptors(),
|
||||
}));
|
||||
return transport.handleRequest(request);
|
||||
}
|
||||
|
||||
export default {
|
||||
async fetch(request, env, ctx) {
|
||||
const path = new URL(request.url).pathname;
|
||||
const challengeResponse = openAiAppsChallengeResponse(request, env);
|
||||
if (challengeResponse) return challengeResponse;
|
||||
const docsResponse = apiDocsResponse(path, apiDocsHtml);
|
||||
if (docsResponse) return docsResponse;
|
||||
if (path === "/.well-known/oauth-protected-resource" ||
|
||||
path === "/.well-known/oauth-protected-resource/mcp") {
|
||||
return protectedResourceMetadata(request, env);
|
||||
}
|
||||
if (path === "/mcp/icon.png") return logseqIconResponse();
|
||||
if (path === "/mcp") return handleMcp(ensureMcpAcceptHeader(request), env, ctx);
|
||||
return worker.fetch(request, env, ctx);
|
||||
},
|
||||
};
|
||||
|
||||
export { SyncDO };
|
||||
311
deps/db-sync/worker/entry_test.mjs
vendored
Normal file
311
deps/db-sync/worker/entry_test.mjs
vendored
Normal file
@@ -0,0 +1,311 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
|
||||
import { chatGptToolDescriptors } from "./chatgpt_app.mjs";
|
||||
import {
|
||||
logseqIconResponse,
|
||||
setLogseqMcpServerInfoIcons,
|
||||
} from "./logseq_icon.mjs";
|
||||
import { ensureMcpAcceptHeader } from "./mcp_request.mjs";
|
||||
import { openAiAppsChallengeResponse } from "./openai_app_challenge.mjs";
|
||||
import { semanticRequestBody, semanticRequestUrl } from "./semantic_request.mjs";
|
||||
import { uploadChatGptAsset } from "./chatgpt_asset_upload.mjs";
|
||||
|
||||
test("OpenAI app domain challenge returns the configured token as plain text", async () => {
|
||||
const token = "openai-domain-verification-token";
|
||||
const response = openAiAppsChallengeResponse(
|
||||
new Request("https://api.logseq.io/.well-known/openai-apps-challenge"),
|
||||
{ OPENAI_APPS_CHALLENGE: token },
|
||||
);
|
||||
|
||||
assert.equal(response.status, 200);
|
||||
assert.equal(response.headers.get("content-type"), "text/plain; charset=utf-8");
|
||||
assert.equal(response.headers.get("cache-control"), "no-store");
|
||||
assert.equal(await response.text(), token);
|
||||
});
|
||||
|
||||
test("OpenAI app domain challenge fails fast when the token is not configured", () => {
|
||||
assert.throws(
|
||||
() => openAiAppsChallengeResponse(
|
||||
new Request("https://api.logseq.io/.well-known/openai-apps-challenge"),
|
||||
{},
|
||||
),
|
||||
/OPENAI_APPS_CHALLENGE is required/,
|
||||
);
|
||||
});
|
||||
|
||||
test("OpenAI app domain challenge route does not match nested paths", async () => {
|
||||
const response = openAiAppsChallengeResponse(
|
||||
new Request("https://api.logseq.io/mcp/.well-known/openai-apps-challenge"),
|
||||
{ OPENAI_APPS_CHALLENGE: "token" },
|
||||
);
|
||||
|
||||
assert.equal(response, null);
|
||||
});
|
||||
|
||||
test("OpenAI app domain challenge route only handles GET", () => {
|
||||
const response = openAiAppsChallengeResponse(
|
||||
new Request("https://api.logseq.io/.well-known/openai-apps-challenge", { method: "POST" }),
|
||||
{ OPENAI_APPS_CHALLENGE: "token" },
|
||||
);
|
||||
|
||||
assert.equal(response, null);
|
||||
});
|
||||
|
||||
test("Code Mode derives decoded asset size instead of trusting the agent", () => {
|
||||
const url = semanticRequestUrl({
|
||||
method: "POST",
|
||||
path: "/api/v1/graphs/graph-1/assets",
|
||||
query: { encoding: "base64", size: 3, checksum: "a".repeat(64) },
|
||||
body: "AQIDBA==",
|
||||
rawBody: true,
|
||||
}, "https://staging.example/mcp");
|
||||
|
||||
assert.equal(url.searchParams.get("size"), "4");
|
||||
});
|
||||
|
||||
test("Code Mode rejects invalid base64 before forwarding", () => {
|
||||
assert.throws(() => semanticRequestUrl({
|
||||
path: "/api/v1/graphs/graph-1/assets",
|
||||
query: { encoding: "base64", size: 1 },
|
||||
body: "not base64!",
|
||||
}, "https://staging.example/mcp"), /invalid base64/);
|
||||
|
||||
});
|
||||
|
||||
test("Code Mode always forwards base64 assets as a raw body", () => {
|
||||
const options = {
|
||||
path: "/api/v1/graphs/graph-1/assets",
|
||||
query: { encoding: "base64", size: 3 },
|
||||
body: "AQIDBA==",
|
||||
};
|
||||
const url = semanticRequestUrl(options, "https://staging.example/mcp");
|
||||
|
||||
assert.equal(semanticRequestBody(options, url), "AQIDBA==");
|
||||
});
|
||||
|
||||
test("ChatGPT tool descriptors declare titles, impact, and per-tool auth", () => {
|
||||
const descriptors = chatGptToolDescriptors();
|
||||
const search = descriptors.find((tool) => tool.name === "search");
|
||||
const execute = descriptors.find((tool) => tool.name === "execute");
|
||||
|
||||
assert.deepEqual(search.annotations, {
|
||||
readOnlyHint: true,
|
||||
openWorldHint: false,
|
||||
destructiveHint: false,
|
||||
idempotentHint: true,
|
||||
});
|
||||
assert.deepEqual(search.securitySchemes, [
|
||||
{ type: "oauth2", scopes: ["logseq/read", "logseq/write"] },
|
||||
]);
|
||||
assert.deepEqual(search._meta.securitySchemes, search.securitySchemes);
|
||||
|
||||
assert.deepEqual(execute.annotations, {
|
||||
readOnlyHint: false,
|
||||
openWorldHint: false,
|
||||
destructiveHint: true,
|
||||
});
|
||||
assert.deepEqual(execute.securitySchemes, [
|
||||
{ type: "oauth2", scopes: ["logseq/read", "logseq/write"] },
|
||||
]);
|
||||
assert.deepEqual(execute._meta.securitySchemes, execute.securitySchemes);
|
||||
});
|
||||
|
||||
test("ChatGPT tools require DB graph property semantics", () => {
|
||||
const descriptors = chatGptToolDescriptors().filter((tool) => ["search", "execute"].includes(tool.name));
|
||||
|
||||
for (const tool of descriptors) {
|
||||
assert.match(tool.description, /DB graph/);
|
||||
assert.match(tool.description, /typed properties/);
|
||||
assert.match(tool.description, /key:: value/);
|
||||
}
|
||||
});
|
||||
|
||||
test("ChatGPT tools describe the DB Task workflow", () => {
|
||||
const descriptors = chatGptToolDescriptors().filter((tool) => ["search", "execute"].includes(tool.name));
|
||||
|
||||
for (const tool of descriptors) {
|
||||
assert.match(tool.description, /\/tasks/);
|
||||
assert.match(tool.description, /never use Markdown TODO/);
|
||||
assert.match(tool.description, /never use graph search to list tasks/);
|
||||
}
|
||||
});
|
||||
|
||||
test("ChatGPT tools direct attachment uploads to the file-aware tool", () => {
|
||||
for (const tool of chatGptToolDescriptors().filter((descriptor) => ["search", "execute"].includes(descriptor.name))) {
|
||||
assert.match(tool.description, /upload_asset/);
|
||||
assert.match(tool.description, /never encode.*base64/i);
|
||||
}
|
||||
});
|
||||
|
||||
test("ChatGPT exposes the asset attachment as an OpenAI file parameter", () => {
|
||||
const tool = chatGptToolDescriptors().find((descriptor) => descriptor.name === "upload_asset");
|
||||
|
||||
assert.equal(tool.title, "Upload Logseq asset");
|
||||
assert.deepEqual(tool._meta["openai/fileParams"], ["file"]);
|
||||
assert.equal(tool.inputSchema.properties.file.type, "object");
|
||||
assert.deepEqual(tool.inputSchema.properties.file.required, ["download_url", "file_id"]);
|
||||
assert.deepEqual(tool.securitySchemes, [{ type: "oauth2", scopes: ["logseq/read", "logseq/write"] }]);
|
||||
assert.deepEqual(tool.annotations, {
|
||||
readOnlyHint: false,
|
||||
openWorldHint: true,
|
||||
destructiveHint: false,
|
||||
idempotentHint: false,
|
||||
});
|
||||
});
|
||||
|
||||
test("ChatGPT asset upload preserves the original bytes and derives metadata", async () => {
|
||||
const original = new Uint8Array(106816);
|
||||
for (let index = 0; index < original.length; index += 1) original[index] = index % 251;
|
||||
const tempObjects = new Map();
|
||||
const bucket = {
|
||||
async put(key, body) {
|
||||
tempObjects.set(key, new Uint8Array(await new Response(body).arrayBuffer()));
|
||||
},
|
||||
async get(key) {
|
||||
const payload = tempObjects.get(key);
|
||||
return payload ? { body: new Response(payload).body } : null;
|
||||
},
|
||||
async delete(key) {
|
||||
tempObjects.delete(key);
|
||||
},
|
||||
};
|
||||
let semanticRequest;
|
||||
const result = await uploadChatGptAsset({
|
||||
graphId: "graph-1",
|
||||
pageId: "page-1",
|
||||
title: "Logseq Logo 2.0",
|
||||
file: {
|
||||
download_url: "https://files.openai.example/logo.jpg",
|
||||
file_id: "file-1",
|
||||
mime_type: "image/jpeg",
|
||||
file_name: "logo.jpg",
|
||||
},
|
||||
}, {
|
||||
origin: "https://api.logseq.io",
|
||||
authorization: "Bearer token",
|
||||
env: { LOGSEQ_SYNC_ASSETS: bucket },
|
||||
fetchFile: async () => new Response(original, {
|
||||
headers: { "content-length": String(original.byteLength), "content-type": "image/jpeg" },
|
||||
}),
|
||||
workerFetch: async (request) => {
|
||||
semanticRequest = request;
|
||||
const uploaded = new Uint8Array(await request.arrayBuffer());
|
||||
assert.deepEqual(uploaded, original);
|
||||
return Response.json({ uuid: "asset-1", size: uploaded.byteLength }, { status: 201 });
|
||||
},
|
||||
});
|
||||
|
||||
const url = new URL(semanticRequest.url);
|
||||
assert.equal(url.pathname, "/api/v1/graphs/graph-1/assets");
|
||||
assert.equal(url.searchParams.get("file-name"), "logo.jpg");
|
||||
assert.equal(url.searchParams.get("page-id"), "page-1");
|
||||
assert.equal(url.searchParams.get("title"), "Logseq Logo 2.0");
|
||||
assert.equal(url.searchParams.get("size"), "106816");
|
||||
assert.equal(url.searchParams.get("checksum"), "2845255da015e951337b70413a1af5b661c59e41cfe5ba4c64dd1b8168143b60");
|
||||
assert.equal(semanticRequest.headers.get("authorization"), "Bearer token");
|
||||
assert.equal(semanticRequest.headers.get("content-type"), "image/jpeg");
|
||||
assert.deepEqual(result, { uuid: "asset-1", size: 106816 });
|
||||
assert.equal(tempObjects.size, 0);
|
||||
});
|
||||
|
||||
test("ChatGPT asset upload rejects missing length without buffering the file", async () => {
|
||||
await assert.rejects(() => uploadChatGptAsset({
|
||||
graphId: "graph-1",
|
||||
file: {
|
||||
download_url: "https://files.openai.example/logo.jpg",
|
||||
file_id: "file-1",
|
||||
file_name: "logo.jpg",
|
||||
},
|
||||
}, {
|
||||
origin: "https://api.logseq.io",
|
||||
env: { LOGSEQ_SYNC_ASSETS: {} },
|
||||
fetchFile: async () => new Response(new Uint8Array([1, 2, 3])),
|
||||
}), /content-length/);
|
||||
});
|
||||
|
||||
test("ChatGPT exposes a read-only image asset tool", () => {
|
||||
const tool = chatGptToolDescriptors().find((descriptor) => descriptor.name === "get_asset_image");
|
||||
|
||||
assert.equal(tool.title, "Display Logseq image asset");
|
||||
assert.deepEqual(tool.annotations, {
|
||||
readOnlyHint: true,
|
||||
openWorldHint: false,
|
||||
destructiveHint: false,
|
||||
idempotentHint: true,
|
||||
});
|
||||
assert.deepEqual(tool.securitySchemes, [{ type: "oauth2", scopes: ["logseq/read"] }]);
|
||||
});
|
||||
|
||||
test("ChatGPT image asset tool declares a UI resource for ordinary chat rendering", () => {
|
||||
const tool = chatGptToolDescriptors().find((descriptor) => descriptor.name === "get_asset_image");
|
||||
|
||||
assert.ok(tool._meta?.ui);
|
||||
assert.equal(tool._meta.ui.resourceUri, "ui://widget/logseq-asset-image.html");
|
||||
assert.equal(tool._meta["openai/outputTemplate"], "ui://widget/logseq-asset-image.html");
|
||||
assert.equal(tool.outputSchema.type, "object");
|
||||
assert.deepEqual(tool.outputSchema.required, ["uuid", "title", "mimeType", "size"]);
|
||||
});
|
||||
|
||||
test("MCP serverInfo advertises the Logseq icon from the worker origin", () => {
|
||||
const server = { server: { _serverInfo: { name: "logseq", version: "1.0.0" } } };
|
||||
|
||||
setLogseqMcpServerInfoIcons(server, new Request("https://staging.example/mcp"));
|
||||
|
||||
assert.deepEqual(server.server._serverInfo.icons, [{
|
||||
src: "https://staging.example/mcp/icon.png",
|
||||
mimeType: "image/png",
|
||||
sizes: ["192x192"],
|
||||
}]);
|
||||
});
|
||||
|
||||
test("MCP icon response serves a PNG logo", async () => {
|
||||
const response = logseqIconResponse();
|
||||
const payload = new Uint8Array(await response.arrayBuffer());
|
||||
|
||||
assert.equal(response.headers.get("content-type"), "image/png");
|
||||
assert.match(response.headers.get("cache-control"), /public/);
|
||||
assert.deepEqual([...payload.slice(0, 8)], [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
|
||||
});
|
||||
|
||||
test("MCP requests with JSON-only Accept are upgraded for streamable HTTP discovery", () => {
|
||||
const request = new Request("https://staging.example/mcp", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
accept: "application/json",
|
||||
"content-type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/list", params: {} }),
|
||||
});
|
||||
|
||||
const upgraded = ensureMcpAcceptHeader(request);
|
||||
|
||||
assert.notEqual(upgraded, request);
|
||||
assert.equal(upgraded.headers.get("accept"), "application/json, text/event-stream");
|
||||
assert.equal(upgraded.headers.get("content-type"), "application/json");
|
||||
});
|
||||
|
||||
test("MCP requests without Accept are upgraded for streamable HTTP discovery", () => {
|
||||
const request = new Request("https://staging.example/mcp", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/list", params: {} }),
|
||||
});
|
||||
|
||||
const upgraded = ensureMcpAcceptHeader(request);
|
||||
|
||||
assert.equal(upgraded.headers.get("accept"), "application/json, text/event-stream");
|
||||
});
|
||||
|
||||
test("MCP requests that already support streamable HTTP are preserved", () => {
|
||||
const request = new Request("https://staging.example/mcp", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
accept: "application/json, text/event-stream",
|
||||
"content-type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/list", params: {} }),
|
||||
});
|
||||
|
||||
assert.equal(ensureMcpAcceptHeader(request), request);
|
||||
});
|
||||
34
deps/db-sync/worker/logseq_icon.mjs
vendored
Normal file
34
deps/db-sync/worker/logseq_icon.mjs
vendored
Normal file
File diff suppressed because one or more lines are too long
12
deps/db-sync/worker/mcp_request.mjs
vendored
Normal file
12
deps/db-sync/worker/mcp_request.mjs
vendored
Normal file
@@ -0,0 +1,12 @@
|
||||
export function ensureMcpAcceptHeader(request) {
|
||||
const accept = request.headers.get("accept") ?? "";
|
||||
const normalizedAccept = accept.toLowerCase();
|
||||
if (normalizedAccept.includes("application/json") &&
|
||||
normalizedAccept.includes("text/event-stream")) {
|
||||
return request;
|
||||
}
|
||||
|
||||
const headers = new Headers(request.headers);
|
||||
headers.set("accept", "application/json, text/event-stream");
|
||||
return new Request(request, { headers });
|
||||
}
|
||||
42
deps/db-sync/worker/migrations/0001_init.sql
vendored
Normal file
42
deps/db-sync/worker/migrations/0001_init.sql
vendored
Normal file
@@ -0,0 +1,42 @@
|
||||
create table if not exists graphs (
|
||||
graph_id text primary key,
|
||||
graph_name text,
|
||||
user_id text,
|
||||
schema_version text,
|
||||
created_at integer,
|
||||
updated_at integer
|
||||
);
|
||||
|
||||
create table if not exists users (
|
||||
id text primary key,
|
||||
email text,
|
||||
email_verified integer,
|
||||
username text
|
||||
);
|
||||
|
||||
create table if not exists user_rsa_keys (
|
||||
user_id text primary key,
|
||||
public_key text,
|
||||
encrypted_private_key text,
|
||||
created_at integer,
|
||||
updated_at integer
|
||||
);
|
||||
|
||||
create table if not exists graph_members (
|
||||
user_id text,
|
||||
graph_id text,
|
||||
role text,
|
||||
invited_by text,
|
||||
created_at integer,
|
||||
primary key (user_id, graph_id),
|
||||
check (role in ('manager', 'member'))
|
||||
);
|
||||
|
||||
create table if not exists graph_aes_keys (
|
||||
graph_id text,
|
||||
user_id text,
|
||||
encrypted_aes_key text,
|
||||
created_at integer,
|
||||
updated_at integer,
|
||||
primary key (graph_id, user_id)
|
||||
);
|
||||
2
deps/db-sync/worker/migrations/0002_add_graph_e2ee.sql
vendored
Normal file
2
deps/db-sync/worker/migrations/0002_add_graph_e2ee.sql
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
alter table graphs
|
||||
add column graph_e2ee integer default 1;
|
||||
8
deps/db-sync/worker/migrations/0003_add_indexes.sql
vendored
Normal file
8
deps/db-sync/worker/migrations/0003_add_indexes.sql
vendored
Normal file
@@ -0,0 +1,8 @@
|
||||
create index if not exists idx_graph_members_graph_id_created_at
|
||||
on graph_members (graph_id, created_at);
|
||||
|
||||
create index if not exists idx_graphs_user_id_updated_at
|
||||
on graphs (user_id, updated_at desc);
|
||||
|
||||
create index if not exists idx_users_email
|
||||
on users (email);
|
||||
2
deps/db-sync/worker/migrations/0004_add_graph_ready_for_use.sql
vendored
Normal file
2
deps/db-sync/worker/migrations/0004_add_graph_ready_for_use.sql
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
alter table graphs
|
||||
add column graph_ready_for_use integer default 1;
|
||||
2
deps/db-sync/worker/migrations/0005_add_user_created_at.sql
vendored
Normal file
2
deps/db-sync/worker/migrations/0005_add_user_created_at.sql
vendored
Normal file
@@ -0,0 +1,2 @@
|
||||
alter table users
|
||||
add column created_at integer;
|
||||
11
deps/db-sync/worker/migrations/0006_add_daily_active_entities.sql
vendored
Normal file
11
deps/db-sync/worker/migrations/0006_add_daily_active_entities.sql
vendored
Normal file
@@ -0,0 +1,11 @@
|
||||
create table if not exists daily_active_entities (
|
||||
day_utc text,
|
||||
entity_type text,
|
||||
entity_id text,
|
||||
first_seen_at integer,
|
||||
primary key (day_utc, entity_type, entity_id),
|
||||
check (entity_type in ('user', 'graph'))
|
||||
);
|
||||
|
||||
create index if not exists idx_daily_active_entities_type_day
|
||||
on daily_active_entities (entity_type, day_utc);
|
||||
16
deps/db-sync/worker/openai_app_challenge.mjs
vendored
Normal file
16
deps/db-sync/worker/openai_app_challenge.mjs
vendored
Normal file
@@ -0,0 +1,16 @@
|
||||
const OPENAI_APPS_CHALLENGE_PATH = "/.well-known/openai-apps-challenge";
|
||||
|
||||
export function openAiAppsChallengeResponse(request, env) {
|
||||
const url = new URL(request.url);
|
||||
if (request.method !== "GET" || url.pathname !== OPENAI_APPS_CHALLENGE_PATH) return null;
|
||||
|
||||
const token = env.OPENAI_APPS_CHALLENGE;
|
||||
if (!token) throw new Error("OPENAI_APPS_CHALLENGE is required");
|
||||
|
||||
return new Response(token, {
|
||||
headers: {
|
||||
"cache-control": "no-store",
|
||||
"content-type": "text/plain; charset=utf-8",
|
||||
},
|
||||
});
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user