Initial import of LogDarioSeq baseline

Establishes version control for the personal Logseq DB fork before
starting the server-client architecture migration and domain features
described in docs/agent-guide and the approved implementation plan.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

deps/db-sync/src/logseq/db_sync/node/api_docs.cljs

@@ -0,0 +1,10 @@
+(ns logseq.db-sync.node.api-docs
+  (:require ["node:fs" :as fs]
+            [logseq.db-sync.worker.routes.semantic :as semantic-routes]))
+
+(defn ^:export main []
+  (let [issuer (or (aget js/process.env "COGNITO_ISSUER") "")
+        document (semantic-routes/openapi-document issuer)]
+    (.mkdirSync fs "worker/dist" #js {:recursive true})
+    (.writeFileSync fs "worker/dist/openapi.json"
+                    (js/JSON.stringify (clj->js document) nil 2))))

deps/db-sync/src/logseq/db_sync/node/assets.cljs

@@ -0,0 +1,58 @@
+(ns logseq.db-sync.node.assets
+  (:require ["fs" :as fs]
+            ["path" :as node-path]
+            [promesa.core :as p]))
+
+(defn- ensure-dir! [dir]
+  (.mkdirSync fs dir #js {:recursive true}))
+
+(defn- meta-path [path]
+  (str path ".meta.json"))
+
+(defn- read-meta [path]
+  (p/let [content (.readFile (.-promises fs) path "utf8")]
+    (js/JSON.parse content)))
+
+(defn- write-meta! [path meta]
+  (.writeFile (.-promises fs) path (js/JSON.stringify meta) "utf8"))
+
+(defn- normalize-bytes [data]
+  (cond
+    (instance? js/Uint8Array data) data
+    (instance? js/ArrayBuffer data) (js/Uint8Array. data)
+    :else (js/Uint8Array. data)))
+
+(defn make-bucket [base-dir]
+  (ensure-dir! base-dir)
+  #js {:get (fn [key]
+              (let [file-path (node-path/join base-dir key)
+                    meta-file (meta-path file-path)]
+                (p/catch
+                 (p/let [buf (.readFile (.-promises fs) file-path)
+                         meta (p/catch (read-meta meta-file) (fn [_] #js {}))]
+                   #js {:body buf
+                        :httpMetadata #js {:contentType (aget meta "contentType")
+                                           :contentEncoding (aget meta "contentEncoding")
+                                           :cacheControl (aget meta "cacheControl")}
+                        :customMetadata (aget meta "customMetadata")})
+                 (fn [_] nil))))
+       :put (fn [key body opts]
+              (let [file-path (node-path/join base-dir key)
+                    meta-file (meta-path file-path)
+                    dir (node-path/dirname file-path)
+                    data (normalize-bytes body)
+                    metadata (or (aget opts "httpMetadata") #js {})
+                    custom (or (aget opts "customMetadata") #js {})]
+                (ensure-dir! dir)
+                (p/let [_ (.writeFile (.-promises fs) file-path data)
+                        _ (write-meta! meta-file #js {:contentType (aget metadata "contentType")
+                                                      :contentEncoding (aget metadata "contentEncoding")
+                                                      :cacheControl (aget metadata "cacheControl")
+                                                      :customMetadata custom})]
+                  #js {:ok true})))
+       :delete (fn [key]
+                 (let [file-path (node-path/join base-dir key)
+                       meta-file (meta-path file-path)]
+                   (p/let [_ (p/catch (.unlink (.-promises fs) file-path) (fn [_] nil))
+                           _ (p/catch (.unlink (.-promises fs) meta-file) (fn [_] nil))]
+                     #js {:ok true})))})

deps/db-sync/src/logseq/db_sync/node/config.cljs

@@ -0,0 +1,44 @@
+(ns logseq.db-sync.node.config
+  (:require [clojure.string :as string]))
+
+(defn- env-value [^js env k]
+  (let [v (aget env k)]
+    (when (seq v) v)))
+
+(defn- parse-int [v default]
+  (let [n (js/parseInt v 10)]
+    (if (js/isNaN n) default n)))
+
+(defn config-from-env []
+  (let [env (.-env js/process)]
+    {:port (when-let [v (env-value env "DB_SYNC_PORT")] (parse-int v 8080))
+     :base-url (env-value env "DB_SYNC_BASE_URL")
+     :data-dir (or (env-value env "DB_SYNC_DATA_DIR") "data/db-sync")
+     :storage-driver (or (env-value env "DB_SYNC_STORAGE_DRIVER") "sqlite")
+     :assets-driver (or (env-value env "DB_SYNC_ASSETS_DRIVER") "filesystem")
+     :log-level (or (env-value env "DB_SYNC_LOG_LEVEL") "info")
+     :cognito-issuer (env-value env "COGNITO_ISSUER")
+     :cognito-client-id (env-value env "COGNITO_CLIENT_ID")
+     :cognito-jwks-url (env-value env "COGNITO_JWKS_URL")}))
+
+(def ^:private allowed-config-keys
+  [:port :base-url :data-dir :storage-driver :assets-driver :log-level
+   :cognito-issuer :cognito-client-id :cognito-jwks-url])
+
+(defn normalize-config [overrides]
+  (let [defaults {:port 8080
+                  :data-dir "data/db-sync"
+                  :storage-driver "sqlite"
+                  :assets-driver "filesystem"
+                  :log-level "info"}
+        merged (merge defaults (config-from-env) overrides)
+        storage-driver (string/lower-case (:storage-driver merged))
+        assets-driver (string/lower-case (:assets-driver merged))]
+    (when-not (#{"sqlite"} storage-driver)
+      (throw (js/Error. (str "unsupported storage driver: " storage-driver))))
+    (when-not (#{"filesystem"} assets-driver)
+      (throw (js/Error. (str "unsupported assets driver: " assets-driver))))
+    (-> merged
+        (select-keys allowed-config-keys)
+        (assoc :storage-driver storage-driver
+               :assets-driver assets-driver))))

deps/db-sync/src/logseq/db_sync/node/dispatch.cljs

@@ -0,0 +1,76 @@
+(ns logseq.db-sync.node.dispatch
+  (:require [clojure.string :as string]
+            [logseq.db-sync.common :as common]
+            [logseq.db-sync.node.graph :as graph]
+            [logseq.db-sync.node.routes :as node-routes]
+            [logseq.db-sync.platform.core :as platform]
+            [logseq.db-sync.worker.handler.assets :as assets-handler]
+            [logseq.db-sync.worker.handler.index :as index-handler]
+            [logseq.db-sync.worker.handler.sync :as sync-handler]
+            [logseq.db-sync.worker.http :as http]
+            [promesa.core :as p]))
+
+(defn- admin-token-valid?
+  [request env]
+  (let [expected (aget env "DB_SYNC_ADMIN_TOKEN")
+        actual (.get (.-headers request) "x-db-sync-admin-token")]
+    (and (string? expected)
+         (seq expected)
+         (= expected actual))))
+
+(defn handle-node-fetch
+  [{:keys [request env registry deps]}]
+  (let [url (platform/request-url request)
+        path (.-pathname url)
+        method (.-method request)
+        index-self #js {:env env :d1 (aget env "DB")}]
+    (cond
+      (= path "/health")
+      (http/json-response :worker/health {:ok true})
+
+      (or (= path "/graphs")
+          (string/starts-with? path "/graphs/"))
+      (index-handler/handle-fetch index-self request)
+
+      (string/starts-with? path "/e2ee")
+      (index-handler/handle-fetch index-self request)
+
+      (string/starts-with? path "/assets/")
+      (if (= method "OPTIONS")
+        (assets-handler/handle request env)
+        (if-let [{:keys [graph-id]} (assets-handler/parse-asset-path path)]
+          (if (admin-token-valid? request env)
+            (assets-handler/handle request env)
+            (p/let [access-resp (index-handler/graph-access-response request env graph-id)]
+              (if (.-ok access-resp)
+                (assets-handler/handle request env)
+                access-resp)))
+          (http/bad-request "invalid asset path")))
+
+      (= method "OPTIONS")
+      (common/options-response)
+
+      (string/starts-with? path "/sync/")
+      (if-let [{:keys [graph-id tail]} (node-routes/parse-sync-path path)]
+        (if (seq graph-id)
+          (if (= method "OPTIONS")
+            (common/options-response)
+            (if (admin-token-valid? request env)
+              (let [ctx (graph/get-or-create-graph registry deps graph-id)
+                    new-url (js/URL. (str (.-origin url) tail (.-search url)))]
+                (.set (.-searchParams new-url) "graph-id" graph-id)
+                (let [rewritten (platform/request (.toString new-url) request)]
+                  (sync-handler/handle-http ctx rewritten)))
+              (p/let [access-resp (index-handler/graph-access-response request env graph-id)]
+                (if (.-ok access-resp)
+                  (let [ctx (graph/get-or-create-graph registry deps graph-id)
+                        new-url (js/URL. (str (.-origin url) tail (.-search url)))]
+                    (.set (.-searchParams new-url) "graph-id" graph-id)
+                    (let [rewritten (platform/request (.toString new-url) request)]
+                      (sync-handler/handle-http ctx rewritten)))
+                  access-resp))))
+          (http/bad-request "missing graph id"))
+        (http/bad-request "missing graph id"))
+
+      :else
+      (http/not-found))))

deps/db-sync/src/logseq/db_sync/node/entry.cljs

@@ -0,0 +1,18 @@
+(ns logseq.db-sync.node.entry
+  (:require [logseq.db-sync.node.config :as config]
+            [logseq.db-sync.node.server :as server]
+            [logseq.db-sync.sentry.node :as sentry]
+            [promesa.core :as p]))
+
+(defonce ^:private *server (atom nil))
+
+(defn main [& _args]
+  (sentry/init!)
+  (let [cfg (config/normalize-config {})]
+    (js/console.log "Starting Logseq sync...")
+    (-> (server/start! cfg)
+        (p/then (fn [result]
+                  (reset! *server result)
+                  (js/console.log (str "Logseq sync listening on port " (:port result)))))
+        (p/catch (fn [error]
+                   (js/console.error "Logseq sync failed to start" error))))))

deps/db-sync/src/logseq/db_sync/node/find_checksum_drift.cljs

@@ -0,0 +1,199 @@
+(ns logseq.db-sync.node.find-checksum-drift
+  (:require ["better-sqlite3" :as sqlite3]
+            ["fs" :as fs]
+            ["path" :as node-path]
+            [clojure.string :as string]
+            [datascript.core :as d]
+            [logseq.db-sync.checksum :as sync-checksum]
+            [logseq.db-sync.common :as common]
+            [logseq.db-sync.storage :as storage]
+            [logseq.db.frontend.schema :as db-schema]))
+
+(def sqlite (if (find-ns 'nbb.core) (aget sqlite3 "default") sqlite3))
+
+(defn- fail!
+  [msg]
+  (binding [*print-fn* *print-err-fn*]
+    (println (str "Error: " msg)))
+  (js/process.exit 1))
+
+(defn- parse-args
+  [argv]
+  (loop [m {}
+         xs (vec argv)]
+    (if-let [x (first xs)]
+      (cond
+        (some #{x} ["-h" "--help"])
+        (recur (assoc m :help? true) (next xs))
+
+        (= "--db" x)
+        (if-let [db (second xs)]
+          (recur (assoc m :db db) (nnext xs))
+          (fail! "Missing value for --db"))
+
+        (= "--diff-out" x)
+        (if-let [diff-out (second xs)]
+          (recur (assoc m :diff-out diff-out) (nnext xs))
+          (fail! "Missing value for --diff-out"))
+
+        :else
+        (fail! (str "Unknown argument: " x)))
+      m)))
+
+(defn- print-help!
+  []
+  (println "Find first tx_log checksum drift in a sqlite graph db.")
+  (println "")
+  (println "Usage:")
+  (println "  node worker/dist/find-checksum-drift.js --db <path/to/db.sqlite>")
+  (println "  node worker/dist/find-checksum-drift.js --db <path/to/db.sqlite> --diff-out <path/to/diff.edn>")
+  (println "")
+  (println "Output:")
+  (println "  - stored/recomputed checksum for current db")
+  (println "  - first tx_log t where incremental != recomputed during replay")
+  (println "  - optional full current-vs-replayed block diff EDN file"))
+
+(defn- normalize-sql
+  [sql]
+  (-> sql string/trim string/lower-case))
+
+(defn- select-sql?
+  [sql]
+  (string/starts-with? (normalize-sql sql) "select"))
+
+(defn- exec-with-args [^js stmt args]
+  (.apply (.-run stmt) stmt (to-array args)))
+
+(defn- all-with-args [^js stmt args]
+  (.apply (.-all stmt) stmt (to-array args)))
+
+(defn- wrap-db [^js db]
+  #js {:exec (fn [sql & args]
+               (if (seq args)
+                 (let [stmt (.prepare db sql)]
+                   (if (select-sql? sql)
+                     (all-with-args stmt args)
+                     (do
+                       (exec-with-args stmt args)
+                       nil)))
+                 (if (select-sql? sql)
+                   (.all (.prepare db sql))
+                   (.exec db sql))))
+       :close (fn [] (.close db))
+       :_db db})
+
+(defn- short-preview
+  [tx-data]
+  (->> tx-data
+       (take 8)
+       vec))
+
+(defn- tx-rows
+  [^js db]
+  (.all (.prepare db "select t, tx, outliner_op from tx_log order by t asc")))
+
+(defn- replay-find-first-mismatch
+  [rows]
+  (let [conn (d/create-conn db-schema/schema)]
+    (loop [idx 0
+           checksum nil]
+      (if (>= idx (count rows))
+        {:ok? true
+         :idx idx
+         :checksum checksum
+         :recomputed (sync-checksum/recompute-checksum @conn)
+         :db @conn}
+        (let [row (nth rows idx)
+              t (aget row "t")
+              outliner-op (aget row "outliner_op")
+              tx-str (aget row "tx")
+              tx-data (common/read-transit tx-str)
+              tx-report (d/transact! conn tx-data)
+              incremental (sync-checksum/update-checksum checksum tx-report)
+              recomputed (sync-checksum/recompute-checksum (:db-after tx-report))]
+          (if (= incremental recomputed)
+            (recur (inc idx) incremental)
+            {:ok? false
+             :idx idx
+             :t t
+             :outliner-op outliner-op
+             :prev-checksum checksum
+             :incremental incremental
+             :recomputed recomputed
+             :tx-size (count tx-data)
+             :tx-preview (short-preview tx-data)
+             :db (:db-after tx-report)}))))))
+
+(defn- block-map-by-uuid
+  [blocks]
+  (into {} (map (juxt :block/uuid identity) blocks)))
+
+(defn- diff-blocks
+  [blocks-a blocks-b]
+  (let [a-by-id (block-map-by-uuid blocks-a)
+        b-by-id (block-map-by-uuid blocks-b)
+        all-ids (set (concat (keys a-by-id) (keys b-by-id)))]
+    (->> all-ids
+         (keep (fn [id]
+                 (let [a (get a-by-id id)
+                       b (get b-by-id id)]
+                   (when (not= a b)
+                     {:block/uuid id
+                      :current a
+                      :replayed b}))))
+         vec)))
+
+(defn- sorted-block-diffs
+  [block-diffs]
+  (sort-by (fn [m]
+             (some-> (:block/uuid m) str))
+           block-diffs))
+
+(defn main
+  [& argv]
+  (let [{:keys [help? db diff-out]} (parse-args argv)]
+    (when help?
+      (print-help!)
+      (js/process.exit 0))
+    (when-not db
+      (fail! "Missing required --db <path> argument"))
+    (let [db-path (node-path/resolve db)]
+      (when-not (.existsSync fs db-path)
+        (fail! (str "SQLite file not found: " db-path)))
+      (let [sqlite-db (new sqlite db-path nil)
+            sql (wrap-db sqlite-db)]
+        (try
+          (storage/init-schema! sql)
+          (let [conn (storage/open-conn sql)
+                db' @conn
+                stored (storage/get-checksum sql)
+                recomputed (sync-checksum/recompute-checksum db')
+                rows (tx-rows sqlite-db)
+                replay-result (replay-find-first-mismatch rows)
+                current-diag (sync-checksum/recompute-checksum-diagnostics db')
+                replay-diag (sync-checksum/recompute-checksum-diagnostics (:db replay-result))
+                block-diffs (-> (diff-blocks (:blocks current-diag) (:blocks replay-diag))
+                                sorted-block-diffs)]
+            (println (str "db: " db-path))
+            (println (str "stored-checksum: " stored))
+            (println (str "recomputed-checksum: " recomputed))
+            (println (str "tx-log-count: " (count rows)))
+            (println (str "current-blocks: " (count (:blocks current-diag))
+                          ", replayed-blocks: " (count (:blocks replay-diag))
+                          ", different-blocks: " (count block-diffs)))
+            (when (seq block-diffs)
+              (println "different-blocks-sample:")
+              (println (pr-str (take 5 block-diffs))))
+            (when (seq diff-out)
+              (let [diff-out-path (node-path/resolve diff-out)]
+                (.writeFileSync fs diff-out-path
+                                (str (pr-str block-diffs) "\n")
+                                "utf8")
+                (println (str "diff-out: " diff-out-path))))
+            (if (:ok? replay-result)
+              (println (pr-str (assoc replay-result
+                                      :stored-checksum stored
+                                      :final-recomputed recomputed)))
+              (println (pr-str replay-result))))
+          (finally
+            (.close sql)))))))

deps/db-sync/src/logseq/db_sync/node/graph.cljs

@@ -0,0 +1,57 @@
+(ns logseq.db-sync.node.graph
+  (:require [logseq.db-sync.node.storage :as storage]))
+
+(defn- make-state []
+  (let [sockets (atom #{})]
+    #js {:getWebSockets (fn [] (to-array @sockets))
+         :addWebSocket (fn [ws] (swap! sockets conj ws))
+         :removeWebSocket (fn [ws] (swap! sockets disj ws))}))
+
+(defn- env-object [cfg index-db assets-bucket]
+  (let [allow-unverified-jwt-claims (some-> js/process .-env (aget "DB_SYNC_ALLOW_UNVERIFIED_JWT_CLAIMS"))
+        env (doto (js-obj)
+              (aset "DB" index-db)
+              (aset "LOGSEQ_SYNC_ASSETS" assets-bucket)
+              ;; Keep node-adapter snapshot stream uncompressed.
+              (aset "DB_SYNC_SNAPSHOT_STREAM_GZIP" "false")
+              (aset "COGNITO_ISSUER" (:cognito-issuer cfg))
+              (aset "COGNITO_CLIENT_ID" (:cognito-client-id cfg))
+              (aset "COGNITO_JWKS_URL" (:cognito-jwks-url cfg)))]
+    (when (some? allow-unverified-jwt-claims)
+      (aset env "DB_SYNC_ALLOW_UNVERIFIED_JWT_CLAIMS" allow-unverified-jwt-claims))
+    env))
+
+(defn graph-context
+  [{:keys [config index-db assets-bucket]} graph-id]
+  (let [{:keys [sql]} (storage/open-graph-db (:data-dir config) graph-id)
+        state (make-state)
+        env (env-object config index-db assets-bucket)]
+    #js {:state state
+         :env env
+         :sql sql
+         :conn nil
+         :schema-ready false}))
+
+(defn get-or-create-graph
+  [registry deps graph-id]
+  (or (get @registry graph-id)
+      (let [ctx (graph-context deps graph-id)]
+        (swap! registry assoc graph-id ctx)
+        ctx)))
+
+(defn- close-graph-context!
+  [^js ctx]
+  (when-let [^js sql (.-sql ctx)]
+    (when-let [close (.-close sql)]
+      (close))))
+
+(defn delete-graph!
+  [registry deps graph-id]
+  (when-let [^js ctx (get @registry graph-id)]
+    (close-graph-context! ctx)
+    (swap! registry dissoc graph-id))
+  (storage/delete-graph-db! (get-in deps [:config :data-dir]) graph-id))
+
+(defn close-graphs! [registry]
+  (doseq [[_ ^js ctx] @registry]
+    (close-graph-context! ctx)))

deps/db-sync/src/logseq/db_sync/node/recompute_log_checksum.cljs

@@ -0,0 +1,119 @@
+(ns logseq.db-sync.node.recompute-log-checksum
+  (:require ["fs" :as fs]
+            ["path" :as node-path]
+            [cljs.reader :as reader]
+            [clojure.string :as string]
+            [datascript.core :as d]
+            [logseq.db :as ldb]
+            [logseq.db-sync.checksum :as sync-checksum]))
+
+(defn- fail!
+  [msg]
+  (binding [*print-fn* *print-err-fn*]
+    (println (str "Error: " msg)))
+  (js/process.exit 1))
+
+(defn- parse-args
+  [argv]
+  (loop [m {}
+         xs (vec argv)]
+    (if-let [x (first xs)]
+      (cond
+        (some #{x} ["-h" "--help"])
+        (recur (assoc m :help? true) (next xs))
+
+        (= "--log" x)
+        (if-let [log-file (second xs)]
+          (recur (assoc m :log-file log-file) (nnext xs))
+          (fail! "Missing value for --log"))
+
+        :else
+        (fail! (str "Unknown argument: " x)))
+      m)))
+
+(defn- print-help!
+  []
+  (println "Recompute checksum from a :before-checksum-error server log line.")
+  (println "")
+  (println "Usage:")
+  (println "  node worker/dist/recompute-log-checksum.js --log /tmp/server1.log")
+  (println "")
+  (println "Output:")
+  (println "  - logged incremental and recomputed checksums")
+  (println "  - replayed incremental and recomputed checksums")
+  (println "  - equality checks"))
+
+(defn- parse-log-payload
+  [line]
+  (let [prefix ":debug :before-checksum-error "]
+    (when-not (string/starts-with? line prefix)
+      (fail! (str "Log does not start with expected prefix: " prefix)))
+    (reader/read-string (subs line (count prefix)))))
+
+(defn- ensure-tx-data
+  [tx-data]
+  (cond
+    (string? tx-data)
+    (let [decoded (ldb/read-transit-str tx-data)]
+      (if (sequential? decoded)
+        decoded
+        (fail! "Decoded transit tx-data is not sequential")))
+
+    (sequential? tx-data)
+    tx-data
+
+    :else
+    (fail! (str "Unsupported tx-data shape: " (type tx-data)))))
+
+(defn main
+  [& argv]
+  (let [{:keys [help? log-file]} (parse-args argv)]
+    (when help?
+      (print-help!)
+      (js/process.exit 0))
+    (when-not log-file
+      (fail! "Missing required --log <path> argument"))
+    (let [log-path (node-path/resolve log-file)]
+      (when-not (.existsSync fs log-path)
+        (fail! (str "Log file not found: " log-path)))
+      (let [line (string/trim (.readFileSync fs log-path "utf8"))
+            payload (parse-log-payload line)
+            db-before (ldb/read-transit-str (:db-before payload))
+            tx-data (vec (ensure-tx-data (:tx-data payload)))
+            tx-report (d/with db-before tx-data)
+            input-checksum (or (:prev-checksum payload)
+                               (:current-checksum payload))
+            logged-prev-full (:prev-full-checksum payload)
+            logged-full-after (or (:recomputed-after-checksum payload)
+                                  (:full-checksum payload))
+            replayed-prev-full (sync-checksum/recompute-checksum db-before)
+            replayed-incremental-from-full-before
+            (sync-checksum/update-checksum replayed-prev-full tx-report)
+            replayed-incremental-from-input
+            (sync-checksum/update-checksum input-checksum tx-report)
+            replayed-recomputed (sync-checksum/recompute-checksum (:db-after tx-report))
+            result {:log-file log-path
+                    :prev-tx (:prev-tx payload)
+                    :tx-meta (:tx-meta payload)
+                    :tx-count (count tx-data)
+                    :input-checksum input-checksum
+                    :logged-prev-full-checksum logged-prev-full
+                    :replayed-prev-full-checksum replayed-prev-full
+                    :input-checksum-eq-replayed-prev-full?
+                    (= input-checksum replayed-prev-full)
+                    :match-logged-prev-full?
+                    (= logged-prev-full replayed-prev-full)
+                    :logged-new-checksum (:new-checksum payload)
+                    :replayed-incremental replayed-incremental-from-input
+                    :replayed-incremental-from-full-before replayed-incremental-from-full-before
+                    :logged-full-after-checksum logged-full-after
+                    :replayed-recomputed-after replayed-recomputed
+                    :match-logged-new?
+                    (= replayed-incremental-from-input (:new-checksum payload))
+                    :match-logged-full-after?
+                    (= replayed-recomputed logged-full-after)
+                    :incremental-eq-full?
+                    (= replayed-incremental-from-input replayed-recomputed)
+                    :incremental-from-full-before-eq-full?
+                    (= replayed-incremental-from-full-before replayed-recomputed)}]
+        (println (pr-str result))))))

deps/db-sync/src/logseq/db_sync/node/routes.cljs

@@ -0,0 +1,16 @@
+(ns logseq.db-sync.node.routes
+  (:require [clojure.string :as string]))
+
+(defn parse-sync-path [path]
+  (when (string/starts-with? path "/sync/")
+    (let [rest-path (subs path (count "/sync/"))
+          rest-path (if (string/starts-with? rest-path "/")
+                      (subs rest-path 1)
+                      rest-path)
+          slash-idx (or (string/index-of rest-path "/") -1)
+          graph-id (if (neg? slash-idx) rest-path (subs rest-path 0 slash-idx))
+          tail (if (neg? slash-idx)
+                 "/"
+                 (subs rest-path slash-idx))]
+      {:graph-id graph-id
+       :tail tail})))

deps/db-sync/src/logseq/db_sync/node/server.cljs

@@ -0,0 +1,181 @@
+(ns logseq.db-sync.node.server
+  (:require ["http" :as http]
+            ["path" :as node-path]
+            ["ws" :as ws]
+            [clojure.string :as string]
+            [lambdaisland.glogi :as log]
+            [logseq.db-sync.index :as index]
+            [logseq.db-sync.logging :as logging]
+            [logseq.db-sync.node.assets :as assets]
+            [logseq.db-sync.node.config :as config]
+            [logseq.db-sync.node.dispatch :as dispatch]
+            [logseq.db-sync.node.graph :as graph]
+            [logseq.db-sync.node.routes :as node-routes]
+            [logseq.db-sync.node.storage :as storage]
+            [logseq.db-sync.platform.core :as platform]
+            [logseq.db-sync.platform.node :as platform-node]
+            [logseq.db-sync.worker.auth :as auth]
+            [logseq.db-sync.worker.handler.sync :as sync-handler]
+            [logseq.db-sync.worker.handler.ws :as ws-handler]
+            [logseq.db-sync.worker.http :as worker-http]
+            [logseq.db-sync.worker.presence :as presence]
+            [promesa.core :as p]))
+
+(logging/install!)
+
+(defn- make-env [cfg index-db assets-bucket]
+  (let [allow-unverified-jwt-claims (some-> js/process .-env (aget "DB_SYNC_ALLOW_UNVERIFIED_JWT_CLAIMS"))
+        env (doto (js-obj)
+              (aset "DB" index-db)
+              (aset "LOGSEQ_SYNC_ASSETS" assets-bucket)
+              ;; Node adapter serves snapshot transit stream without gzip to avoid
+              ;; browser/adapter content-encoding mismatches during graph download.
+              (aset "DB_SYNC_SNAPSHOT_STREAM_GZIP" "false")
+              (aset "COGNITO_ISSUER" (:cognito-issuer cfg))
+              (aset "COGNITO_CLIENT_ID" (:cognito-client-id cfg))
+              (aset "COGNITO_JWKS_URL" (:cognito-jwks-url cfg)))]
+    (when (some? allow-unverified-jwt-claims)
+      (aset env "DB_SYNC_ALLOW_UNVERIFIED_JWT_CLAIMS" allow-unverified-jwt-claims))
+    env))
+
+(defn- request-origin-opts
+  [cfg]
+  (if-let [base-url (:base-url cfg)]
+    (let [url (js/URL. base-url)
+          protocol (.-protocol url)
+          scheme (if (string/ends-with? protocol ":")
+                   (subs protocol 0 (dec (count protocol)))
+                   protocol)]
+      {:scheme scheme
+       :host (.-host url)})
+    {:scheme "http"}))
+
+(defn- access-allowed?
+  [env graph-id request]
+  (p/let [claims (auth/auth-claims request env)
+          user-id (when claims (aget claims "sub"))
+          db (aget env "DB")]
+    (if (string? user-id)
+      (index/<user-has-access-to-graph? db graph-id user-id)
+      false)))
+
+(defn- attach-ws! [^js ctx ^js socket]
+  (let [state (.-state ctx)]
+    (when-let [add-ws (.-addWebSocket state)]
+      (add-ws socket))
+    (set! (.-serializeAttachment socket) (fn [_] nil))
+    (set! (.-deserializeAttachment socket) (fn [] nil))))
+
+(defn- detach-ws! [^js ctx ^js socket]
+  (let [state (.-state ctx)]
+    (when-let [remove-ws (.-removeWebSocket state)]
+      (remove-ws socket))))
+
+(defn- reject-ws-upgrade!
+  [^js socket status reason]
+  (.write socket
+          (str "HTTP/1.1 " status " Conflict\r\n"
+               "Connection: close\r\n"
+               "Content-Type: application/json\r\n\r\n"
+               "{\"error\":\"" reason "\"}"))
+  (.destroy socket))
+
+(defn- handle-ws-connection
+  [ctx env request ^js socket]
+  (p/let [claims (auth/auth-claims request env)
+          user (presence/claims->user claims)]
+    (when user
+      (presence/add-presence! ctx socket user))
+    (presence/broadcast-online-users! ctx))
+  (.on socket "message"
+       (fn [data]
+         (let [text (if (string? data) data (.toString data))]
+           (try
+             (ws-handler/handle-ws-message! ctx socket text)
+             (catch :default e
+               (log/error :db-sync/ws-error e)
+               (.send socket (js/JSON.stringify #js {:type "error" :message "server error"})))))))
+  (.on socket "close"
+       (fn []
+         (presence/remove-presence! ctx socket)
+         (presence/broadcast-online-users! ctx)
+         (detach-ws! ctx socket)))
+  (.on socket "error"
+       (fn [error]
+         (presence/remove-presence! ctx socket)
+         (presence/broadcast-online-users! ctx)
+         (detach-ws! ctx socket)
+         (log/error :db-sync/ws-error error))))
+
+(defn start!
+  [overrides]
+  (let [cfg (config/normalize-config overrides)
+        request-origin (request-origin-opts cfg)
+        index-db (storage/open-index-db (:data-dir cfg))
+        assets-bucket (assets/make-bucket (node-path/join (:data-dir cfg) "assets"))
+        registry (atom {})
+        deps {:config cfg
+              :index-db index-db
+              :assets-bucket assets-bucket}
+        env (doto (make-env cfg index-db assets-bucket)
+              (aset "DB_SYNC_DELETE_GRAPH"
+                    (fn [graph-id]
+                      (graph/delete-graph! registry deps graph-id))))
+        server (.createServer http
+                              (fn [req res]
+                                (-> (p/let [request (platform-node/request-from-node req request-origin)
+                                            response (dispatch/handle-node-fetch {:request request
+                                                                                  :env env
+                                                                                  :registry registry
+                                                                                  :deps deps})]
+                                      (platform-node/send-response! res response))
+                                    (p/catch
+                                     (fn [e]
+                                       (log/error :db-sync/node-request-failed {:error e})
+                                       (js/console.error ":db-sync/node-request-failed" e)
+                                       (platform-node/send-response! res (worker-http/error-response "server error" 500)))))))
+        WSS (or (.-WebSocketServer ws) (.-Server ws))
+        ^js wss (new WSS #js {:noServer true})]
+    (.on server "error" (fn [error] (log/error :db-sync/node-server-error {:error error})))
+    (.on wss "error" (fn [error] (log/error :db-sync/node-ws-error {:error error})))
+    (p/let [_ (index/<index-init! index-db)]
+      (.on server "upgrade"
+           (fn [req ^js socket head]
+             (let [request (platform-node/request-from-node req request-origin)
+                   url (platform/request-url request)
+                   path (.-pathname url)
+                   parsed (node-routes/parse-sync-path path)
+                   graph-id (:graph-id parsed)]
+               (if (and graph-id (seq graph-id))
+                 (p/let [allowed? (access-allowed? env graph-id request)]
+                   (if allowed?
+                     (let [ctx (graph/get-or-create-graph registry deps graph-id)]
+                       (p/let [ready-for-sync? (sync-handler/<ready-for-sync? ctx graph-id)]
+                         (if ready-for-sync?
+                           (.handleUpgrade wss req socket head
+                                           (fn [ws-socket]
+                                             (attach-ws! ctx ws-socket)
+                                             (handle-ws-connection ctx env request ws-socket)))
+                           (reject-ws-upgrade! socket 409 "graph not ready"))))
+                     (.destroy socket)))
+                 (.destroy socket)))))
+      (p/let [_ (js/Promise.
+                 (fn [resolve]
+                   (.listen server (:port cfg)
+                            (fn [] (resolve nil)))))
+              address (.address server)
+              port (if (number? address) address (.-port address))
+              base-url (or (:base-url cfg) (str "http://localhost:" port))]
+        {:server server
+         :wss wss
+         :env env
+         :registry registry
+         :port port
+         :base-url base-url
+         :stop! (fn []
+                  (graph/close-graphs! registry)
+                  (when-let [close (.-close index-db)]
+                    (close))
+                  (js/Promise.
+                   (fn [resolve]
+                     (.close server (fn [] (resolve nil))))))}))))

deps/db-sync/src/logseq/db_sync/node/show_checksum.cljs

@@ -0,0 +1,105 @@
+(ns logseq.db-sync.node.show-checksum
+  (:require ["better-sqlite3" :as sqlite3]
+            ["fs" :as fs]
+            ["path" :as node-path]
+            [clojure.string :as string]
+            [datascript.core :as d]
+            [logseq.db-sync.checksum :as sync-checksum]
+            [logseq.db-sync.storage :as storage]))
+
+(def sqlite (if (find-ns 'nbb.core) (aget sqlite3 "default") sqlite3))
+
+(defn- fail!
+  [msg]
+  (binding [*print-fn* *print-err-fn*]
+    (println (str "Error: " msg)))
+  (js/process.exit 1))
+
+(defn- parse-args
+  [argv]
+  (let [args (vec argv)]
+    (cond
+      (some #{"-h" "--help"} args)
+      {:help? true}
+
+      :else
+      (loop [m {}
+             xs args]
+        (if-let [x (first xs)]
+          (cond
+            (= "--db" x)
+            (if-let [db (second xs)]
+              (recur (assoc m :db db) (nnext xs))
+              (fail! "Missing value for --db"))
+
+            :else
+            (fail! (str "Unknown argument: " x)))
+          m)))))
+
+(defn- print-help!
+  []
+  (println "Show checksums for a sqlite db.")
+  (println "")
+  (println "Usage:")
+  (println "  node worker/dist/show-sqlite-checksum.js --db <path/to/db.sqlite>")
+  (println "")
+  (println "Output:")
+  (println "  - stored checksum (sync_meta.checksum, if present)")
+  (println "  - recomputed checksum (from datascript graph)"))
+
+(defn- normalize-sql
+  [sql]
+  (-> sql string/trim string/lower-case))
+
+(defn- select-sql?
+  [sql]
+  (string/starts-with? (normalize-sql sql) "select"))
+
+(defn- exec-with-args [^js stmt args]
+  (.apply (.-run stmt) stmt (to-array args)))
+
+(defn- all-with-args [^js stmt args]
+  (.apply (.-all stmt) stmt (to-array args)))
+
+(defn- wrap-db [^js db]
+  #js {:exec (fn [sql & args]
+               (if (seq args)
+                 (let [stmt (.prepare db sql)]
+                   (if (select-sql? sql)
+                     (all-with-args stmt args)
+                     (do
+                       (exec-with-args stmt args)
+                       nil)))
+                 (if (select-sql? sql)
+                   (.all (.prepare db sql))
+                   (.exec db sql))))
+       :close (fn [] (.close db))
+       :_db db})
+
+(defn- datom-count
+  [db]
+  (count (d/datoms db :eavt)))
+
+(defn main
+  [& argv]
+  (let [{:keys [help? db]} (parse-args argv)]
+    (when help?
+      (print-help!)
+      (js/process.exit 0))
+    (when-not db
+      (fail! "Missing required --db <path> argument"))
+    (let [db-path (node-path/resolve db)]
+      (when-not (.existsSync fs db-path)
+        (fail! (str "SQLite file not found: " db-path)))
+      (let [sqlite-db (new sqlite db-path nil)
+            sql (wrap-db sqlite-db)]
+        (try
+          (storage/init-schema! sql)
+          (let [conn (storage/open-conn sql)
+                db' @conn
+                recomputed-checksum (sync-checksum/recompute-checksum db')]
+            (println (str "db: " db-path))
+            (println (str "datoms: " (datom-count db')))
+            (println (str "recomputed-checksum: " recomputed-checksum)))
+          (finally
+            (.close sql)))))))

deps/db-sync/src/logseq/db_sync/node/storage.cljs

@@ -0,0 +1,58 @@
+(ns logseq.db-sync.node.storage
+  (:require ["better-sqlite3" :as sqlite3]
+            ["fs" :as fs]
+            ["path" :as node-path]
+            [clojure.string :as string]
+            [logseq.db.common.sqlite :as common-sqlite]))
+
+(def sqlite (if (find-ns 'nbb.core) (aget sqlite3 "default") sqlite3))
+
+(defn- ensure-dir! [dir]
+  (.mkdirSync fs dir #js {:recursive true}))
+
+(defn- normalize-sql [sql]
+  (-> sql string/trim string/lower-case))
+
+(defn- select-sql? [sql]
+  (string/starts-with? (normalize-sql sql) "select"))
+
+(defn- exec-with-args [^js stmt args]
+  (.apply (.-run stmt) stmt (to-array args)))
+
+(defn- all-with-args [^js stmt args]
+  (.apply (.-all stmt) stmt (to-array args)))
+
+(defn- wrap-db [^js db]
+  #js {:exec (fn [sql & args]
+               (if (seq args)
+                 (let [stmt (.prepare db sql)]
+                   (if (select-sql? sql)
+                     (all-with-args stmt args)
+                     (do
+                       (exec-with-args stmt args)
+                       nil)))
+                 (if (select-sql? sql)
+                   (.all (.prepare db sql))
+                   (.exec db sql))))
+       :prepare (fn [sql] (.prepare db sql))
+       :close (fn [] (.close db))
+       :_db db})
+
+(defn open-index-db [data-dir]
+  (let [db-path (node-path/join data-dir "index.sqlite")]
+    (ensure-dir! (node-path/dirname db-path))
+    (wrap-db (new sqlite db-path nil))))
+
+(defn open-graph-db [data-dir graph-id]
+  (let [[graph-name db-path] (common-sqlite/get-db-full-path (node-path/join data-dir "graphs") graph-id)]
+    (ensure-dir! (node-path/dirname db-path))
+    {:graph-name graph-name
+     :sql (wrap-db (new sqlite db-path nil))}))
+
+(defn delete-graph-db!
+  [data-dir graph-id]
+  (let [[_graph-name db-path] (common-sqlite/get-db-full-path (node-path/join data-dir "graphs") graph-id)
+        graph-dir (node-path/dirname db-path)]
+    (when (.existsSync fs graph-dir)
+      (.rmSync fs graph-dir #js {:recursive true
+                                 :force true}))))