Initial import of LogDarioSeq baseline

Establishes version control for the personal Logseq DB fork before
starting the server-client architecture migration and domain features
described in docs/agent-guide and the approved implementation plan.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

deps/db/script/create_graph.cljs

@@ -0,0 +1,76 @@
+(ns create-graph
+  "A script that creates or updates a DB graph given a sqlite.build EDN file.
+   If the given graph already exists, the EDN file updates the graph."
+  (:require ["fs" :as fs]
+            ["path" :as node-path]
+            [babashka.cli :as cli]
+            [clojure.edn :as edn]
+            [logseq.db :as ldb]
+            [logseq.db.common.sqlite-cli :as sqlite-cli]
+            [logseq.db.frontend.validate :as db-validate]
+            [logseq.db.sqlite.export :as sqlite-export]
+            [logseq.outliner.cli :as outliner-cli]
+            [clojure.pprint :as pprint]
+            [nbb.classpath :as cp]
+            [nbb.core :as nbb]))
+
+(defn- resolve-path
+  "If relative path, resolve with $ORIGINAL_PWD"
+  [path]
+  (if (node-path/isAbsolute path)
+    path
+    (node-path/join (or js/process.env.ORIGINAL_PWD ".") path)))
+
+(defn- validate-db [db db-name options]
+  (if-let [errors (:errors
+                   (db-validate/validate-local-db!
+                    db
+                    (merge options {:db-name db-name :verbose true})))]
+    (do
+      (println "Found" (count errors)
+               (if (= 1 (count errors)) "entity" "entities")
+               "with errors:")
+      (pprint/pprint errors)
+      (js/process.exit 1))
+    (println "Valid!")))
+
+(def spec
+  "Options spec"
+  {:help {:alias :h
+          :desc "Print help"}
+   :validate {:alias :v
+              :desc "Validate db after creation"}
+   :import {:alias :i
+            :desc "Import edn file using sqlite-export"}})
+
+(defn -main [args]
+  (let [{options :opts args' :args} (cli/parse-args args {:spec spec})
+        [graph-dir edn-path] args'
+        _ (when (or (nil? graph-dir) (nil? edn-path) (:help options))
+            (println (str "Usage: $0 GRAPH-NAME EDN-PATH [OPTIONS]\nOptions:\n"
+                          (cli/format-opts {:spec spec})))
+            (js/process.exit 1))
+        init-conn-args (conj (sqlite-cli/->open-db-args graph-dir))
+        sqlite-build-edn (merge (if (:import options) {} {:auto-create-ontology? true})
+                                (-> (resolve-path edn-path) fs/readFileSync str edn/read-string))
+        graph-exists? (fs/existsSync (apply node-path/join init-conn-args))
+        db-name (if (= 1 (count init-conn-args)) (first init-conn-args) (second init-conn-args))
+        conn (apply outliner-cli/init-conn
+                    (conj init-conn-args {:classpath (cp/get-classpath) :import-type :cli/create-graph}))
+        {:keys [init-tx block-props-tx misc-tx] :as _txs}
+        (if (:import options)
+          (sqlite-export/build-import sqlite-build-edn @conn {})
+          (outliner-cli/build-blocks-tx sqlite-build-edn))]
+    (println "Generating" (count (filter :block/name init-tx)) "pages and"
+             (count (filter :block/title init-tx)) "blocks ...")
+    ;; (fs/writeFileSync "txs.edn" (with-out-str (cljs.pprint/pprint _txs)))
+    ;; (cljs.pprint/pprint _txs)
+    (ldb/transact! conn init-tx)
+    (when (seq block-props-tx) (ldb/transact! conn block-props-tx))
+    (when (seq misc-tx) (ldb/transact! conn misc-tx))
+    (println (if graph-exists? "Updated graph" "Created graph") (str db-name "!"))
+    (when (:validate options)
+      (validate-db @conn db-name {}))))
+
+(when (= nbb/*file* (nbb/invoked-file))
+  (-main *command-line-args*))

deps/db/script/create_graph/inferred.edn

@@ -0,0 +1,23 @@
+;; Script that generates classes and properties for a demo of inferring properties.
+;; To generate this graph:
+;; bb dev:create inferred deps/db/create_graph/inferred.edn
+;;
+;; To try the demo in the UI, in any page type:
+;; - Good Will Hunting #Movie #Ben-Affleck
+;; or
+;; - DB 3 #Meeting #Tienson
+{:auto-create-ontology? true
+ :classes {:Movie {:build/class-properties [:actor :comment]}
+           :Meeting {:build/class-properties [:attendee :duration]}}
+ :properties
+ {:actor {:logseq.property/type :node
+          :db/cardinality :many
+          :build/property-classes [:Person]}
+  :attendee {:logseq.property/type :node
+             :db/cardinality :many
+             :build/property-classes [:Person]}}
+ :pages-and-blocks
+ [{:page {:block/title "Matt-Damon" :build/tags [:Person]}}
+  {:page {:block/title "Ben-Affleck" :build/tags [:Person]}}
+  {:page {:block/title "Tienson" :build/tags [:Person]}}
+  {:page {:block/title "Zhiyuan" :build/tags [:Person]}}]}

deps/db/script/dump_datoms.cljs

@@ -0,0 +1,25 @@
+  (ns dump-datoms
+    "A script that dumps all eavt datoms to a specified edn file
+
+     $ pnpm exec nbb-logseq script/dump_datoms.cljs db-name datoms.edn"
+    (:require ["fs" :as fs]
+              ["path" :as node-path]
+              [clojure.pprint :as pprint]
+              [datascript.core :as d]
+              [logseq.db.common.sqlite-cli :as sqlite-cli]
+              [nbb.core :as nbb]))
+
+(defn -main [args]
+  (when (< (count args) 2)
+    (println "Usage: $0 GRAPH FILE")
+    (js/process.exit 1))
+  (let [[graph-name file*] args
+        conn (apply sqlite-cli/open-db! (sqlite-cli/->open-db-args graph-name))
+        datoms (mapv #(vec %) (d/datoms @conn :eavt))
+        parent-dir (or js/process.env.ORIGINAL_PWD ".")
+        file (node-path/join parent-dir file*)]
+    (println "Writing" (count datoms) "datoms to" file)
+    (fs/writeFileSync file (with-out-str (pprint/pprint datoms)))))
+
+(when (= nbb/*file* (nbb/invoked-file))
+  (-main *command-line-args*))

deps/db/script/export_client_ops_ops.cljs

@@ -0,0 +1,93 @@
+(ns export-client-ops-ops
+  "Export client-ops tx entities to an EDN file sorted by :db-sync/created-at asc.
+
+  Examples:
+  $ pnpm exec nbb-logseq script/export_client_ops_ops.cljs /path/to/client-ops.sqlite
+  $ pnpm exec nbb-logseq script/export_client_ops_ops.cljs /path/to/client-ops.sqlite --out client-ops-ops.edn"
+  (:require ["fs" :as fs]
+            ["path" :as node-path]
+            [babashka.cli :as cli]
+            [clojure.pprint :as pprint]
+            [datascript.core :as d]
+            [logseq.db.common.sqlite-cli :as sqlite-cli]
+            [nbb.core :as nbb]))
+
+(def spec
+  {:help {:alias :h
+          :desc "Show help"}
+   :out {:alias :o
+         :desc "Output EDN file path"
+         :coerce :string}})
+
+(def op-keys
+  [:db-sync/tx-id
+   :db-sync/created-at
+   :db-sync/pending?
+   :db-sync/failed?
+   :db-sync/outliner-op
+   :db-sync/forward-outliner-ops
+   :db-sync/inverse-outliner-ops
+   :db-sync/inferred-outliner-ops?
+   :db-sync/normalized-tx-data
+   :db-sync/reversed-tx-data])
+
+(defn usage []
+  (str "Usage: pnpm exec nbb-logseq script/export_client_ops_ops.cljs <client-ops-db> [--out <file.edn>]\n"
+       "Options:\n"
+       (cli/format-opts {:spec spec})))
+
+(defn resolve-path
+  [path*]
+  (if (node-path/isAbsolute path*)
+    path*
+    (node-path/join (or js/process.env.ORIGINAL_PWD ".") path*)))
+
+(defn default-output-path
+  [input-path]
+  (let [resolved (resolve-path input-path)
+        dir (node-path/dirname resolved)
+        base (node-path/basename resolved)
+        name (node-path/parse base)
+        stem (or (.-name name) base)]
+    (node-path/join dir (str stem "-ops.edn"))))
+
+(defn entity->op
+  [db eid]
+  (let [ent (d/entity db eid)]
+    (merge
+     {:db/id (:db/id ent)}
+     (select-keys (into {} ent) op-keys))))
+
+(defn read-ops
+  [conn]
+  (let [db @conn]
+    (->> (d/datoms db :avet :db-sync/created-at)
+         (map (fn [datom] (entity->op db (:e datom))))
+         (sort-by (juxt :db-sync/created-at :db/id))
+         vec)))
+
+(defn -main
+  [argv]
+  (let [{:keys [opts args]} (cli/parse-args argv {:spec spec})
+        [input-path] args]
+    (when (or (:help opts) (nil? input-path))
+      (println (usage))
+      (js/process.exit (if (:help opts) 0 1)))
+    (let [db-path (resolve-path input-path)
+          _ (when-not (fs/existsSync db-path)
+              (println "Client ops db file does not exist:" db-path)
+              (js/process.exit 1))
+          out-path (resolve-path (or (:out opts) (default-output-path db-path)))
+          open-args (sqlite-cli/->open-db-args db-path)
+          {:keys [conn sqlite]} (apply sqlite-cli/open-sqlite-datascript! open-args)]
+      (try
+        (let [ops (read-ops conn)]
+          (println "Read" (count ops) "ops from" db-path)
+          (println "Writing ops EDN to" out-path)
+          (fs/writeFileSync out-path (with-out-str (pprint/pprint ops))))
+        (finally
+          (when sqlite
+            (.close sqlite)))))))
+
+(when (= nbb/*file* (nbb/invoked-file))
+  (-main *command-line-args*))

deps/db/script/query.cljs

@@ -0,0 +1,70 @@
+(ns query
+  "A script that queries any db graph from the commandline e.g.
+
+  $ pnpm exec nbb-logseq script/query.cljs db-name '[:find (pull ?b [:block/name :block/title]) :where [?b :block/created-at]]'"
+  (:require ["child_process" :as child-process]
+            [babashka.cli :as cli]
+            [clojure.edn :as edn]
+            [clojure.pprint :as pprint]
+            [datascript.core :as d]
+            [logseq.db.common.sqlite-cli :as sqlite-cli]
+            [logseq.db.frontend.rules :as rules]
+            [nbb.core :as nbb]))
+
+(defn- sh
+  "Run shell cmd synchronously and print to inherited streams by default. Aims
+  to be similar to babashka.tasks/shell"
+  [cmd opts]
+  (child-process/spawnSync (first cmd)
+                           (clj->js (rest cmd))
+                           (clj->js (merge {:stdio "inherit"} opts))))
+
+(def spec
+  "Options spec"
+  {:help {:alias :h
+          :desc "Print help"}
+   :verbose {:alias :v
+             :desc "Print more info"}
+   :raw {:alias :r
+         :desc "Print results plainly. Useful when piped to bb"}
+   :additional-graphs {:alias :a
+                       :coerce []
+                       :desc "Additional graphs to query"}
+   :entity {:alias :e
+            :coerce []
+            :desc "Lookup entities instead of query"}})
+
+(defn query-graph [graph args'' options]
+  (let [conn (apply sqlite-cli/open-db! (sqlite-cli/->open-db-args graph))
+        results (if (:entity options)
+                  (map #(when-let [ent (d/entity @conn
+                                                 (if (string? %) (edn/read-string %) %))]
+                          (into {:db/id (:db/id ent)} ent))
+                       (:entity options))
+                  ;; assumes no :in are in queries
+                  (let [query (into (edn/read-string (first args'')) [:in '$ '%])
+                        res (d/q query @conn (rules/extract-rules rules/db-query-dsl-rules))]
+                    ;; Remove nesting for most queries which just have one :find binding
+                    (if (= 1 (count (first res))) (mapv first res) res)))]
+    (when (:verbose options) (println "DB contains" (count (d/datoms @conn :eavt)) "datoms"))
+    (if (:raw options)
+      (prn results)
+      (if (zero? (.-status (child-process/spawnSync "which" #js ["puget"])))
+        (sh ["puget"] {:input (pr-str results) :stdio ["pipe" "inherit" "inherit"]})
+        (pprint/pprint results)))))
+
+(defn -main [args]
+  (let [{options :opts args' :args} (cli/parse-args args {:spec spec})
+        [graph-dir & args''] args'
+        _ (when (or (nil? graph-dir) (:help options))
+            (println (str "Usage: $0 GRAPH-NAME [& ARGS] [OPTIONS]\nOptions:\n"
+                          (cli/format-opts {:spec spec})))
+            (js/process.exit 1))
+        graph-dirs (cond-> [graph-dir]
+                     (:additional-graphs options)
+                     (into (:additional-graphs options)))]
+    (doseq [graph-dir graph-dirs]
+      (query-graph graph-dir args'' options))))
+
+(when (= nbb/*file* (nbb/invoked-file))
+  (-main *command-line-args*))

deps/db/script/replay_sync_artifact.cljs

@@ -0,0 +1,208 @@
+(ns replay-sync-artifact
+  "Replay sync simulation artifacts with DataScript + checksum logic in nbb."
+  (:require ["node:fs" :as fs]
+            ["node:path" :as node-path]
+            [babashka.cli :as cli]
+            [clojure.string :as string]
+            [datascript.core :as d]
+            [logseq.db-sync.checksum :as sync-checksum]
+            [logseq.db.frontend.schema :as db-schema]
+            [nbb.core :as nbb]))
+
+(def cli-spec
+  {:help {:alias :h
+          :desc "Show help"}
+   :artifact {:alias :a
+              :desc "Path to artifact.json"
+              :coerce :string}
+   :client {:alias :c
+            :desc "Only replay one client instance index"
+            :coerce :long}
+   :round {:alias :r
+           :desc "Only replay one round index"
+           :coerce :long}
+   :pretty {:desc "Pretty-print JSON output"}})
+
+(def checksum-attr-map
+  {"uuid" :block/uuid
+   "title" :block/title
+   "name" :block/name
+   "parent" :block/parent
+   "page" :block/page
+   "order" :block/order
+   ":block/uuid" :block/uuid
+   ":block/title" :block/title
+   ":block/name" :block/name
+   ":block/parent" :block/parent
+   ":block/page" :block/page
+   ":block/order" :block/order
+   "block/uuid" :block/uuid
+   "block/title" :block/title
+   "block/name" :block/name
+   "block/parent" :block/parent
+   "block/page" :block/page
+   "block/order" :block/order})
+
+(def checksum-attrs (set (vals checksum-attr-map)))
+
+(defn usage []
+  (str "Usage: pnpm exec nbb-logseq -cp src:script:../db-sync/src script/replay_sync_artifact.cljs --artifact <path> [--client <n>] [--round <n>] [--pretty]\n"
+       "Options:\n"
+       (cli/format-opts {:spec cli-spec})))
+
+(defn parse-uuid
+  [value]
+  (when (string? value)
+    (try
+      (uuid value)
+      (catch :default _
+        nil))))
+
+(defn parse-int
+  [value]
+  (when-not (nil? value)
+    (let [n (js/Number value)]
+      (when (js/Number.isInteger n)
+        (int n)))))
+
+(defn parse-attr
+  [value]
+  (cond
+    (keyword? value)
+    value
+
+    (string? value)
+    (let [trimmed (string/trim value)]
+      (or (get checksum-attr-map trimmed)
+          (get checksum-attr-map (string/lower-case trimmed))
+          (when (string/starts-with? trimmed ":")
+            (get checksum-attr-map (subs trimmed 1)))))
+
+    :else
+    nil))
+
+(defn snapshot-block->entity
+  [block]
+  (let [id (parse-int (get block "id"))
+        uuid-value (parse-uuid (get block "uuid"))]
+    (when (and id uuid-value)
+      (cond-> {:db/id id
+               :block/uuid uuid-value}
+        (string? (get block "title"))
+        (assoc :block/title (get block "title"))
+
+        (string? (get block "name"))
+        (assoc :block/name (get block "name"))
+
+        (string? (get block "order"))
+        (assoc :block/order (get block "order"))
+
+        (parse-int (get block "parentId"))
+        (assoc :block/parent (parse-int (get block "parentId")))
+
+        (parse-int (get block "pageId"))
+        (assoc :block/page (parse-int (get block "pageId")))))))
+
+(defn datom->tx
+  [datom]
+  (let [e (parse-int (get datom "e"))
+        attr (parse-attr (get datom "a"))
+        added? (not= false (get datom "added"))
+        value (get datom "v")]
+    (when (and e (contains? checksum-attrs attr))
+      (let [value' (case attr
+                     :block/uuid (or (parse-uuid value) value)
+                     :block/parent (or (parse-int value) value)
+                     :block/page (or (parse-int value) value)
+                     value)]
+        [(if added? :db/add :db/retract) e attr value']))))
+
+(defn replay-round
+  [client round]
+  (let [initial-db (get round "initialDb")
+        blocks (if (map? initial-db) (get initial-db "blocks") nil)
+        tx-log (-> (get round "txCapture") (get "txLog"))
+        initial-entities (if (sequential? blocks)
+                           (keep snapshot-block->entity blocks)
+                           [])
+        db0 (d/db-with (d/empty-db db-schema/schema) initial-entities)
+        checksum0 (sync-checksum/recompute-checksum db0)
+        replay-result
+        (reduce
+         (fn [{:keys [db checksum mismatches tx-count] :as acc} tx-entry]
+           (let [datoms (if (map? tx-entry) (get tx-entry "datoms") nil)
+                 tx-data (if (sequential? datoms) (keep datom->tx datoms) [])
+                 op-index (when (map? tx-entry) (parse-int (get tx-entry "opIndex")))]
+             (if (empty? tx-data)
+               (assoc acc :tx-count (inc tx-count))
+               (let [report (d/with db tx-data)
+                     db' (:db-after report)
+                     incremental (sync-checksum/update-checksum checksum report)
+                     full (sync-checksum/recompute-checksum db')]
+                 {:db db'
+                  :checksum incremental
+                  :tx-count (inc tx-count)
+                  :mismatches
+                  (cond-> mismatches
+                    (not= incremental full)
+                    (conj {:txIndex tx-count
+                           :opIndex op-index
+                           :incremental incremental
+                           :full full
+                           :txSize (count tx-data)}))}))))
+         {:db db0
+          :checksum checksum0
+          :mismatches []
+          :tx-count 0}
+         (if (sequential? tx-log) tx-log []))]
+    {:instanceIndex (parse-int (get client "instanceIndex"))
+     :round (parse-int (get round "round"))
+     :initialBlockCount (count initial-entities)
+     :txCount (:tx-count replay-result)
+     :mismatchCount (count (:mismatches replay-result))
+     :firstMismatch (first (:mismatches replay-result))
+     :finalChecksum (:checksum replay-result)}))
+
+(defn select-rounds
+  [client opts]
+  (let [rounds (if (sequential? (get client "rounds")) (get client "rounds") [])]
+    (if-let [round-index (:round opts)]
+      (filter #(= round-index (parse-int (get % "round"))) rounds)
+      rounds)))
+
+(defn select-clients
+  [artifact opts]
+  (let [clients (if (sequential? (get artifact "clients")) (get artifact "clients") [])]
+    (if-let [instance-index (:client opts)]
+      (filter #(= instance-index (parse-int (get % "instanceIndex"))) clients)
+      clients)))
+
+(defn -main
+  [argv]
+  (let [{:keys [opts]} (cli/parse-args argv {:spec cli-spec})
+        artifact-path (:artifact opts)]
+    (when (or (:help opts) (string/blank? artifact-path))
+      (println (usage))
+      (js/process.exit (if (:help opts) 0 1)))
+    (let [resolved-path (.resolve node-path artifact-path)
+          content (.readFileSync fs resolved-path "utf8")
+          artifact (js->clj (js/JSON.parse content))
+          selected-clients (vec (select-clients artifact opts))
+          results (->> selected-clients
+                       (mapcat (fn [client]
+                                 (map #(replay-round client %)
+                                      (select-rounds client opts))))
+                       (vec))
+          payload {:artifactPath resolved-path
+                   :runId (get artifact "runId")
+                   :createdAt (get artifact "createdAt")
+                   :selectedClient (or (:client opts) nil)
+                   :selectedRound (or (:round opts) nil)
+                   :resultCount (count results)
+                   :results results}]
+      (if (:pretty opts)
+        (println (js/JSON.stringify (clj->js payload) nil 2))
+        (println (js/JSON.stringify (clj->js payload)))))))
+
+(when (= nbb/*file* (nbb/invoked-file))
+  (-main *command-line-args*))