dasemu/Synapse
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
1e7e9a3c133b5fed0d846e59835616013753fffa
Synapse/deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs
Dasemu 40631362b3 Initial import of LogDarioSeq baseline 
Establishes version control for the personal Logseq DB fork before
starting the server-client architecture migration and domain features
described in docs/agent-guide and the approved implementation plan.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-08-25 15:32:15 +02:00

2485 lines
131 KiB
Clojure
Raw Blame History

(ns ^:node-only logseq.graph-parser.exporter-test
(:require ["fs" :as fs]
["os" :as os]
["path" :as node-path]
[cljs.test :refer [are deftest is testing]]
[clojure.set :as set]
[clojure.string :as string]
[datascript.core :as d]
[logseq.common.config :as common-config]
[logseq.common.graph :as common-graph]
[logseq.common.path :as path]
[logseq.common.util.block-ref :as block-ref]
[logseq.common.util.date-time :as date-time-util]
[logseq.common.util.page-ref :as page-ref]
[logseq.common.uuid :as common-uuid]
[logseq.db :as ldb]
[logseq.db.common.entity-plus :as entity-plus]
[logseq.db.frontend.asset :as db-asset]
[logseq.db.frontend.content :as db-content]
[logseq.db.frontend.malli-schema :as db-malli-schema]
[logseq.db.frontend.property :as db-property]
[logseq.db.frontend.rules :as rules]
[logseq.db.frontend.validate :as db-validate]
[logseq.db.test.helper :as db-test]
[logseq.graph-parser.block :as gp-block]
[logseq.graph-parser.exporter :as gp-exporter]
[logseq.graph-parser.test.docs-graph-helper :as docs-graph-helper]
[logseq.graph-parser.test.helper :as test-helper :include-macros true :refer [deftest-async]]
[logseq.outliner.db-pipeline :as db-pipeline]
[promesa.core :as p]))
;; Helpers
;; =======
;; some have been copied from db-import script
(defn- extract-rules
[rules]
(rules/extract-rules rules/db-query-dsl-rules
rules
{:deps rules/rules-dependencies}))
(defn- find-block-by-property [db property]
(d/q '[:find [?b ...]
:in $ ?prop %
:where (has-property ?b ?prop)]
db property (extract-rules [:has-property])))
(defn- find-block-by-property-value [db property property-value]
(->> (d/q '[:find [?b ...]
:in $ ?prop ?prop-value %
:where (property ?b ?prop ?prop-value)]
db property property-value (extract-rules [:property]))
first
(d/entity db)))
(defn- ordered-children
[block]
(->> (:block/_parent block)
(remove :logseq.property/created-from-property)
(sort-by :block/order)
vec))
(defn- block-tree-with-properties
[block]
{:title (:block/title block)
:properties (dissoc (db-test/readable-properties block) :block/tags)
:children (mapv block-tree-with-properties (ordered-children block))})
(defn- find-template-by-title
[db title]
(some->> (d/q '[:find [?b ...]
:in $ ?title
:where
[?b :block/title ?title]
[?b :block/tags :logseq.class/Template]]
db title)
first
(d/entity db)))
(defn- template-content-trees
[db title]
(some->> (find-template-by-title db title)
ordered-children
(mapv block-tree-with-properties)))
(defn- block-status
[db content]
(:logseq.property/status (db-test/find-block-by-content db content)))
(defn- status-content
[db content]
(db-property/closed-value-content (block-status db content)))
(defn- status-closed-value-contents
[db]
(set (map db-property/closed-value-content
(db-property/get-closed-property-values db :logseq.property/status))))
(defn- status-closed-value-content-frequencies
[db]
(frequencies (map db-property/closed-value-content
(db-property/get-closed-property-values db :logseq.property/status))))
(defn- blocks-by-title
[db title]
(->> (d/q '[:find [?b ...]
:in $ ?title
:where
[?b :block/page]
[?b :block/title ?title]]
db title)
(map #(d/entity db %))))
(defn- task-blocks-by-title
[db title]
(->> (d/q '[:find [?b ...]
:in $ ?title
:where
[?b :block/page]
[?b :block/title ?title]
[?b :block/tags :logseq.class/Task]]
db title)
(map #(d/entity db %))))
(defn- report-retracts-block-uuid?
[tx-report block-uuid]
(boolean
(some (fn [datom]
(and (= :block/uuid (:a datom))
(= block-uuid (:v datom))
(false? (:added datom))))
(:tx-data tx-report))))
(defn- imported-favorite-titles
[db]
(->> (ldb/get-page-blocks db (:db/id (ldb/get-page db common-config/favorites-page-name)))
(keep :block/link)
(map #(d/entity db (:db/id %)))
(map ldb/get-title-with-parents)))
(defn- build-graph-files
"Given a file graph directory, return all files including assets and adds relative paths
on ::rpath since paths are absolute by default and exporter needs relative paths for
some operations"
[dir*]
(let [dir (node-path/resolve dir*)]
(->> (common-graph/get-files dir)
(concat (when (fs/existsSync (path/path-join dir* "assets"))
(common-graph/readdir (path/path-join dir* "assets"))))
(mapv #(hash-map :path %
::rpath (node-path/relative dir* %))))))
(defn- <read-file
[file]
(p/let [s (fs/readFileSync (:path file))]
(str s)))
(defn- notify-user [m]
(println (:msg m))
(when (:ex-data m)
(println "Ex-data:" (pr-str (merge (dissoc (:ex-data m) :error)
(when-let [err (get-in m [:ex-data :error])]
{:original-error (ex-data (.-cause err))}))))
(println "Stacktrace:")
(if-let [stack (some-> (get-in m [:ex-data :error]) ex-data :sci.impl/callstack deref)]
(println (string/join
"\n"
(map
#(str (:file %)
(when (:line %) (str ":" (:line %)))
(when (:sci.impl/f-meta %)
(str " calls #'" (get-in % [:sci.impl/f-meta :ns]) "/" (get-in % [:sci.impl/f-meta :name]))))
(reverse stack))))
(println (some-> (get-in m [:ex-data :error]) .-stack))))
(when (= :error (:level m))
(js/process.exit 1)))
(def default-export-options
{;; common options
:rpath-key ::rpath
:notify-user notify-user
:<read-file <read-file
;; :set-ui-state prn
;; config file options
;; TODO: Add actual default
:default-config {}})
;; tweaked from db-import
(defn- <read-and-copy-asset [file assets buffer-handler *asset-ids]
(p/let [buffer (fs/readFileSync (:path file))
checksum (db-asset/<get-file-array-buffer-checksum buffer)
asset-id (d/squuid)
asset-name (gp-exporter/asset-path->name (:path file))
asset-type (db-asset/asset-path->type (:path file))
{:keys [with-edn-content pdf-annotation?]} (buffer-handler buffer)]
(when-not pdf-annotation?
(swap! *asset-ids conj asset-id))
(swap! assets assoc asset-name
(with-edn-content
{:size (.-length buffer)
:type asset-type
:path (:path file)
:checksum checksum
:asset-id asset-id}))
buffer))
;; Copied from db-import script and tweaked for an in-memory import
(defn- import-file-graph-to-db
"Import a file graph dir just like UI does. However, unlike the UI the
exporter receives file maps containing keys :path and ::rpath since :path
are full paths"
[file-graph-dir conn {:keys [assets] :or {assets (atom [])} :as options}]
(let [*files (build-graph-files file-graph-dir)
config-file (first (filter #(string/ends-with? (:path %) "logseq/config.edn") *files))
_ (assert config-file "No 'logseq/config.edn' found for file graph dir")
options' (merge default-export-options
{:user-options (merge {:convert-all-tags? false} (dissoc options :assets :verbose))
;; asset file options
:<get-file-stat (fn [path]
(let [abs-path (if (node-path/isAbsolute path)
path
(node-path/resolve file-graph-dir path))]
;; inline require to allow cljs tests to run
(.stat (js/require "fs/promises") abs-path)))
:<read-and-copy-asset (fn [file *assets buffer-handler]
(<read-and-copy-asset file *assets buffer-handler assets))}
(select-keys options [:verbose]))]
(gp-exporter/export-file-graph conn conn config-file *files options')))
(defn- import-files-to-db
"Import specific doc files for dev purposes"
[files conn options]
(reset! gp-block/*export-to-db-graph? true)
(-> (p/let [doc-options (gp-exporter/build-doc-options (merge {:macros {} :file/name-format :triple-lowbar}
(:user-config options))
(merge default-export-options
{:user-options (merge {:convert-all-tags? false}
(dissoc options :user-config :verbose))}
(select-keys options [:verbose])))
files' (mapv #(hash-map :path %) files)
_ (gp-exporter/export-doc-files conn files' <read-file doc-options)]
{:import-state (:import-state doc-options)})
(p/finally (fn [_]
(reset! gp-block/*export-to-db-graph? false)))))
(defn- write-temp-graph-file
[relative-path content]
(let [dir (fs/mkdtempSync (node-path/join (os/tmpdir) "logseq-graph-parser-test-"))
file-path (node-path/join dir relative-path)]
(fs/mkdirSync (node-path/dirname file-path) #js {:recursive true})
(fs/writeFileSync file-path content)
file-path))
(defn- write-temp-file-graph
[files]
(let [dir (fs/mkdtempSync (node-path/join (os/tmpdir) "logseq-graph-parser-test-"))]
(doseq [[relative-path content] files]
(let [file-path (node-path/join dir relative-path)]
(fs/mkdirSync (node-path/dirname file-path) #js {:recursive true})
(fs/writeFileSync file-path content)))
dir))
(defn- next-random!
[*seed n]
(let [next-seed (mod (* 48271 @*seed) 2147483647)]
(reset! *seed next-seed)
(mod next-seed n)))
(defn- random-choice!
[*seed choices]
(nth choices (next-random! *seed (count choices))))
(defn- generated-uuid
[n]
(str "10000000-0000-4000-8000-" (.padStart (.toString n 16) 12 "0")))
(def generated-file-graph-page-names
["Generated Alpha"
"Generated Beta"
"Generated Gamma"
"Generated Delta"
"Generated Epsilon"
"Generated Zeta"])
(def generated-file-graph-ref-page-names
(into generated-file-graph-page-names
["Generated/Missing Namespace"
"Generated Nested/Child"
"Missing Alias Target"
"Missing Tag Target"
"Generated PDF"]))
(def generated-file-graph-task-markers
[nil "TODO" "DOING" "DONE" "LATER" "NOW" "WAITING"])
(def generated-file-graph-tags
["generated"
"import"
"file-graph"
"edge-case"
"db-test"])
(def generated-file-graph-test-seeds
[1309 42 8675309])
(defn- generated-page-preamble
[*seed page-name]
(let [alias-name (str page-name " Alias " (next-random! *seed 30))
tag-page (random-choice! *seed generated-file-graph-ref-page-names)]
(case (next-random! *seed 5)
0 (str "alias:: [[" alias-name "]], [[Missing Alias Target]]\n"
"tags:: [[" tag-page "]], #generated-page\n"
"generated-page-rank:: " (next-random! *seed 100) "\n\n")
1 (str "title:: " page-name "\n"
"public:: true\n\n")
"")))
(defn- generated-title-suffix
[*seed]
(case (next-random! *seed 8)
0 (str " [missing asset](../assets/missing-" (next-random! *seed 50) ".pdf)")
1 (str " ![missing image](../assets/missing-" (next-random! *seed 50) ".png)")
2 (str " [[" (random-choice! *seed generated-file-graph-ref-page-names) "]]")
3 " #[[generated multi tag]]"
""))
(defn- generated-extra-lines
[*seed index]
(case (next-random! *seed 8)
0 (str " collapsed:: true\n"
" background-color:: yellow\n")
1 (str " alias:: [[Generated Block Alias " (next-random! *seed 100) "]]\n")
2 (str " | generated | table |\n"
" | row | " index " |\n")
3 (str " ```clojure\n"
" (def generated-" index " " (next-random! *seed 100) ")\n"
" ```\n")
""))
(defn- generated-md-block
[*seed index]
(let [block-id (generated-uuid (inc index))
duplicate-id (generated-uuid 1)
missing-ref-id (generated-uuid (+ 2000 index))
task-marker (random-choice! *seed generated-file-graph-task-markers)
page-name (random-choice! *seed generated-file-graph-ref-page-names)
missing-page-name (str "Generated Missing " (next-random! *seed 1000))
tag (random-choice! *seed generated-file-graph-tags)
ref-id (case (next-random! *seed 5)
0 block-id
1 duplicate-id
2 missing-ref-id
(generated-uuid (inc (next-random! *seed 90))))
id-value (case (next-random! *seed 11)
0 "broken-generated-id"
1 duplicate-id
block-id)
page-ref (if (zero? (next-random! *seed 3))
missing-page-name
page-name)
title-prefix (if task-marker (str task-marker " ") "")
title-suffix (generated-title-suffix *seed)
temporal-line (case (next-random! *seed 6)
0 " SCHEDULED: <2026-01-05 Mon .+1w>\n"
1 " DEADLINE: <2026-01-09 Fri +2d>\n"
"")
extra-lines (generated-extra-lines *seed index)
nested-line (when (zero? (next-random! *seed 3))
(str " - nested generated block " index
" [[Generated Nested " (next-random! *seed 30) "]]\n"))]
(str "- " title-prefix "generated block " index
" [[" page-ref "]] ((" ref-id ")) #" tag title-suffix "\n"
temporal-line
" id:: " id-value "\n"
" generated-ref:: [[" page-name "]]\n"
" generated-rank:: " (next-random! *seed 100) "\n"
extra-lines
nested-line)))
(defn- generated-md-file-content
[*seed file-index page-name block-count]
(str (generated-page-preamble *seed page-name)
(apply str
(map #(generated-md-block *seed (+ (* file-index 100) %))
(range block-count)))))
(defn- generated-md-file-graph
[seed]
(let [*seed (atom seed)
pages (map-indexed
(fn [index page-name]
[(str "pages/" (string/replace (string/lower-case page-name) " " "_") ".md")
(generated-md-file-content *seed index page-name (+ 8 (next-random! *seed 8)))])
generated-file-graph-page-names)
journals [["journals/2026_01_05.md"
(generated-md-file-content *seed 20 "2026_01_05" (+ 8 (next-random! *seed 8)))]
["journals/2026_01_06.md"
(generated-md-file-content *seed 21 "2026_01_06" (+ 8 (next-random! *seed 8)))]
["assets/generated.md"
"Generated asset content\n"]]]
(into {"logseq/config.edn" "{:preferred-format :markdown\n :journal/page-title-format \"yyyy_MM_dd\"}\n"}
(concat pages journals))))
(defn- assert-generated-md-file-graph-imports
[seed]
(let [graph-dir (write-temp-file-graph (generated-md-file-graph seed))]
(p/let [conn (db-test/create-conn)
_ (db-pipeline/add-listener conn)
_ (import-file-graph-to-db graph-dir conn {})
generated-block-count (->> (d/q '[:find [?title ...]
:where [?b :block/title ?title]]
@conn)
(filter #(string/includes? % "generated block"))
count)
validation-errors (map :entity (:errors (db-validate/validate-local-db! @conn)))]
(is (<= 60 generated-block-count)
(str "Seed " seed " imports the generated block corpus"))
(is (empty? validation-errors)
(str "Seed " seed " generated Markdown file graph validates")))))
(defn- assert-generated-md-file-graphs-import
[seeds]
(p/loop [remaining-seeds (seq seeds)]
(when remaining-seeds
(p/let [_ (assert-generated-md-file-graph-imports (first remaining-seeds))]
(p/recur (next remaining-seeds))))))
;; Tests
;; =====
(deftest-async import-block-with-journal-ref-and-time-property-value
(p/let [file (write-temp-graph-file
"journals/2023_06_21.md"
"- DONE foo bar #sometag1 #sometag2\n completed:: [[Sun, 06.08.2023]] *14:42*\n")
conn (db-test/create-conn)
_ (db-pipeline/add-listener conn)
_ (import-files-to-db [file] conn {:user-config {:journal/page-title-format "EEE, dd.MM.yyyy"}})]
(is (some? (db-test/find-block-by-content @conn #"foo bar"))
"Block with a journal reference plus time in a property value imports")
(is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn))))
"Imported graph validates")))
(deftest-async import-quote-with-email-address
(p/let [file (write-temp-graph-file
"pages/email.md"
"- > \"CachyOS <admin@cachyos.org>\"\n")
conn (db-test/create-conn)
_ (db-pipeline/add-listener conn)
_ (import-files-to-db [file] conn {})]
(is (= "\"CachyOS <admin@cachyos.org>\""
(:block/title (db-test/find-block-by-content @conn #"CachyOS")))
"Email addresses inside quotes are preserved during import")
(is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn))))
"Imported graph validates")))
(deftest-async import-org-page-title-when-property-appears-in-middle
(p/let [file (write-temp-graph-file
"pages/20230410145300-end_to_end_note.org"
":PROPERTIES:
:ID: c537c812-1ec9-4f13-adaf-1a39fd7da967
:END:
#+title: end_to_end_note
#+date: <2023-04-10 Mon 14:53>
#+filetags: :PUBLIC:
abc
#+hugo: more
123
")
conn (db-test/create-conn)
_ (db-pipeline/add-listener conn)
_ (import-files-to-db [file] conn {})]
(is (some? (db-test/find-page-by-title @conn "end_to_end_note"))
"Org #+title is imported when another property appears in the middle of the file")
(is (nil? (db-test/find-page-by-title @conn "20230410145300-end_to_end_note"))
"Importer should not fall back to the org-roam file stem")
(is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn))))
"Imported graph validates")))
(deftest-async import-empty-journal-file
(p/let [file (write-temp-graph-file "journals/2025_11_11.md" "\n")
conn (db-test/create-conn)
_ (db-pipeline/add-listener conn)
_ (import-files-to-db [file] conn {})]
(is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn))))
"Empty imported files do not transact nil block refs")))
(deftest-async import-repeated-deadline-and-scheduled
(p/let [file (write-temp-graph-file
"pages/repeated-tasks.md"
(str "- TODO wish [[name]] a happy birthday\n"
" SCHEDULED: <2025-11-01 Sat 08:00 .+1y>\n"
"- TODO prepare weekly report\n"
" DEADLINE: <2025-11-07 Fri +2w>\n"
"- TODO plan release\n"
" DEADLINE: <2025-11-08 Sat>\n"
" SCHEDULED: <2025-11-07 Fri .+1w>\n"))
conn (db-test/create-conn)
_ (db-pipeline/add-listener conn)
_ (import-files-to-db [file] conn {})]
(let [birthday-properties (db-test/readable-properties
(db-test/find-block-by-content @conn #"happy birthday"))
report-properties (db-test/readable-properties
(db-test/find-block-by-content @conn #"weekly report"))
mixed-properties (db-test/readable-properties
(db-test/find-block-by-content @conn #"plan release"))
birthday-scheduled (:logseq.property/scheduled birthday-properties)
birthday-date (js/Date. birthday-scheduled)]
(is (= 20251101 (date-time-util/ms->journal-day birthday-scheduled))
"Repeated scheduled timestamp keeps its scheduled date")
(is (= [8 0] [(.getHours birthday-date) (.getMinutes birthday-date)])
"Repeated scheduled timestamp keeps its time")
(is (= {:logseq.property.repeat/repeated? true
:logseq.property.repeat/temporal-property :logseq.property/scheduled
:logseq.property.repeat/repeat-type :logseq.property.repeat/repeat-type.dotted-plus
:logseq.property.repeat/recur-frequency 1
:logseq.property.repeat/recur-unit :logseq.property.repeat/recur-unit.year}
(select-keys birthday-properties
[:logseq.property.repeat/repeated?
:logseq.property.repeat/temporal-property
:logseq.property.repeat/repeat-type
:logseq.property.repeat/recur-frequency
:logseq.property.repeat/recur-unit]))
"Repeated scheduled timestamp keeps its repeat properties including the `.+` cookie kind")
(is (= {:logseq.property/deadline 20251107
:logseq.property.repeat/repeated? true
:logseq.property.repeat/temporal-property :logseq.property/deadline
:logseq.property.repeat/repeat-type :logseq.property.repeat/repeat-type.plus
:logseq.property.repeat/recur-frequency 2
:logseq.property.repeat/recur-unit :logseq.property.repeat/recur-unit.week}
(-> report-properties
(update :logseq.property/deadline date-time-util/ms->journal-day)
(select-keys [:logseq.property/deadline
:logseq.property.repeat/repeated?
:logseq.property.repeat/temporal-property
:logseq.property.repeat/repeat-type
:logseq.property.repeat/recur-frequency
:logseq.property.repeat/recur-unit])))
"Repeated deadline timestamp keeps its repeat properties including the `+` cookie kind")
(is (= {:logseq.property/deadline 20251108
:logseq.property/scheduled 20251107
:logseq.property.repeat/repeated? true
:logseq.property.repeat/temporal-property :logseq.property/scheduled
:logseq.property.repeat/repeat-type :logseq.property.repeat/repeat-type.dotted-plus
:logseq.property.repeat/recur-frequency 1
:logseq.property.repeat/recur-unit :logseq.property.repeat/recur-unit.week}
(-> mixed-properties
(update :logseq.property/deadline date-time-util/ms->journal-day)
(update :logseq.property/scheduled date-time-util/ms->journal-day)
(select-keys [:logseq.property/deadline
:logseq.property/scheduled
:logseq.property.repeat/repeated?
:logseq.property.repeat/temporal-property
:logseq.property.repeat/repeat-type
:logseq.property.repeat/recur-frequency
:logseq.property.repeat/recur-unit])))
"Mixed deadline and scheduled timestamps keep both dates and the repeated temporal property")
(is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn))))
"Imported graph validates"))))
(deftest-async import-preserves-legacy-task-markers-as-status-choices
(p/let [file (write-temp-graph-file
"pages/tasks.md"
(str "- TODO\n"
"- DONE\n"
"- I recorded a [[voice note]].\n"
" - TODO\n"
"- TODO todo item\n"
"- LATER later item\n"
"- NOW now item\n"
"- DOING doing item\n"
"- WAIT waiting item\n"
"- WAITING waiting full item\n"
"- IN-PROGRESS in-progress item\n"
"- DONE done item\n"))
conn (db-test/create-conn)
_ (db-pipeline/add-listener conn)
_ (import-files-to-db [file] conn {})]
(is (= :logseq.property/status.todo
(:db/ident (block-status @conn "todo item")))
"TODO still imports to the built-in Todo status")
(is (= :logseq.property/status.doing
(:db/ident (block-status @conn "doing item")))
"DOING still imports to the built-in Doing status")
(is (= :logseq.property/status.done
(:db/ident (block-status @conn "done item")))
"DONE still imports to the built-in Done status")
(is (= {:logseq.property/status.todo 2
:logseq.property/status.done 1}
(frequencies (map #(-> % db-test/readable-properties :logseq.property/status)
(task-blocks-by-title @conn ""))))
"Built-in task markers without titles still import as tasks")
(let [nested-empty-task (first (ordered-children (db-test/find-block-by-content @conn #"recorded")))]
(is (= "" (:block/title nested-empty-task))
"Nested nameless task keeps an empty title")
(is (= {:logseq.property/status :logseq.property/status.todo
:block/tags [:logseq.class/Task]}
(select-keys (db-test/readable-properties nested-empty-task)
[:logseq.property/status :block/tags]))
"Nested nameless TODO keeps its task properties"))
(is (= :logseq.property/status.todo
(:db/ident (block-status @conn "later item")))
"LATER imports to the built-in Todo status")
(is (= :logseq.property/status.doing
(:db/ident (block-status @conn "now item")))
"NOW imports to the built-in Doing status")
(is (= "WAIT" (status-content @conn "waiting item"))
"WAIT imports as its own status choice")
(is (= "WAITING" (status-content @conn "waiting full item"))
"WAITING imports as its own status choice")
(is (= "IN-PROGRESS" (status-content @conn "in-progress item"))
"IN-PROGRESS imports as its own status choice")
(is (set/subset? #{"WAIT" "WAITING" "IN-PROGRESS"}
(status-closed-value-contents @conn))
"Custom imported markers are added to Status closed values")
(is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn))))
"Imported graph validates")))
(deftest-async import-custom-task-marker-across-multiple-files
(p/let [first-file (write-temp-graph-file
"pages/custom-status-a.md"
"- WAITING first custom status item\n")
second-file (write-temp-graph-file
"pages/custom-status-b.md"
"- WAITING second custom status item\n")
conn (db-test/create-conn)
_ (db-pipeline/add-listener conn)
_ (import-files-to-db [first-file second-file] conn {})]
(is (= "WAITING" (status-content @conn "first custom status item"))
"Custom status marker imports from the first file")
(is (= "WAITING" (status-content @conn "second custom status item"))
"Custom status marker imports from the second file")
(is (= 1 (get (status-closed-value-content-frequencies @conn) "WAITING"))
"Custom status closed value is shared across imported files")
(is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn))))
"Imported graph validates")))
(deftest-async import-repairs-duplicated-block-ids
(let [duplicated-uuid #uuid "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"]
(p/let [file (write-temp-graph-file
"pages/duplicated-ids.md"
(str "- First duplicated id\n"
" id:: " duplicated-uuid "\n"
"- Second duplicated id\n"
" id:: " duplicated-uuid "\n"))
conn (db-test/create-conn)
_ (db-pipeline/add-listener conn)
_ (import-files-to-db [file] conn {})
first-block (db-test/find-block-by-content @conn "First duplicated id")
second-block (db-test/find-block-by-content @conn "Second duplicated id")]
(is (= duplicated-uuid (:block/uuid first-block))
"The first imported block keeps the original id")
(is (some? (:block/uuid second-block))
"The duplicate imported block gets a replacement id")
(is (not= duplicated-uuid (:block/uuid second-block))
"The duplicate imported block does not keep the conflicting id")
(is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn))))
"Imported graph validates"))))
(deftest-async import-removes-pre-block-marker-and-missing-block-refs
(let [missing-uuid #uuid "11111111-1111-1111-1111-111111111111"
missing-embed-uuid #uuid "55555555-5555-5555-5555-555555555555"
target-uuid #uuid "22222222-2222-2222-2222-222222222222"
empty-title-property-uuid #uuid "33333333-3333-3333-3333-333333333333"
empty-title-parent-uuid #uuid "44444444-4444-4444-4444-444444444444"]
(p/let [source-file (write-temp-graph-file
"pages/A.md"
(str "Plain pre-block\n"
"- Missing ref ((" missing-uuid "))\n"
"- {{embed ((" missing-embed-uuid "))}}\n"
"- Existing ref ((" target-uuid "))\n"
"- ((" empty-title-property-uuid "))\n"
" heading:: true\n"
" background-color:: yellow\n"
"- ((" empty-title-parent-uuid "))\n"
" - Child survives\n"))
target-file (write-temp-graph-file
"pages/Z.md"
(str "- Target block\n"
" id:: " target-uuid "\n"))
conn (db-test/create-conn)
_ (db-pipeline/add-listener conn)
_ (import-files-to-db [source-file target-file] conn {})
missing-block (db-test/find-block-by-content @conn #"Missing ref")
existing-block (db-test/find-block-by-content @conn #"Existing ref")
target-block (db-test/find-block-by-content @conn "Target block")
empty-title-blocks (blocks-by-title @conn "")
empty-title-property-block (some #(when (:logseq.property/heading %) %) empty-title-blocks)
empty-title-parent-block (some #(when (some (fn [child]
(= "Child survives" (:block/title child)))
(ordered-children %))
%)
empty-title-blocks)]
(is (empty? (filter #(= :block/pre-block? (:a %))
(d/datoms @conn :eavt)))
"Legacy pre-block markers are never transacted")
(is (= "Missing ref" (:block/title missing-block))
"Missing OG block refs are removed from imported content")
(is (empty? (:block/refs missing-block))
"Missing OG block refs are removed from imported refs")
(is (nil? (d/entity @conn [:block/uuid missing-uuid]))
"Missing OG block refs do not leave placeholder entities")
(is (nil? (d/entity @conn [:block/uuid missing-embed-uuid]))
"Missing OG block embeds do not leave placeholder entities")
(is (empty? (d/q '[:find ?b
:where [?b :block/link ?target]
[(missing? $ ?target :block/uuid)]]
@conn))
"Missing OG block embeds do not leave dangling block links")
(is (nil? (d/entity @conn [:block/uuid empty-title-property-uuid]))
"Missing OG block refs in empty-title property blocks do not leave placeholder entities")
(is (nil? (d/entity @conn [:block/uuid empty-title-parent-uuid]))
"Missing OG block refs in empty-title parent blocks do not leave placeholder entities")
(is (= 3 (count empty-title-blocks))
"Blocks whose titles become empty after cleanup are preserved")
(is (true? (:logseq.property/heading empty-title-property-block))
"Empty-title blocks keep imported heading properties")
(is (= "yellow"
(:logseq.property/background-color (db-test/readable-properties empty-title-property-block)))
"Empty-title blocks keep imported background colors")
(is (= ["Child survives"] (mapv :block/title (ordered-children empty-title-parent-block)))
"Empty-title parent blocks keep their children")
(is (= [(:db/id target-block)] (mapv :db/id (:block/refs existing-block)))
"Existing block refs are preserved, including forward refs from later files")
(is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn))))
"Imported graph validates"))))
(deftest-async import-generated-markdown-file-graph
(assert-generated-md-file-graphs-import generated-file-graph-test-seeds))
(deftest-async ^:integration import-generated-markdown-file-graph-fuzz
(assert-generated-md-file-graphs-import (range 1 101)))
(deftest-async export-doc-files-propagates-missing-block-ref-cleanup-report
(let [missing-uuid #uuid "55555555-5555-5555-5555-555555555555"
tx-reports (atom [])]
(p/let [file (write-temp-graph-file
"pages/A.md"
(str "- Missing ref ((" missing-uuid "))\n"))
conn (db-test/create-conn)
_ (db-pipeline/add-listener conn)
doc-options (gp-exporter/build-doc-options
{:macros {} :file/name-format :triple-lowbar}
(merge default-export-options
{:user-options {:convert-all-tags? false}
:on-tx-report #(swap! tx-reports conj %)}))
_ (gp-exporter/export-doc-files conn [{:path file}] <read-file doc-options)]
(is (some #(report-retracts-block-uuid? % missing-uuid) @tx-reports)
"Missing block ref cleanup tx-report is propagated to import callers")
(is (nil? (d/entity @conn [:block/uuid missing-uuid]))
"Missing OG block ref placeholder is removed"))))
(deftest export-doc-files-aborts-on-export-file-failure
(cljs.test/async
done
(let [attempted-paths (atom [])
failed-error (ex-info "worker transact failed" {:code :worker-transact-failed})
graph-dir (write-temp-file-graph [["pages/A.md" "- first\n"]
["pages/B.md" "- second\n"]])
first-file (node-path/join graph-dir "pages/A.md")
second-file (node-path/join graph-dir "pages/B.md")
conn (db-test/create-conn)
doc-options (gp-exporter/build-doc-options
{:macros {} :file/name-format :triple-lowbar}
(merge default-export-options
{:notify-user (constantly nil)
:user-options {:convert-all-tags? false}
:<export-file (fn [_conn {:file/keys [path]} _opts]
(swap! attempted-paths conj path)
(p/rejected failed-error))}))
assert-failure (fn [result]
(is (= :worker-transact-failed
(or (:code (ex-data result))
(:code (ex-data (.-cause result)))))
"Export file failure is propagated to the caller")
(is (= [first-file] @attempted-paths)
"Import stops after the first export file failure")
(done))]
(try
(-> (gp-exporter/export-doc-files
conn
[{:path first-file} {:path second-file}]
<read-file
doc-options)
(.then (fn [_]
(is false "Export file failure should reject")
(done)))
(.catch assert-failure))
(catch :default e
(assert-failure e))))))
(deftest-async export-doc-files-preserves-filesystem-timestamps
(let [created-at (js/Date. "2020-01-02T03:04:05.000Z")
modified-at (js/Date. "2021-06-07T08:09:10.000Z")]
(p/let [file (write-temp-graph-file "pages/timestamps.md" "- timestamped\n")
conn (db-test/create-conn)
_ (db-pipeline/add-listener conn)
doc-options (gp-exporter/build-doc-options
{:macros {} :file/name-format :triple-lowbar}
(merge default-export-options
{:user-options {:convert-all-tags? false}
:<get-file-stat (fn [_]
{:birthtime created-at
:mtime modified-at})
:<export-file (fn [conn' file-map opts]
(gp-exporter/<add-file-to-db-graph
conn' (:file/path file-map) (:file/content file-map) opts))}))
_ (gp-exporter/export-doc-files conn [{:path file}] <read-file doc-options)
page (ldb/get-page @conn "timestamps")
block (db-test/find-block-by-content @conn "timestamped")]
(is (= (.getTime created-at) (:block/created-at page) (:block/created-at block)))
(is (= (.getTime modified-at) (:block/updated-at page) (:block/updated-at block)))
(is (empty? (:errors (db-validate/validate-local-db! @conn)))))))
(deftest update-asset-links-in-block-title
(are [x y]
(= y (@#'gp-exporter/update-asset-links-in-block-title (first x) {(second x) "UUID"} (atom {})))
;; Standard image link with metadata
["![greg-popovich-thumbs-up.png](../assets/greg-popovich-thumbs-up_1704749687791_0.png){:height 288, :width 100} says pop"
"assets/greg-popovich-thumbs-up_1704749687791_0.png"]
"[[UUID]] says pop"
;; Image link with no metadata
["![some-title](../assets/CleanShot_2022-10-12_at_15.53.20@2x_1665561216083_0.png)"
"assets/CleanShot_2022-10-12_at_15.53.20@2x_1665561216083_0.png"]
"[[UUID]]"
;; 2nd link
["[[FIRST UUID]] and ![dino!](assets/subdir/partydino.gif)"
"assets/subdir/partydino.gif"]
"[[FIRST UUID]] and [[UUID]]"))
(deftest-async import-missing-local-pdf-asset-link-is-ignored-quietly
(let [graph-dir (write-temp-file-graph
{"logseq/config.edn" "{}"
"pages/missing-asset.md" "- Missing local PDF [paper](../assets/missing-paper.pdf)\n"})
console-errors (atom [])
original-stderr-write (.-write (.-stderr js/process))]
(set! (.-write (.-stderr js/process))
(fn [& args]
(swap! console-errors conj (first args))
true))
(-> (p/let [conn (db-test/create-conn)
assets (atom [])
{:keys [import-state]} (import-file-graph-to-db graph-dir conn {:assets assets})]
(is (= [{:reason "No asset data found for this asset path"
:path "../assets/missing-paper.pdf"
:location {:block "Missing local PDF [paper](../assets/missing-paper.pdf)"}}]
@(:ignored-assets import-state))
"Missing local PDF asset links are reported through ignored assets")
(is (empty? @console-errors)
"Missing local PDF asset links are ignored without noisy console errors")
(is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn))))
"Imported graph validates"))
(p/finally (fn [_]
(set! (.-write (.-stderr js/process)) original-stderr-write))))))
(deftest extract-template-blocks
(let [page-uuid (random-uuid)
parent-uuid (random-uuid)
child-uuid (random-uuid)
include-children-only-uuid (random-uuid)
child-only-1-uuid (random-uuid)
child-only-2-uuid (random-uuid)
blocks [{:block/uuid parent-uuid
:block/title "source parent"
:block/page [:block/uuid page-uuid]
:block/parent {:block/uuid page-uuid}
:block/order "a"
:block/properties {:template " trimmed template "
:name ""}
:block/properties-text-values {:template " trimmed template "
:name ""}
:block/properties-order [:template :name]}
{:block/uuid child-uuid
:block/title "child"
:block/page [:block/uuid page-uuid]
:block/parent [:block/uuid parent-uuid]
:block/order "b"
:block/properties {:template "nested child"
:name "child default"}
:block/properties-text-values {:template "nested child"
:name "child default"}
:block/properties-order [:template :name]}
{:block/uuid include-children-only-uuid
:block/title "exclude source block"
:block/page [:block/uuid page-uuid]
:block/parent {:block/uuid page-uuid}
:block/order "c"
:block/properties {:template "children only"
:template-including-parent false}
:block/properties-text-values {:template "children only"
:template-including-parent "false"}
:block/properties-order [:template :template-including-parent]}
{:block/uuid child-only-1-uuid
:block/title "first child"
:block/page [:block/uuid page-uuid]
:block/parent [:block/uuid include-children-only-uuid]
:block/order "d"}
{:block/uuid child-only-2-uuid
:block/title "second child"
:block/page [:block/uuid page-uuid]
:block/parent [:block/uuid include-children-only-uuid]
:block/order "e"}]
{:keys [blocks preserve-empty-properties-uuids]}
(#'gp-exporter/handle-template-blocks blocks)]
(testing "template roots replace source blocks"
(is (= ["trimmed template"
"source parent"
"nested child"
"child"
"children only"
"first child"
"second child"]
(mapv :block/title blocks)))
(is (= #{parent-uuid child-uuid include-children-only-uuid child-only-1-uuid child-only-2-uuid}
(set/intersection preserve-empty-properties-uuids
#{parent-uuid child-uuid include-children-only-uuid child-only-1-uuid child-only-2-uuid}))))
(testing "template roots use trimmed names and include parent content when configured"
(is (= #{"trimmed template" "nested child" "children only"}
(->> blocks
(filter #(some #{:logseq.class/Template} (:block/tags %)))
(map :block/title)
set)))
(is (= ["source parent"]
(->> blocks
(remove #(some #{:logseq.class/Template} (:block/tags %)))
(filter #(= [:block/uuid parent-uuid] (:block/parent %)))
(map :block/title))))
(is (= 2
(count (set/difference preserve-empty-properties-uuids
#{parent-uuid child-uuid include-children-only-uuid child-only-1-uuid child-only-2-uuid})))
"in-place template content blocks are marked to preserve empty properties"))))
(deftest-async ^:integration export-docs-graph-with-convert-all-tags
(p/let [file-graph-dir "test/resources/docs-0.10.12"
_ (docs-graph-helper/clone-docs-repo-if-not-exists file-graph-dir "v0.10.12")
start-time (cljs.core/system-time)
conn (db-test/create-conn)
_ (db-pipeline/add-listener conn)
{:keys [import-state]}
(import-file-graph-to-db file-graph-dir conn {:convert-all-tags? true})
end-time (cljs.core/system-time)]
;; Add multiplicative factor for CI as it runs about twice as slow
(let [max-time (-> 25 (* (if js/process.env.CI 2 1)))]
(is (< (-> end-time (- start-time) (/ 1000)) max-time)
(str "Importing large graph takes less than " max-time "s")))
(is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn))))
"Created graph has no validation errors")
(is (= 0 (count @(:ignored-properties import-state))) "No ignored properties")
(is (= 0 (count @(:ignored-assets import-state))) "No ignored assets")
(is (= []
(->> (d/q '[:find (pull ?b [:block/title {:block/tags [:db/ident]}])
:where [?b :block/tags :logseq.class/Tag]]
@conn)
(map first)
(remove #(= [{:db/ident :logseq.class/Tag}] (:block/tags %)))))
"All classes only have :logseq.class/Tag as their tag (and don't have Page)")))
(deftest-async import-linked-file-pdf-annotations
(let [annotation-id #uuid "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"
area-annotation-id #uuid "cccccccc-cccc-cccc-cccc-cccccccccccc"
dir (fs/mkdtempSync (node-path/join (os/tmpdir) "logseq-graph-parser-test-"))
external-pdf-path (node-path/join dir "external/Linked Paper.pdf")
graph-dir (node-path/join dir "graph")
area-image-stamp "area-stamp"
encoded-pdf-uri (str "file://" (string/replace external-pdf-path " " "%20"))]
(fs/mkdirSync (node-path/dirname external-pdf-path) #js {:recursive true})
(fs/writeFileSync external-pdf-path "pdf")
(doseq [[relative-path content]
{"logseq/config.edn" "{}"
"pages/source.md" (str "- ![Linked Paper.pdf](" encoded-pdf-uri ")\n")
"pages/hls__Linked Paper.md" (str "file:: [Linked Paper.pdf](" encoded-pdf-uri ")\n"
"file-path:: " encoded-pdf-uri "\n\n"
"- External highlight from linked pdf\n"
" ls-type:: annotation\n"
" hl-page:: 3\n"
" hl-color:: yellow\n"
" id:: " annotation-id "\n"
"- External area highlight from linked pdf\n"
" ls-type:: annotation\n"
" hl-page:: 4\n"
" hl-color:: yellow\n"
" id:: " area-annotation-id "\n")
"assets/Linked Paper.edn" (str "{:highlights [{:id #uuid \"" annotation-id "\","
" :page 3,"
" :position {:bounding {:x1 1 :y1 2 :x2 3 :y2 4 :width 10 :height 20},"
" :rects (),"
" :page 3},"
" :content {:text \"External highlight from linked pdf\"},"
" :properties {:color \"yellow\"}}"
" {:id #uuid \"" area-annotation-id "\","
" :page 4,"
" :position {:bounding {:x1 11 :y1 12 :x2 13 :y2 14 :width 10 :height 20},"
" :rects (),"
" :page 4},"
" :content {:image \"" area-image-stamp "\"},"
" :properties {:color \"yellow\"}}]}")
(str "assets/Linked Paper/4_" area-annotation-id "_" area-image-stamp ".png") "png"}]
(let [file-path (node-path/join graph-dir relative-path)]
(fs/mkdirSync (node-path/dirname file-path) #js {:recursive true})
(fs/writeFileSync file-path content)))
(p/let [conn (db-test/create-conn)
assets (atom [])
{:keys [import-state]} (import-file-graph-to-db graph-dir conn {:assets assets})
asset (db-test/find-block-by-content @conn "Linked Paper")
annotation (db-test/find-block-by-content @conn "External highlight from linked pdf")
area-annotation (db-test/find-block-by-content @conn "External area highlight from linked pdf")]
(is (some? asset)
"Linked file PDF imports as an external Asset")
(is (= {:block/tags [:logseq.class/Asset]
:logseq.property.asset/type "pdf"
:logseq.property.asset/external-url encoded-pdf-uri}
(select-keys (db-test/readable-properties asset)
[:block/tags
:logseq.property.asset/type
:logseq.property.asset/external-url]))
"Linked file PDF keeps the file URI as external asset metadata")
(is (= {:block/tags [:logseq.class/Pdf-annotation]
:logseq.property/asset "Linked Paper"
:logseq.property.pdf/hl-page 3}
(select-keys (db-test/readable-properties annotation)
[:block/tags
:logseq.property/asset
:logseq.property.pdf/hl-page]))
"Linked file PDF annotations import and point at the external Asset")
(is (= {:block/tags [:logseq.class/Pdf-annotation]
:logseq.property/asset "Linked Paper"
:logseq.property.pdf/hl-page 4
:logseq.property.pdf/hl-image "pdf area highlight"
:logseq.property.pdf/hl-type :area}
(select-keys (db-test/readable-properties area-annotation)
[:block/tags
:logseq.property/asset
:logseq.property.pdf/hl-page
:logseq.property.pdf/hl-image
:logseq.property.pdf/hl-type]))
"Linked file PDF area highlights import their image assets")
(is (= 0 (count @(:ignored-assets import-state))) "No ignored assets"))))
(deftest-async import-linked-file-pdf-annotations-with-uppercase-extension
(let [annotation-id #uuid "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb"
dir (fs/mkdtempSync (node-path/join (os/tmpdir) "logseq-graph-parser-test-"))
external-pdf-path (node-path/join dir "external/Linked Paper.PDF")
graph-dir (node-path/join dir "graph")
encoded-pdf-uri (str "file://" (string/replace external-pdf-path " " "%20"))]
(fs/mkdirSync (node-path/dirname external-pdf-path) #js {:recursive true})
(fs/writeFileSync external-pdf-path "pdf")
(doseq [[relative-path content]
{"logseq/config.edn" "{}"
"pages/source.md" (str "- ![Linked Paper.PDF](" encoded-pdf-uri ")\n")
"pages/hls__Linked Paper.md" (str "file:: [Linked Paper.PDF](" encoded-pdf-uri ")\n"
"file-path:: " encoded-pdf-uri "\n\n"
"- External highlight from linked pdf\n"
" ls-type:: annotation\n"
" hl-page:: 3\n"
" hl-color:: yellow\n"
" id:: " annotation-id "\n")
"assets/Linked Paper.edn" (str "{:highlights [{:id #uuid \"" annotation-id "\","
" :page 3,"
" :position {:bounding {:x1 1 :y1 2 :x2 3 :y2 4 :width 10 :height 20},"
" :rects (),"
" :page 3},"
" :content {:text \"External highlight from linked pdf\"},"
" :properties {:color \"yellow\"}}]}")}]
(let [file-path (node-path/join graph-dir relative-path)]
(fs/mkdirSync (node-path/dirname file-path) #js {:recursive true})
(fs/writeFileSync file-path content)))
(p/let [conn (db-test/create-conn)
assets (atom [])
{:keys [import-state]} (import-file-graph-to-db graph-dir conn {:assets assets})
asset (db-test/find-block-by-content @conn "Linked Paper")
annotation (db-test/find-block-by-content @conn "External highlight from linked pdf")]
(is (some? asset)
"Linked file PDF imports as an external Asset")
(is (= {:block/tags [:logseq.class/Asset]
:logseq.property.asset/type "pdf"
:logseq.property.asset/external-url encoded-pdf-uri}
(select-keys (db-test/readable-properties asset)
[:block/tags
:logseq.property.asset/type
:logseq.property.asset/external-url]))
"Linked file PDF keeps the file URI as external asset metadata")
(is (= {:block/tags [:logseq.class/Pdf-annotation]
:logseq.property/asset "Linked Paper"
:logseq.property.pdf/hl-page 3}
(select-keys (db-test/readable-properties annotation)
[:block/tags
:logseq.property/asset
:logseq.property.pdf/hl-page]))
"Linked file PDF annotations import and keep highlight positions from the EDN file")
(is (= 0 (count @(:ignored-assets import-state))) "No ignored assets"))))
(deftest-async ^:integration import-large-flat-file-without-stack-overflow
(p/let [file (write-temp-graph-file
"pages/large.md"
(apply str (map #(str "- large line " % " #tag\n") (range 45000))))
conn (db-test/create-conn)
_ (db-pipeline/add-listener conn)
_ (import-files-to-db [file] conn {:convert-all-tags? true})]
(is (= 45000
(->> (d/q '[:find [?title ...]
:where [?b :block/title ?title]]
@conn)
(filter #(string/starts-with? % "large line "))
count))
"Large flat files import without overflowing the stack")
(is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn))))
"Imported graph validates")))
(deftest-async export-basic-graph-with-convert-all-tags
;; This graph will contain basic examples of different features to import
(p/let [file-graph-dir "test/resources/exporter-test-graph"
conn (db-test/create-conn)
;; Calculate refs like frontend
_ (db-pipeline/add-listener conn)
assets (atom [])
{:keys [import-state]} (import-file-graph-to-db file-graph-dir conn {:assets assets :convert-all-tags? true})]
(testing "whole graph"
(is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn))))
"Created graph has no validation errors")
;; Counts
;; Includes journals as property values e.g. :logseq.property/deadline
(is (= 34 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Journal]] @conn))))
(is (= 9 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Asset]] @conn))))
(is (= 6 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Task]] @conn))))
(is (= 4 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Query]] @conn))))
(is (= 2 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Card]] @conn))))
(is (= 1 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Cards]] @conn))))
(is (= 2 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Code-block]] @conn))))
(is (= 1 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Math-block]] @conn))))
(is (= 9 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Template]] @conn))))
(is (= 6 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Quote-block]] @conn))))
(is (= 8 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Pdf-annotation]] @conn))))
;; Properties and tags aren't included in this count as they aren't a Page
(is (= 13
(->> (d/q '[:find [?b ...]
:where
[?b :block/title]
[_ :block/page ?b]
(not [?b :logseq.property/built-in?])] @conn)
(map #(d/entity @conn %))
(filter ldb/internal-page?)
#_(map #(select-keys % [:block/title :block/tags]))
count))
"Correct number of pages with block content")
(is (= 16 (->> @conn
(d/q '[:find [?ident ...]
:where [?b :block/tags :logseq.class/Tag] [?b :db/ident ?ident] (not [?b :logseq.property/built-in?])])
count))
"Correct number of user classes")
(is (= 0 (count @(:ignored-properties import-state))) "No ignored properties")
(is (= 0 (count @(:ignored-assets import-state))) "No ignored assets")
(is (= 1 (count @(:ignored-files import-state))) "Ignore .edn for now")
;; 2 zotero pdf are external files so not counted here
(is (= 7 (count @assets))))
(testing "logseq files"
(is (= ".foo {}\n"
(ffirst (d/q '[:find ?content :where [?b :file/path "logseq/custom.css"] [?b :file/content ?content]] @conn))))
(is (= "logseq.api.show_msg('hello good sir!');\n"
(ffirst (d/q '[:find ?content :where [?b :file/path "logseq/custom.js"] [?b :file/content ?content]] @conn)))))
(testing "favorites"
(is (= #{"Interstellar" "some page" "new page" "n1/x/y"}
(set (imported-favorite-titles @conn)))))
(testing "user properties"
(is (= 23
(->> @conn
(d/q '[:find [(pull ?b [:db/ident]) ...]
:where [?b :block/tags :logseq.class/Property]])
(remove #(db-malli-schema/internal-ident? (:db/ident %)))
count))
"Correct number of user properties")
(is (= #{{:db/ident :user.property/prop-bool :logseq.property/type :checkbox}
{:db/ident :user.property/prop-string :logseq.property/type :default}
{:db/ident :user.property/prop-num :logseq.property/type :number}
{:db/ident :user.property/sameas :logseq.property/type :url}
{:db/ident :user.property/rangeincludes :logseq.property/type :node}
{:db/ident :user.property/startedat :logseq.property/type :date}}
(->> @conn
(d/q '[:find [(pull ?b [:db/ident :logseq.property/type]) ...]
:where [?b :block/tags :logseq.class/Property]])
(filter #(contains? #{:prop-bool :prop-string :prop-num :rangeincludes :sameas :startedat}
(keyword (name (:db/ident %)))))
set))
"Main property types have correct inferred :type")
(is (= :default
(:logseq.property/type (d/entity @conn :user.property/description)))
"Property value consisting of text and refs is inferred as :default")
(is (= :url
(:logseq.property/type (d/entity @conn :user.property/url)))
"Property value with a macro correctly inferred as :url")
(is (= {:user.property/prop-bool true
:user.property/prop-num 5
:user.property/prop-string "woot"}
(db-test/readable-properties (db-test/find-block-by-content @conn "b1")))
"Basic block has correct properties")
(is (= #{"prop-num" "prop-string" "prop-bool"}
(->> (db-test/find-block-by-content @conn "b1")
:block/refs
(map :block/title)
set))
"Block with properties has correct refs")
(is (= {:user.property/prop-num2 10
:block/tags [:logseq.class/Page]}
(db-test/readable-properties (db-test/find-page-by-title @conn "new page")))
"New page has correct properties")
(is (= {:user.property/prop-bool true
:user.property/prop-num 5
:user.property/prop-string "yeehaw"
:block/tags [:logseq.class/Page :user.class/SomeNamespace]}
(db-test/readable-properties (db-test/find-page-by-title @conn "some page")))
"Existing page has correct properties")
(is (= {:user.property/rating 5.5}
(db-test/readable-properties (db-test/find-block-by-content @conn ":rating float")))
"Block with float property imports as a float")
(is (= []
(->> (d/q '[:find (pull ?b [:block/title {:block/tags [:db/ident]}])
:where [?b :block/tags :logseq.class/Property]]
@conn)
(map first)
(remove #(= [{:db/ident :logseq.class/Property}] (:block/tags %)))))
"All properties only have :logseq.class/Property as their tag (and don't have Page)"))
(testing "built-in properties"
(is (= [(:db/id (db-test/find-block-by-content @conn "original block"))]
(mapv :db/id (:block/refs (db-test/find-block-by-content @conn #"ref to"))))
"block with a block-ref has correct :block/refs")
(is (= "ref to [[65cbb772-fb79-462d-87c8-6f0dad751dee]]"
(:block/title (db-test/find-block-by-content @conn #"ref to")))
"block-ref ((uuid)) is converted to page-ref [[uuid]] in block title on import")
(is (= 20221126
(-> (db-test/readable-properties (db-test/find-block-by-content @conn "only deadline"))
:logseq.property/deadline
date-time-util/ms->journal-day))
"deadline block has correct journal as property value")
(is (= {:logseq.property/scheduled 20221125
:logseq.property/deadline 20221125}
(-> (db-test/readable-properties (db-test/find-block-by-content @conn #"deadline and scheduled"))
(select-keys [:logseq.property/scheduled :logseq.property/deadline])
(update-vals date-time-util/ms->journal-day)))
"scheduled block converted to correct deadline")
(is (= 1 (count (d/q '[:find [(pull ?b [*]) ...]
:in $ ?content
:where [?b :block/title ?content]]
@conn "Apr 1st, 2024")))
"Only one journal page exists when deadline is on same day as journal")
(is (= {:logseq.property/priority :logseq.property/priority.high}
(db-test/readable-properties (db-test/find-block-by-content @conn "high priority")))
"priority block has correct property")
(is (= {:logseq.property/status :logseq.property/status.doing
:logseq.property/priority :logseq.property/priority.medium
:block/tags [:logseq.class/Task]}
(db-test/readable-properties (db-test/find-block-by-content @conn "status test")))
"status block has correct task properties and class")
(let [empty-title-task (first (task-blocks-by-title @conn ""))]
(is (= {:logseq.property/status :logseq.property/status.todo
:block/tags [:logseq.class/Task]}
(select-keys (db-test/readable-properties empty-title-task)
[:logseq.property/status :block/tags]))
"Empty-title TODO from file graph imports as a task"))
(is (= #{:logseq.property/status :block/tags}
(set (keys (db-test/readable-properties (db-test/find-block-by-content @conn "old todo block")))))
"old task properties like 'todo' are ignored")
(is (= {:logseq.property/order-list-type "number"}
(db-test/readable-properties (db-test/find-block-by-content @conn "list one")))
"numered block has correct property")
(is (= #{"gpt"}
(:block/alias (db-test/readable-properties (db-test/find-page-by-title @conn "chat-gpt"))))
"alias set correctly")
(is (= ["y"]
(->> (d/q '[:find [?b ...] :where [?b :block/title "y"] [?b :block/parent]]
@conn)
first
(d/entity @conn)
:block/alias
(map :block/title)))
"alias set correctly on namespaced page")
(is (= {:logseq.property.linked-references/includes #{"Oct 9th, 2024"}
:logseq.property.linked-references/excludes #{"ref2"}}
(select-keys (db-test/readable-properties (db-test/find-page-by-title @conn "chat-gpt"))
[:logseq.property.linked-references/excludes :logseq.property.linked-references/includes]))
"linked ref filters set correctly"))
(testing "built-in classes and their properties"
;; Queries
(is (= {:logseq.property.table/sorting [{:id :user.property/prop-num, :asc? false}]
:logseq.property.view/type :logseq.property.view/type.table
:logseq.property.table/ordered-columns [:block/title :user.property/prop-string :user.property/prop-num]
:logseq.property/query "(property :prop-string)"
:block/tags [:logseq.class/Query]}
(db-test/readable-properties (find-block-by-property-value @conn :logseq.property/query "(property :prop-string)")))
"simple query block has correct query properties")
(is (= "For example, here's a query with title text:"
(:block/title (db-test/find-block-by-content @conn #"query with title text")))
"Text around a simple query block is set as a query's title")
(is (= {:logseq.property.view/type :logseq.property.view/type.list
:logseq.property/query "{:query (task todo doing)}"
:block/tags [:logseq.class/Query]
:logseq.property.table/ordered-columns [:block/title]}
(db-test/readable-properties (db-test/find-block-by-content @conn #"tasks with todo")))
"Advanced query has correct query properties")
(is (= "tasks with todo and doing"
(:block/title (db-test/find-block-by-content @conn #"tasks with todo")))
"Advanced query has custom title migrated")
;; Card
(is (= {:block/tags [:logseq.class/Card]}
(db-test/readable-properties (db-test/find-block-by-content @conn "card 1")))
"None of the card properties are imported since they are deprecated")
;; Cards (flashcard browser)
(is (= {:block/tags [:logseq.class/Cards]
:logseq.property/query "(tags #Card)"}
(db-test/readable-properties (find-block-by-property-value @conn :logseq.property/query "(tags #Card)")))
"cards macro block has correct Cards class and query property")
;; Math blocks
(is (= {:block/tags [:logseq.class/Math-block]
:logseq.property.node/display-type :math}
(db-test/readable-properties (db-test/find-block-by-content @conn "E=mc^2")))
"Math block has correct Math-block class and display-type")
(is (= "E=mc^2" (:block/title (db-test/find-block-by-content @conn "E=mc^2")))
"Math block title has delimiters stripped")
;; Templates
(is (= #{"meeting"
"title-only-no-children"
"properties-only-no-children"
"title-only-with-children"
"empty-title-with-children"
"children-only"
"nested-father"
"nested-child-1"
"nested-child-2"}
(->> (d/q '[:find [?title ...]
:where
[?b :block/tags :logseq.class/Template]
[?b :block/title ?title]]
@conn)
set))
"All template definitions are imported as Template blocks")
(let [journal-uuid (:block/uuid (db-test/find-journal-by-journal-day @conn 20240216))
template-page-uuids (->> (d/q '[:find [?page-uuid ...]
:where
[?b :block/tags :logseq.class/Template]
[?b :block/page ?page]
[?page :block/uuid ?page-uuid]]
@conn)
set)]
(is (= #{journal-uuid} template-page-uuids)
"All template blocks are created on their source journal page"))
(is (= [{:title "MEETING TITLE"
:properties {:user.property/participants #{"TODO"}}
:children []}]
(template-content-trees @conn "meeting")))
(is (= [{:title "TITLE"
:properties {}
:children []}]
(template-content-trees @conn "title-only-no-children")))
(is (= [{:title ""
:properties {:user.property/name ""
:user.property/author ""}
:children []}]
(template-content-trees @conn "properties-only-no-children")))
(is (= [{:title "TITLE"
:properties {}
:children [{:title "intro" :properties {} :children []}
{:title "notes" :properties {} :children []}]}]
(template-content-trees @conn "title-only-with-children")))
(is (= [{:title ""
:properties {}
:children [{:title "intro" :properties {} :children []}
{:title "notes" :properties {} :children []}]}]
(template-content-trees @conn "empty-title-with-children")))
(is (= [{:title "intro" :properties {} :children []}
{:title "notes" :properties {} :children []}]
(template-content-trees @conn "children-only")))
(is (= [{:title "it's a template with nested templates"
:properties {:user.property/name "you named it"}
:children [{:title "nested-child-1"
:properties {}
:children [{:title "child-1"
:properties {:user.property/name ""}
:children [{:title "child-1-1"
:properties {:user.property/name ""}
:children []}]}]}
{:title "nested-child-2"
:properties {}
:children [{:title "child-2-1"
:properties {:user.property/name ""}
:children []}]}
{:title "child-3"
:properties {:user.property/name ""}
:children []}]}]
(template-content-trees @conn "nested-father")))
(is (= [{:title "child-1"
:properties {:user.property/name ""}
:children [{:title "child-1-1"
:properties {:user.property/name ""}
:children []}]}]
(template-content-trees @conn "nested-child-1")))
(is (= [{:title "child-2-1"
:properties {:user.property/name ""}
:children []}]
(template-content-trees @conn "nested-child-2")))
;; Assets
(is (= {:block/tags [:logseq.class/Asset]
:logseq.property.asset/type "png"
:logseq.property.asset/checksum "3d5e620cac62159d8196c118574bfea7a16e86fa86efd1c3fa15a00a0a08792d"
:logseq.property.asset/size 753471
:logseq.property.asset/resize-metadata {:height 288, :width 252}}
(db-test/readable-properties (db-test/find-block-by-content @conn "greg-popovich-thumbs-up_1704749687791_0")))
"Asset has correct properties")
(is (= {:block/tags [:logseq.class/Asset]
:logseq.property.asset/type "pdf"
:logseq.property.asset/external-url "zotero://select/library/items/QDM8H6EH"
:logseq.property.asset/external-file-name "zotero-link://it/Understanding EXPLAIN.pdf"}
(select-keys
(db-test/readable-properties (db-test/find-block-by-content @conn "Understanding EXPLAIN"))
[:block/tags
:logseq.property.asset/type
:logseq.property.asset/external-url
:logseq.property.asset/external-file-name]))
"Zotero linked pdf asset has correct external path info")
(is (= {:block/tags [:logseq.class/Asset]
:logseq.property.asset/type "pdf"
:logseq.property.asset/external-url "zotero://select/library/items/RX5JS7SY"
:logseq.property.asset/external-file-name "zotero-path://RX5JS7SY/zlib.pdf"}
(select-keys
(db-test/readable-properties (db-test/find-block-by-content @conn "zlib"))
[:block/tags
:logseq.property.asset/type
:logseq.property.asset/external-url
:logseq.property.asset/external-file-name]))
"Zotero imported pdf asset has correct external path info")
(is (= (d/entity @conn :logseq.class/Asset)
(:block/page (db-test/find-block-by-content @conn "greg-popovich-thumbs-up_1704749687791_0")))
"Imported into Asset page")
;; Annotations
(is (= {:logseq.property.pdf/hl-color :logseq.property/color.blue
:logseq.property.pdf/hl-page 8
:block/tags [:logseq.class/Pdf-annotation]
:logseq.property/asset "Sina_de_Capoeria_Batizado_2025_-_Program_Itinerary_1752179325104_0"}
(dissoc (db-test/readable-properties (db-test/find-block-by-content @conn #"Duke School - modified"))
:logseq.property.pdf/hl-value :logseq.property/ls-type))
"Pdf text highlight has correct properties")
(is (= ["note about duke" "sub note"]
(mapv :block/title (rest (ldb/get-block-and-children @conn (:block/uuid (db-test/find-block-by-content @conn #"Duke School - modified"))))))
"Pdf text highlight has correct children blocks")
(is (= {:logseq.property.pdf/hl-color :logseq.property/color.yellow
:logseq.property.pdf/hl-page 1
:block/tags [:logseq.class/Pdf-annotation]
:logseq.property/asset "Sina_de_Capoeria_Batizado_2025_-_Program_Itinerary_1752179325104_0"
:logseq.property.pdf/hl-image "pdf area highlight"
:logseq.property.pdf/hl-type :area}
(dissoc (->> (d/q '[:find [?b ...]
:where [?b :block/tags :logseq.class/Pdf-annotation] [?b :block/title ""]] @conn)
first
(d/entity @conn)
db-test/readable-properties)
:logseq.property.pdf/hl-value :logseq.property/ls-type))
"Pdf area highlight has correct properties")
(is (= ""
(:block/title
(d/entity @conn [:block/uuid #uuid "68702499-159a-4a14-a0cf-cf5f015535c2"])))
"Pdf annotation without text imports with an empty title")
(is (= {:block/tags [:logseq.class/Pdf-annotation]
:logseq.property/asset "Understanding EXPLAIN"
:logseq.property.pdf/hl-color :logseq.property/color.yellow
:logseq.property.pdf/hl-page 6}
(select-keys
(db-test/readable-properties (db-test/find-block-by-content @conn #"EXPLAIN is a really nice command"))
[:block/tags
:logseq.property/asset
:logseq.property.pdf/hl-color
:logseq.property.pdf/hl-page]))
"Zotero linked pdf text highlight links to correct asset")
(is (= {:block/tags [:logseq.class/Pdf-annotation]
:logseq.property/asset "zlib"
:logseq.property.pdf/hl-color :logseq.property/color.red
:logseq.property.pdf/hl-page 1}
(select-keys
(db-test/readable-properties (db-test/find-block-by-content @conn #"The zlib library is a general purpose data compression library"))
[:block/tags
:logseq.property/asset
:logseq.property.pdf/hl-color
:logseq.property.pdf/hl-page]))
"Zotero imported pdf text highlight links to correct asset")
(let [area-hl (d/q '[:find (pull ?b [:block/title
{:block/tags [:db/ident]}
{:logseq.property/asset [:block/title]}
{:logseq.property.pdf/hl-image [:block/title]}
{:logseq.property.pdf/hl-color [:db/ident]}
:logseq.property.pdf/hl-type
:logseq.property.pdf/hl-page]) .
:where
[?asset :block/title "Understanding EXPLAIN"]
[?asset :block/tags :logseq.class/Asset]
[?b :block/title "[:span]"]
[?b :logseq.property/asset ?asset]]
@conn)]
(is (= {:logseq.property.pdf/hl-color :logseq.property/color.green
:logseq.property.pdf/hl-page 8
:block/tags [:logseq.class/Pdf-annotation]
:logseq.property/asset "Understanding EXPLAIN"
:logseq.property.pdf/hl-image "pdf area highlight"
:logseq.property.pdf/hl-type :area}
(-> area-hl
(update :block/tags #(mapv :db/ident %))
(update :logseq.property/asset #(:block/title %))
(update :logseq.property.pdf/hl-color #(:db/ident %))
(update :logseq.property.pdf/hl-image #(:block/title %))
(select-keys [:block/tags
:logseq.property/asset
:logseq.property.pdf/hl-color
:logseq.property.pdf/hl-page
:logseq.property.pdf/hl-image
:logseq.property.pdf/hl-type])))
"Zotero linked pdf area highlight links to correct asset"))
(let [area-hl (d/q '[:find (pull ?b [:block/title
{:block/tags [:db/ident]}
{:logseq.property/asset [:block/title]}
{:logseq.property.pdf/hl-image [:block/title]}
{:logseq.property.pdf/hl-color [:db/ident]}
:logseq.property.pdf/hl-type
:logseq.property.pdf/hl-page]) .
:where
[?asset :block/title "zlib"]
[?asset :block/tags :logseq.class/Asset]
[?b :block/title "[:span]"]
[?b :logseq.property/asset ?asset]]
@conn)]
(is (= {:logseq.property.pdf/hl-color :logseq.property/color.blue
:logseq.property.pdf/hl-page 1
:block/tags [:logseq.class/Pdf-annotation]
:logseq.property/asset "zlib"
:logseq.property.pdf/hl-image "pdf area highlight"
:logseq.property.pdf/hl-type :area}
(-> area-hl
(update :block/tags #(mapv :db/ident %))
(update :logseq.property/asset #(:block/title %))
(update :logseq.property.pdf/hl-color #(:db/ident %))
(update :logseq.property.pdf/hl-image #(:block/title %))
(select-keys [:block/tags
:logseq.property/asset
:logseq.property.pdf/hl-color
:logseq.property.pdf/hl-page
:logseq.property.pdf/hl-image
:logseq.property.pdf/hl-type])))
"Zotero imported pdf area highlight links to correct asset"))
;; Quotes
(is (string/starts-with? (:block/title (db-test/find-block-by-content @conn #"Saito")) "From Inception:\n> Saito:")
"Mixed #+BEGIN_QUOTE block: heading retained and quote content prefixed with '>'")
(is (nil? (:logseq.property.node/display-type (db-test/find-block-by-content @conn #"Saito")))
"Mixed #+BEGIN_QUOTE block is not converted to a Quote-block")
(is (= "markdown quote\n[[wut]]\nline 3"
(:block/title (db-test/find-block-by-content @conn #"markdown quote")))
"Markdown quote imports as full multi-line quote")
(is (string/starts-with? (:block/title (db-test/find-block-by-content @conn #"Learn Datalog"))
"Test of various ast types:\n> *Italic*")
"Mixed #+BEGIN_QUOTE block retains heading and quote content in block title")
(is (= "Blockquotes\n> Nested Blockquotes"
(:block/title (db-test/find-block-by-content @conn #"Nested Blockquotes")))
"Nested '>> quote' is preserved as '> ' prefix in Quote-block title")
(is (= :quote (:logseq.property.node/display-type (db-test/find-block-by-content @conn #"Nested Blockquotes")))
"Nested markdown quote block is tagged as Quote-block")
(is (= "it's a\n\norg blockquote"
(:block/title (db-test/find-block-by-content @conn #"org blockquote")))
"#+BEGIN_QUOTE pure block title has no '> ' prefix — display-type provides blockquote styling")
(is (= :quote (:logseq.property.node/display-type (db-test/find-block-by-content @conn #"org blockquote")))
"#+BEGIN_QUOTE pure block is tagged as Quote-block")
(is (= "> Blockquotes\n> and\n\nsomething else"
(:block/title (db-test/find-block-by-content @conn #"Blockquotes\n> and")))
"Mixed #+BEGIN_QUOTE at start of block: quote content prefixed with '>' and blank line separates following text")
(is (nil? (:logseq.property.node/display-type (db-test/find-block-by-content @conn #"Blockquotes\n> and")))
"Mixed #+BEGIN_QUOTE block at start is not converted to a Quote-block")
(let [block (db-test/find-block-by-content @conn #"Question 1")]
(is (string/includes? (:block/title block) "\n>> nested")
"Mixed markdown quote block preserves nested quote depth")
(is (nil? (:logseq.property.node/display-type block))
"Mixed markdown quote block is not converted to a Quote-block"))
(let [block (db-test/find-block-by-content @conn #"Question 2")]
(is (string/includes? (:block/title block) "> Question 3")
"Mixed markdown quote child block preserves separated quote lines")
(is (string/includes? (:block/title block) "> Answer 3")
"Mixed markdown quote child block preserves quote content after blank quote line")))
(testing "embeds"
(is (= {:block/title ""}
(-> (d/q '[:find [(pull ?b [*]) ...]
:in $ ?title
:where [?b :block/link ?l] [?b :block/page ?bp] [?bp :block/journal-day 20250612] [?l :block/title ?title]]
@conn
"page embed")
first
(select-keys [:block/title])))
"Page embed linked correctly")
(is (= {:block/title ""}
(-> (d/q '[:find [(pull ?b [*]) ...]
:in $ ?title
:where [?b :block/link ?l] [?b :block/page ?bp] [?bp :block/journal-day 20250612] [?l :block/title ?title]]
@conn
"test block embed")
first
(select-keys [:block/title])))
"Block embed linked correctly"))
(testing "tags convert to classes"
(is (= :user.class/Quotes___life
(:db/ident (db-test/find-page-by-title @conn "life")))
"Namespaced tag's ident has hierarchy to make it unique")
(is (= [:logseq.class/Tag]
(map :db/ident (:block/tags (db-test/find-page-by-title @conn "life"))))
"When a class is used and referenced on the same page, there should only be one instance of it")
(is (= [:user.class/Quotes___life]
(mapv :db/ident (:block/tags (db-test/find-block-by-content @conn #"with namespace tag"))))
"Block tagged with namespace tag is only associated with leaf child tag")
(is (= #{:user.class/ai :user.class/block-tag :user.class/p1}
(set (map :db/ident (:block/tags (db-test/find-block-by-content @conn #"Block tags")))))
"Block with tags through tags property")
(is (= []
(->> (d/q '[:find (pull ?b [:block/title {:block/tags [:db/ident]}])
:where [?b :block/tags :logseq.class/Tag]]
@conn)
(map first)
(remove #(= [{:db/ident :logseq.class/Tag}] (:block/tags %)))))
"All classes only have :logseq.class/Tag as their tag (and don't have Page)"))
(testing "namespaces"
(let [expand-children (fn expand-children [ent parent]
(if-let [children (:block/_parent ent)]
(cons {:parent (:block/title parent) :child (:block/title ent)}
(mapcat #(expand-children % ent) children))
[{:parent (:block/title parent) :child (:block/title ent)}]))]
;; check pages only
(is (= [{:parent "n1" :child "x"}
{:parent "x" :child "z"}
{:parent "x" :child "y"}]
(take 3 (rest (expand-children (db-test/find-page-by-title @conn "n1") nil))))
"First namespace tests duplicate parent page name")
(is (= [{:parent "n2" :child "x"}
{:parent "x" :child "z"}
{:parent "n2" :child "alias"}]
(rest (expand-children (db-test/find-page-by-title @conn "n2") nil)))
"First namespace tests duplicate child page name and built-in page name")))
(testing "journal timestamps"
(is (= (date-time-util/journal-day->ms 20240207)
(:block/created-at (db-test/find-page-by-title @conn "Feb 7th, 2024")))
"journal pages are created on their journal day")
(is (= (date-time-util/journal-day->ms 20240207)
(:block/created-at (db-test/find-block-by-content @conn #"Inception")))
"journal blocks are created on their page's journal day"))
(testing "db attributes"
(is (= true
(:block/collapsed? (db-test/find-block-by-content @conn "collapsed block")))
"Collapsed blocks are imported"))
(testing "property :type changes"
(is (= :node
(:logseq.property/type (d/entity @conn :user.property/finishedat)))
":date property to :node value changes to :node")
(is (= :default
(:logseq.property/type (d/entity @conn :user.property/participants)))
"template values cause participants to remain a :default property")
(is (= #{"[[Feb 7th, 2024]]"}
(:user.property/participants (db-test/readable-properties (db-test/find-block-by-content @conn #"test :node -> :date"))))
":default participants property keeps the imported text value")
(is (= :default
(:logseq.property/type (d/entity @conn :user.property/description)))
":default property to :node (or any non :default value) remains :default")
(is (= "[[Jakob]]"
(:user.property/description (db-test/readable-properties (db-test/find-block-by-content @conn #":default to :node"))))
":default to :node property saves :default property value default with full text")
(testing "with changes to upstream/existing property value"
(is (= :default
(:logseq.property/type (d/entity @conn :user.property/duration)))
":number property to :default value changes to :default")
(is (= "20"
(:user.property/duration (db-test/readable-properties (db-test/find-block-by-content @conn "existing :number to :default"))))
"existing :number property value correctly saved as :default")
(is (= :default
(:logseq.property/type (d/entity @conn :user.property/people2)))
":node property changes to :default when :node is defined in same file")
;; tests :node :many to :default transition after :node is defined in separate file
(is (= {:logseq.property/type :default :db/cardinality :db.cardinality/many}
(select-keys (d/entity @conn :user.property/people) [:logseq.property/type :db/cardinality]))
":node property to :default value changes to :default and keeps existing cardinality")
(is (= #{"[[Jakob]] [[Gabriel]]"}
(:user.property/people (db-test/readable-properties (db-test/find-block-by-content @conn ":node people"))))
"existing :node property value correctly saved as :default with full text")
(is (= #{"[[Gabriel]] [[Jakob]]"}
(:user.property/people (db-test/readable-properties (db-test/find-block-by-content @conn #"pending block for :node"))))
"pending :node property value correctly saved as :default with full text")
(is (some? (db-test/find-page-by-title @conn "Jakob"))
"Previous :node property value still exists")
(is (= 3 (count (find-block-by-property @conn :user.property/people)))
"Converted property has correct number of property values")))
(testing "imported concepts can have names of new-built concepts"
(is (= #{:logseq.property/description :user.property/description}
(set (d/q '[:find [?ident ...] :where [?b :db/ident ?ident] [?b :block/name "description"]] @conn)))
"user description property is separate from built-in one")
(is (= #{"Page" "Tag"}
(set (d/q '[:find [?t-title ...] :where
[?b :block/tags ?t]
[?b :block/name "task"]
[?t :block/title ?t-title]] @conn)))
"user page is separate from built-in class"))
(testing "multiline blocks"
(is (= "|markdown| table|\n|some|thing|" (:block/title (db-test/find-block-by-content @conn #"markdown.*table"))))
(is (= "normal multiline block\na 2nd\nand a 3rd" (:block/title (db-test/find-block-by-content @conn #"normal multiline block"))))
(is (= "colored multiline block\nlast line" (:block/title (db-test/find-block-by-content @conn #"colored multiline block"))))
(let [block (db-test/find-block-by-content @conn #"multiline block with prop and deadline")]
(is (= "multiline block with prop and deadline\nlast line" (:block/title block)))
(is (= 20221126
(-> (db-test/readable-properties block)
:logseq.property/deadline
date-time-util/ms->journal-day))
"multiline block has correct journal as property value")
(is (= "red"
(-> (db-test/readable-properties block)
:logseq.property/background-color))
"multiline block has correct background color as property value"))
(let [block (db-test/find-block-by-content @conn #"multiline block with deadline and scheduled in 1 line and sth else")]
(is (= "multiline block with deadline and scheduled in 1 line and sth else\nsomething else\nlast line" (:block/title block)))
(is (= 20221126
(-> (db-test/readable-properties block)
:logseq.property/deadline
date-time-util/ms->journal-day))
"multiline block with deadline and scheduled has correct deadline journal as property value")
(is (= 20221126
(-> (db-test/readable-properties block)
:logseq.property/scheduled
date-time-util/ms->journal-day))
"multiline block with deadline and scheduled has correct scheduled journal as property value"))
(is (= "logbook block" (:block/title (db-test/find-block-by-content @conn #"^logbook block"))))
(is (= "multiline logbook block\nlast line" (:block/title (db-test/find-block-by-content @conn #"multiline logbook block")))))
(testing ":block/refs"
(let [page (db-test/find-page-by-title @conn "chat-gpt")]
(is (set/subset?
#{"type" "LargeLanguageModel"}
(->> page :block/refs (map #(:block/title (d/entity @conn (:db/id %)))) set))
"Page has correct property and property value :block/refs"))
(let [block (db-test/find-block-by-content @conn "old todo block")]
(is (set/subset?
#{:logseq.property/status :logseq.class/Task}
(->> block
:block/refs
(map #(:db/ident (d/entity @conn (:db/id %))))
set))
"Block has correct task tag and property :block/refs")))))
(deftest-async export-basic-graph-with-convert-all-tags-option-disabled
(p/let [file-graph-dir "test/resources/exporter-test-graph"
conn (db-test/create-conn)
{:keys [import-state]}
(import-file-graph-to-db file-graph-dir conn {:convert-all-tags? false})]
(is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn))))
"Created graph has no validation errors")
(is (= 0 (count @(:ignored-properties import-state))) "No ignored properties")
(is (= 0 (->> @conn
(d/q '[:find [?ident ...]
:where [?b :block/tags :logseq.class/Tag] [?b :db/ident ?ident] (not [?b :logseq.property/built-in?])])
count))
"Correct number of user classes")
(is (= 6 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Task]] @conn))))
(is (= 4 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Query]] @conn))))
(is (= 2 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Card]] @conn))))
(testing "replacing refs in :block/title when :remove-inline-tags? set"
(is (= 2
(->> (entity-plus/lookup-kv-then-entity
(db-test/find-block-by-content @conn #"replace with same start string")
:block/raw-title)
(re-seq db-content/id-ref-pattern)
distinct
count))
"A block with ref names that start with same string has 2 distinct refs")
(is (= 1
(->> (entity-plus/lookup-kv-then-entity
(db-test/find-block-by-content @conn #"replace case insensitive")
:block/raw-title)
(re-seq db-content/id-ref-pattern)
distinct
count))
"A block with different case of same ref names has 1 distinct ref"))
(testing "tags convert to page, refs and page-tags"
(let [block (db-test/find-block-by-content @conn #"Inception")
tag-page (db-test/find-page-by-title @conn "Movie")
tagged-page (db-test/find-page-by-title @conn "Interstellar")]
(is (string/starts-with? (str (:block/title block)) "Inception [[")
"tagged block tag converts tag to page ref")
(is (= [(:db/id tag-page)] (map :db/id (:block/refs block)))
"tagged block has correct refs")
(is (and tag-page (not (ldb/class? tag-page)))
"tag page is not a class")
(is (= #{"Movie"}
(:logseq.property/page-tags (db-test/readable-properties tagged-page)))
"tagged page has existing page imported as a tag to page-tags")
(is (= #{"LargeLanguageModel" "fun" "ai"}
(:logseq.property/page-tags (db-test/readable-properties (db-test/find-page-by-title @conn "chat-gpt"))))
"tagged page has new page and other pages marked with '#' and '[[]]` imported as tags to page-tags")))))
(deftest-async import-journals-use-standard-uuids-and-keep-uuid-refs
(p/let [file-graph-dir "test/resources/exporter-test-graph"
files (mapv #(path/path-join file-graph-dir %) ["journals/2026_01_27.md"])
conn (db-test/create-conn)
_ (import-files-to-db files conn {})]
(let [journal (db-test/find-journal-by-journal-day @conn 20260127)
ref-journal (db-test/find-journal-by-journal-day @conn 20260101)
ref-block (some->> (d/q '[:find [?b ...]
:in $ ?page ?ref-page
:where
[?b :block/page ?page]
[?b :block/refs ?ref-page]]
@conn (:db/id journal) (:db/id ref-journal))
first
(d/entity @conn))]
(is (= (common-uuid/gen-uuid :journal-page-uuid 20260127)
(:block/uuid journal))
"Imported journal page keeps the standard journal uuid")
(is (= (common-uuid/gen-uuid :journal-page-uuid 20260101)
(:block/uuid ref-journal))
"Referenced journal page keeps the standard journal uuid")
(is (= #{(:block/uuid ref-journal)}
(set (map :block/uuid (:block/refs ref-block))))
"Journal refs point at the standard journal uuid"))))
(deftest-async import-journal-with-slash-title-format-does-not-create-namespace-pages
(p/let [file-graph-dir "test/resources/exporter-test-graph"
files (mapv #(path/path-join file-graph-dir %) ["journals/2026_01_27.md"])
conn (db-test/create-conn)
_ (import-files-to-db files conn {:user-config {:journal/page-title-format "yyyy/MM/dd"}})
journal (db-test/find-journal-by-journal-day @conn 20260127)]
(is (= "2026/01/27" (:block/title journal))
"Journal title follows slash title format")
(is (= (common-uuid/gen-uuid :journal-page-uuid 20260127)
(:block/uuid journal))
"Slash-formatted journal keeps the standard journal uuid")
(is (nil? (:block/namespace journal))
"Slash-formatted journal does not keep a namespace attribute")
(is (nil? (db-test/find-page-by-title @conn "2026"))
"Journal title is not split into a year namespace page")
(is (nil? (db-test/find-page-by-title @conn "01"))
"Journal title is not split into a month namespace page")
(is (nil? (db-test/find-page-by-title @conn "27"))
"Journal title is not split into a day namespace page")))
(deftest-async import-slash-journal-ref-does-not-create-namespace-pages
(p/let [file (write-temp-graph-file "journals/2026_05_18.md" "- yes\n- [[Sun, 2026/05/17]]\n")
conn (db-test/create-conn)
_ (import-files-to-db [file] conn {:user-config {:journal/page-title-format "EEE, yyyy/MM/dd"}})
ref-journal (db-test/find-journal-by-journal-day @conn 20260517)]
(is (= "Sun, 2026/05/17" (:block/title ref-journal))
"Journal reference is imported as a journal page")
(is (nil? (:block/namespace ref-journal))
"Referenced slash-formatted journal does not keep a namespace attribute")
(is (nil? (db-test/find-page-by-title @conn "Sun, 2026"))
"Journal reference is not split into a parent namespace page")
(is (nil? (db-test/find-page-by-title @conn "05"))
"Journal reference is not split into a child namespace page")))
(deftest-async import-legacy-journal-file-name-refs-as-journals
(p/let [source-file (write-temp-graph-file
"journals/2026_04_01.md"
"- legacy journal ref [[2026_04_02]]\n")
target-file (write-temp-graph-file
"journals/2026_04_02.md"
"- target journal\n")
conn (db-test/create-conn)
_ (import-files-to-db [source-file target-file] conn {})
legacy-journal (db-test/find-journal-by-journal-day @conn 20260402)
legacy-ref-block (db-test/find-block-by-content @conn #"legacy journal ref")]
(is (some? legacy-journal)
"Legacy yyyy_MM_dd journal page refs resolve imported journal files")
(is (= #{(:block/uuid legacy-journal)}
(set (map :block/uuid (:block/refs legacy-ref-block))))
"Legacy journal page ref points at the journal page")
(is (nil? (db-test/find-page-by-title @conn "2026_04_02"))
"Legacy journal page ref does not create an ordinary page")))
(deftest-async import-default-format-journal-refs-with-custom-title-format
(p/let [dir (write-temp-file-graph
{"pages/source.md"
"- existing journal [[May 18th, 2021]]\n- missing journal [[May 19th, 2021]]\n"
"journals/2021_05_18.md"
"- journal entry\n"
"pages/2021_05_19.md"
"- ordinary date-named page\n"})
source-file (-> (node-path/join dir "pages/source.md")
(string/replace "\\" "/"))
journal-file (-> (node-path/join dir "journals/2021_05_18.md")
(string/replace "\\" "/"))
ordinary-date-file (-> (node-path/join dir "pages/2021_05_19.md")
(string/replace "\\" "/"))
conn (db-test/create-conn)
_ (import-files-to-db [source-file journal-file ordinary-date-file] conn
{:user-config {:journal/page-title-format "EEEE, dd-MM-yyyy"}})
existing-journal (db-test/find-journal-by-journal-day @conn 20210518)
missing-page (db-test/find-page-by-title @conn "May 19th, 2021")
existing-ref-block (db-test/find-block-by-content @conn #"existing journal")
missing-ref-block (db-test/find-block-by-content @conn #"missing journal")]
(is (= "Tuesday, 18-05-2021" (:block/title existing-journal))
"Default-format ref resolves the journal file using the configured title")
(is (= #{:logseq.class/Journal}
(set (map :db/ident (:block/tags existing-journal))))
"Existing journal does not retain the ordinary page tag")
(is (= #{(:block/uuid existing-journal)}
(set (map :block/uuid (:block/refs existing-ref-block))))
"Existing journal ref points at the journal page")
(is (= #{:logseq.class/Page}
(set (map :db/ident (:block/tags missing-page))))
"Default-format ref without a file under journals remains an ordinary page")
(is (nil? (db-test/find-journal-by-journal-day @conn 20210519))
"Date-named file outside journals does not create a journal")
(is (= #{(:block/uuid missing-page)}
(set (map :block/uuid (:block/refs missing-ref-block))))
"Missing journal ref points at the ordinary page")))
(deftest-async import-creates-missing-ordinary-page-refs
(p/let [file (write-temp-graph-file
"pages/source.md"
"- missing page ref [[Missing Page]]\n")
conn (db-test/create-conn)
_ (import-files-to-db [file] conn {})
missing-page (db-test/find-page-by-title @conn "Missing Page")
source-block (db-test/find-block-by-content @conn #"missing page ref")]
(is (some? missing-page)
"Missing ordinary page refs create ordinary pages")
(is (= #{(:block/uuid missing-page)}
(set (map :block/uuid (:block/refs source-block))))
"Missing ordinary page ref points at the created page")))
(deftest-async import-favorites-from-og-config-edn
(p/let [dir (write-temp-file-graph
{"logseq/config.edn"
(str "{:favorites [\"Projects\" \"[[Projects]]\" \"foo/bar\"]\n"
" :file/name-format :triple-lowbar}\n")
"pages/Projects.md" "- project work\n- [[foo/bar]]\n"})
conn (db-test/create-conn)
_ (db-pipeline/add-listener conn)
_ (import-file-graph-to-db dir conn {})
favorite-titles (imported-favorite-titles @conn)]
(is (= 3 (count favorite-titles))
"Bare names, bracketed page refs, and namespaced pages each become a favorite link")
(is (= #{"Projects" "foo/bar"}
(set favorite-titles))
"Imported favorites resolve to the original pages including flattened namespaces")))
(deftest-async import-normalizes-existing-random-journal-uuid-and-text-refs
(let [old-journal-uuid (random-uuid)
standard-journal-uuid (common-uuid/gen-uuid :journal-page-uuid 20260127)
title (str "refs " (page-ref/->page-ref old-journal-uuid)
" and " (block-ref/->block-ref old-journal-uuid))
conn (db-test/create-conn-with-blocks
{:pages-and-blocks
[{:page {:build/journal 20260127
:block/uuid old-journal-uuid
:build/keep-uuid? true}
:blocks [{:block/title title}]}]})
file (write-temp-graph-file "pages/trigger-normalize.md" "- trigger normalize\n")]
(p/let [_ (import-files-to-db [file] conn {})
journal (db-test/find-journal-by-journal-day @conn 20260127)
ref-block (db-test/find-block-by-content @conn #"refs")]
(is (= standard-journal-uuid (:block/uuid journal))
"Existing random journal uuid is normalized to the standard journal uuid")
(is (nil? (d/entity @conn [:block/uuid old-journal-uuid]))
"Old journal uuid no longer resolves after normalization")
(is (= (str "refs " (page-ref/->page-ref standard-journal-uuid)
" and " (block-ref/->block-ref standard-journal-uuid))
(:block/title ref-block))
"Text references are rewritten to the standard journal uuid")
(is (= (:db/id journal) (get-in ref-block [:block/page :db/id]))
"Structured block page reference still points to the same journal entity"))))
(deftest-async export-files-with-tag-classes-option
(p/let [file-graph-dir "test/resources/exporter-test-graph"
files (mapv #(path/path-join file-graph-dir %) ["journals/2024_02_07.md" "pages/Interstellar.md"])
conn (db-test/create-conn)
_ (import-files-to-db files conn {:tag-classes ["movie"]})]
(is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn))))
"Created graph has no validation errors")
(let [block (db-test/find-block-by-content @conn #"Inception")
tag-page (db-test/find-page-by-title @conn "Movie")
another-tag-page (db-test/find-page-by-title @conn "p0")]
(is (= (:block/title block) "Inception")
"tagged block with configured tag strips tag from content")
(is (= [:user.class/Movie]
(:block/tags (db-test/readable-properties block)))
"tagged block has configured tag imported as a class")
(is (= [:logseq.class/Tag] (mapv :db/ident (:block/tags tag-page)))
"configured tag page in :tag-classes is a class")
(is (and another-tag-page (not (ldb/class? another-tag-page)))
"unconfigured tag page is not a class")
(is (= {:block/tags [:logseq.class/Page :user.class/Movie]}
(db-test/readable-properties (db-test/find-page-by-title @conn "Interstellar")))
"tagged page has configured tag imported as a class"))))
(deftest-async export-files-with-property-classes-option
(p/let [file-graph-dir "test/resources/exporter-test-graph"
files (mapv #(path/path-join file-graph-dir %)
["journals/2024_02_23.md" "pages/url.md" "pages/Whiteboard___Tool.md"
"pages/Whiteboard___Arrow_head_toggle.md"
"pages/Library.md"])
conn (db-test/create-conn)
_ (import-files-to-db files conn {:property-classes ["type"]})
_ (@#'gp-exporter/export-class-properties conn conn)]
(is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn))))
"Created graph has no validation errors")
(is (= #{:user.class/Property :user.class/Movie :user.class/Class :user.class/Tool}
(->> @conn
(d/q '[:find [?ident ...]
:where [?b :block/tags :logseq.class/Tag] [?b :db/ident ?ident] (not [?b :logseq.property/built-in?])])
set))
"All classes are correctly defined by :type")
(is (= #{:user.property/url :user.property/sameas :user.property/rangeincludes}
(->> (d/entity @conn :user.class/Property)
:logseq.property.class/properties
(map :db/ident)
set))
"Properties are correctly inferred for a class")
(let [block (db-test/find-block-by-content @conn #"The Creator")
tag-page (db-test/find-page-by-title @conn "Movie")]
(is (= (:block/title block) "The Creator")
"tagged block with configured tag strips tag from content")
(is (= [:user.class/Movie]
(:block/tags (db-test/readable-properties block)))
"tagged block has configured tag imported as a class")
(is (= (:user.property/testtagclass block) (:block/tags block))
"tagged block can have another property that references the same class it is tagged with,
without creating a duplicate class")
(is (= [:logseq.class/Tag] (map :db/ident (:block/tags tag-page)))
"configured tag page derived from :property-classes is a class")
(is (nil? (db-test/find-page-by-title @conn "type"))
"No page exists for configured property")
(is (= #{:user.class/Property :logseq.class/Property}
(set (:block/tags (db-test/readable-properties (db-test/find-page-by-title @conn "url")))))
"tagged page has correct tags including one from option"))))
(deftest-async export-files-with-remove-inline-tags
(p/let [file-graph-dir "test/resources/exporter-test-graph"
files (mapv #(path/path-join file-graph-dir %) ["journals/2024_02_07.md"
"journals/2026_01_27.md"])
conn (db-test/create-conn)
_ (import-files-to-db files conn {:remove-inline-tags? false :convert-all-tags? true})
namespaced-file (write-temp-graph-file
"pages/namespace-inline-tag.md"
"- #parent/child\n")
namespaced-conn (db-test/create-conn)
_ (import-files-to-db [namespaced-file] namespaced-conn {:remove-inline-tags? false :convert-all-tags? true})
[block tag] (->> (d/q '[:find ?b ?t
:where
[?b :block/tags ?t]
[?b :block/page]
[?t :db/ident :user.class/parent___child]]
@namespaced-conn)
first
(map #(d/entity @namespaced-conn %)))
raw-title (:block/title block)]
(is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn))))
"Created graph has no validation errors")
(is (string/starts-with? (:block/title (db-test/find-block-by-content @conn #"Inception"))
"Inception #Movie")
"block with tag preserves inline tag")
(is (string/includes? (:block/title (db-test/find-block-by-content @conn #"block with multi word tag"))
"#[[another test]]")
"block with multi word tag preserves inline tag")
(testing "namespaced inline tag on first line is preserved as inline tag"
(is (some? block)
"imported first-line namespaced tag block")
(is (string? raw-title)
"imported block has raw title")
(when (string? raw-title)
(is (ldb/inline-tag? raw-title tag)
"first-line namespaced tag is stored as an inline tag")))))
(deftest-async export-files-with-ignored-properties
(p/let [file-graph-dir "test/resources/exporter-test-graph"
files (mapv #(path/path-join file-graph-dir %) ["ignored/icon-page.md"])
conn (db-test/create-conn)
{:keys [import-state]} (import-files-to-db files conn {})]
(is (= 2
(count (filter #(= :icon (:property %)) @(:ignored-properties import-state))))
"icon properties are visibly ignored in order to not fail import")))
(deftest-async export-files-with-property-parent-classes-option
(p/let [file-graph-dir "test/resources/exporter-test-graph"
files (mapv #(path/path-join file-graph-dir %) ["journals/2024_11_26.md"
"pages/CreativeWork.md" "pages/Movie.md" "pages/type.md"
"pages/Whiteboard___Tool.md" "pages/Whiteboard___Arrow_head_toggle.md"
"pages/Property.md" "pages/url.md"])
conn (db-test/create-conn)
_ (import-files-to-db files conn {:property-parent-classes ["parent"]
;; Also add this option to trigger some edge cases with namespace pages
:property-classes ["type"]})]
(is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn))))
"Created graph has no validation errors")
(is (= #{:user.class/Movie :user.class/CreativeWork :user.class/Thing :user.class/Feature
:user.class/Class :user.class/Tool :user.class/Whiteboard___Tool :user.class/Property}
(->> @conn
(d/q '[:find [?ident ...]
:where [?b :block/tags :logseq.class/Tag] [?b :db/ident ?ident] (not [?b :logseq.property/built-in?])])
set))
"All classes are correctly defined by :type")
(is (= ["CreativeWork"] (map :block/title (:logseq.property.class/extends (d/entity @conn :user.class/Movie))))
"Existing page correctly set as class parent")
(is (= ["Thing"] (map :block/title (:logseq.property.class/extends (d/entity @conn :user.class/CreativeWork))))
"New page correctly set as class parent")))
(deftest-async export-files-with-property-pages-disabled
(p/let [file-graph-dir "test/resources/exporter-test-graph"
;; any page with properties
files (mapv #(path/path-join file-graph-dir %) ["journals/2024_01_17.md"])
conn (db-test/create-conn)
_ (import-files-to-db files conn {:user-config {:property-pages/enabled? false
:property-pages/excludelist #{:prop-string}}})]
(is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn))))
"Created graph has no validation errors")))
(deftest-async export-config-file-sets-title-format
(p/let [conn (db-test/create-conn)
read-file #(p/do! "{:journal/page-title-format \"yyyy-MM-dd\"}")
_ (gp-exporter/export-config-file conn "logseq/config.edn" read-file {})]
(is (= "yyyy-MM-dd"
(:logseq.property.journal/title-format (d/entity @conn :logseq.class/Journal)))
"title format set correctly by config")))
(deftest split-title-by-code-fences
(let [split-fn #'gp-exporter/split-title-by-code-fences]
(testing "standalone code fence with language"
(is (= {:text-parts []
:code-segs [{:text "it's an individual code snippet with language tag"
:lang "markdown"}]}
(split-fn "```markdown\nit's an individual code snippet with language tag\n```"))))
(testing "standalone code fence without language"
(is (= {:text-parts []
:code-segs [{:text "it's an individual code snippet without language tag"
:lang nil}]}
(split-fn "```\nit's an individual code snippet without language tag\n```"))))
(testing "one code fence with leading text"
(is (= {:text-parts ["before code snippet"]
:code-segs [{:text "echo \"ok\"\nexit"
:lang nil}]}
(split-fn "before code snippet\n```\necho \"ok\"\nexit\n```"))))
(testing "one code fence with leading and trailing text"
(is (= {:text-parts ["before code snippet" "after code snippet"]
:code-segs [{:text "echo \"ok\"\nexit"
:lang "bash"}]}
(split-fn "before code snippet\n```bash\necho \"ok\"\nexit\n```\nafter code snippet"))))
(testing "one code fence followed by trailing text"
(is (= {:text-parts ["after code snippet"]
:code-segs [{:text "echo \"ok\"\nexit"
:lang "bash"}]}
(split-fn "```bash\necho \"ok\"\nexit\n```\nafter code snippet"))))
(testing "multiple code fences mixed with text"
(is (= {:text-parts ["before code snippet" "middle" "after code snippet"]
:code-segs [{:text "echo \"ok\"\nexit"
:lang "bash"}
{:text "echo \"bye\"\nexit"
:lang "bash"}]}
(split-fn "before code snippet\n```bash\necho \"ok\"\nexit\n```\nmiddle\n```bash\necho \"bye\"\nexit\n```\nafter code snippet"))))
(testing "edge: one code fence followed by opening fence without closing fence"
(is (= {:text-parts ["echo \"missing end fence\""] ;; no "```bash" ahead, it's fine as is; let's leave it
:code-segs [{:text "echo \"ok\"\nexit"
:lang "bash"}]}
(split-fn "```bash\necho \"ok\"\nexit\n```\n```bash\necho \"missing end fence\""))))
(testing "edge: pure multiple code fences with no extra text"
(let [{:keys [text-parts code-segs]} (split-fn "```markdown\n1st code snippet with language tag\n```\n```\n2nd code snippet without language tag\n```")]
(is (and (empty? text-parts) (> (count code-segs) 1)) "not pure single code and no mixed content")))
(testing "edge: opening fence without closing fence"
(let [title "```bash\necho \"missing end fence\""
{:keys [text-parts code-segs]} (split-fn title)]
(is (and (= (count text-parts) 1) (not= (first text-parts) title) (empty? code-segs)) "not pure single code and no mixed content")))
(testing "edge: plain text without any code fence"
(is (= {:text-parts ["plain text only"]
:code-segs []}
(split-fn "plain text only"))))
(testing "edge: empty title"
(is (= {:text-parts [""]
:code-segs []}
(split-fn ""))))))
(deftest-async export-files-with-extract-code-snippet
(p/let [file-graph-dir "test/resources/exporter-test-graph"
files (mapv #(path/path-join file-graph-dir %) ["journals/2026_03_01.md"])
conn (db-test/create-conn)
_ (import-files-to-db files conn {:extract-code-snippets? true})
journal-page-eid (d/q '[:find ?p . :where [?p :block/journal-day 20260301]] @conn)
top-blocks (->> (d/q '[:find [?b ...]
:in $ ?page
:where
[?b :block/page ?page]
[?b :block/parent ?page]]
@conn journal-page-eid)
(map #(d/entity @conn %))
(sort-by :block/order)
vec)
get-direct-children (fn [block]
(->> (d/q '[:find [?c ...]
:in $ ?parent
:where [?c :block/parent ?parent]]
@conn (:db/id block))
(map #(d/entity @conn %))))]
(testing "standalone code block with language tag"
(let [b (nth top-blocks 0)]
(is (= "it's an individual code snippet with language tag" (:block/title b))
"Standalone code block title has fences stripped")
(is (= 0 (count (get-direct-children b)))
"Standalone code block has no children")
(is (= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags b))))
"Standalone code block is tagged as Code-block")
(is (= "markdown" (:logseq.property.code/lang b))
"Standalone code block has markdown language property")))
(testing "standalone code block without language tag"
(let [b (nth top-blocks 1)]
(is (= "it's an individual code snippet without language tag" (:block/title b))
"Standalone code block title has fences stripped")
(is (= 0 (count (get-direct-children b)))
"Standalone code block has no children")
(is (= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags b))))
"Standalone code block is tagged as Code-block")
(is (= nil (:logseq.property.code/lang b))
"Standalone code block has no language property")))
(testing "text before code snippet"
(let [b (nth top-blocks 2)
children (get-direct-children b)]
(is (= "before code snippet" (:block/title b))
"Block title has text only without code")
(is (= 1 (count children))
"Block has 1 code child")
(is (= "echo \"ok\"\nexit" (:block/title (first children)))
"Child code block has correct content without fence markers")
(is (= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags (first children)))))
"Child block is tagged as Code-block")
(is (= nil (:logseq.property.code/lang (first children)))
"Child block has no language property")))
(testing "text before and after code snippet"
(let [b (nth top-blocks 3)
children (get-direct-children b)]
(is (= "before code snippet\nafter code snippet" (:block/title b))
"Block title has text only without code")
(is (= 1 (count children))
"Block has 1 code child")
(is (= "echo \"ok\"\nexit" (:block/title (first children)))
"Child code block has correct content without fence markers")
(is (= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags (first children)))))
"Child block is tagged as Code-block")
(is (= "bash" (:logseq.property.code/lang (first children)))
"Child block has bash language property")))
(testing "code snippet before text"
(let [b (nth top-blocks 4)
children (get-direct-children b)]
(is (= "after code snippet" (:block/title b))
"Block title has text only without code")
(is (= 1 (count children))
"Block has 1 code child")
(is (= "echo \"ok\"\nexit" (:block/title (first children)))
"Child code block has correct content without fence markers")
(is (= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags (first children)))))
"Child block is tagged as Code-block")
(is (= "bash" (:logseq.property.code/lang (first children)))
"Child block has bash language property")))
(testing "multiple code snippets mixed with text"
(let [b (nth top-blocks 5)
children (sort-by :block/order (get-direct-children b))]
(is (= "before code snippet\nmiddle\nafter code snippet" (:block/title b))
"Block title has all text parts without code")
(is (= 2 (count children))
"Block has 2 code children")
(is (= "echo \"ok\"\nexit" (:block/title (first children)))
"First child code block has correct content without fence markers")
(is (= "echo \"bye\"\nexit" (:block/title (second children)))
"Second child code block has correct content without fence markers")
(is (every? #(= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags %)))) children)
"Both child blocks are tagged as Code-block")
(is (every? #(= "bash" (:logseq.property.code/lang %)) children)
"Both child blocks have bash language property")))))
(deftest-async export-files-without-extract-code-snippet
(p/let [file-graph-dir "test/resources/exporter-test-graph"
files (mapv #(path/path-join file-graph-dir %) ["journals/2026_03_01.md"])
conn (db-test/create-conn)
_ (import-files-to-db files conn {:extract-code-snippets? false})
journal-page-eid (d/q '[:find ?p . :where [?p :block/journal-day 20260301]] @conn)
top-blocks (->> (d/q '[:find [?b ...]
:in $ ?page
:where
[?b :block/page ?page]
[?b :block/parent ?page]]
@conn journal-page-eid)
(map #(d/entity @conn %))
(sort-by :block/order)
vec)
get-direct-children (fn [block]
(->> (d/q '[:find [?c ...]
:in $ ?parent
:where [?c :block/parent ?parent]]
@conn (:db/id block))
(map #(d/entity @conn %))))]
(testing "standalone code block with language tag is still tagged as Code-block"
(let [b (nth top-blocks 0)]
(is (= "it's an individual code snippet with language tag" (:block/title b))
"Standalone code block title has fences stripped")
(is (= 0 (count (get-direct-children b)))
"Standalone code block has no children")
(is (= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags b))))
"Standalone code block is tagged as Code-block")
(is (= "markdown" (:logseq.property.code/lang b))
"Standalone code block has markdown language property")))
(testing "standalone code block without language tag is still tagged as Code-block"
(let [b (nth top-blocks 1)]
(is (= "it's an individual code snippet without language tag" (:block/title b))
"Standalone code block title has fences stripped")
(is (= 0 (count (get-direct-children b)))
"Standalone code block has no children")
(is (= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags b))))
"Standalone code block is tagged as Code-block")
(is (= nil (:logseq.property.code/lang b))
"Standalone code block has no language property")))
(testing "mixed-content block is NOT extracted into children when extract-code-snippets? is false"
(let [b (nth top-blocks 2)]
(is (= 0 (count (get-direct-children b)))
"Block with text before code has no children extracted")
(is (string/includes? (:block/title b) "```")
"Block title retains raw code fence markup")))
(testing "another mixed-content block is NOT extracted when extract-code-snippets? is false"
(let [b (nth top-blocks 3)]
(is (= 0 (count (get-direct-children b)))
"Block with text surrounding code has no children extracted")
(is (string/includes? (:block/title b) "```")
"Block title retains raw code fence markup")))))
(deftest page-alias-sanitise-for-import
(testing "duplicate-owner alias is dropped and reported"
(let [alias-owners (atom {})
ignored-props (atom [])
pages [{:block/name "p1" :block/alias [{:block/name "shared"}]}
{:block/name "p2" :block/alias [{:block/name "shared"}]}]
result (gp-exporter/sanitize-page-aliases-for-import!
pages alias-owners ignored-props)]
(is (= "p1" (get @alias-owners "shared"))
"first declarer wins ownership")
(is (= 1 (count (filter #(= :alias/duplicate-owner (:reason %)) @ignored-props)))
"duplicate alias is reported in ignored-properties")
(is (nil? (:block/alias (second result)))
"second page's conflicting alias is removed")))
(testing "alias-of-alias is dropped and reported"
(let [alias-owners (atom {})
ignored-props (atom [])
pages [{:block/name "root" :block/alias [{:block/name "mid"}]}
{:block/name "mid" :block/alias [{:block/name "leaf"}]}]
result (gp-exporter/sanitize-page-aliases-for-import!
pages alias-owners ignored-props)]
(is (= 1 (count (filter #(= :alias/alias-owns-aliases (:reason %)) @ignored-props)))
"alias-of-alias is reported in ignored-properties")
(is (nil? (:block/alias (first result)))
"alias pointing to a page that owns aliases is removed")))
(testing "self-alias is dropped and reported"
(let [alias-owners (atom {})
ignored-props (atom [])
pages [{:block/name "self" :block/alias [{:block/name "self"}]}]
result (gp-exporter/sanitize-page-aliases-for-import!
pages alias-owners ignored-props)]
(is (nil? (:block/alias (first result)))
"self-alias declaration is removed")
(is (some #(= :alias/self (:reason %)) @ignored-props)
"self-alias is reported in ignored-properties"))))
(deftest page-alias-sanitise-for-import-cross-file
(testing "source-is-alias caught across files (root->mid in file 1, mid->leaf in file 2)"
(let [alias-owners (atom {})
ignored-props (atom [])]
;; File 1: root claims mid as alias → registers "mid" -> "root"
(gp-exporter/sanitize-page-aliases-for-import!
[{:block/name "root" :block/alias [{:block/name "mid"}]}]
alias-owners ignored-props)
;; File 2: mid tries to declare leaf as its alias — but mid is already an alias
(let [result (gp-exporter/sanitize-page-aliases-for-import!
[{:block/name "mid" :block/alias [{:block/name "leaf"}]}]
alias-owners ignored-props)]
(is (nil? (:block/alias (first result)))
"mid's alias declaration dropped: mid is already an alias")
(is (some #(= :alias/source-is-alias (:reason %)) @ignored-props)
"source-is-alias reason recorded"))))
(testing "alias-owns-aliases caught across files (mid->leaf in file 1, root->mid in file 2)"
(let [alias-owners (atom {})
ignored-props (atom [])]
;; File 1: mid claims leaf as alias → registers "leaf" -> "mid"; mid is now an owner
(gp-exporter/sanitize-page-aliases-for-import!
[{:block/name "mid" :block/alias [{:block/name "leaf"}]}]
alias-owners ignored-props)
;; File 2: root tries to use mid as alias — but mid already owns aliases
(let [result (gp-exporter/sanitize-page-aliases-for-import!
[{:block/name "root" :block/alias [{:block/name "mid"}]}]
alias-owners ignored-props)]
(is (nil? (:block/alias (first result)))
"root's alias pointing to mid dropped: mid already owns aliases")
(is (some #(= :alias/alias-owns-aliases (:reason %)) @ignored-props)
"alias-owns-aliases reason recorded")))))
Reference in New Issue View Git Blame Copy Permalink