(ns ^:node-only logseq.graph-parser.exporter-test (:require ["fs" :as fs] ["os" :as os] ["path" :as node-path] [cljs.test :refer [are deftest is testing]] [clojure.set :as set] [clojure.string :as string] [datascript.core :as d] [logseq.common.config :as common-config] [logseq.common.graph :as common-graph] [logseq.common.path :as path] [logseq.common.util.block-ref :as block-ref] [logseq.common.util.date-time :as date-time-util] [logseq.common.util.page-ref :as page-ref] [logseq.common.uuid :as common-uuid] [logseq.db :as ldb] [logseq.db.common.entity-plus :as entity-plus] [logseq.db.frontend.asset :as db-asset] [logseq.db.frontend.content :as db-content] [logseq.db.frontend.malli-schema :as db-malli-schema] [logseq.db.frontend.property :as db-property] [logseq.db.frontend.rules :as rules] [logseq.db.frontend.validate :as db-validate] [logseq.db.test.helper :as db-test] [logseq.graph-parser.block :as gp-block] [logseq.graph-parser.exporter :as gp-exporter] [logseq.graph-parser.test.docs-graph-helper :as docs-graph-helper] [logseq.graph-parser.test.helper :as test-helper :include-macros true :refer [deftest-async]] [logseq.outliner.db-pipeline :as db-pipeline] [promesa.core :as p])) ;; Helpers ;; ======= ;; some have been copied from db-import script (defn- extract-rules [rules] (rules/extract-rules rules/db-query-dsl-rules rules {:deps rules/rules-dependencies})) (defn- find-block-by-property [db property] (d/q '[:find [?b ...] :in $ ?prop % :where (has-property ?b ?prop)] db property (extract-rules [:has-property]))) (defn- find-block-by-property-value [db property property-value] (->> (d/q '[:find [?b ...] :in $ ?prop ?prop-value % :where (property ?b ?prop ?prop-value)] db property property-value (extract-rules [:property])) first (d/entity db))) (defn- ordered-children [block] (->> (:block/_parent block) (remove :logseq.property/created-from-property) (sort-by :block/order) vec)) (defn- block-tree-with-properties [block] {:title (:block/title block) :properties (dissoc (db-test/readable-properties block) :block/tags) :children (mapv block-tree-with-properties (ordered-children block))}) (defn- find-template-by-title [db title] (some->> (d/q '[:find [?b ...] :in $ ?title :where [?b :block/title ?title] [?b :block/tags :logseq.class/Template]] db title) first (d/entity db))) (defn- template-content-trees [db title] (some->> (find-template-by-title db title) ordered-children (mapv block-tree-with-properties))) (defn- block-status [db content] (:logseq.property/status (db-test/find-block-by-content db content))) (defn- status-content [db content] (db-property/closed-value-content (block-status db content))) (defn- status-closed-value-contents [db] (set (map db-property/closed-value-content (db-property/get-closed-property-values db :logseq.property/status)))) (defn- status-closed-value-content-frequencies [db] (frequencies (map db-property/closed-value-content (db-property/get-closed-property-values db :logseq.property/status)))) (defn- blocks-by-title [db title] (->> (d/q '[:find [?b ...] :in $ ?title :where [?b :block/page] [?b :block/title ?title]] db title) (map #(d/entity db %)))) (defn- task-blocks-by-title [db title] (->> (d/q '[:find [?b ...] :in $ ?title :where [?b :block/page] [?b :block/title ?title] [?b :block/tags :logseq.class/Task]] db title) (map #(d/entity db %)))) (defn- report-retracts-block-uuid? [tx-report block-uuid] (boolean (some (fn [datom] (and (= :block/uuid (:a datom)) (= block-uuid (:v datom)) (false? (:added datom)))) (:tx-data tx-report)))) (defn- imported-favorite-titles [db] (->> (ldb/get-page-blocks db (:db/id (ldb/get-page db common-config/favorites-page-name))) (keep :block/link) (map #(d/entity db (:db/id %))) (map ldb/get-title-with-parents))) (defn- build-graph-files "Given a file graph directory, return all files including assets and adds relative paths on ::rpath since paths are absolute by default and exporter needs relative paths for some operations" [dir*] (let [dir (node-path/resolve dir*)] (->> (common-graph/get-files dir) (concat (when (fs/existsSync (path/path-join dir* "assets")) (common-graph/readdir (path/path-join dir* "assets")))) (mapv #(hash-map :path % ::rpath (node-path/relative dir* %)))))) (defn- (get-in m [:ex-data :error]) ex-data :sci.impl/callstack deref)] (println (string/join "\n" (map #(str (:file %) (when (:line %) (str ":" (:line %))) (when (:sci.impl/f-meta %) (str " calls #'" (get-in % [:sci.impl/f-meta :ns]) "/" (get-in % [:sci.impl/f-meta :name])))) (reverse stack)))) (println (some-> (get-in m [:ex-data :error]) .-stack)))) (when (= :error (:level m)) (js/process.exit 1))) (def default-export-options {;; common options :rpath-key ::rpath :notify-user notify-user :name (:path file)) asset-type (db-asset/asset-path->type (:path file)) {:keys [with-edn-content pdf-annotation?]} (buffer-handler buffer)] (when-not pdf-annotation? (swap! *asset-ids conj asset-id)) (swap! assets assoc asset-name (with-edn-content {:size (.-length buffer) :type asset-type :path (:path file) :checksum checksum :asset-id asset-id})) buffer)) ;; Copied from db-import script and tweaked for an in-memory import (defn- import-file-graph-to-db "Import a file graph dir just like UI does. However, unlike the UI the exporter receives file maps containing keys :path and ::rpath since :path are full paths" [file-graph-dir conn {:keys [assets] :or {assets (atom [])} :as options}] (let [*files (build-graph-files file-graph-dir) config-file (first (filter #(string/ends-with? (:path %) "logseq/config.edn") *files)) _ (assert config-file "No 'logseq/config.edn' found for file graph dir") options' (merge default-export-options {:user-options (merge {:convert-all-tags? false} (dissoc options :assets :verbose)) ;; asset file options : (p/let [doc-options (gp-exporter/build-doc-options (merge {:macros {} :file/name-format :triple-lowbar} (:user-config options)) (merge default-export-options {:user-options (merge {:convert-all-tags? false} (dissoc options :user-config :verbose))} (select-keys options [:verbose]))) files' (mapv #(hash-map :path %) files) _ (gp-exporter/export-doc-files conn files' \n" 1 " DEADLINE: <2026-01-09 Fri +2d>\n" "") extra-lines (generated-extra-lines *seed index) nested-line (when (zero? (next-random! *seed 3)) (str " - nested generated block " index " [[Generated Nested " (next-random! *seed 30) "]]\n"))] (str "- " title-prefix "generated block " index " [[" page-ref "]] ((" ref-id ")) #" tag title-suffix "\n" temporal-line " id:: " id-value "\n" " generated-ref:: [[" page-name "]]\n" " generated-rank:: " (next-random! *seed 100) "\n" extra-lines nested-line))) (defn- generated-md-file-content [*seed file-index page-name block-count] (str (generated-page-preamble *seed page-name) (apply str (map #(generated-md-block *seed (+ (* file-index 100) %)) (range block-count))))) (defn- generated-md-file-graph [seed] (let [*seed (atom seed) pages (map-indexed (fn [index page-name] [(str "pages/" (string/replace (string/lower-case page-name) " " "_") ".md") (generated-md-file-content *seed index page-name (+ 8 (next-random! *seed 8)))]) generated-file-graph-page-names) journals [["journals/2026_01_05.md" (generated-md-file-content *seed 20 "2026_01_05" (+ 8 (next-random! *seed 8)))] ["journals/2026_01_06.md" (generated-md-file-content *seed 21 "2026_01_06" (+ 8 (next-random! *seed 8)))] ["assets/generated.md" "Generated asset content\n"]]] (into {"logseq/config.edn" "{:preferred-format :markdown\n :journal/page-title-format \"yyyy_MM_dd\"}\n"} (concat pages journals)))) (defn- assert-generated-md-file-graph-imports [seed] (let [graph-dir (write-temp-file-graph (generated-md-file-graph seed))] (p/let [conn (db-test/create-conn) _ (db-pipeline/add-listener conn) _ (import-file-graph-to-db graph-dir conn {}) generated-block-count (->> (d/q '[:find [?title ...] :where [?b :block/title ?title]] @conn) (filter #(string/includes? % "generated block")) count) validation-errors (map :entity (:errors (db-validate/validate-local-db! @conn)))] (is (<= 60 generated-block-count) (str "Seed " seed " imports the generated block corpus")) (is (empty? validation-errors) (str "Seed " seed " generated Markdown file graph validates"))))) (defn- assert-generated-md-file-graphs-import [seeds] (p/loop [remaining-seeds (seq seeds)] (when remaining-seeds (p/let [_ (assert-generated-md-file-graph-imports (first remaining-seeds))] (p/recur (next remaining-seeds)))))) ;; Tests ;; ===== (deftest-async import-block-with-journal-ref-and-time-property-value (p/let [file (write-temp-graph-file "journals/2023_06_21.md" "- DONE foo bar #sometag1 #sometag2\n completed:: [[Sun, 06.08.2023]] *14:42*\n") conn (db-test/create-conn) _ (db-pipeline/add-listener conn) _ (import-files-to-db [file] conn {:user-config {:journal/page-title-format "EEE, dd.MM.yyyy"}})] (is (some? (db-test/find-block-by-content @conn #"foo bar")) "Block with a journal reference plus time in a property value imports") (is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn)))) "Imported graph validates"))) (deftest-async import-quote-with-email-address (p/let [file (write-temp-graph-file "pages/email.md" "- > \"CachyOS \"\n") conn (db-test/create-conn) _ (db-pipeline/add-listener conn) _ (import-files-to-db [file] conn {})] (is (= "\"CachyOS \"" (:block/title (db-test/find-block-by-content @conn #"CachyOS"))) "Email addresses inside quotes are preserved during import") (is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn)))) "Imported graph validates"))) (deftest-async import-org-page-title-when-property-appears-in-middle (p/let [file (write-temp-graph-file "pages/20230410145300-end_to_end_note.org" ":PROPERTIES: :ID: c537c812-1ec9-4f13-adaf-1a39fd7da967 :END: #+title: end_to_end_note #+date: <2023-04-10 Mon 14:53> #+filetags: :PUBLIC: abc #+hugo: more 123 ") conn (db-test/create-conn) _ (db-pipeline/add-listener conn) _ (import-files-to-db [file] conn {})] (is (some? (db-test/find-page-by-title @conn "end_to_end_note")) "Org #+title is imported when another property appears in the middle of the file") (is (nil? (db-test/find-page-by-title @conn "20230410145300-end_to_end_note")) "Importer should not fall back to the org-roam file stem") (is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn)))) "Imported graph validates"))) (deftest-async import-empty-journal-file (p/let [file (write-temp-graph-file "journals/2025_11_11.md" "\n") conn (db-test/create-conn) _ (db-pipeline/add-listener conn) _ (import-files-to-db [file] conn {})] (is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn)))) "Empty imported files do not transact nil block refs"))) (deftest-async import-repeated-deadline-and-scheduled (p/let [file (write-temp-graph-file "pages/repeated-tasks.md" (str "- TODO wish [[name]] a happy birthday\n" " SCHEDULED: <2025-11-01 Sat 08:00 .+1y>\n" "- TODO prepare weekly report\n" " DEADLINE: <2025-11-07 Fri +2w>\n" "- TODO plan release\n" " DEADLINE: <2025-11-08 Sat>\n" " SCHEDULED: <2025-11-07 Fri .+1w>\n")) conn (db-test/create-conn) _ (db-pipeline/add-listener conn) _ (import-files-to-db [file] conn {})] (let [birthday-properties (db-test/readable-properties (db-test/find-block-by-content @conn #"happy birthday")) report-properties (db-test/readable-properties (db-test/find-block-by-content @conn #"weekly report")) mixed-properties (db-test/readable-properties (db-test/find-block-by-content @conn #"plan release")) birthday-scheduled (:logseq.property/scheduled birthday-properties) birthday-date (js/Date. birthday-scheduled)] (is (= 20251101 (date-time-util/ms->journal-day birthday-scheduled)) "Repeated scheduled timestamp keeps its scheduled date") (is (= [8 0] [(.getHours birthday-date) (.getMinutes birthday-date)]) "Repeated scheduled timestamp keeps its time") (is (= {:logseq.property.repeat/repeated? true :logseq.property.repeat/temporal-property :logseq.property/scheduled :logseq.property.repeat/repeat-type :logseq.property.repeat/repeat-type.dotted-plus :logseq.property.repeat/recur-frequency 1 :logseq.property.repeat/recur-unit :logseq.property.repeat/recur-unit.year} (select-keys birthday-properties [:logseq.property.repeat/repeated? :logseq.property.repeat/temporal-property :logseq.property.repeat/repeat-type :logseq.property.repeat/recur-frequency :logseq.property.repeat/recur-unit])) "Repeated scheduled timestamp keeps its repeat properties including the `.+` cookie kind") (is (= {:logseq.property/deadline 20251107 :logseq.property.repeat/repeated? true :logseq.property.repeat/temporal-property :logseq.property/deadline :logseq.property.repeat/repeat-type :logseq.property.repeat/repeat-type.plus :logseq.property.repeat/recur-frequency 2 :logseq.property.repeat/recur-unit :logseq.property.repeat/recur-unit.week} (-> report-properties (update :logseq.property/deadline date-time-util/ms->journal-day) (select-keys [:logseq.property/deadline :logseq.property.repeat/repeated? :logseq.property.repeat/temporal-property :logseq.property.repeat/repeat-type :logseq.property.repeat/recur-frequency :logseq.property.repeat/recur-unit]))) "Repeated deadline timestamp keeps its repeat properties including the `+` cookie kind") (is (= {:logseq.property/deadline 20251108 :logseq.property/scheduled 20251107 :logseq.property.repeat/repeated? true :logseq.property.repeat/temporal-property :logseq.property/scheduled :logseq.property.repeat/repeat-type :logseq.property.repeat/repeat-type.dotted-plus :logseq.property.repeat/recur-frequency 1 :logseq.property.repeat/recur-unit :logseq.property.repeat/recur-unit.week} (-> mixed-properties (update :logseq.property/deadline date-time-util/ms->journal-day) (update :logseq.property/scheduled date-time-util/ms->journal-day) (select-keys [:logseq.property/deadline :logseq.property/scheduled :logseq.property.repeat/repeated? :logseq.property.repeat/temporal-property :logseq.property.repeat/repeat-type :logseq.property.repeat/recur-frequency :logseq.property.repeat/recur-unit]))) "Mixed deadline and scheduled timestamps keep both dates and the repeated temporal property") (is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn)))) "Imported graph validates")))) (deftest-async import-preserves-legacy-task-markers-as-status-choices (p/let [file (write-temp-graph-file "pages/tasks.md" (str "- TODO\n" "- DONE\n" "- I recorded a [[voice note]].\n" " - TODO\n" "- TODO todo item\n" "- LATER later item\n" "- NOW now item\n" "- DOING doing item\n" "- WAIT waiting item\n" "- WAITING waiting full item\n" "- IN-PROGRESS in-progress item\n" "- DONE done item\n")) conn (db-test/create-conn) _ (db-pipeline/add-listener conn) _ (import-files-to-db [file] conn {})] (is (= :logseq.property/status.todo (:db/ident (block-status @conn "todo item"))) "TODO still imports to the built-in Todo status") (is (= :logseq.property/status.doing (:db/ident (block-status @conn "doing item"))) "DOING still imports to the built-in Doing status") (is (= :logseq.property/status.done (:db/ident (block-status @conn "done item"))) "DONE still imports to the built-in Done status") (is (= {:logseq.property/status.todo 2 :logseq.property/status.done 1} (frequencies (map #(-> % db-test/readable-properties :logseq.property/status) (task-blocks-by-title @conn "")))) "Built-in task markers without titles still import as tasks") (let [nested-empty-task (first (ordered-children (db-test/find-block-by-content @conn #"recorded")))] (is (= "" (:block/title nested-empty-task)) "Nested nameless task keeps an empty title") (is (= {:logseq.property/status :logseq.property/status.todo :block/tags [:logseq.class/Task]} (select-keys (db-test/readable-properties nested-empty-task) [:logseq.property/status :block/tags])) "Nested nameless TODO keeps its task properties")) (is (= :logseq.property/status.todo (:db/ident (block-status @conn "later item"))) "LATER imports to the built-in Todo status") (is (= :logseq.property/status.doing (:db/ident (block-status @conn "now item"))) "NOW imports to the built-in Doing status") (is (= "WAIT" (status-content @conn "waiting item")) "WAIT imports as its own status choice") (is (= "WAITING" (status-content @conn "waiting full item")) "WAITING imports as its own status choice") (is (= "IN-PROGRESS" (status-content @conn "in-progress item")) "IN-PROGRESS imports as its own status choice") (is (set/subset? #{"WAIT" "WAITING" "IN-PROGRESS"} (status-closed-value-contents @conn)) "Custom imported markers are added to Status closed values") (is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn)))) "Imported graph validates"))) (deftest-async import-custom-task-marker-across-multiple-files (p/let [first-file (write-temp-graph-file "pages/custom-status-a.md" "- WAITING first custom status item\n") second-file (write-temp-graph-file "pages/custom-status-b.md" "- WAITING second custom status item\n") conn (db-test/create-conn) _ (db-pipeline/add-listener conn) _ (import-files-to-db [first-file second-file] conn {})] (is (= "WAITING" (status-content @conn "first custom status item")) "Custom status marker imports from the first file") (is (= "WAITING" (status-content @conn "second custom status item")) "Custom status marker imports from the second file") (is (= 1 (get (status-closed-value-content-frequencies @conn) "WAITING")) "Custom status closed value is shared across imported files") (is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn)))) "Imported graph validates"))) (deftest-async import-repairs-duplicated-block-ids (let [duplicated-uuid #uuid "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"] (p/let [file (write-temp-graph-file "pages/duplicated-ids.md" (str "- First duplicated id\n" " id:: " duplicated-uuid "\n" "- Second duplicated id\n" " id:: " duplicated-uuid "\n")) conn (db-test/create-conn) _ (db-pipeline/add-listener conn) _ (import-files-to-db [file] conn {}) first-block (db-test/find-block-by-content @conn "First duplicated id") second-block (db-test/find-block-by-content @conn "Second duplicated id")] (is (= duplicated-uuid (:block/uuid first-block)) "The first imported block keeps the original id") (is (some? (:block/uuid second-block)) "The duplicate imported block gets a replacement id") (is (not= duplicated-uuid (:block/uuid second-block)) "The duplicate imported block does not keep the conflicting id") (is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn)))) "Imported graph validates")))) (deftest-async import-removes-pre-block-marker-and-missing-block-refs (let [missing-uuid #uuid "11111111-1111-1111-1111-111111111111" missing-embed-uuid #uuid "55555555-5555-5555-5555-555555555555" target-uuid #uuid "22222222-2222-2222-2222-222222222222" empty-title-property-uuid #uuid "33333333-3333-3333-3333-333333333333" empty-title-parent-uuid #uuid "44444444-4444-4444-4444-444444444444"] (p/let [source-file (write-temp-graph-file "pages/A.md" (str "Plain pre-block\n" "- Missing ref ((" missing-uuid "))\n" "- {{embed ((" missing-embed-uuid "))}}\n" "- Existing ref ((" target-uuid "))\n" "- ((" empty-title-property-uuid "))\n" " heading:: true\n" " background-color:: yellow\n" "- ((" empty-title-parent-uuid "))\n" " - Child survives\n")) target-file (write-temp-graph-file "pages/Z.md" (str "- Target block\n" " id:: " target-uuid "\n")) conn (db-test/create-conn) _ (db-pipeline/add-listener conn) _ (import-files-to-db [source-file target-file] conn {}) missing-block (db-test/find-block-by-content @conn #"Missing ref") existing-block (db-test/find-block-by-content @conn #"Existing ref") target-block (db-test/find-block-by-content @conn "Target block") empty-title-blocks (blocks-by-title @conn "") empty-title-property-block (some #(when (:logseq.property/heading %) %) empty-title-blocks) empty-title-parent-block (some #(when (some (fn [child] (= "Child survives" (:block/title child))) (ordered-children %)) %) empty-title-blocks)] (is (empty? (filter #(= :block/pre-block? (:a %)) (d/datoms @conn :eavt))) "Legacy pre-block markers are never transacted") (is (= "Missing ref" (:block/title missing-block)) "Missing OG block refs are removed from imported content") (is (empty? (:block/refs missing-block)) "Missing OG block refs are removed from imported refs") (is (nil? (d/entity @conn [:block/uuid missing-uuid])) "Missing OG block refs do not leave placeholder entities") (is (nil? (d/entity @conn [:block/uuid missing-embed-uuid])) "Missing OG block embeds do not leave placeholder entities") (is (empty? (d/q '[:find ?b :where [?b :block/link ?target] [(missing? $ ?target :block/uuid)]] @conn)) "Missing OG block embeds do not leave dangling block links") (is (nil? (d/entity @conn [:block/uuid empty-title-property-uuid])) "Missing OG block refs in empty-title property blocks do not leave placeholder entities") (is (nil? (d/entity @conn [:block/uuid empty-title-parent-uuid])) "Missing OG block refs in empty-title parent blocks do not leave placeholder entities") (is (= 3 (count empty-title-blocks)) "Blocks whose titles become empty after cleanup are preserved") (is (true? (:logseq.property/heading empty-title-property-block)) "Empty-title blocks keep imported heading properties") (is (= "yellow" (:logseq.property/background-color (db-test/readable-properties empty-title-property-block))) "Empty-title blocks keep imported background colors") (is (= ["Child survives"] (mapv :block/title (ordered-children empty-title-parent-block))) "Empty-title parent blocks keep their children") (is (= [(:db/id target-block)] (mapv :db/id (:block/refs existing-block))) "Existing block refs are preserved, including forward refs from later files") (is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn)))) "Imported graph validates")))) (deftest-async import-generated-markdown-file-graph (assert-generated-md-file-graphs-import generated-file-graph-test-seeds)) (deftest-async ^:integration import-generated-markdown-file-graph-fuzz (assert-generated-md-file-graphs-import (range 1 101))) (deftest-async export-doc-files-propagates-missing-block-ref-cleanup-report (let [missing-uuid #uuid "55555555-5555-5555-5555-555555555555" tx-reports (atom [])] (p/let [file (write-temp-graph-file "pages/A.md" (str "- Missing ref ((" missing-uuid "))\n")) conn (db-test/create-conn) _ (db-pipeline/add-listener conn) doc-options (gp-exporter/build-doc-options {:macros {} :file/name-format :triple-lowbar} (merge default-export-options {:user-options {:convert-all-tags? false} :on-tx-report #(swap! tx-reports conj %)})) _ (gp-exporter/export-doc-files conn [{:path file}] (gp-exporter/export-doc-files conn [{:path first-file} {:path second-file}] (p/let [conn (db-test/create-conn) assets (atom []) {:keys [import-state]} (import-file-graph-to-db graph-dir conn {:assets assets})] (is (= [{:reason "No asset data found for this asset path" :path "../assets/missing-paper.pdf" :location {:block "Missing local PDF [paper](../assets/missing-paper.pdf)"}}] @(:ignored-assets import-state)) "Missing local PDF asset links are reported through ignored assets") (is (empty? @console-errors) "Missing local PDF asset links are ignored without noisy console errors") (is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn)))) "Imported graph validates")) (p/finally (fn [_] (set! (.-write (.-stderr js/process)) original-stderr-write)))))) (deftest extract-template-blocks (let [page-uuid (random-uuid) parent-uuid (random-uuid) child-uuid (random-uuid) include-children-only-uuid (random-uuid) child-only-1-uuid (random-uuid) child-only-2-uuid (random-uuid) blocks [{:block/uuid parent-uuid :block/title "source parent" :block/page [:block/uuid page-uuid] :block/parent {:block/uuid page-uuid} :block/order "a" :block/properties {:template " trimmed template " :name ""} :block/properties-text-values {:template " trimmed template " :name ""} :block/properties-order [:template :name]} {:block/uuid child-uuid :block/title "child" :block/page [:block/uuid page-uuid] :block/parent [:block/uuid parent-uuid] :block/order "b" :block/properties {:template "nested child" :name "child default"} :block/properties-text-values {:template "nested child" :name "child default"} :block/properties-order [:template :name]} {:block/uuid include-children-only-uuid :block/title "exclude source block" :block/page [:block/uuid page-uuid] :block/parent {:block/uuid page-uuid} :block/order "c" :block/properties {:template "children only" :template-including-parent false} :block/properties-text-values {:template "children only" :template-including-parent "false"} :block/properties-order [:template :template-including-parent]} {:block/uuid child-only-1-uuid :block/title "first child" :block/page [:block/uuid page-uuid] :block/parent [:block/uuid include-children-only-uuid] :block/order "d"} {:block/uuid child-only-2-uuid :block/title "second child" :block/page [:block/uuid page-uuid] :block/parent [:block/uuid include-children-only-uuid] :block/order "e"}] {:keys [blocks preserve-empty-properties-uuids]} (#'gp-exporter/handle-template-blocks blocks)] (testing "template roots replace source blocks" (is (= ["trimmed template" "source parent" "nested child" "child" "children only" "first child" "second child"] (mapv :block/title blocks))) (is (= #{parent-uuid child-uuid include-children-only-uuid child-only-1-uuid child-only-2-uuid} (set/intersection preserve-empty-properties-uuids #{parent-uuid child-uuid include-children-only-uuid child-only-1-uuid child-only-2-uuid})))) (testing "template roots use trimmed names and include parent content when configured" (is (= #{"trimmed template" "nested child" "children only"} (->> blocks (filter #(some #{:logseq.class/Template} (:block/tags %))) (map :block/title) set))) (is (= ["source parent"] (->> blocks (remove #(some #{:logseq.class/Template} (:block/tags %))) (filter #(= [:block/uuid parent-uuid] (:block/parent %))) (map :block/title)))) (is (= 2 (count (set/difference preserve-empty-properties-uuids #{parent-uuid child-uuid include-children-only-uuid child-only-1-uuid child-only-2-uuid}))) "in-place template content blocks are marked to preserve empty properties")))) (deftest-async ^:integration export-docs-graph-with-convert-all-tags (p/let [file-graph-dir "test/resources/docs-0.10.12" _ (docs-graph-helper/clone-docs-repo-if-not-exists file-graph-dir "v0.10.12") start-time (cljs.core/system-time) conn (db-test/create-conn) _ (db-pipeline/add-listener conn) {:keys [import-state]} (import-file-graph-to-db file-graph-dir conn {:convert-all-tags? true}) end-time (cljs.core/system-time)] ;; Add multiplicative factor for CI as it runs about twice as slow (let [max-time (-> 25 (* (if js/process.env.CI 2 1)))] (is (< (-> end-time (- start-time) (/ 1000)) max-time) (str "Importing large graph takes less than " max-time "s"))) (is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn)))) "Created graph has no validation errors") (is (= 0 (count @(:ignored-properties import-state))) "No ignored properties") (is (= 0 (count @(:ignored-assets import-state))) "No ignored assets") (is (= [] (->> (d/q '[:find (pull ?b [:block/title {:block/tags [:db/ident]}]) :where [?b :block/tags :logseq.class/Tag]] @conn) (map first) (remove #(= [{:db/ident :logseq.class/Tag}] (:block/tags %))))) "All classes only have :logseq.class/Tag as their tag (and don't have Page)"))) (deftest-async import-linked-file-pdf-annotations (let [annotation-id #uuid "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb" area-annotation-id #uuid "cccccccc-cccc-cccc-cccc-cccccccccccc" dir (fs/mkdtempSync (node-path/join (os/tmpdir) "logseq-graph-parser-test-")) external-pdf-path (node-path/join dir "external/Linked Paper.pdf") graph-dir (node-path/join dir "graph") area-image-stamp "area-stamp" encoded-pdf-uri (str "file://" (string/replace external-pdf-path " " "%20"))] (fs/mkdirSync (node-path/dirname external-pdf-path) #js {:recursive true}) (fs/writeFileSync external-pdf-path "pdf") (doseq [[relative-path content] {"logseq/config.edn" "{}" "pages/source.md" (str "- ![Linked Paper.pdf](" encoded-pdf-uri ")\n") "pages/hls__Linked Paper.md" (str "file:: [Linked Paper.pdf](" encoded-pdf-uri ")\n" "file-path:: " encoded-pdf-uri "\n\n" "- External highlight from linked pdf\n" " ls-type:: annotation\n" " hl-page:: 3\n" " hl-color:: yellow\n" " id:: " annotation-id "\n" "- External area highlight from linked pdf\n" " ls-type:: annotation\n" " hl-page:: 4\n" " hl-color:: yellow\n" " id:: " area-annotation-id "\n") "assets/Linked Paper.edn" (str "{:highlights [{:id #uuid \"" annotation-id "\"," " :page 3," " :position {:bounding {:x1 1 :y1 2 :x2 3 :y2 4 :width 10 :height 20}," " :rects ()," " :page 3}," " :content {:text \"External highlight from linked pdf\"}," " :properties {:color \"yellow\"}}" " {:id #uuid \"" area-annotation-id "\"," " :page 4," " :position {:bounding {:x1 11 :y1 12 :x2 13 :y2 14 :width 10 :height 20}," " :rects ()," " :page 4}," " :content {:image \"" area-image-stamp "\"}," " :properties {:color \"yellow\"}}]}") (str "assets/Linked Paper/4_" area-annotation-id "_" area-image-stamp ".png") "png"}] (let [file-path (node-path/join graph-dir relative-path)] (fs/mkdirSync (node-path/dirname file-path) #js {:recursive true}) (fs/writeFileSync file-path content))) (p/let [conn (db-test/create-conn) assets (atom []) {:keys [import-state]} (import-file-graph-to-db graph-dir conn {:assets assets}) asset (db-test/find-block-by-content @conn "Linked Paper") annotation (db-test/find-block-by-content @conn "External highlight from linked pdf") area-annotation (db-test/find-block-by-content @conn "External area highlight from linked pdf")] (is (some? asset) "Linked file PDF imports as an external Asset") (is (= {:block/tags [:logseq.class/Asset] :logseq.property.asset/type "pdf" :logseq.property.asset/external-url encoded-pdf-uri} (select-keys (db-test/readable-properties asset) [:block/tags :logseq.property.asset/type :logseq.property.asset/external-url])) "Linked file PDF keeps the file URI as external asset metadata") (is (= {:block/tags [:logseq.class/Pdf-annotation] :logseq.property/asset "Linked Paper" :logseq.property.pdf/hl-page 3} (select-keys (db-test/readable-properties annotation) [:block/tags :logseq.property/asset :logseq.property.pdf/hl-page])) "Linked file PDF annotations import and point at the external Asset") (is (= {:block/tags [:logseq.class/Pdf-annotation] :logseq.property/asset "Linked Paper" :logseq.property.pdf/hl-page 4 :logseq.property.pdf/hl-image "pdf area highlight" :logseq.property.pdf/hl-type :area} (select-keys (db-test/readable-properties area-annotation) [:block/tags :logseq.property/asset :logseq.property.pdf/hl-page :logseq.property.pdf/hl-image :logseq.property.pdf/hl-type])) "Linked file PDF area highlights import their image assets") (is (= 0 (count @(:ignored-assets import-state))) "No ignored assets")))) (deftest-async import-linked-file-pdf-annotations-with-uppercase-extension (let [annotation-id #uuid "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb" dir (fs/mkdtempSync (node-path/join (os/tmpdir) "logseq-graph-parser-test-")) external-pdf-path (node-path/join dir "external/Linked Paper.PDF") graph-dir (node-path/join dir "graph") encoded-pdf-uri (str "file://" (string/replace external-pdf-path " " "%20"))] (fs/mkdirSync (node-path/dirname external-pdf-path) #js {:recursive true}) (fs/writeFileSync external-pdf-path "pdf") (doseq [[relative-path content] {"logseq/config.edn" "{}" "pages/source.md" (str "- ![Linked Paper.PDF](" encoded-pdf-uri ")\n") "pages/hls__Linked Paper.md" (str "file:: [Linked Paper.PDF](" encoded-pdf-uri ")\n" "file-path:: " encoded-pdf-uri "\n\n" "- External highlight from linked pdf\n" " ls-type:: annotation\n" " hl-page:: 3\n" " hl-color:: yellow\n" " id:: " annotation-id "\n") "assets/Linked Paper.edn" (str "{:highlights [{:id #uuid \"" annotation-id "\"," " :page 3," " :position {:bounding {:x1 1 :y1 2 :x2 3 :y2 4 :width 10 :height 20}," " :rects ()," " :page 3}," " :content {:text \"External highlight from linked pdf\"}," " :properties {:color \"yellow\"}}]}")}] (let [file-path (node-path/join graph-dir relative-path)] (fs/mkdirSync (node-path/dirname file-path) #js {:recursive true}) (fs/writeFileSync file-path content))) (p/let [conn (db-test/create-conn) assets (atom []) {:keys [import-state]} (import-file-graph-to-db graph-dir conn {:assets assets}) asset (db-test/find-block-by-content @conn "Linked Paper") annotation (db-test/find-block-by-content @conn "External highlight from linked pdf")] (is (some? asset) "Linked file PDF imports as an external Asset") (is (= {:block/tags [:logseq.class/Asset] :logseq.property.asset/type "pdf" :logseq.property.asset/external-url encoded-pdf-uri} (select-keys (db-test/readable-properties asset) [:block/tags :logseq.property.asset/type :logseq.property.asset/external-url])) "Linked file PDF keeps the file URI as external asset metadata") (is (= {:block/tags [:logseq.class/Pdf-annotation] :logseq.property/asset "Linked Paper" :logseq.property.pdf/hl-page 3} (select-keys (db-test/readable-properties annotation) [:block/tags :logseq.property/asset :logseq.property.pdf/hl-page])) "Linked file PDF annotations import and keep highlight positions from the EDN file") (is (= 0 (count @(:ignored-assets import-state))) "No ignored assets")))) (deftest-async ^:integration import-large-flat-file-without-stack-overflow (p/let [file (write-temp-graph-file "pages/large.md" (apply str (map #(str "- large line " % " #tag\n") (range 45000)))) conn (db-test/create-conn) _ (db-pipeline/add-listener conn) _ (import-files-to-db [file] conn {:convert-all-tags? true})] (is (= 45000 (->> (d/q '[:find [?title ...] :where [?b :block/title ?title]] @conn) (filter #(string/starts-with? % "large line ")) count)) "Large flat files import without overflowing the stack") (is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn)))) "Imported graph validates"))) (deftest-async export-basic-graph-with-convert-all-tags ;; This graph will contain basic examples of different features to import (p/let [file-graph-dir "test/resources/exporter-test-graph" conn (db-test/create-conn) ;; Calculate refs like frontend _ (db-pipeline/add-listener conn) assets (atom []) {:keys [import-state]} (import-file-graph-to-db file-graph-dir conn {:assets assets :convert-all-tags? true})] (testing "whole graph" (is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn)))) "Created graph has no validation errors") ;; Counts ;; Includes journals as property values e.g. :logseq.property/deadline (is (= 34 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Journal]] @conn)))) (is (= 9 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Asset]] @conn)))) (is (= 6 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Task]] @conn)))) (is (= 4 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Query]] @conn)))) (is (= 2 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Card]] @conn)))) (is (= 1 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Cards]] @conn)))) (is (= 2 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Code-block]] @conn)))) (is (= 1 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Math-block]] @conn)))) (is (= 9 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Template]] @conn)))) (is (= 6 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Quote-block]] @conn)))) (is (= 8 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Pdf-annotation]] @conn)))) ;; Properties and tags aren't included in this count as they aren't a Page (is (= 13 (->> (d/q '[:find [?b ...] :where [?b :block/title] [_ :block/page ?b] (not [?b :logseq.property/built-in?])] @conn) (map #(d/entity @conn %)) (filter ldb/internal-page?) #_(map #(select-keys % [:block/title :block/tags])) count)) "Correct number of pages with block content") (is (= 16 (->> @conn (d/q '[:find [?ident ...] :where [?b :block/tags :logseq.class/Tag] [?b :db/ident ?ident] (not [?b :logseq.property/built-in?])]) count)) "Correct number of user classes") (is (= 0 (count @(:ignored-properties import-state))) "No ignored properties") (is (= 0 (count @(:ignored-assets import-state))) "No ignored assets") (is (= 1 (count @(:ignored-files import-state))) "Ignore .edn for now") ;; 2 zotero pdf are external files so not counted here (is (= 7 (count @assets)))) (testing "logseq files" (is (= ".foo {}\n" (ffirst (d/q '[:find ?content :where [?b :file/path "logseq/custom.css"] [?b :file/content ?content]] @conn)))) (is (= "logseq.api.show_msg('hello good sir!');\n" (ffirst (d/q '[:find ?content :where [?b :file/path "logseq/custom.js"] [?b :file/content ?content]] @conn))))) (testing "favorites" (is (= #{"Interstellar" "some page" "new page" "n1/x/y"} (set (imported-favorite-titles @conn))))) (testing "user properties" (is (= 23 (->> @conn (d/q '[:find [(pull ?b [:db/ident]) ...] :where [?b :block/tags :logseq.class/Property]]) (remove #(db-malli-schema/internal-ident? (:db/ident %))) count)) "Correct number of user properties") (is (= #{{:db/ident :user.property/prop-bool :logseq.property/type :checkbox} {:db/ident :user.property/prop-string :logseq.property/type :default} {:db/ident :user.property/prop-num :logseq.property/type :number} {:db/ident :user.property/sameas :logseq.property/type :url} {:db/ident :user.property/rangeincludes :logseq.property/type :node} {:db/ident :user.property/startedat :logseq.property/type :date}} (->> @conn (d/q '[:find [(pull ?b [:db/ident :logseq.property/type]) ...] :where [?b :block/tags :logseq.class/Property]]) (filter #(contains? #{:prop-bool :prop-string :prop-num :rangeincludes :sameas :startedat} (keyword (name (:db/ident %))))) set)) "Main property types have correct inferred :type") (is (= :default (:logseq.property/type (d/entity @conn :user.property/description))) "Property value consisting of text and refs is inferred as :default") (is (= :url (:logseq.property/type (d/entity @conn :user.property/url))) "Property value with a macro correctly inferred as :url") (is (= {:user.property/prop-bool true :user.property/prop-num 5 :user.property/prop-string "woot"} (db-test/readable-properties (db-test/find-block-by-content @conn "b1"))) "Basic block has correct properties") (is (= #{"prop-num" "prop-string" "prop-bool"} (->> (db-test/find-block-by-content @conn "b1") :block/refs (map :block/title) set)) "Block with properties has correct refs") (is (= {:user.property/prop-num2 10 :block/tags [:logseq.class/Page]} (db-test/readable-properties (db-test/find-page-by-title @conn "new page"))) "New page has correct properties") (is (= {:user.property/prop-bool true :user.property/prop-num 5 :user.property/prop-string "yeehaw" :block/tags [:logseq.class/Page :user.class/SomeNamespace]} (db-test/readable-properties (db-test/find-page-by-title @conn "some page"))) "Existing page has correct properties") (is (= {:user.property/rating 5.5} (db-test/readable-properties (db-test/find-block-by-content @conn ":rating float"))) "Block with float property imports as a float") (is (= [] (->> (d/q '[:find (pull ?b [:block/title {:block/tags [:db/ident]}]) :where [?b :block/tags :logseq.class/Property]] @conn) (map first) (remove #(= [{:db/ident :logseq.class/Property}] (:block/tags %))))) "All properties only have :logseq.class/Property as their tag (and don't have Page)")) (testing "built-in properties" (is (= [(:db/id (db-test/find-block-by-content @conn "original block"))] (mapv :db/id (:block/refs (db-test/find-block-by-content @conn #"ref to")))) "block with a block-ref has correct :block/refs") (is (= "ref to [[65cbb772-fb79-462d-87c8-6f0dad751dee]]" (:block/title (db-test/find-block-by-content @conn #"ref to"))) "block-ref ((uuid)) is converted to page-ref [[uuid]] in block title on import") (is (= 20221126 (-> (db-test/readable-properties (db-test/find-block-by-content @conn "only deadline")) :logseq.property/deadline date-time-util/ms->journal-day)) "deadline block has correct journal as property value") (is (= {:logseq.property/scheduled 20221125 :logseq.property/deadline 20221125} (-> (db-test/readable-properties (db-test/find-block-by-content @conn #"deadline and scheduled")) (select-keys [:logseq.property/scheduled :logseq.property/deadline]) (update-vals date-time-util/ms->journal-day))) "scheduled block converted to correct deadline") (is (= 1 (count (d/q '[:find [(pull ?b [*]) ...] :in $ ?content :where [?b :block/title ?content]] @conn "Apr 1st, 2024"))) "Only one journal page exists when deadline is on same day as journal") (is (= {:logseq.property/priority :logseq.property/priority.high} (db-test/readable-properties (db-test/find-block-by-content @conn "high priority"))) "priority block has correct property") (is (= {:logseq.property/status :logseq.property/status.doing :logseq.property/priority :logseq.property/priority.medium :block/tags [:logseq.class/Task]} (db-test/readable-properties (db-test/find-block-by-content @conn "status test"))) "status block has correct task properties and class") (let [empty-title-task (first (task-blocks-by-title @conn ""))] (is (= {:logseq.property/status :logseq.property/status.todo :block/tags [:logseq.class/Task]} (select-keys (db-test/readable-properties empty-title-task) [:logseq.property/status :block/tags])) "Empty-title TODO from file graph imports as a task")) (is (= #{:logseq.property/status :block/tags} (set (keys (db-test/readable-properties (db-test/find-block-by-content @conn "old todo block"))))) "old task properties like 'todo' are ignored") (is (= {:logseq.property/order-list-type "number"} (db-test/readable-properties (db-test/find-block-by-content @conn "list one"))) "numered block has correct property") (is (= #{"gpt"} (:block/alias (db-test/readable-properties (db-test/find-page-by-title @conn "chat-gpt")))) "alias set correctly") (is (= ["y"] (->> (d/q '[:find [?b ...] :where [?b :block/title "y"] [?b :block/parent]] @conn) first (d/entity @conn) :block/alias (map :block/title))) "alias set correctly on namespaced page") (is (= {:logseq.property.linked-references/includes #{"Oct 9th, 2024"} :logseq.property.linked-references/excludes #{"ref2"}} (select-keys (db-test/readable-properties (db-test/find-page-by-title @conn "chat-gpt")) [:logseq.property.linked-references/excludes :logseq.property.linked-references/includes])) "linked ref filters set correctly")) (testing "built-in classes and their properties" ;; Queries (is (= {:logseq.property.table/sorting [{:id :user.property/prop-num, :asc? false}] :logseq.property.view/type :logseq.property.view/type.table :logseq.property.table/ordered-columns [:block/title :user.property/prop-string :user.property/prop-num] :logseq.property/query "(property :prop-string)" :block/tags [:logseq.class/Query]} (db-test/readable-properties (find-block-by-property-value @conn :logseq.property/query "(property :prop-string)"))) "simple query block has correct query properties") (is (= "For example, here's a query with title text:" (:block/title (db-test/find-block-by-content @conn #"query with title text"))) "Text around a simple query block is set as a query's title") (is (= {:logseq.property.view/type :logseq.property.view/type.list :logseq.property/query "{:query (task todo doing)}" :block/tags [:logseq.class/Query] :logseq.property.table/ordered-columns [:block/title]} (db-test/readable-properties (db-test/find-block-by-content @conn #"tasks with todo"))) "Advanced query has correct query properties") (is (= "tasks with todo and doing" (:block/title (db-test/find-block-by-content @conn #"tasks with todo"))) "Advanced query has custom title migrated") ;; Card (is (= {:block/tags [:logseq.class/Card]} (db-test/readable-properties (db-test/find-block-by-content @conn "card 1"))) "None of the card properties are imported since they are deprecated") ;; Cards (flashcard browser) (is (= {:block/tags [:logseq.class/Cards] :logseq.property/query "(tags #Card)"} (db-test/readable-properties (find-block-by-property-value @conn :logseq.property/query "(tags #Card)"))) "cards macro block has correct Cards class and query property") ;; Math blocks (is (= {:block/tags [:logseq.class/Math-block] :logseq.property.node/display-type :math} (db-test/readable-properties (db-test/find-block-by-content @conn "E=mc^2"))) "Math block has correct Math-block class and display-type") (is (= "E=mc^2" (:block/title (db-test/find-block-by-content @conn "E=mc^2"))) "Math block title has delimiters stripped") ;; Templates (is (= #{"meeting" "title-only-no-children" "properties-only-no-children" "title-only-with-children" "empty-title-with-children" "children-only" "nested-father" "nested-child-1" "nested-child-2"} (->> (d/q '[:find [?title ...] :where [?b :block/tags :logseq.class/Template] [?b :block/title ?title]] @conn) set)) "All template definitions are imported as Template blocks") (let [journal-uuid (:block/uuid (db-test/find-journal-by-journal-day @conn 20240216)) template-page-uuids (->> (d/q '[:find [?page-uuid ...] :where [?b :block/tags :logseq.class/Template] [?b :block/page ?page] [?page :block/uuid ?page-uuid]] @conn) set)] (is (= #{journal-uuid} template-page-uuids) "All template blocks are created on their source journal page")) (is (= [{:title "MEETING TITLE" :properties {:user.property/participants #{"TODO"}} :children []}] (template-content-trees @conn "meeting"))) (is (= [{:title "TITLE" :properties {} :children []}] (template-content-trees @conn "title-only-no-children"))) (is (= [{:title "" :properties {:user.property/name "" :user.property/author ""} :children []}] (template-content-trees @conn "properties-only-no-children"))) (is (= [{:title "TITLE" :properties {} :children [{:title "intro" :properties {} :children []} {:title "notes" :properties {} :children []}]}] (template-content-trees @conn "title-only-with-children"))) (is (= [{:title "" :properties {} :children [{:title "intro" :properties {} :children []} {:title "notes" :properties {} :children []}]}] (template-content-trees @conn "empty-title-with-children"))) (is (= [{:title "intro" :properties {} :children []} {:title "notes" :properties {} :children []}] (template-content-trees @conn "children-only"))) (is (= [{:title "it's a template with nested templates" :properties {:user.property/name "you named it"} :children [{:title "nested-child-1" :properties {} :children [{:title "child-1" :properties {:user.property/name ""} :children [{:title "child-1-1" :properties {:user.property/name ""} :children []}]}]} {:title "nested-child-2" :properties {} :children [{:title "child-2-1" :properties {:user.property/name ""} :children []}]} {:title "child-3" :properties {:user.property/name ""} :children []}]}] (template-content-trees @conn "nested-father"))) (is (= [{:title "child-1" :properties {:user.property/name ""} :children [{:title "child-1-1" :properties {:user.property/name ""} :children []}]}] (template-content-trees @conn "nested-child-1"))) (is (= [{:title "child-2-1" :properties {:user.property/name ""} :children []}] (template-content-trees @conn "nested-child-2"))) ;; Assets (is (= {:block/tags [:logseq.class/Asset] :logseq.property.asset/type "png" :logseq.property.asset/checksum "3d5e620cac62159d8196c118574bfea7a16e86fa86efd1c3fa15a00a0a08792d" :logseq.property.asset/size 753471 :logseq.property.asset/resize-metadata {:height 288, :width 252}} (db-test/readable-properties (db-test/find-block-by-content @conn "greg-popovich-thumbs-up_1704749687791_0"))) "Asset has correct properties") (is (= {:block/tags [:logseq.class/Asset] :logseq.property.asset/type "pdf" :logseq.property.asset/external-url "zotero://select/library/items/QDM8H6EH" :logseq.property.asset/external-file-name "zotero-link://it/Understanding EXPLAIN.pdf"} (select-keys (db-test/readable-properties (db-test/find-block-by-content @conn "Understanding EXPLAIN")) [:block/tags :logseq.property.asset/type :logseq.property.asset/external-url :logseq.property.asset/external-file-name])) "Zotero linked pdf asset has correct external path info") (is (= {:block/tags [:logseq.class/Asset] :logseq.property.asset/type "pdf" :logseq.property.asset/external-url "zotero://select/library/items/RX5JS7SY" :logseq.property.asset/external-file-name "zotero-path://RX5JS7SY/zlib.pdf"} (select-keys (db-test/readable-properties (db-test/find-block-by-content @conn "zlib")) [:block/tags :logseq.property.asset/type :logseq.property.asset/external-url :logseq.property.asset/external-file-name])) "Zotero imported pdf asset has correct external path info") (is (= (d/entity @conn :logseq.class/Asset) (:block/page (db-test/find-block-by-content @conn "greg-popovich-thumbs-up_1704749687791_0"))) "Imported into Asset page") ;; Annotations (is (= {:logseq.property.pdf/hl-color :logseq.property/color.blue :logseq.property.pdf/hl-page 8 :block/tags [:logseq.class/Pdf-annotation] :logseq.property/asset "Sina_de_Capoeria_Batizado_2025_-_Program_Itinerary_1752179325104_0"} (dissoc (db-test/readable-properties (db-test/find-block-by-content @conn #"Duke School - modified")) :logseq.property.pdf/hl-value :logseq.property/ls-type)) "Pdf text highlight has correct properties") (is (= ["note about duke" "sub note"] (mapv :block/title (rest (ldb/get-block-and-children @conn (:block/uuid (db-test/find-block-by-content @conn #"Duke School - modified")))))) "Pdf text highlight has correct children blocks") (is (= {:logseq.property.pdf/hl-color :logseq.property/color.yellow :logseq.property.pdf/hl-page 1 :block/tags [:logseq.class/Pdf-annotation] :logseq.property/asset "Sina_de_Capoeria_Batizado_2025_-_Program_Itinerary_1752179325104_0" :logseq.property.pdf/hl-image "pdf area highlight" :logseq.property.pdf/hl-type :area} (dissoc (->> (d/q '[:find [?b ...] :where [?b :block/tags :logseq.class/Pdf-annotation] [?b :block/title ""]] @conn) first (d/entity @conn) db-test/readable-properties) :logseq.property.pdf/hl-value :logseq.property/ls-type)) "Pdf area highlight has correct properties") (is (= "" (:block/title (d/entity @conn [:block/uuid #uuid "68702499-159a-4a14-a0cf-cf5f015535c2"]))) "Pdf annotation without text imports with an empty title") (is (= {:block/tags [:logseq.class/Pdf-annotation] :logseq.property/asset "Understanding EXPLAIN" :logseq.property.pdf/hl-color :logseq.property/color.yellow :logseq.property.pdf/hl-page 6} (select-keys (db-test/readable-properties (db-test/find-block-by-content @conn #"EXPLAIN is a really nice command")) [:block/tags :logseq.property/asset :logseq.property.pdf/hl-color :logseq.property.pdf/hl-page])) "Zotero linked pdf text highlight links to correct asset") (is (= {:block/tags [:logseq.class/Pdf-annotation] :logseq.property/asset "zlib" :logseq.property.pdf/hl-color :logseq.property/color.red :logseq.property.pdf/hl-page 1} (select-keys (db-test/readable-properties (db-test/find-block-by-content @conn #"The zlib library is a general purpose data compression library")) [:block/tags :logseq.property/asset :logseq.property.pdf/hl-color :logseq.property.pdf/hl-page])) "Zotero imported pdf text highlight links to correct asset") (let [area-hl (d/q '[:find (pull ?b [:block/title {:block/tags [:db/ident]} {:logseq.property/asset [:block/title]} {:logseq.property.pdf/hl-image [:block/title]} {:logseq.property.pdf/hl-color [:db/ident]} :logseq.property.pdf/hl-type :logseq.property.pdf/hl-page]) . :where [?asset :block/title "Understanding EXPLAIN"] [?asset :block/tags :logseq.class/Asset] [?b :block/title "[:span]"] [?b :logseq.property/asset ?asset]] @conn)] (is (= {:logseq.property.pdf/hl-color :logseq.property/color.green :logseq.property.pdf/hl-page 8 :block/tags [:logseq.class/Pdf-annotation] :logseq.property/asset "Understanding EXPLAIN" :logseq.property.pdf/hl-image "pdf area highlight" :logseq.property.pdf/hl-type :area} (-> area-hl (update :block/tags #(mapv :db/ident %)) (update :logseq.property/asset #(:block/title %)) (update :logseq.property.pdf/hl-color #(:db/ident %)) (update :logseq.property.pdf/hl-image #(:block/title %)) (select-keys [:block/tags :logseq.property/asset :logseq.property.pdf/hl-color :logseq.property.pdf/hl-page :logseq.property.pdf/hl-image :logseq.property.pdf/hl-type]))) "Zotero linked pdf area highlight links to correct asset")) (let [area-hl (d/q '[:find (pull ?b [:block/title {:block/tags [:db/ident]} {:logseq.property/asset [:block/title]} {:logseq.property.pdf/hl-image [:block/title]} {:logseq.property.pdf/hl-color [:db/ident]} :logseq.property.pdf/hl-type :logseq.property.pdf/hl-page]) . :where [?asset :block/title "zlib"] [?asset :block/tags :logseq.class/Asset] [?b :block/title "[:span]"] [?b :logseq.property/asset ?asset]] @conn)] (is (= {:logseq.property.pdf/hl-color :logseq.property/color.blue :logseq.property.pdf/hl-page 1 :block/tags [:logseq.class/Pdf-annotation] :logseq.property/asset "zlib" :logseq.property.pdf/hl-image "pdf area highlight" :logseq.property.pdf/hl-type :area} (-> area-hl (update :block/tags #(mapv :db/ident %)) (update :logseq.property/asset #(:block/title %)) (update :logseq.property.pdf/hl-color #(:db/ident %)) (update :logseq.property.pdf/hl-image #(:block/title %)) (select-keys [:block/tags :logseq.property/asset :logseq.property.pdf/hl-color :logseq.property.pdf/hl-page :logseq.property.pdf/hl-image :logseq.property.pdf/hl-type]))) "Zotero imported pdf area highlight links to correct asset")) ;; Quotes (is (string/starts-with? (:block/title (db-test/find-block-by-content @conn #"Saito")) "From Inception:\n> Saito:") "Mixed #+BEGIN_QUOTE block: heading retained and quote content prefixed with '>'") (is (nil? (:logseq.property.node/display-type (db-test/find-block-by-content @conn #"Saito"))) "Mixed #+BEGIN_QUOTE block is not converted to a Quote-block") (is (= "markdown quote\n[[wut]]\nline 3" (:block/title (db-test/find-block-by-content @conn #"markdown quote"))) "Markdown quote imports as full multi-line quote") (is (string/starts-with? (:block/title (db-test/find-block-by-content @conn #"Learn Datalog")) "Test of various ast types:\n> *Italic*") "Mixed #+BEGIN_QUOTE block retains heading and quote content in block title") (is (= "Blockquotes\n> Nested Blockquotes" (:block/title (db-test/find-block-by-content @conn #"Nested Blockquotes"))) "Nested '>> quote' is preserved as '> ' prefix in Quote-block title") (is (= :quote (:logseq.property.node/display-type (db-test/find-block-by-content @conn #"Nested Blockquotes"))) "Nested markdown quote block is tagged as Quote-block") (is (= "it's a\n\norg blockquote" (:block/title (db-test/find-block-by-content @conn #"org blockquote"))) "#+BEGIN_QUOTE pure block title has no '> ' prefix — display-type provides blockquote styling") (is (= :quote (:logseq.property.node/display-type (db-test/find-block-by-content @conn #"org blockquote"))) "#+BEGIN_QUOTE pure block is tagged as Quote-block") (is (= "> Blockquotes\n> and\n\nsomething else" (:block/title (db-test/find-block-by-content @conn #"Blockquotes\n> and"))) "Mixed #+BEGIN_QUOTE at start of block: quote content prefixed with '>' and blank line separates following text") (is (nil? (:logseq.property.node/display-type (db-test/find-block-by-content @conn #"Blockquotes\n> and"))) "Mixed #+BEGIN_QUOTE block at start is not converted to a Quote-block") (let [block (db-test/find-block-by-content @conn #"Question 1")] (is (string/includes? (:block/title block) "\n>> nested") "Mixed markdown quote block preserves nested quote depth") (is (nil? (:logseq.property.node/display-type block)) "Mixed markdown quote block is not converted to a Quote-block")) (let [block (db-test/find-block-by-content @conn #"Question 2")] (is (string/includes? (:block/title block) "> Question 3") "Mixed markdown quote child block preserves separated quote lines") (is (string/includes? (:block/title block) "> Answer 3") "Mixed markdown quote child block preserves quote content after blank quote line"))) (testing "embeds" (is (= {:block/title ""} (-> (d/q '[:find [(pull ?b [*]) ...] :in $ ?title :where [?b :block/link ?l] [?b :block/page ?bp] [?bp :block/journal-day 20250612] [?l :block/title ?title]] @conn "page embed") first (select-keys [:block/title]))) "Page embed linked correctly") (is (= {:block/title ""} (-> (d/q '[:find [(pull ?b [*]) ...] :in $ ?title :where [?b :block/link ?l] [?b :block/page ?bp] [?bp :block/journal-day 20250612] [?l :block/title ?title]] @conn "test block embed") first (select-keys [:block/title]))) "Block embed linked correctly")) (testing "tags convert to classes" (is (= :user.class/Quotes___life (:db/ident (db-test/find-page-by-title @conn "life"))) "Namespaced tag's ident has hierarchy to make it unique") (is (= [:logseq.class/Tag] (map :db/ident (:block/tags (db-test/find-page-by-title @conn "life")))) "When a class is used and referenced on the same page, there should only be one instance of it") (is (= [:user.class/Quotes___life] (mapv :db/ident (:block/tags (db-test/find-block-by-content @conn #"with namespace tag")))) "Block tagged with namespace tag is only associated with leaf child tag") (is (= #{:user.class/ai :user.class/block-tag :user.class/p1} (set (map :db/ident (:block/tags (db-test/find-block-by-content @conn #"Block tags"))))) "Block with tags through tags property") (is (= [] (->> (d/q '[:find (pull ?b [:block/title {:block/tags [:db/ident]}]) :where [?b :block/tags :logseq.class/Tag]] @conn) (map first) (remove #(= [{:db/ident :logseq.class/Tag}] (:block/tags %))))) "All classes only have :logseq.class/Tag as their tag (and don't have Page)")) (testing "namespaces" (let [expand-children (fn expand-children [ent parent] (if-let [children (:block/_parent ent)] (cons {:parent (:block/title parent) :child (:block/title ent)} (mapcat #(expand-children % ent) children)) [{:parent (:block/title parent) :child (:block/title ent)}]))] ;; check pages only (is (= [{:parent "n1" :child "x"} {:parent "x" :child "z"} {:parent "x" :child "y"}] (take 3 (rest (expand-children (db-test/find-page-by-title @conn "n1") nil)))) "First namespace tests duplicate parent page name") (is (= [{:parent "n2" :child "x"} {:parent "x" :child "z"} {:parent "n2" :child "alias"}] (rest (expand-children (db-test/find-page-by-title @conn "n2") nil))) "First namespace tests duplicate child page name and built-in page name"))) (testing "journal timestamps" (is (= (date-time-util/journal-day->ms 20240207) (:block/created-at (db-test/find-page-by-title @conn "Feb 7th, 2024"))) "journal pages are created on their journal day") (is (= (date-time-util/journal-day->ms 20240207) (:block/created-at (db-test/find-block-by-content @conn #"Inception"))) "journal blocks are created on their page's journal day")) (testing "db attributes" (is (= true (:block/collapsed? (db-test/find-block-by-content @conn "collapsed block"))) "Collapsed blocks are imported")) (testing "property :type changes" (is (= :node (:logseq.property/type (d/entity @conn :user.property/finishedat))) ":date property to :node value changes to :node") (is (= :default (:logseq.property/type (d/entity @conn :user.property/participants))) "template values cause participants to remain a :default property") (is (= #{"[[Feb 7th, 2024]]"} (:user.property/participants (db-test/readable-properties (db-test/find-block-by-content @conn #"test :node -> :date")))) ":default participants property keeps the imported text value") (is (= :default (:logseq.property/type (d/entity @conn :user.property/description))) ":default property to :node (or any non :default value) remains :default") (is (= "[[Jakob]]" (:user.property/description (db-test/readable-properties (db-test/find-block-by-content @conn #":default to :node")))) ":default to :node property saves :default property value default with full text") (testing "with changes to upstream/existing property value" (is (= :default (:logseq.property/type (d/entity @conn :user.property/duration))) ":number property to :default value changes to :default") (is (= "20" (:user.property/duration (db-test/readable-properties (db-test/find-block-by-content @conn "existing :number to :default")))) "existing :number property value correctly saved as :default") (is (= :default (:logseq.property/type (d/entity @conn :user.property/people2))) ":node property changes to :default when :node is defined in same file") ;; tests :node :many to :default transition after :node is defined in separate file (is (= {:logseq.property/type :default :db/cardinality :db.cardinality/many} (select-keys (d/entity @conn :user.property/people) [:logseq.property/type :db/cardinality])) ":node property to :default value changes to :default and keeps existing cardinality") (is (= #{"[[Jakob]] [[Gabriel]]"} (:user.property/people (db-test/readable-properties (db-test/find-block-by-content @conn ":node people")))) "existing :node property value correctly saved as :default with full text") (is (= #{"[[Gabriel]] [[Jakob]]"} (:user.property/people (db-test/readable-properties (db-test/find-block-by-content @conn #"pending block for :node")))) "pending :node property value correctly saved as :default with full text") (is (some? (db-test/find-page-by-title @conn "Jakob")) "Previous :node property value still exists") (is (= 3 (count (find-block-by-property @conn :user.property/people))) "Converted property has correct number of property values"))) (testing "imported concepts can have names of new-built concepts" (is (= #{:logseq.property/description :user.property/description} (set (d/q '[:find [?ident ...] :where [?b :db/ident ?ident] [?b :block/name "description"]] @conn))) "user description property is separate from built-in one") (is (= #{"Page" "Tag"} (set (d/q '[:find [?t-title ...] :where [?b :block/tags ?t] [?b :block/name "task"] [?t :block/title ?t-title]] @conn))) "user page is separate from built-in class")) (testing "multiline blocks" (is (= "|markdown| table|\n|some|thing|" (:block/title (db-test/find-block-by-content @conn #"markdown.*table")))) (is (= "normal multiline block\na 2nd\nand a 3rd" (:block/title (db-test/find-block-by-content @conn #"normal multiline block")))) (is (= "colored multiline block\nlast line" (:block/title (db-test/find-block-by-content @conn #"colored multiline block")))) (let [block (db-test/find-block-by-content @conn #"multiline block with prop and deadline")] (is (= "multiline block with prop and deadline\nlast line" (:block/title block))) (is (= 20221126 (-> (db-test/readable-properties block) :logseq.property/deadline date-time-util/ms->journal-day)) "multiline block has correct journal as property value") (is (= "red" (-> (db-test/readable-properties block) :logseq.property/background-color)) "multiline block has correct background color as property value")) (let [block (db-test/find-block-by-content @conn #"multiline block with deadline and scheduled in 1 line and sth else")] (is (= "multiline block with deadline and scheduled in 1 line and sth else\nsomething else\nlast line" (:block/title block))) (is (= 20221126 (-> (db-test/readable-properties block) :logseq.property/deadline date-time-util/ms->journal-day)) "multiline block with deadline and scheduled has correct deadline journal as property value") (is (= 20221126 (-> (db-test/readable-properties block) :logseq.property/scheduled date-time-util/ms->journal-day)) "multiline block with deadline and scheduled has correct scheduled journal as property value")) (is (= "logbook block" (:block/title (db-test/find-block-by-content @conn #"^logbook block")))) (is (= "multiline logbook block\nlast line" (:block/title (db-test/find-block-by-content @conn #"multiline logbook block"))))) (testing ":block/refs" (let [page (db-test/find-page-by-title @conn "chat-gpt")] (is (set/subset? #{"type" "LargeLanguageModel"} (->> page :block/refs (map #(:block/title (d/entity @conn (:db/id %)))) set)) "Page has correct property and property value :block/refs")) (let [block (db-test/find-block-by-content @conn "old todo block")] (is (set/subset? #{:logseq.property/status :logseq.class/Task} (->> block :block/refs (map #(:db/ident (d/entity @conn (:db/id %)))) set)) "Block has correct task tag and property :block/refs"))))) (deftest-async export-basic-graph-with-convert-all-tags-option-disabled (p/let [file-graph-dir "test/resources/exporter-test-graph" conn (db-test/create-conn) {:keys [import-state]} (import-file-graph-to-db file-graph-dir conn {:convert-all-tags? false})] (is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn)))) "Created graph has no validation errors") (is (= 0 (count @(:ignored-properties import-state))) "No ignored properties") (is (= 0 (->> @conn (d/q '[:find [?ident ...] :where [?b :block/tags :logseq.class/Tag] [?b :db/ident ?ident] (not [?b :logseq.property/built-in?])]) count)) "Correct number of user classes") (is (= 6 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Task]] @conn)))) (is (= 4 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Query]] @conn)))) (is (= 2 (count (d/q '[:find ?b :where [?b :block/tags :logseq.class/Card]] @conn)))) (testing "replacing refs in :block/title when :remove-inline-tags? set" (is (= 2 (->> (entity-plus/lookup-kv-then-entity (db-test/find-block-by-content @conn #"replace with same start string") :block/raw-title) (re-seq db-content/id-ref-pattern) distinct count)) "A block with ref names that start with same string has 2 distinct refs") (is (= 1 (->> (entity-plus/lookup-kv-then-entity (db-test/find-block-by-content @conn #"replace case insensitive") :block/raw-title) (re-seq db-content/id-ref-pattern) distinct count)) "A block with different case of same ref names has 1 distinct ref")) (testing "tags convert to page, refs and page-tags" (let [block (db-test/find-block-by-content @conn #"Inception") tag-page (db-test/find-page-by-title @conn "Movie") tagged-page (db-test/find-page-by-title @conn "Interstellar")] (is (string/starts-with? (str (:block/title block)) "Inception [[") "tagged block tag converts tag to page ref") (is (= [(:db/id tag-page)] (map :db/id (:block/refs block))) "tagged block has correct refs") (is (and tag-page (not (ldb/class? tag-page))) "tag page is not a class") (is (= #{"Movie"} (:logseq.property/page-tags (db-test/readable-properties tagged-page))) "tagged page has existing page imported as a tag to page-tags") (is (= #{"LargeLanguageModel" "fun" "ai"} (:logseq.property/page-tags (db-test/readable-properties (db-test/find-page-by-title @conn "chat-gpt")))) "tagged page has new page and other pages marked with '#' and '[[]]` imported as tags to page-tags"))))) (deftest-async import-journals-use-standard-uuids-and-keep-uuid-refs (p/let [file-graph-dir "test/resources/exporter-test-graph" files (mapv #(path/path-join file-graph-dir %) ["journals/2026_01_27.md"]) conn (db-test/create-conn) _ (import-files-to-db files conn {})] (let [journal (db-test/find-journal-by-journal-day @conn 20260127) ref-journal (db-test/find-journal-by-journal-day @conn 20260101) ref-block (some->> (d/q '[:find [?b ...] :in $ ?page ?ref-page :where [?b :block/page ?page] [?b :block/refs ?ref-page]] @conn (:db/id journal) (:db/id ref-journal)) first (d/entity @conn))] (is (= (common-uuid/gen-uuid :journal-page-uuid 20260127) (:block/uuid journal)) "Imported journal page keeps the standard journal uuid") (is (= (common-uuid/gen-uuid :journal-page-uuid 20260101) (:block/uuid ref-journal)) "Referenced journal page keeps the standard journal uuid") (is (= #{(:block/uuid ref-journal)} (set (map :block/uuid (:block/refs ref-block)))) "Journal refs point at the standard journal uuid")))) (deftest-async import-journal-with-slash-title-format-does-not-create-namespace-pages (p/let [file-graph-dir "test/resources/exporter-test-graph" files (mapv #(path/path-join file-graph-dir %) ["journals/2026_01_27.md"]) conn (db-test/create-conn) _ (import-files-to-db files conn {:user-config {:journal/page-title-format "yyyy/MM/dd"}}) journal (db-test/find-journal-by-journal-day @conn 20260127)] (is (= "2026/01/27" (:block/title journal)) "Journal title follows slash title format") (is (= (common-uuid/gen-uuid :journal-page-uuid 20260127) (:block/uuid journal)) "Slash-formatted journal keeps the standard journal uuid") (is (nil? (:block/namespace journal)) "Slash-formatted journal does not keep a namespace attribute") (is (nil? (db-test/find-page-by-title @conn "2026")) "Journal title is not split into a year namespace page") (is (nil? (db-test/find-page-by-title @conn "01")) "Journal title is not split into a month namespace page") (is (nil? (db-test/find-page-by-title @conn "27")) "Journal title is not split into a day namespace page"))) (deftest-async import-slash-journal-ref-does-not-create-namespace-pages (p/let [file (write-temp-graph-file "journals/2026_05_18.md" "- yes\n- [[Sun, 2026/05/17]]\n") conn (db-test/create-conn) _ (import-files-to-db [file] conn {:user-config {:journal/page-title-format "EEE, yyyy/MM/dd"}}) ref-journal (db-test/find-journal-by-journal-day @conn 20260517)] (is (= "Sun, 2026/05/17" (:block/title ref-journal)) "Journal reference is imported as a journal page") (is (nil? (:block/namespace ref-journal)) "Referenced slash-formatted journal does not keep a namespace attribute") (is (nil? (db-test/find-page-by-title @conn "Sun, 2026")) "Journal reference is not split into a parent namespace page") (is (nil? (db-test/find-page-by-title @conn "05")) "Journal reference is not split into a child namespace page"))) (deftest-async import-legacy-journal-file-name-refs-as-journals (p/let [source-file (write-temp-graph-file "journals/2026_04_01.md" "- legacy journal ref [[2026_04_02]]\n") target-file (write-temp-graph-file "journals/2026_04_02.md" "- target journal\n") conn (db-test/create-conn) _ (import-files-to-db [source-file target-file] conn {}) legacy-journal (db-test/find-journal-by-journal-day @conn 20260402) legacy-ref-block (db-test/find-block-by-content @conn #"legacy journal ref")] (is (some? legacy-journal) "Legacy yyyy_MM_dd journal page refs resolve imported journal files") (is (= #{(:block/uuid legacy-journal)} (set (map :block/uuid (:block/refs legacy-ref-block)))) "Legacy journal page ref points at the journal page") (is (nil? (db-test/find-page-by-title @conn "2026_04_02")) "Legacy journal page ref does not create an ordinary page"))) (deftest-async import-default-format-journal-refs-with-custom-title-format (p/let [dir (write-temp-file-graph {"pages/source.md" "- existing journal [[May 18th, 2021]]\n- missing journal [[May 19th, 2021]]\n" "journals/2021_05_18.md" "- journal entry\n" "pages/2021_05_19.md" "- ordinary date-named page\n"}) source-file (-> (node-path/join dir "pages/source.md") (string/replace "\\" "/")) journal-file (-> (node-path/join dir "journals/2021_05_18.md") (string/replace "\\" "/")) ordinary-date-file (-> (node-path/join dir "pages/2021_05_19.md") (string/replace "\\" "/")) conn (db-test/create-conn) _ (import-files-to-db [source-file journal-file ordinary-date-file] conn {:user-config {:journal/page-title-format "EEEE, dd-MM-yyyy"}}) existing-journal (db-test/find-journal-by-journal-day @conn 20210518) missing-page (db-test/find-page-by-title @conn "May 19th, 2021") existing-ref-block (db-test/find-block-by-content @conn #"existing journal") missing-ref-block (db-test/find-block-by-content @conn #"missing journal")] (is (= "Tuesday, 18-05-2021" (:block/title existing-journal)) "Default-format ref resolves the journal file using the configured title") (is (= #{:logseq.class/Journal} (set (map :db/ident (:block/tags existing-journal)))) "Existing journal does not retain the ordinary page tag") (is (= #{(:block/uuid existing-journal)} (set (map :block/uuid (:block/refs existing-ref-block)))) "Existing journal ref points at the journal page") (is (= #{:logseq.class/Page} (set (map :db/ident (:block/tags missing-page)))) "Default-format ref without a file under journals remains an ordinary page") (is (nil? (db-test/find-journal-by-journal-day @conn 20210519)) "Date-named file outside journals does not create a journal") (is (= #{(:block/uuid missing-page)} (set (map :block/uuid (:block/refs missing-ref-block)))) "Missing journal ref points at the ordinary page"))) (deftest-async import-creates-missing-ordinary-page-refs (p/let [file (write-temp-graph-file "pages/source.md" "- missing page ref [[Missing Page]]\n") conn (db-test/create-conn) _ (import-files-to-db [file] conn {}) missing-page (db-test/find-page-by-title @conn "Missing Page") source-block (db-test/find-block-by-content @conn #"missing page ref")] (is (some? missing-page) "Missing ordinary page refs create ordinary pages") (is (= #{(:block/uuid missing-page)} (set (map :block/uuid (:block/refs source-block)))) "Missing ordinary page ref points at the created page"))) (deftest-async import-favorites-from-og-config-edn (p/let [dir (write-temp-file-graph {"logseq/config.edn" (str "{:favorites [\"Projects\" \"[[Projects]]\" \"foo/bar\"]\n" " :file/name-format :triple-lowbar}\n") "pages/Projects.md" "- project work\n- [[foo/bar]]\n"}) conn (db-test/create-conn) _ (db-pipeline/add-listener conn) _ (import-file-graph-to-db dir conn {}) favorite-titles (imported-favorite-titles @conn)] (is (= 3 (count favorite-titles)) "Bare names, bracketed page refs, and namespaced pages each become a favorite link") (is (= #{"Projects" "foo/bar"} (set favorite-titles)) "Imported favorites resolve to the original pages including flattened namespaces"))) (deftest-async import-normalizes-existing-random-journal-uuid-and-text-refs (let [old-journal-uuid (random-uuid) standard-journal-uuid (common-uuid/gen-uuid :journal-page-uuid 20260127) title (str "refs " (page-ref/->page-ref old-journal-uuid) " and " (block-ref/->block-ref old-journal-uuid)) conn (db-test/create-conn-with-blocks {:pages-and-blocks [{:page {:build/journal 20260127 :block/uuid old-journal-uuid :build/keep-uuid? true} :blocks [{:block/title title}]}]}) file (write-temp-graph-file "pages/trigger-normalize.md" "- trigger normalize\n")] (p/let [_ (import-files-to-db [file] conn {}) journal (db-test/find-journal-by-journal-day @conn 20260127) ref-block (db-test/find-block-by-content @conn #"refs")] (is (= standard-journal-uuid (:block/uuid journal)) "Existing random journal uuid is normalized to the standard journal uuid") (is (nil? (d/entity @conn [:block/uuid old-journal-uuid])) "Old journal uuid no longer resolves after normalization") (is (= (str "refs " (page-ref/->page-ref standard-journal-uuid) " and " (block-ref/->block-ref standard-journal-uuid)) (:block/title ref-block)) "Text references are rewritten to the standard journal uuid") (is (= (:db/id journal) (get-in ref-block [:block/page :db/id])) "Structured block page reference still points to the same journal entity")))) (deftest-async export-files-with-tag-classes-option (p/let [file-graph-dir "test/resources/exporter-test-graph" files (mapv #(path/path-join file-graph-dir %) ["journals/2024_02_07.md" "pages/Interstellar.md"]) conn (db-test/create-conn) _ (import-files-to-db files conn {:tag-classes ["movie"]})] (is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn)))) "Created graph has no validation errors") (let [block (db-test/find-block-by-content @conn #"Inception") tag-page (db-test/find-page-by-title @conn "Movie") another-tag-page (db-test/find-page-by-title @conn "p0")] (is (= (:block/title block) "Inception") "tagged block with configured tag strips tag from content") (is (= [:user.class/Movie] (:block/tags (db-test/readable-properties block))) "tagged block has configured tag imported as a class") (is (= [:logseq.class/Tag] (mapv :db/ident (:block/tags tag-page))) "configured tag page in :tag-classes is a class") (is (and another-tag-page (not (ldb/class? another-tag-page))) "unconfigured tag page is not a class") (is (= {:block/tags [:logseq.class/Page :user.class/Movie]} (db-test/readable-properties (db-test/find-page-by-title @conn "Interstellar"))) "tagged page has configured tag imported as a class")))) (deftest-async export-files-with-property-classes-option (p/let [file-graph-dir "test/resources/exporter-test-graph" files (mapv #(path/path-join file-graph-dir %) ["journals/2024_02_23.md" "pages/url.md" "pages/Whiteboard___Tool.md" "pages/Whiteboard___Arrow_head_toggle.md" "pages/Library.md"]) conn (db-test/create-conn) _ (import-files-to-db files conn {:property-classes ["type"]}) _ (@#'gp-exporter/export-class-properties conn conn)] (is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn)))) "Created graph has no validation errors") (is (= #{:user.class/Property :user.class/Movie :user.class/Class :user.class/Tool} (->> @conn (d/q '[:find [?ident ...] :where [?b :block/tags :logseq.class/Tag] [?b :db/ident ?ident] (not [?b :logseq.property/built-in?])]) set)) "All classes are correctly defined by :type") (is (= #{:user.property/url :user.property/sameas :user.property/rangeincludes} (->> (d/entity @conn :user.class/Property) :logseq.property.class/properties (map :db/ident) set)) "Properties are correctly inferred for a class") (let [block (db-test/find-block-by-content @conn #"The Creator") tag-page (db-test/find-page-by-title @conn "Movie")] (is (= (:block/title block) "The Creator") "tagged block with configured tag strips tag from content") (is (= [:user.class/Movie] (:block/tags (db-test/readable-properties block))) "tagged block has configured tag imported as a class") (is (= (:user.property/testtagclass block) (:block/tags block)) "tagged block can have another property that references the same class it is tagged with, without creating a duplicate class") (is (= [:logseq.class/Tag] (map :db/ident (:block/tags tag-page))) "configured tag page derived from :property-classes is a class") (is (nil? (db-test/find-page-by-title @conn "type")) "No page exists for configured property") (is (= #{:user.class/Property :logseq.class/Property} (set (:block/tags (db-test/readable-properties (db-test/find-page-by-title @conn "url"))))) "tagged page has correct tags including one from option")))) (deftest-async export-files-with-remove-inline-tags (p/let [file-graph-dir "test/resources/exporter-test-graph" files (mapv #(path/path-join file-graph-dir %) ["journals/2024_02_07.md" "journals/2026_01_27.md"]) conn (db-test/create-conn) _ (import-files-to-db files conn {:remove-inline-tags? false :convert-all-tags? true}) namespaced-file (write-temp-graph-file "pages/namespace-inline-tag.md" "- #parent/child\n") namespaced-conn (db-test/create-conn) _ (import-files-to-db [namespaced-file] namespaced-conn {:remove-inline-tags? false :convert-all-tags? true}) [block tag] (->> (d/q '[:find ?b ?t :where [?b :block/tags ?t] [?b :block/page] [?t :db/ident :user.class/parent___child]] @namespaced-conn) first (map #(d/entity @namespaced-conn %))) raw-title (:block/title block)] (is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn)))) "Created graph has no validation errors") (is (string/starts-with? (:block/title (db-test/find-block-by-content @conn #"Inception")) "Inception #Movie") "block with tag preserves inline tag") (is (string/includes? (:block/title (db-test/find-block-by-content @conn #"block with multi word tag")) "#[[another test]]") "block with multi word tag preserves inline tag") (testing "namespaced inline tag on first line is preserved as inline tag" (is (some? block) "imported first-line namespaced tag block") (is (string? raw-title) "imported block has raw title") (when (string? raw-title) (is (ldb/inline-tag? raw-title tag) "first-line namespaced tag is stored as an inline tag"))))) (deftest-async export-files-with-ignored-properties (p/let [file-graph-dir "test/resources/exporter-test-graph" files (mapv #(path/path-join file-graph-dir %) ["ignored/icon-page.md"]) conn (db-test/create-conn) {:keys [import-state]} (import-files-to-db files conn {})] (is (= 2 (count (filter #(= :icon (:property %)) @(:ignored-properties import-state)))) "icon properties are visibly ignored in order to not fail import"))) (deftest-async export-files-with-property-parent-classes-option (p/let [file-graph-dir "test/resources/exporter-test-graph" files (mapv #(path/path-join file-graph-dir %) ["journals/2024_11_26.md" "pages/CreativeWork.md" "pages/Movie.md" "pages/type.md" "pages/Whiteboard___Tool.md" "pages/Whiteboard___Arrow_head_toggle.md" "pages/Property.md" "pages/url.md"]) conn (db-test/create-conn) _ (import-files-to-db files conn {:property-parent-classes ["parent"] ;; Also add this option to trigger some edge cases with namespace pages :property-classes ["type"]})] (is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn)))) "Created graph has no validation errors") (is (= #{:user.class/Movie :user.class/CreativeWork :user.class/Thing :user.class/Feature :user.class/Class :user.class/Tool :user.class/Whiteboard___Tool :user.class/Property} (->> @conn (d/q '[:find [?ident ...] :where [?b :block/tags :logseq.class/Tag] [?b :db/ident ?ident] (not [?b :logseq.property/built-in?])]) set)) "All classes are correctly defined by :type") (is (= ["CreativeWork"] (map :block/title (:logseq.property.class/extends (d/entity @conn :user.class/Movie)))) "Existing page correctly set as class parent") (is (= ["Thing"] (map :block/title (:logseq.property.class/extends (d/entity @conn :user.class/CreativeWork)))) "New page correctly set as class parent"))) (deftest-async export-files-with-property-pages-disabled (p/let [file-graph-dir "test/resources/exporter-test-graph" ;; any page with properties files (mapv #(path/path-join file-graph-dir %) ["journals/2024_01_17.md"]) conn (db-test/create-conn) _ (import-files-to-db files conn {:user-config {:property-pages/enabled? false :property-pages/excludelist #{:prop-string}}})] (is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn)))) "Created graph has no validation errors"))) (deftest-async export-config-file-sets-title-format (p/let [conn (db-test/create-conn) read-file #(p/do! "{:journal/page-title-format \"yyyy-MM-dd\"}") _ (gp-exporter/export-config-file conn "logseq/config.edn" read-file {})] (is (= "yyyy-MM-dd" (:logseq.property.journal/title-format (d/entity @conn :logseq.class/Journal))) "title format set correctly by config"))) (deftest split-title-by-code-fences (let [split-fn #'gp-exporter/split-title-by-code-fences] (testing "standalone code fence with language" (is (= {:text-parts [] :code-segs [{:text "it's an individual code snippet with language tag" :lang "markdown"}]} (split-fn "```markdown\nit's an individual code snippet with language tag\n```")))) (testing "standalone code fence without language" (is (= {:text-parts [] :code-segs [{:text "it's an individual code snippet without language tag" :lang nil}]} (split-fn "```\nit's an individual code snippet without language tag\n```")))) (testing "one code fence with leading text" (is (= {:text-parts ["before code snippet"] :code-segs [{:text "echo \"ok\"\nexit" :lang nil}]} (split-fn "before code snippet\n```\necho \"ok\"\nexit\n```")))) (testing "one code fence with leading and trailing text" (is (= {:text-parts ["before code snippet" "after code snippet"] :code-segs [{:text "echo \"ok\"\nexit" :lang "bash"}]} (split-fn "before code snippet\n```bash\necho \"ok\"\nexit\n```\nafter code snippet")))) (testing "one code fence followed by trailing text" (is (= {:text-parts ["after code snippet"] :code-segs [{:text "echo \"ok\"\nexit" :lang "bash"}]} (split-fn "```bash\necho \"ok\"\nexit\n```\nafter code snippet")))) (testing "multiple code fences mixed with text" (is (= {:text-parts ["before code snippet" "middle" "after code snippet"] :code-segs [{:text "echo \"ok\"\nexit" :lang "bash"} {:text "echo \"bye\"\nexit" :lang "bash"}]} (split-fn "before code snippet\n```bash\necho \"ok\"\nexit\n```\nmiddle\n```bash\necho \"bye\"\nexit\n```\nafter code snippet")))) (testing "edge: one code fence followed by opening fence without closing fence" (is (= {:text-parts ["echo \"missing end fence\""] ;; no "```bash" ahead, it's fine as is; let's leave it :code-segs [{:text "echo \"ok\"\nexit" :lang "bash"}]} (split-fn "```bash\necho \"ok\"\nexit\n```\n```bash\necho \"missing end fence\"")))) (testing "edge: pure multiple code fences with no extra text" (let [{:keys [text-parts code-segs]} (split-fn "```markdown\n1st code snippet with language tag\n```\n```\n2nd code snippet without language tag\n```")] (is (and (empty? text-parts) (> (count code-segs) 1)) "not pure single code and no mixed content"))) (testing "edge: opening fence without closing fence" (let [title "```bash\necho \"missing end fence\"" {:keys [text-parts code-segs]} (split-fn title)] (is (and (= (count text-parts) 1) (not= (first text-parts) title) (empty? code-segs)) "not pure single code and no mixed content"))) (testing "edge: plain text without any code fence" (is (= {:text-parts ["plain text only"] :code-segs []} (split-fn "plain text only")))) (testing "edge: empty title" (is (= {:text-parts [""] :code-segs []} (split-fn "")))))) (deftest-async export-files-with-extract-code-snippet (p/let [file-graph-dir "test/resources/exporter-test-graph" files (mapv #(path/path-join file-graph-dir %) ["journals/2026_03_01.md"]) conn (db-test/create-conn) _ (import-files-to-db files conn {:extract-code-snippets? true}) journal-page-eid (d/q '[:find ?p . :where [?p :block/journal-day 20260301]] @conn) top-blocks (->> (d/q '[:find [?b ...] :in $ ?page :where [?b :block/page ?page] [?b :block/parent ?page]] @conn journal-page-eid) (map #(d/entity @conn %)) (sort-by :block/order) vec) get-direct-children (fn [block] (->> (d/q '[:find [?c ...] :in $ ?parent :where [?c :block/parent ?parent]] @conn (:db/id block)) (map #(d/entity @conn %))))] (testing "standalone code block with language tag" (let [b (nth top-blocks 0)] (is (= "it's an individual code snippet with language tag" (:block/title b)) "Standalone code block title has fences stripped") (is (= 0 (count (get-direct-children b))) "Standalone code block has no children") (is (= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags b)))) "Standalone code block is tagged as Code-block") (is (= "markdown" (:logseq.property.code/lang b)) "Standalone code block has markdown language property"))) (testing "standalone code block without language tag" (let [b (nth top-blocks 1)] (is (= "it's an individual code snippet without language tag" (:block/title b)) "Standalone code block title has fences stripped") (is (= 0 (count (get-direct-children b))) "Standalone code block has no children") (is (= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags b)))) "Standalone code block is tagged as Code-block") (is (= nil (:logseq.property.code/lang b)) "Standalone code block has no language property"))) (testing "text before code snippet" (let [b (nth top-blocks 2) children (get-direct-children b)] (is (= "before code snippet" (:block/title b)) "Block title has text only without code") (is (= 1 (count children)) "Block has 1 code child") (is (= "echo \"ok\"\nexit" (:block/title (first children))) "Child code block has correct content without fence markers") (is (= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags (first children))))) "Child block is tagged as Code-block") (is (= nil (:logseq.property.code/lang (first children))) "Child block has no language property"))) (testing "text before and after code snippet" (let [b (nth top-blocks 3) children (get-direct-children b)] (is (= "before code snippet\nafter code snippet" (:block/title b)) "Block title has text only without code") (is (= 1 (count children)) "Block has 1 code child") (is (= "echo \"ok\"\nexit" (:block/title (first children))) "Child code block has correct content without fence markers") (is (= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags (first children))))) "Child block is tagged as Code-block") (is (= "bash" (:logseq.property.code/lang (first children))) "Child block has bash language property"))) (testing "code snippet before text" (let [b (nth top-blocks 4) children (get-direct-children b)] (is (= "after code snippet" (:block/title b)) "Block title has text only without code") (is (= 1 (count children)) "Block has 1 code child") (is (= "echo \"ok\"\nexit" (:block/title (first children))) "Child code block has correct content without fence markers") (is (= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags (first children))))) "Child block is tagged as Code-block") (is (= "bash" (:logseq.property.code/lang (first children))) "Child block has bash language property"))) (testing "multiple code snippets mixed with text" (let [b (nth top-blocks 5) children (sort-by :block/order (get-direct-children b))] (is (= "before code snippet\nmiddle\nafter code snippet" (:block/title b)) "Block title has all text parts without code") (is (= 2 (count children)) "Block has 2 code children") (is (= "echo \"ok\"\nexit" (:block/title (first children))) "First child code block has correct content without fence markers") (is (= "echo \"bye\"\nexit" (:block/title (second children))) "Second child code block has correct content without fence markers") (is (every? #(= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags %)))) children) "Both child blocks are tagged as Code-block") (is (every? #(= "bash" (:logseq.property.code/lang %)) children) "Both child blocks have bash language property"))))) (deftest-async export-files-without-extract-code-snippet (p/let [file-graph-dir "test/resources/exporter-test-graph" files (mapv #(path/path-join file-graph-dir %) ["journals/2026_03_01.md"]) conn (db-test/create-conn) _ (import-files-to-db files conn {:extract-code-snippets? false}) journal-page-eid (d/q '[:find ?p . :where [?p :block/journal-day 20260301]] @conn) top-blocks (->> (d/q '[:find [?b ...] :in $ ?page :where [?b :block/page ?page] [?b :block/parent ?page]] @conn journal-page-eid) (map #(d/entity @conn %)) (sort-by :block/order) vec) get-direct-children (fn [block] (->> (d/q '[:find [?c ...] :in $ ?parent :where [?c :block/parent ?parent]] @conn (:db/id block)) (map #(d/entity @conn %))))] (testing "standalone code block with language tag is still tagged as Code-block" (let [b (nth top-blocks 0)] (is (= "it's an individual code snippet with language tag" (:block/title b)) "Standalone code block title has fences stripped") (is (= 0 (count (get-direct-children b))) "Standalone code block has no children") (is (= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags b)))) "Standalone code block is tagged as Code-block") (is (= "markdown" (:logseq.property.code/lang b)) "Standalone code block has markdown language property"))) (testing "standalone code block without language tag is still tagged as Code-block" (let [b (nth top-blocks 1)] (is (= "it's an individual code snippet without language tag" (:block/title b)) "Standalone code block title has fences stripped") (is (= 0 (count (get-direct-children b))) "Standalone code block has no children") (is (= #{:logseq.class/Code-block} (set (map :db/ident (:block/tags b)))) "Standalone code block is tagged as Code-block") (is (= nil (:logseq.property.code/lang b)) "Standalone code block has no language property"))) (testing "mixed-content block is NOT extracted into children when extract-code-snippets? is false" (let [b (nth top-blocks 2)] (is (= 0 (count (get-direct-children b))) "Block with text before code has no children extracted") (is (string/includes? (:block/title b) "```") "Block title retains raw code fence markup"))) (testing "another mixed-content block is NOT extracted when extract-code-snippets? is false" (let [b (nth top-blocks 3)] (is (= 0 (count (get-direct-children b))) "Block with text surrounding code has no children extracted") (is (string/includes? (:block/title b) "```") "Block title retains raw code fence markup"))))) (deftest page-alias-sanitise-for-import (testing "duplicate-owner alias is dropped and reported" (let [alias-owners (atom {}) ignored-props (atom []) pages [{:block/name "p1" :block/alias [{:block/name "shared"}]} {:block/name "p2" :block/alias [{:block/name "shared"}]}] result (gp-exporter/sanitize-page-aliases-for-import! pages alias-owners ignored-props)] (is (= "p1" (get @alias-owners "shared")) "first declarer wins ownership") (is (= 1 (count (filter #(= :alias/duplicate-owner (:reason %)) @ignored-props))) "duplicate alias is reported in ignored-properties") (is (nil? (:block/alias (second result))) "second page's conflicting alias is removed"))) (testing "alias-of-alias is dropped and reported" (let [alias-owners (atom {}) ignored-props (atom []) pages [{:block/name "root" :block/alias [{:block/name "mid"}]} {:block/name "mid" :block/alias [{:block/name "leaf"}]}] result (gp-exporter/sanitize-page-aliases-for-import! pages alias-owners ignored-props)] (is (= 1 (count (filter #(= :alias/alias-owns-aliases (:reason %)) @ignored-props))) "alias-of-alias is reported in ignored-properties") (is (nil? (:block/alias (first result))) "alias pointing to a page that owns aliases is removed"))) (testing "self-alias is dropped and reported" (let [alias-owners (atom {}) ignored-props (atom []) pages [{:block/name "self" :block/alias [{:block/name "self"}]}] result (gp-exporter/sanitize-page-aliases-for-import! pages alias-owners ignored-props)] (is (nil? (:block/alias (first result))) "self-alias declaration is removed") (is (some #(= :alias/self (:reason %)) @ignored-props) "self-alias is reported in ignored-properties")))) (deftest page-alias-sanitise-for-import-cross-file (testing "source-is-alias caught across files (root->mid in file 1, mid->leaf in file 2)" (let [alias-owners (atom {}) ignored-props (atom [])] ;; File 1: root claims mid as alias → registers "mid" -> "root" (gp-exporter/sanitize-page-aliases-for-import! [{:block/name "root" :block/alias [{:block/name "mid"}]}] alias-owners ignored-props) ;; File 2: mid tries to declare leaf as its alias — but mid is already an alias (let [result (gp-exporter/sanitize-page-aliases-for-import! [{:block/name "mid" :block/alias [{:block/name "leaf"}]}] alias-owners ignored-props)] (is (nil? (:block/alias (first result))) "mid's alias declaration dropped: mid is already an alias") (is (some #(= :alias/source-is-alias (:reason %)) @ignored-props) "source-is-alias reason recorded")))) (testing "alias-owns-aliases caught across files (mid->leaf in file 1, root->mid in file 2)" (let [alias-owners (atom {}) ignored-props (atom [])] ;; File 1: mid claims leaf as alias → registers "leaf" -> "mid"; mid is now an owner (gp-exporter/sanitize-page-aliases-for-import! [{:block/name "mid" :block/alias [{:block/name "leaf"}]}] alias-owners ignored-props) ;; File 2: root tries to use mid as alias — but mid already owns aliases (let [result (gp-exporter/sanitize-page-aliases-for-import! [{:block/name "root" :block/alias [{:block/name "mid"}]}] alias-owners ignored-props)] (is (nil? (:block/alias (first result))) "root's alias pointing to mid dropped: mid already owns aliases") (is (some #(= :alias/alias-owns-aliases (:reason %)) @ignored-props) "alias-owns-aliases reason recorded")))))