dasemu/Synapse
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial import of LogDarioSeq baseline

Browse Source 
Establishes version control for the personal Logseq DB fork before
starting the server-client architecture migration and domain features
described in docs/agent-guide and the approved implementation plan.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Darío Sevilla
2026-08-25 15:32:15 +02:00
commit 40631362b3
2016 changed files with 543898 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

906
deps/graph-parser/src/logseq/graph_parser/block.cljs vendored Normal file
View File

@@ -0,0 +1,906 @@
(ns logseq.graph-parser.block
"Given mldoc ast, prepares block data in preparation for db transaction.
Used by file and DB graphs"
(:require [clojure.set :as set]
[clojure.string :as string]
[clojure.walk :as walk]
[datascript.core :as d]
[logseq.common.config :as common-config]
[logseq.common.date :as common-date]
[logseq.common.util :as common-util]
[logseq.common.util.block-ref :as block-ref]
[logseq.common.util.date-time :as date-time-util]
[logseq.common.util.page-ref :as page-ref]
[logseq.common.uuid :as common-uuid]
[logseq.db :as ldb]
[logseq.db.common.entity-plus :as entity-plus]
[logseq.db.common.order :as db-order]
[logseq.db.frontend.class :as db-class]
[logseq.db.frontend.entity-util :as entity-util]
[logseq.graph-parser.mldoc :as gp-mldoc]
[logseq.graph-parser.property :as gp-property]
[logseq.graph-parser.text :as text]
[logseq.graph-parser.utf8 :as utf8]))
(defn heading-block?
[block]
(and
(vector? block)
(= "Heading" (first block))))
(defn get-tag
[block]
(when-let [tag-value (and (vector? block)
(= "Tag" (first block))
(second block))]
(->> tag-value
(map (fn [[elem value]]
(case elem
"Plain" value
"Link" (:full_text value)
"Nested_link" (:content value)
"")))
(string/join))))
(defn get-page-reference
[block format]
(let [page (cond
(and (vector? block) (= "Link" (first block)))
(let [url-type (first (:url (second block)))
value (second (:url (second block)))]
;; {:url ["File" "file:../pages/hello_world.org"], :label [["Plain" "hello world"]], :title nil}
(or
(and
(= url-type "Page_ref")
(and (string? value)
(not (common-config/local-relative-asset? value)))
value)
(and
(= url-type "Search")
(page-ref/page-ref? value)
(text/page-ref-un-brackets! value))
(and (= url-type "Search")
(= format :org)
(not (common-config/local-relative-asset? value))
value)
(and
(= url-type "File")
(second (first (:label (second block)))))))
(and (vector? block) (= "Nested_link" (first block)))
(let [content (:content (last block))]
(subs content 2 (- (count content) 2)))
(and (vector? block)
(= "Macro" (first block)))
(let [{:keys [name arguments]} (second block)
argument (string/join ", " arguments)]
(if (= name "embed")
(when (page-ref/page-ref? argument)
(text/page-ref-un-brackets! argument))
{:type "macro"
:name name
:arguments arguments}))
(and (vector? block)
(= "Tag" (first block)))
(let [text (get-tag block)]
(text/page-ref-un-brackets! text))
:else
nil)]
(when page (or (when (string? page)
(block-ref/get-block-ref-id page))
page))))
(defn get-block-reference
[block]
(when-let [block-id (cond
(and (vector? block)
(= "Block_reference" (first block)))
(last block)
(and (vector? block)
(= "Link" (first block))
(map? (second block))
(= "Block_ref" (first (:url (second block)))))
(second (:url (second block)))
(and (vector? block)
(= "Macro" (first block)))
(let [{:keys [name arguments]} (second block)]
(when (and (= name "embed")
(string? (first arguments))
(block-ref/string-block-ref? (first arguments)))
(block-ref/get-string-block-ref-id (first arguments))))
(and (vector? block)
(= "Link" (first block))
(map? (second block)))
(if (= "id" (:protocol (second (:url (second block)))))
(:link (second (:url (second block))))
(let [id (second (:url (second block)))]
;; these can be maps
(when (string? id)
(or (block-ref/get-block-ref-id id) id))))
:else
nil)]
(when (some-> block-id parse-uuid)
block-id)))
(defn paragraph-block?
[block]
(and
(vector? block)
(= "Paragraph" (first block))))
(defn timestamp-block?
[block]
(and
(vector? block)
(= "Timestamp" (first block))))
(defn- get-page-refs-from-property-names
[properties {:property-pages/keys [enabled? excludelist]}]
(if (contains? #{true nil} enabled?)
(sequence
(comp (map (comp name first))
(remove string/blank?)
(remove (set (map name excludelist)))
;; Remove built-in properties as we don't want pages
;; created for them by default
(remove (into #{}
(map name)
(apply conj
(apply disj
(gp-property/editable-built-in-properties)
gp-property/editable-linkable-built-in-properties)
(gp-property/hidden-built-in-properties))))
(distinct))
properties)
[]))
(defn- extract-refs-from-property-value
[value format]
(cond
(coll? value)
(filter (fn [v] (and (string? v) (not (string/blank? v)))) value)
(and (string? value) (= \" (first value) (last value)))
nil
(string? value)
(let [ast (gp-mldoc/inline->edn value (gp-mldoc/default-config format))]
(text/extract-refs-from-mldoc-ast ast))
:else
nil))
(defn- get-page-ref-names-from-properties
[properties user-config]
(let [page-refs (->>
properties
(remove (fn [[k _]]
(contains?
(set/union (apply disj
(gp-property/editable-built-in-properties)
gp-property/editable-linkable-built-in-properties)
(gp-property/hidden-built-in-properties))
(keyword k))))
;; get links ast
(map last)
(mapcat (fn [value]
(extract-refs-from-property-value value (get user-config :format :markdown))))
;; comma separated collections
(concat (->> (map second properties)
(filter coll?)
(apply concat))))
page-refs-from-property-names (get-page-refs-from-property-names properties user-config)]
(->> (concat page-refs page-refs-from-property-names)
(remove string/blank?)
distinct)))
(defn- extract-block-refs
[nodes]
(let [ref-blocks (atom nil)]
(walk/postwalk
(fn [form]
(when-let [block (get-block-reference form)]
(swap! ref-blocks conj block))
form)
nodes)
(keep (fn [block]
(when-let [id (parse-uuid block)]
[:block/uuid id]))
@ref-blocks)))
(defn extract-properties
[properties user-config]
(when (seq properties)
(let [properties (seq properties)
*invalid-properties (atom #{})
properties (->> properties
(map (fn [[k v mldoc-ast]]
(let [k (if (or (keyword? k) (symbol? k))
(subs (str k) 1)
k)
k (-> (string/lower-case k)
(string/replace "/" "-")
(string/replace " " "-")
(string/replace "_" "-"))]
(if (gp-property/valid-property-name? (str ":" k))
(let [k' (keyword
(if (contains? #{"custom_id" "custom-id"} k)
"id"
k))
v' (text/parse-property k v mldoc-ast user-config)]
[k' v' mldoc-ast v])
(do (swap! *invalid-properties conj k)
nil)))))
(remove #(nil? (second %))))
page-refs (get-page-ref-names-from-properties properties user-config)
block-refs (extract-block-refs properties)
properties-text-values (->> (map (fn [[k _v _refs original-text]] [k original-text]) properties)
(into {}))
properties (map (fn [[k v _]] [k v]) properties)
properties' (into {} properties)]
{:properties properties'
:properties-order (map first properties)
:properties-text-values properties-text-values
:invalid-properties @*invalid-properties
:page-refs page-refs
:block-refs block-refs})))
(defn- paragraph-timestamp-block?
[block]
(and (paragraph-block? block)
(or (timestamp-block? (first (second block)))
(timestamp-block? (second (second block))))))
(defn- extract-timestamps
[block]
(some->>
(second block)
(filter timestamp-block?)
(map last)
(into {})))
(defn- timestamp->scheduled-or-deadline-value
[{:keys [date repetition] timestamp-time :time}]
(let [{:keys [year month day]} date
day (js/parseInt (str year (common-util/zero-pad month) (common-util/zero-pad day)))]
(if (or timestamp-time repetition)
(cond-> {:date-int day}
timestamp-time
(assoc :time timestamp-time)
repetition
(assoc :repetition repetition))
day)))
;; {"Deadline" {:date {:year 2020, :month 10, :day 20}, :wday "Tue", :time {:hour 8, :min 0}, :repetition [["DoublePlus"] ["Day"] 1], :active true}}
(defn timestamps->scheduled-and-deadline
[timestamps]
(let [timestamps (update-keys timestamps (comp keyword string/lower-case))
m (some->> (select-keys timestamps [:scheduled :deadline])
(map (fn [[k v]]
(let [value (timestamp->scheduled-or-deadline-value v)]
(case k
:scheduled
{:scheduled value}
:deadline
{:deadline value})))))]
(apply merge m)))
(defn- convert-page-if-journal-impl
"Convert journal file name to user' custom date format"
[original-page-name date-formatter & {:keys [export-to-db-graph?]}]
(when original-page-name
(let [page-name (common-util/page-name-sanity-lc original-page-name)
day (when date-formatter
(date-time-util/journal-title->int
page-name
;; When exporting, only use the configured date-formatter. Allowing for other date formatters allows
;; for page names to change which breaks looking up journal refs for unconfigured journal pages
(if export-to-db-graph? [date-formatter] (date-time-util/safe-journal-title-formatters date-formatter))))]
(if day
(let [original-page-name' (date-time-util/int->journal-title day date-formatter)
default-journal-page-name (date-time-util/int->journal-title day date-time-util/default-journal-title-formatter)]
[original-page-name' (common-util/page-name-sanity-lc default-journal-page-name) day])
[original-page-name page-name day]))))
(def convert-page-if-journal (memoize convert-page-if-journal-impl))
;; Hack to detect export as some fns are too deeply nested to be refactored to get explicit option
(def *export-to-db-graph? (atom false))
(defn- get-page
"Similar to get-page but only for file graphs"
[db page-name]
(when (and db (string? page-name))
(d/entity db
(first (sort (map :e (entity-util/get-pages-by-name db page-name)))))))
(defn- page-name-string->map
[original-page-name db date-formatter
{:keys [with-timestamp? page-uuid from-page class? skip-existing-page-check? skip-journal?]}]
(let [db-based? (entity-plus/db-based-graph? db)
original-page-name (common-util/remove-boundary-slashes original-page-name)
[original-page-name' page-name journal-day]
(if skip-journal?
[original-page-name (common-util/page-name-sanity-lc original-page-name) nil]
(convert-page-if-journal original-page-name date-formatter {:export-to-db-graph? @*export-to-db-graph?}))
namespace? (and (or (not db-based?) @*export-to-db-graph?)
(not journal-day)
(not (boolean (text/get-nested-page-name original-page-name')))
(text/namespace-page? original-page-name'))
page-entity (when (and db (not skip-existing-page-check?))
(cond
journal-day
(ldb/get-journal-page-by-day db journal-day)
(and class? db-based?)
(some->> (ldb/page-exists? db original-page-name' #{:logseq.class/Tag})
first
(d/entity db))
:else
(get-page db original-page-name')))
original-page-name' (or from-page (:block/title page-entity) original-page-name')
page-name' (if (and journal-day page-entity)
(:block/name page-entity)
page-name)
page (merge
{:block/name page-name'
:block/title original-page-name'}
(when (and original-page-name
(not= (string/lower-case original-page-name)
(string/lower-case original-page-name'))
(not @*export-to-db-graph?))
{:block.temp/original-page-name original-page-name})
(if (and class? page-entity (:db/ident page-entity))
{:block/uuid (:block/uuid page-entity)
:db/ident (:db/ident page-entity)}
(let [new-uuid* (if (uuid? page-uuid)
page-uuid
(if journal-day
(common-uuid/gen-uuid :journal-page-uuid journal-day)
(common-uuid/gen-uuid)))
new-uuid (if skip-existing-page-check?
new-uuid*
(or
(cond page-entity (:block/uuid page-entity)
(uuid? page-uuid) page-uuid)
new-uuid*))]
{:block/uuid new-uuid}))
(when namespace?
(let [namespace' (first (common-util/split-last "/" original-page-name))]
(when-not (string/blank? namespace')
{:block/namespace {:block/name (string/trim (common-util/page-name-sanity-lc namespace'))}})))
(when (and with-timestamp? (or skip-existing-page-check? (not page-entity))) ;; Only assign timestamp on creating new entity
(let [current-ms (common-util/time-ms)]
{:block/created-at current-ms
:block/updated-at current-ms}))
(if journal-day
(cond-> {:block/journal-day journal-day}
db-based?
(assoc :block/tags [:logseq.class/Journal])
(not db-based?)
(assoc :block/type "journal"))
{}))]
[page page-entity]))
(defn sanitize-hashtag-name
"This must be kept in sync with its reverse operation in logseq.db.frontend.content"
[s]
(string/replace s "#" "HashTag-"))
(defn- page-entity?
"Support DB or file graphs because of exporter"
[entity]
(or (entity-util/page? entity)
(contains? #{"page" "journal"} (:block/type entity))))
;; TODO: refactor
(defn page-name->map
"Create a page's map structure given a original page name (string).
map as input is supported for legacy compatibility.
`with-timestamp?`: assign timestampes to the map structure.
Useful when creating new pages from references or namespaces,
as there's no chance to introduce timestamps via editing in page
`skip-existing-page-check?`: if true, allows pages to have the same name"
[original-page-name db with-timestamp? date-formatter
& {:keys [page-uuid class?] :as options}]
(when-not (and db (common-util/uuid-string? original-page-name)
(not (page-entity? (d/entity db [:block/uuid (uuid original-page-name)]))))
(let [db-based? (entity-plus/db-based-graph? db)
original-page-name (cond-> (string/trim original-page-name)
db-based?
sanitize-hashtag-name)
[page _page-entity] (cond
(and original-page-name (string? original-page-name))
(page-name-string->map original-page-name db date-formatter
(assoc options :with-timestamp? with-timestamp?))
:else
(let [page (cond (and (map? original-page-name) (:block/uuid original-page-name))
original-page-name
(map? original-page-name)
(assoc original-page-name :block/uuid (or page-uuid (d/squuid)))
:else
nil)]
[page nil]))]
(when page
(if db-based?
(let [tags (if class? [:logseq.class/Tag]
(or (:block/tags page)
[:logseq.class/Page]))]
(assoc page :block/tags tags))
(assoc page :block/type (or (:block/type page) "page")))))))
(defn- db-namespace-page?
"Namespace page that're not journal pages"
[db-based? page]
(and db-based?
(text/namespace-page? page)
(not (common-date/valid-journal-title-with-slash? page))))
(defn- ref->map
[db *col {:keys [date-formatter *name->id tag? db-based? structured-tags]}]
(let [col (distinct (remove string/blank? @*col))
children-pages (->> (mapcat (fn [p]
(let [p (if (map? p)
(:block/title p)
p)]
(when (string? p)
(let [p (or (text/get-nested-page-name p) p)]
(if (and (text/namespace-page? p)
(not (common-date/valid-journal-title-with-slash? p))
(not tag?))
(common-util/split-namespace-pages p)
[p])))))
col)
(remove string/blank?)
(distinct))
col (->> (distinct (concat col children-pages))
(remove nil?))
export-to-db-graph? @*export-to-db-graph?]
(map
(fn [item]
(let [macro? (and (map? item)
(= "macro" (:type item)))
tag? (if export-to-db-graph?
tag?
(or (contains? structured-tags item) tag?))]
(when-not macro?
(let [m (page-name->map item db true date-formatter {:class? tag?})
result (cond->> m
(and db-based? tag? (not (:db/ident m)))
(db-class/build-new-class db))
page-name (if db-based? (:block/title result) (:block/name result))
id (get @*name->id page-name)]
(when (nil? id)
(swap! *name->id assoc page-name (:block/uuid result)))
;; Changing a :block/uuid should be done cautiously here as it can break
;; the identity of built-in concepts in db graphs
(if (and id
(or (when-let [ident (:db/ident result)]
(nil? (d/entity db ident)))
export-to-db-graph?))
(assoc result :block/uuid id)
result))))) col)))
(defn- with-page-refs-and-tags
[{:keys [title body tags refs marker priority] :as block} db date-formatter {:keys [structured-tags]
:or {structured-tags #{}}}]
(let [db-based? (and (entity-plus/db-based-graph? db) (not @*export-to-db-graph?))
refs (->> (concat tags refs (when-not db-based? [marker priority]))
(remove string/blank?)
(distinct))
*refs (atom refs)
*structured-tags (atom (set structured-tags))]
(walk/prewalk
(fn [form]
;; skip custom queries
(when-not (and (vector? form)
(= (first form) "Custom")
(= (second form) "query"))
(when-let [page (get-page-reference form (get block :format :markdown))]
(when-let [page' (when-not (db-namespace-page? db-based? page)
page)]
(swap! *refs conj page')))
(when-let [tag (get-tag form)]
(let [tag (text/page-ref-un-brackets! tag)]
(when-let [tag' (when-not (db-namespace-page? db-based? tag)
tag)]
(when (common-util/tag-valid? tag')
(swap! *refs conj tag')
(swap! *structured-tags conj tag')))))
form))
(concat title body))
(swap! *refs #(remove string/blank? %))
(let [*name->id (atom {})
ref->map-options {:db-based? db-based?
:date-formatter date-formatter
:*name->id *name->id
:structured-tags (set @*structured-tags)}
refs (->> (ref->map db *refs ref->map-options)
(remove nil?)
(map (fn [ref]
(cond-> ref
(:block.temp/original-page-name ref)
(assoc :block.temp/original-page-name (:block.temp/original-page-name ref))))))
tags (ref->map db *structured-tags (assoc ref->map-options :tag? true))]
(assoc block
:refs refs
:tags tags))))
(defn- with-block-refs
[{:keys [title body] :as block}]
(let [ref-blocks (extract-block-refs (concat title body))
refs (distinct (concat (:refs block) ref-blocks))]
(assoc block :refs refs)))
(defn block-keywordize
[block]
(update-keys
block
(fn [k]
(if (namespace k)
k
(keyword "block" k)))))
(defn- sanity-blocks-data
"Clean up blocks data and add `block` ns to all keys"
[blocks]
(map (fn [block]
(if (map? block)
(block-keywordize (common-util/remove-nils-non-nested block))
block))
blocks))
(defn get-block-content
[utf8-content block format meta' block-pattern]
(let [content (if-let [end-pos (:end_pos meta')]
(utf8/substring utf8-content
(:start_pos meta')
end-pos)
(utf8/substring utf8-content
(:start_pos meta')))
content (when content
(let [content (text/remove-level-spaces content format block-pattern)]
(if (or (:pre-block? block)
(= (get block :format :markdown) :org))
content
(gp-mldoc/remove-indentation-spaces content (inc (:level block)) false))))]
(if (= format :org)
content
(gp-property/->new-properties content))))
(defn get-custom-id-or-new-id
[properties]
(or (when-let [custom-id (or (get-in properties [:properties :custom-id])
(get-in properties [:properties :custom_id])
(get-in properties [:properties :id]))]
;; guard against non-string custom-ids
(when-let [custom-id (and (string? custom-id) (string/trim custom-id))]
(some-> custom-id parse-uuid)))
(d/squuid)))
(defn get-page-refs-from-properties
[properties db date-formatter user-config]
(let [page-refs (get-page-ref-names-from-properties properties user-config)]
(map (fn [page] (page-name->map page db true date-formatter)) page-refs)))
(defn- with-page-block-refs
[block db date-formatter opts]
(some-> block
(with-page-refs-and-tags db date-formatter opts)
with-block-refs
(update :refs (fn [col] (remove nil? col)))))
(defn- macro->block
"macro: {:name \"\" arguments [\"\"]}"
[macro]
{:block/uuid (common-uuid/gen-uuid)
:block/type "macro"
:block/properties {:logseq.macro-name (:name macro)
:logseq.macro-arguments (:arguments macro)}})
(defn extract-macros-from-ast
[ast]
(let [*result (atom #{})]
(walk/postwalk
(fn [f]
(if (and (vector? f) (= (first f) "Macro"))
(do
(swap! *result conj (second f))
nil)
f))
ast)
(mapv macro->block @*result)))
(defn with-pre-block-if-exists
[blocks body pre-block-properties encoded-content {:keys [db date-formatter user-config]}]
(let [first-block (first blocks)
first-block-start-pos (get-in first-block [:block/meta :start_pos])
;; Add pre-block
blocks (if (or (> first-block-start-pos 0)
(empty? blocks))
(cons
(merge
(let [content (utf8/substring encoded-content 0 first-block-start-pos)
{:keys [properties properties-order properties-text-values invalid-properties]} pre-block-properties
id (get-custom-id-or-new-id {:properties properties})
property-refs (->> (get-page-refs-from-properties
properties db date-formatter
user-config)
(map :block/title))
pre-block? (if (:heading properties) false true)
block {:block/uuid id
:block/title content
:block/level 1
:block/properties properties
:block/properties-order (vec properties-order)
:block/properties-text-values properties-text-values
:block/invalid-properties invalid-properties
:block/pre-block? pre-block?
:block/macros (extract-macros-from-ast body)
:block.temp/ast-body body}
{:keys [tags refs]}
(with-page-block-refs {:body body :refs property-refs} db date-formatter {})]
(cond-> block
tags
(assoc :block/tags tags)
true
(assoc :block/refs (concat refs (:block-refs pre-block-properties)))))
(select-keys first-block [:block/format :block/page]))
blocks)
blocks)]
blocks))
(defn- with-heading-property
[properties markdown-heading? size]
(if markdown-heading?
(assoc properties :heading size)
properties))
(defn- construct-block
[block properties* timestamps body encoded-content format pos-meta {:keys [block-pattern db date-formatter remove-properties? remove-logbook? remove-deadline-scheduled? db-graph-mode? export-to-db-graph?]}]
(let [id (get-custom-id-or-new-id properties*)
block-tags (and export-to-db-graph? (get-in properties* [:properties :tags]))
;; For export, remove tags from properties as they are being converted to classes
properties (if (seq block-tags)
(-> properties*
(update :properties #(dissoc % :tags))
(update :properties-text-values #(dissoc % :tags))
(update :properties-order (fn [v] (remove #(= :tags %) v)))
(update :page-refs (fn [v] (remove #(= "tags" %) v))))
properties*)
ref-pages-in-properties (->> (:page-refs properties)
(remove string/blank?))
block (second block)
unordered? (:unordered block)
markdown-heading? (and (:size block) (= :markdown format))
block (if markdown-heading?
(assoc block
:level (if unordered? (:level block) 1))
block)
block (cond->
(-> (assoc block
:uuid id
:refs ref-pages-in-properties
:format format
:meta pos-meta)
(dissoc :size :unordered))
(or (seq (:properties properties)) markdown-heading?)
(assoc :properties (with-heading-property (:properties properties) markdown-heading? (:size block))
:properties-text-values (:properties-text-values properties)
:properties-order (vec (:properties-order properties)))
(seq (:invalid-properties properties))
(assoc :invalid-properties (:invalid-properties properties)))
block (if (get-in block [:properties :collapsed])
(-> (assoc block :collapsed? true)
(update :properties (fn [m] (dissoc m :collapsed)))
(update :properties-text-values dissoc :collapsed)
(update :properties-order (fn [keys'] (vec (remove #{:collapsed} keys')))))
block)
title (cond->> (get-block-content encoded-content block format pos-meta block-pattern)
remove-properties?
(gp-property/remove-properties (get block :format :markdown))
remove-logbook?
(gp-property/remove-logbook)
remove-deadline-scheduled?
(gp-property/remove-deadline-scheduled))
block (assoc block :block/title title)
block (if (seq timestamps)
(merge block (timestamps->scheduled-and-deadline timestamps))
block)
db-based? (or db-graph-mode? export-to-db-graph?)
block (-> block
(assoc :body body)
(with-page-block-refs db date-formatter
(cond-> {} (seq block-tags) (assoc :structured-tags block-tags))))
block (if db-based? block
(-> block
(update :tags (fn [tags] (map #(assoc % :block/format format) tags)))
(update :refs (fn [refs] (map #(if (map? %) (assoc % :block/format format) %) refs)))))
block (update block :refs concat (:block-refs properties))
{:keys [created-at updated-at]} (:properties properties)
block (cond-> block
(and created-at (integer? created-at))
(assoc :block/created-at created-at)
(and updated-at (integer? updated-at))
(assoc :block/updated-at updated-at))]
(dissoc block :title :body :anchor)))
(defn fix-duplicate-id
[block]
(println "Logseq will assign a new id for block with content:" (pr-str (:block/title block)))
(-> block
(assoc :block/uuid (d/squuid))
(update :block/properties dissoc :id)
(update :block/properties-text-values dissoc :id)
(update :block/properties-order #(vec (remove #{:id} %)))
(update :block/title (fn [c]
(let [replace-str (re-pattern
(str
"\n*\\s*"
(if (= :markdown (get block :block/format :markdown))
(str "id" gp-property/colons " " (:block/uuid block))
(str (gp-property/colons-org "id") " " (:block/uuid block)))))]
(string/replace-first c replace-str ""))))))
(defn fix-block-id-if-duplicated!
"If the block exists in another page or the current page, we need to fix it"
[db page-name *extracted-block-ids block]
(let [block-page-name (:block/name (:block/page (d/entity db [:block/uuid (:block/uuid block)])))
block (if (or (and block-page-name (not= block-page-name page-name))
(contains? @*extracted-block-ids (:block/uuid block)))
(fix-duplicate-id block)
block)]
(swap! *extracted-block-ids conj (:block/uuid block))
block))
(defn extract-blocks
"Extract headings from mldoc ast. Args:
* `ast`: mldoc ast.
* `content`: markdown or org-mode text.
* `format`: content's format, it could be either :markdown or :org-mode.
* `options`: Options are :user-config, :block-pattern, :date-formatter, :db and
* :db-graph-mode? : Set when a db graph in the frontend
* :export-to-db-graph? : Set when exporting to a db graph"
[ast content format {:keys [user-config db-graph-mode? export-to-db-graph?] :as options}]
{:pre [(seq ast) (string? content) (contains? #{:markdown :org} format)]}
(let [encoded-content (utf8/encode content)
all-blocks (vec (reverse ast))
[blocks body pre-block-properties]
(loop [headings []
ast-blocks (reverse ast)
block-idx 0
timestamps {}
properties {}
body []
prev-block-num 0]
(if (seq ast-blocks)
(let [[ast-block pos-meta] (first ast-blocks)]
(cond
(paragraph-timestamp-block? ast-block)
(let [ts (extract-timestamps ast-block)
timestamps' (merge timestamps ts)]
(recur headings (rest ast-blocks) (inc block-idx) timestamps' properties body (inc prev-block-num)))
(gp-property/properties-ast? ast-block)
(let [properties (extract-properties (second ast-block) (assoc user-config :format format))]
(recur headings (rest ast-blocks) (inc block-idx) timestamps properties body (inc prev-block-num)))
(heading-block? ast-block)
(let [cut-multiline? (and export-to-db-graph? (= prev-block-num 0))
prev-blocks (map first (subvec all-blocks (max 0 (- block-idx prev-block-num)) block-idx))
pos-meta' (if cut-multiline?
pos-meta
;; fix start_pos
(assoc pos-meta :end_pos
(if (seq headings)
(get-in (last headings) [:meta :start_pos])
nil)))
;; Remove properties, deadline/scheduled and logbook text from title in db graphs
options' (assoc options
:remove-properties?
(and export-to-db-graph? (some gp-property/properties-ast? prev-blocks))
:remove-logbook?
(and export-to-db-graph? (some #(= ["Drawer" "logbook"] (take 2 %)) prev-blocks))
:remove-deadline-scheduled?
(and export-to-db-graph? (some #(seq (set/intersection (set (flatten %)) #{"Deadline" "Scheduled"})) prev-blocks)))
block' (construct-block ast-block properties timestamps body encoded-content format pos-meta' options')
block'' (cond
db-graph-mode?
block'
export-to-db-graph?
(assoc block' :block.temp/ast-blocks (cons ast-block body))
:else
(assoc block' :macros (extract-macros-from-ast (cons ast-block body))))]
(recur (conj headings block'') (rest ast-blocks) (inc block-idx) {} {} [] 0))
:else
(recur headings (rest ast-blocks) (inc block-idx) timestamps properties (conj body ast-block) (inc prev-block-num))))
[(-> (reverse headings)
sanity-blocks-data)
body
properties]))
result (with-pre-block-if-exists blocks body pre-block-properties encoded-content options)]
(map #(dissoc % :block/meta) result)))
(defn with-parent-and-order
[page-id blocks]
(let [[blocks other-blocks] (split-with
(fn [b]
(not= "macro" (:block/type b)))
blocks)
result (loop [blocks (map (fn [block] (assoc block :block/level-spaces (:block/level block))) blocks)
parents [{:page/id page-id ; db id or a map {:block/name "xxx"}
:block/level 0
:block/level-spaces 0}]
result []]
(if (empty? blocks)
(map #(dissoc % :block/level-spaces) result)
(let [[block & others] blocks
level-spaces (:block/level-spaces block)
{uuid' :block/uuid :block/keys [level parent] :as last-parent} (last parents)
parent-spaces (:block/level-spaces last-parent)
[blocks parents result]
(cond
(= level-spaces parent-spaces) ; sibling
(let [block (assoc block
:block/parent parent
:block/level level)
parents' (conj (vec (butlast parents)) block)
result' (conj result block)]
[others parents' result'])
(> level-spaces parent-spaces) ; child
(let [parent (if uuid' [:block/uuid uuid'] (:page/id last-parent))
block (cond->
(assoc block
:block/parent parent)
;; fix block levels with wrong order
;; For example:
;; - a
;; - b
;; What if the input indentation is two spaces instead of 4 spaces
(>= (- level-spaces parent-spaces) 1)
(assoc :block/level (inc level)))
parents' (conj parents block)
result' (conj result block)]
[others parents' result'])
(< level-spaces parent-spaces)
(cond
(some #(= (:block/level-spaces %) (:block/level-spaces block)) parents) ; outdent
(let [parents' (vec (filter (fn [p] (<= (:block/level-spaces p) level-spaces)) parents))
blocks (cons (assoc (first blocks)
:block/level (dec level))
(rest blocks))]
[blocks parents' result])
:else
(let [[f r] (split-with (fn [p] (<= (:block/level-spaces p) level-spaces)) parents)
left (first r)
parent-id (if-let [block-id (:block/uuid (last f))]
[:block/uuid block-id]
page-id)
block (assoc block
:block/parent parent-id
:block/level (:block/level left)
:block/level-spaces (:block/level-spaces left))
parents' (->> (concat f [block]) vec)
result' (conj result block)]
[others parents' result'])))]
(recur blocks parents result))))
result' (map (fn [block] (assoc block :block/order (db-order/gen-key))) result)]
(concat result' other-blocks)))

3169
deps/graph-parser/src/logseq/graph_parser/exporter.cljs vendored Normal file
View File

File diff suppressed because it is too large Load Diff

327
deps/graph-parser/src/logseq/graph_parser/extract.cljc vendored Normal file
View File

@@ -0,0 +1,327 @@
(ns logseq.graph-parser.extract
"For file graphs, handles extraction of blocks, pages and mldoc ast in
preparation for db transaction"
;; Disable clj linters since we don't support clj
#?(:clj {:clj-kondo/config {:linters {:unresolved-namespace {:level :off}
:unresolved-symbol {:level :off}}}})
(:require #?(:org.babashka/nbb [logseq.common.log :as log]
:default [lambdaisland.glogi :as log])
[clojure.set :as set]
[clojure.string :as string]
[clojure.walk :as walk]
[datascript.core :as d]
[logseq.common.util :as common-util]
[logseq.common.uuid :as common-uuid]
[logseq.db :as ldb]
[logseq.graph-parser.block :as gp-block]
[logseq.graph-parser.mldoc :as gp-mldoc]
[logseq.graph-parser.property :as gp-property]
[logseq.graph-parser.text :as text]))
(defn- mldoc-support?
[format']
(contains? #{:org :markdown :md} (keyword format')))
(defn- path->file-name
;; Only for internal paths, as they are converted to POXIS already
;; https://github.com/logseq/logseq/blob/48b8e54e0fdd8fbd2c5d25b7f1912efef8814714/deps/graph-parser/src/logseq/graph_parser/extract.cljc#L32
;; Should be converted to POXIS first for external paths
[path]
(if (string/includes? path "/")
(last (common-util/split-last "/" path))
path))
(defn- path->file-body
[path]
(when-let [file-name (path->file-name path)]
(if (string/includes? file-name ".")
(first (common-util/split-last "." file-name))
file-name)))
(defn- safe-url-decode
[string]
(if (string/includes? string "%")
(some-> string str common-util/safe-decode-uri-component)
string))
(defn- decode-namespace-underlines
"Decode namespace underlines to slashed;
If continuous underlines, only decode at start;
Having empty namespace is invalid."
[string]
(string/replace string "___" "/"))
(defn- make-valid-namespaces
"Remove those empty namespaces from title to make it a valid page name."
[title]
(->> (string/split title "/")
(remove empty?)
(string/join "/")))
(defn- tri-lb-title-parsing
"Parsing file name under the new file name format
Avoid calling directly"
[file-name]
(some-> file-name
(decode-namespace-underlines)
(string/replace common-util/url-encoded-pattern safe-url-decode)
(make-valid-namespaces)))
;; Keep for backward compatibility
;; Rule of dir-ver 0
;; Source: https://github.com/logseq/logseq/blob/e7110eea6790eda5861fdedb6b02c2a78b504cd9/deps/graph-parser/src/logseq/graph_parser/extract.cljc#L35
(defn- legacy-title-parsing
[file-name-body]
(let [title (string/replace file-name-body "." "/")]
(or (common-util/safe-decode-uri-component title) title)))
(defn title-parsing
"Convert file name in the given file name format to page title"
[file-name-body filename-format]
(case filename-format
:triple-lowbar (tri-lb-title-parsing file-name-body)
(legacy-title-parsing file-name-body)))
(defn- get-page-name
"Get page name with overridden order of
`title::` property
file name parsing
first block content
note: `page-name-order` is deprecated on Apr. 2021
uri-encoded? - since paths on mobile are uri-encoded, need to decode them first
filename-format - the format used to parse file name
"
[file-path ast uri-encoded? filename-format]
;; headline
(let [ast (map first ast)
file (if uri-encoded? (js/decodeURI file-path) file-path)]
;; check backward compatibility?
;; FIXME: use pre-config dir
(if (string/starts-with? file "pages/contents.")
"Contents"
(let [first-block (last (first (filter gp-block/heading-block? ast)))
property-name (some (fn [ast-node]
(when (contains? #{"Properties" "Property_Drawer"} (first ast-node))
(let [properties-ast (second ast-node)
properties (zipmap (map (comp keyword string/lower-case first) properties-ast)
(map second properties-ast))]
(:title properties))))
ast)
first-block-name (let [title (last (first (:title first-block)))]
(and first-block
(string? title)
title))
file-name (when-let [result (path->file-body file)]
(if (mldoc-support? (common-util/get-file-ext file))
(title-parsing result filename-format)
result))]
(or property-name
file-name
first-block-name)))))
(defn- extract-page-alias-and-tags
[page-m page-name properties]
(let [alias (:alias properties)
alias' (if (coll? alias) alias [(str alias)])
aliases (and alias'
(seq (remove #(or (= page-name (common-util/page-name-sanity-lc %))
(string/blank? %)) ;; disable blank alias
alias')))
aliases' (keep
(fn [alias]
(let [page-name (common-util/page-name-sanity-lc alias)]
{:block/name page-name
:block/title alias}))
aliases)
result (cond-> page-m
(seq aliases')
(assoc :block/alias aliases')
(:tags properties)
(assoc :block/tags (concat
(:block/tags page-m)
(let [tags (:tags properties)
tags (if (coll? tags) tags [(str tags)])
tags (remove string/blank? tags)]
(map (fn [tag] {:block/name (common-util/page-name-sanity-lc tag)
:block/title tag})
tags)))))]
(update result :block/properties #(apply dissoc % gp-property/editable-linkable-built-in-properties))))
(defn- build-page-map
[properties invalid-properties properties-text-values file page page-name
{:keys [date-formatter db from-page skip-journal?]}]
(let [[*valid-properties *invalid-properties]
((juxt filter remove)
(fn [[k _v]] (gp-property/valid-property-name? (str k))) properties)
valid-properties (into {} *valid-properties)
invalid-properties (set (->> (map (comp name first) *invalid-properties)
(concat invalid-properties)))
page-m (->
(common-util/remove-nils-non-nested
(assoc
(gp-block/page-name->map page db true date-formatter
:from-page from-page
:skip-journal? skip-journal?)
:block/file {:file/path (common-util/path-normalize file)}))
(extract-page-alias-and-tags page-name properties))]
(cond->
page-m
(seq valid-properties)
(assoc :block/properties valid-properties
:block/properties-text-values (select-keys properties-text-values (keys valid-properties)))
(seq invalid-properties)
(assoc :block/invalid-properties invalid-properties))))
(defn- attach-block-ids-if-match
"If block-ids are provided and match the number of blocks, attach them to blocks
If block-ids are provided but don't match the number of blocks, WARN and ignore
If block-ids are not provided (nil), just ignore"
[block-ids blocks]
(or (when block-ids
(if (= (count block-ids) (count blocks))
(mapv (fn [block-id block]
(if (some? block-id)
(assoc block :block/uuid (uuid block-id))
block))
block-ids blocks)
(log/error :gp-extract/attach-block-ids-not-match "attach-block-ids-if-match: block-ids provided, but doesn't match the number of blocks, ignoring")))
blocks))
(defn- build-pages-aux
[db page-map ref-pages date-formatter format]
(let [namespace-pages (let [page (:block/title page-map)]
(when (and (not (:block/journal-day page-map))
(text/namespace-page? page))
(->> (common-util/split-namespace-pages page)
(map (fn [page]
(-> (gp-block/page-name->map page db true date-formatter)
(assoc :block/format format)))))))
pages (->> (concat
[page-map]
@ref-pages
namespace-pages)
;; remove block references
(remove vector?)
(remove nil?)
(filter :block/name))
pages (common-util/distinct-by :block/name pages)
pages (remove nil? pages)]
(map (fn [page]
(let [page-id (if-let [journal-day (:block/journal-day page)]
(common-uuid/gen-uuid :journal-page-uuid journal-day)
(or (when db
(:block/uuid (ldb/get-page db (:block/name page))))
(d/squuid)))]
(assoc page :block/uuid page-id)))
pages)))
;; TODO: performance improvement
(defn- extract-pages-and-blocks
"uri-encoded? - if is true, apply URL decode on the file path
options -
:resolve-uuid-fn - Optional fn which is called to resolve uuids of each block. Enables diff-merge
(2 ways diff) based uuid resolution upon external editing.
returns a list of the uuids, given the receiving ast, or nil if not able to resolve.
Implemented in reset-file-handler/diff-merge-uuids-2ways for IoC
Called in gp-extract/extract as AST is being parsed and properties are extracted there"
[format ast properties file content {:keys [date-formatter db filename-format resolve-uuid-fn skip-journal?]
:or {resolve-uuid-fn (constantly nil)}
:as options}]
(assert db "Datascript DB is required")
(try
(let [page (get-page-name file ast false filename-format)
[page page-name _journal-day]
(if skip-journal?
[page (common-util/page-name-sanity-lc page) nil]
(gp-block/convert-page-if-journal page date-formatter))
options' (assoc options :page-name page-name)
;; In case of diff-merge (2way) triggered, use the uuids to override the ones extracted from the AST
override-uuids (resolve-uuid-fn format ast content options')
*extracted-block-ids (atom #{})
blocks (->> (gp-block/extract-blocks ast content format options')
(attach-block-ids-if-match override-uuids)
(mapv #(gp-block/fix-block-id-if-duplicated! db page-name *extracted-block-ids %))
;; FIXME: use page uuid
(gp-block/with-parent-and-order {:block/name page-name})
(vec))
ref-pages (atom #{})
blocks (map (fn [block]
(if (= (:block/type block) "macro")
block
(let [block-ref-pages (seq (:block/refs block))]
(when block-ref-pages
(swap! ref-pages set/union (set block-ref-pages)))
(-> block
(dissoc :ref-pages)
(assoc :block/page [:block/name page-name]
:block/refs block-ref-pages
:block/format format)))))
blocks)
[properties invalid-properties properties-text-values]
(if (:block/pre-block? (first blocks))
[(:block/properties (first blocks))
(:block/invalid-properties (first blocks))
(:block/properties-text-values (first blocks))]
[properties [] {}])
page-map (build-page-map properties invalid-properties properties-text-values file page page-name (assoc options' :from-page page))
pages (build-pages-aux db page-map ref-pages date-formatter format)
blocks (->> (remove nil? blocks)
(map (fn [b] (dissoc b :block.temp/ast-title :block.temp/ast-body :block/level :block/children :block/meta))))]
[pages blocks])
(catch :default e
(log/error :exception e))))
(defn extract
"Extracts pages, blocks and ast from given file"
[file-path content {:keys [user-config verbose] :or {verbose true} :as options}]
(if (string/blank? content)
{}
(let [format (common-util/get-format file-path)
_ (when verbose (println "Parsing start: " file-path))
ast (gp-mldoc/->edn content (gp-mldoc/default-config format
;; {:parse_outline_only? true}
))]
(when verbose (println "Parsing finished: " file-path))
(let [first-block (ffirst ast)
properties (let [properties (and (gp-property/properties-ast? first-block)
(->> (last first-block)
(map (fn [[x y mldoc-ast]]
(let [k (if (keyword? x)
(subs (str x) 1)
x)]
[(string/lower-case k) (text/parse-property k y mldoc-ast (assoc user-config :format format))])))
(into {})
(walk/keywordize-keys)))]
(when (and properties (seq properties))
(if (:filters properties)
(update properties :filters
(fn [v]
(string/replace (or v "") "\\" "")))
properties)))
[pages blocks] (extract-pages-and-blocks format ast properties file-path content options)]
{:pages pages
:blocks blocks
:ast ast}))))
(defn- with-block-uuid
[pages]
(->> (common-util/distinct-by :block/name pages)
(map (fn [page]
(if-let [journal-day (:block/journal-day page)]
(assoc page :block/uuid (common-uuid/gen-uuid :journal-page-uuid journal-day))
(cond-> page
(nil? (:block/uuid page))
(assoc :block/uuid (d/squuid))))))))
(defn with-ref-pages
[pages blocks]
(let [ref-pages (->> (mapcat :block/refs blocks)
(filter :block/name))]
(->> (concat pages ref-pages)
(group-by :block/name)
vals
(map (partial apply merge))
(with-block-uuid))))

417
deps/graph-parser/src/logseq/graph_parser/mldoc.cljc vendored Normal file
View File

@@ -0,0 +1,417 @@
(ns logseq.graph-parser.mldoc
"Wraps https://github.com/logseq/mldoc to parse files into mldoc ast. This ns
encapsulates mldoc's json api by only taking and returning edn"
;; Disable clj linters since we don't support clj
#?(:clj {:clj-kondo/config {:linters {:unresolved-namespace {:level :off}
:unresolved-symbol {:level :off}}}})
(:require #?(:org.babashka/nbb ["mldoc$default" :refer [Mldoc]]
:default ["mldoc" :refer [Mldoc]])
#?(:org.babashka/nbb [logseq.common.log :as log]
:default [lambdaisland.glogi :as log])
[cljs-bean.core :as bean]
[clojure.string :as string]
[clojure.walk :as walk]
[goog.object :as gobj]
[logseq.common.config :as common-config]
[logseq.common.util :as common-util]
[logseq.db.sqlite.util :as sqlite-util]
[logseq.graph-parser.utf8 :as utf8]))
(defonce parseJson (gobj/get Mldoc "parseJson"))
(defonce parseInlineJson (gobj/get Mldoc "parseInlineJson"))
(defonce astExportMarkdown (gobj/get Mldoc "astExportMarkdown"))
(defonce getReferences (gobj/get Mldoc "getReferences"))
(def default-references
(js/JSON.stringify
(clj->js {:embed_blocks []
:embed_pages []})))
(defn- convert-export-md-remove-options [opts]
(->> opts
(mapv (fn [opt]
(case opt
:page-ref ["Page_ref"]
:emphasis ["Emphasis"]
[])))
(remove empty?)))
(defn parse-json
[content config]
(parseJson content config))
(defn inline-parse-json
[text config]
(parseInlineJson text config))
(defn get-references
[text config]
(when-not (string/blank? text)
(common-util/json->clj (getReferences text config))))
(defn ast-export-markdown
[ast config references]
(astExportMarkdown ast
config
(or references default-references)))
(defn default-config-map
([format]
(default-config-map format {:export-heading-to-list? false}))
([format {:keys [export-heading-to-list? export-keep-properties? export-md-indent-style export-md-remove-options parse_outline_only?]}]
(let [format (string/capitalize (name (or format :markdown)))]
(->> {:toc false
:parse_outline_only (or parse_outline_only? false)
:heading_number false
:keep_line_break true
:format format
:heading_to_list (or export-heading-to-list? false)
:exporting_keep_properties export-keep-properties?
:export_md_indent_style export-md-indent-style
:export_md_remove_options
(convert-export-md-remove-options export-md-remove-options)}
(filter #(not (nil? (second %))))
(into {})))))
(defn default-config
([format]
(default-config format {:export-heading-to-list? false}))
([format opts]
(->> (default-config-map format opts)
bean/->js
js/JSON.stringify)))
(defn remove-indentation-spaces
"Remove the indentation spaces from the content. Only for markdown.
level - ast level + 1 (2 for the first level, 3 for the second level, etc., as the non-first line of multi-line block has 2 more space
Ex.
- level 1 multiline block first line
level 1 multiline block second line
\t- level 2 multiline block first line
\t level 2 multiline block second line
remove-first-line? - apply the indentation removal to the first line or not"
[s level remove-first-line?]
(let [lines (string/split-lines s)
[f & r] lines
body (map (fn [line]
;; Check if the indentation area only contains white spaces
;; Level = ast level + 1, 1-based indentation level
;; For markdown in Logseq, the indentation area for the non-first line of multi-line block is (ast level - 1) * "\t" + 2 * "(space)"
(if (string/blank? (common-util/safe-subs line 0 level))
;; If valid, then remove the indentation area spaces. Keep the rest of the line (might contain leading spaces)
(common-util/safe-subs line level)
;; Otherwise, trim these invalid spaces
(string/triml line)))
(if remove-first-line? lines r))
content (if remove-first-line? body (cons f body))]
(string/join "\n" content)))
(defn- update-src-full-content
[ast content]
(let [content (utf8/encode content)]
(map (fn [[block pos-meta]]
(if (and (vector? block)
(= "Src" (first block)))
(let [{:keys [start_pos end_pos]} pos-meta
content (utf8/substring content start_pos end_pos)
spaces (re-find #"^[\t ]+" (first (string/split-lines content)))
content (if spaces (remove-indentation-spaces content (count spaces) true)
content)
block ["Src" (assoc (second block) :full_content content)]]
[block pos-meta])
[block pos-meta])) ast)))
(def ^:private inline-ast-types
#{"Plain" "Spaces" "Link" "Nested_link" "Target" "Subscript" "Superscript"
"Footnote_Reference" "Cookie" "Latex_Fragment" "Macro" "Entity" "Timestamp"
"Radio_Target" "Export_Snippet" "Inline_Source_Block" "Email" "Inline_Hiccup"
"Inline_Html" "Emphasis" "Verbatim" "Code" "Break_Line" "Hard_Break_Line"})
(defn- inline-coll?
[x]
(and (vector? x)
(seq x)
(every? #(and (vector? %)
(string? (first %))
(contains? inline-ast-types (first %)))
x)))
(defn- inline-ast->source
;; Only reconstructs the source text for the node types that appear when
;; mldoc splits a macro containing ^{}/{_{}. Nested rich-inline types
;; (Emphasis, Code, etc.) inside a Superscript/Subscript will produce nil,
;; which (apply str ...) silently drops. This is intentional: the function
;; is only called during macro-recovery, not for general rendering.
[[typ content]]
(case typ
"Plain" content
"Spaces" content
"Link" (:full_text content)
;; Nested_link is the AST node for [[page name]]; :content is the page name
;; string. Without this case, collect-macro-source aborts via when-let
;; whenever a page ref appears inside a fragmented macro.
"Nested_link" (str "[[" (:content content) "]]")
"Superscript" (str "^{" (apply str (map inline-ast->source content)) "}")
"Subscript" (str "_{" (apply str (map inline-ast->source content)) "}")
nil))
(defn- starts-with-at?
[s prefix idx]
(let [end (+ idx (count prefix))]
(and (<= end (count s))
(= prefix (subs s idx end)))))
(defn- unclosed-script-markup?
[s]
(loop [idx 0
depth 0]
(if (< idx (count s))
(cond
(or (starts-with-at? s "^{" idx)
(starts-with-at? s "_{" idx))
(recur (+ idx 2) (inc depth))
(and (pos? depth) (= \} (nth s idx)))
(recur (inc idx) (dec depth))
:else
(recur (inc idx) depth))
(pos? depth))))
(defn- split-macro-arguments
[s]
(if (string/blank? s)
[]
(loop [idx 0
start 0
page-ref-depth 0
script-depth 0
quoted? false
escaped? false
result []]
(if (< idx (count s))
(let [c (nth s idx)]
(cond
escaped?
(recur (inc idx) start page-ref-depth script-depth quoted? false result)
(= \\ c)
(recur (inc idx) start page-ref-depth script-depth quoted? true result)
(= \" c)
(recur (inc idx) start page-ref-depth script-depth (not quoted?) false result)
quoted?
(recur (inc idx) start page-ref-depth script-depth quoted? false result)
(starts-with-at? s "[[" idx)
(recur (+ idx 2) start (inc page-ref-depth) script-depth quoted? false result)
(and (pos? page-ref-depth) (starts-with-at? s "]]" idx))
(recur (+ idx 2) start (dec page-ref-depth) script-depth quoted? false result)
(or (starts-with-at? s "^{" idx)
(starts-with-at? s "_{" idx))
(recur (+ idx 2) start page-ref-depth (inc script-depth) quoted? false result)
(and (pos? script-depth) (= \} c))
(recur (inc idx) start page-ref-depth (dec script-depth) quoted? false result)
(and (zero? page-ref-depth) (zero? script-depth) (= \, c))
(recur (inc idx) (inc idx) page-ref-depth script-depth quoted? false
(conj result (string/trim (subs s start idx))))
:else
(recur (inc idx) start page-ref-depth script-depth quoted? false result)))
(conj result (string/trim (subs s start)))))))
(defn- macro-source->ast
[s]
(when (and (string/starts-with? s "{{")
(string/ends-with? s "}}"))
(let [content (-> s
(subs 2 (- (count s) 2))
string/triml)
[_ name arguments] (re-matches #"([^\s]+)(?:\s+([\s\S]*))?" content)]
(when name
["Macro" {:name name
:arguments (split-macro-arguments arguments)}]))))
(defn- collect-macro-source
[items]
(loop [remaining items
source ""]
(when-let [[item & more] (seq remaining)]
(let [[typ content] item]
(cond
(= "Plain" typ)
(if-let [idx (string/index-of content "}}")]
(let [macro-source (str source (subs content 0 (+ idx 2)))]
(when-let [macro (macro-source->ast macro-source)]
(let [suffix (subs content (+ idx 2))]
{:macro macro
:suffix (when-not (string/blank? suffix) ["Plain" suffix])
:remaining more})))
(recur more (str source content)))
:else
(when-let [item-source (inline-ast->source item)]
(recur more (str source item-source))))))))
(defn- close-script-markup-in-macro
;; Appends a single "}" to the last argument to close the outermost
;; unmatched ^{/{_ bracket. Handles only depth-1 cases; deeper nesting
;; (e.g. Ca^{^{2}}) is not supported and is not expected in practice.
[macro]
(update-in macro [1 :arguments]
(fn [arguments]
(update arguments (dec (count arguments)) str "}"))))
(defn- recover-inline-macros
[inline-list]
(loop [remaining inline-list
result []]
(if-let [[item & more] (seq remaining)]
(let [[typ content] item]
(cond
(and (= "Plain" typ)
(string/includes? content "{{"))
(let [idx (string/index-of content "{{")
prefix (subs content 0 idx)
start-item ["Plain" (subs content idx)]]
(if-let [{:keys [macro suffix remaining]} (collect-macro-source (cons start-item more))]
(recur (if suffix
(cons suffix remaining)
remaining)
(cond-> result
(not (string/blank? prefix)) (conj ["Plain" prefix])
true (conj macro)))
(recur more (conj result item))))
(and (= "Macro" typ)
(seq (:arguments content))
(unclosed-script-markup? (last (:arguments content)))
(= "Plain" (ffirst more))
(string/starts-with? (second (first more)) "}"))
(let [[_ plain] (first more)
suffix (subs plain 1)]
(recur (if (string/blank? suffix)
(rest more)
(cons ["Plain" suffix] (rest more)))
(conj result (close-script-markup-in-macro item))))
:else
(recur more (conj result item))))
result)))
(defn- normalize-macro-asts
[ast]
(walk/postwalk
(fn [x]
(if (inline-coll? x)
(recover-inline-macros x)
x))
ast))
(defn- macro-with-script-markup?
[content]
(and (string/includes? content "{{")
(or (string/includes? content "^{")
(string/includes? content "_{"))))
(defn collect-page-properties
[ast config]
(when (seq ast)
(let [original-ast ast
directive? (fn [[item _]] (= "directive" (string/lower-case (first item))))
grouped-ast (group-by directive? original-ast)
[properties-ast other-ast] [(->> (get grouped-ast true)
(map first))
(get grouped-ast false)]
properties (map (fn [[_directive k v]]
[k v (get-references v config)])
properties-ast)]
(if (seq properties)
(cons [["Properties" properties] nil] other-ast)
original-ast))))
(defn get-default-config
"Gets a mldoc default config for the given format. Works for DB and file graphs"
[repo format]
(let [db-based? (sqlite-util/db-based-graph? repo)]
(->>
(cond-> (default-config-map format)
db-based?
(assoc :enable_drawers false
:parse_marker false
:parse_priority false))
bean/->js
js/JSON.stringify)))
(defn ->edn
;; TODO: Re-enable schema
;; {:malli/schema [:=> [:cat :string :string] mldoc-schema/block-ast-with-pos-coll-schema]}
([content config]
(if (string? content)
(try
(if (string/blank? content)
[]
(-> content
(parse-json config)
(common-util/json->clj)
(cond-> (macro-with-script-markup? content)
(normalize-macro-asts))
(update-src-full-content content)
(collect-page-properties config)))
(catch :default e
(log/error :unexpected-error e)
[]))
(log/error :edn/wrong-content-type content)))
([repo content format]
(->edn content (get-default-config repo format))))
(defn ->db-edn
"Wrapper around ->edn for DB graphs"
[content format]
(->edn "logseq_db_repo_stub" content format))
(defn inline->edn
[text config]
(try
(if (string/blank? text)
{}
(-> text
(inline-parse-json config)
(common-util/json->clj)
(cond-> (macro-with-script-markup? text)
(normalize-macro-asts))))
(catch :default _e
[])))
(defn ast-link?
[[type link]]
(let [[ref-type ref-value] (:url link)]
(and (= "Link" type)
(or
;; 1. url
(not (contains? #{"Page_ref" "Block_ref"} ref-type))
(and (contains? #{"Page_ref"} ref-type)
(boolean (common-config/local-relative-asset? ref-value)))))))
(defn mldoc-link?
"Check whether s is a link (including page/block refs)."
[format s]
(let [result (inline->edn s (default-config format))]
(and
(= 1 (count result))
(let [result' (first result)]
(or (contains? #{"Nested_link"} (first result'))
(contains? #{"Page_ref" "Block_ref" "Complex"} (first (:url (second result')))))))))
(defn block-with-title?
[type]
(contains? #{"Paragraph"
"Raw_Html"
"Hiccup"
"Heading"} type))

221
deps/graph-parser/src/logseq/graph_parser/property.cljs vendored Normal file
View File

@@ -0,0 +1,221 @@
(ns logseq.graph-parser.property
"For file graphs, core vars and util fns for properties"
(:require [clojure.set :as set]
[clojure.string :as string]
[goog.string :as gstring]
[goog.string.format]
[logseq.common.util :as common-util]))
(def colons "Property delimiter for markdown mode" "::")
(defn colons-org
"Property delimiter for org mode"
[property]
(str ":" property ":"))
(defn properties-ast?
[block]
(and
(vector? block)
(contains? #{"Property_Drawer" "Properties"}
(first block))))
(defn valid-property-name?
[s]
{:pre [(string? s)]}
(and (common-util/valid-edn-keyword? s)
(not (re-find #"[\"|^|(|)|{|}]+" s))
;; Disallow tags as property names
(not (re-find #"^:#" s))))
;; Built-in properties are properties that logseq uses for its features. Most of
;; these properties are hidden from the user but a few like the editable ones
;; are visible for the user to edit.
(def editable-linkable-built-in-properties
"Properties used by logseq that user can edit and that can have linkable property values"
#{:alias :aliases :tags})
(defn editable-built-in-properties
"Properties used by logseq that user can edit"
[]
(set/union #{:title :icon :template :template-including-parent :public :filters :exclude-from-graph-view
:logseq.query/nlp-date
;; org-mode only
:macro :filetags}
editable-linkable-built-in-properties))
(defn hidden-built-in-properties
"Properties used by logseq that user can't edit or see"
[]
#{:custom-id :background_color :created_at :last_modified_at ; backward compatibility only
:id :background-color :heading :collapsed
:created-at :updated-at :last-modified-at
:query-table :query-properties :query-sort-by :query-sort-desc :ls-type
:hl-type :hl-page :hl-stamp :hl-color :hl-value :logseq.macro-name :logseq.macro-arguments
:logseq.order-list-type
; task markers
:todo :doing :now :later :done})
(def built-in-property-types
"Types for built-in properties. Built-in properties whose values are to be
parsed by gp-text/parse-non-string-property-value should be added here"
{:template-including-parent :boolean
:public :boolean
:exclude-from-graph-view :boolean
:logseq.query/nlp-date :boolean
:heading :boolean ; FIXME: or integer
:collapsed :boolean
:created-at :integer
:created_at :integer
:updated-at :integer
:last-modified-at :integer
:last_modified_at :integer
:query-table :boolean
:query-sort-desc :boolean
:hl-page :integer
:hl-stamp :integer
:todo :integer
:doing :integer
:now :integer
:later :integer
:done :integer})
(assert (set/subset? (set (keys built-in-property-types))
(set/union (hidden-built-in-properties)
(editable-built-in-properties)))
"Keys of built-in-property-types must be valid built-in properties")
(defn unparsed-built-in-properties
"Properties whose values will not be parsed by gp-text/parse-property"
[]
(set/difference (set/union (hidden-built-in-properties)
(editable-built-in-properties))
;; Refs need to be parsed
editable-linkable-built-in-properties
;; All these should be parsed by gp-text/parse-non-string-property-value
(set (keys built-in-property-types))))
(defonce properties-start ":PROPERTIES:")
(defonce properties-end ":END:")
(defonce properties-end-pattern
(re-pattern (gstring/format "%s[\t\r ]*\n|(%s\\s*$)" properties-end properties-end)))
(defn- contains-properties?
[content]
(when content
(and (string/includes? content properties-start)
(re-find properties-end-pattern content))))
(defn ->new-properties
"New syntax: key:: value"
[content]
(if (contains-properties? content)
(let [lines (string/split-lines content)
start-idx (.indexOf lines properties-start)
end-idx (.indexOf lines properties-end)]
(if (and (>= start-idx 0) (> end-idx 0) (> end-idx start-idx))
(let [before (subvec lines 0 start-idx)
middle (->> (subvec lines (inc start-idx) end-idx)
(map (fn [text]
(let [[k v] (common-util/split-first ":" (subs text 1))]
(if (and k v)
(let [k (string/replace k "_" "-")
compare-k (keyword (string/lower-case k))
k (if (contains? #{:id :custom_id :custom-id} compare-k) "id" k)
k (if (contains? #{:last-modified-at} compare-k) "updated-at" k)]
(str k colons " " (string/trim v)))
text)))))
after (subvec lines (inc end-idx))
lines (concat before middle after)]
(string/join "\n" lines))
content))
content))
(defn simplified-property?
[line]
(boolean
(and (string? line)
(re-find (re-pattern (str "^\\s?[^ ]+" colons)) line))))
(defn remove-properties
[format content]
(cond
(contains-properties? content)
(let [lines (string/split-lines content)
[title-lines properties&body] (split-with #(-> (string/triml %)
string/upper-case
(string/starts-with? properties-start)
not)
lines)
body (drop-while #(-> (string/trim %)
string/upper-case
(string/starts-with? properties-end)
not
(or (string/blank? %)))
properties&body)
body (if (and (seq body)
(-> (first body)
string/triml
string/upper-case
(string/starts-with? properties-end)))
(let [line (string/replace (first body) #"(?i):END:\s?" "")]
(if (string/blank? line)
(rest body)
(cons line (rest body))))
body)]
(->> (concat title-lines body)
(string/join "\n")))
(not= format :org)
(let [lines (string/split-lines content)
lines (if (simplified-property? (first lines))
(drop-while simplified-property? lines)
(cons (first lines)
(drop-while simplified-property? (rest lines))))]
(string/join "\n" lines))
:else
content))
(defn remove-logbook
[content]
(when (string? content)
(let [lines (string/split-lines content)
[result _in-logbook?]
(reduce (fn [[acc in-logbook?] line]
(let [trimmed (string/trim line)
upper (string/upper-case trimmed)]
(cond
(string/starts-with? upper ":LOGBOOK:")
[acc true]
(and in-logbook? (string/starts-with? upper ":END:"))
[acc false]
in-logbook?
[acc true]
:else
[(conj acc line) in-logbook?])))
[[] false]
lines)]
(string/join "\n" result))))
(defn remove-deadline-scheduled
[content]
(when (string? content)
(let [lines (string/split-lines content)]
(if (= 1 (count lines))
content
(let [first-line (first lines)
rest-lines (rest lines)
rest-lines (keep (fn [line]
(let [upper (string/upper-case (string/triml line))]
(if (or (string/starts-with? upper "DEADLINE: ")
(string/starts-with? upper "SCHEDULED: "))
(let [cleaned (-> line (string/replace #"(?i)(?:^|\s)(DEADLINE|SCHEDULED):\s+<[^>]*>" "") string/trim)]
(when-not (string/blank? cleaned)
cleaned))
line)))
rest-lines)]
(string/join "\n" (cons first-line rest-lines)))))))

217
deps/graph-parser/src/logseq/graph_parser/schema/mldoc.cljc vendored Normal file
View File

@@ -0,0 +1,217 @@
(ns logseq.graph-parser.schema.mldoc
"Malli schema for mldoc AST")
(defn- field-optional-and-maybe-nil
[k v]
[k {:optional true} [:maybe v]])
(def pos-schema
[:map
[:start_pos :int]
[:end_pos :int]])
(def nested-link-schema
[:schema {:registry {::nested-link
[:map
[:content :string]
[:children [:sequential [:or
[:tuple [:= "Label"] :string]
[:tuple [:= "Nested_link"] [:ref ::nested-link]]]]]]}}
::nested-link])
(def timestamp-schema
[:map
[:date [:map
[:year :int]
[:month :int]
[:day :int]]]
[:wday :string]
(field-optional-and-maybe-nil
:time [:map
[:hour :int]
[:min :int]])
(field-optional-and-maybe-nil
:repetition
:any)
[:active :boolean]])
(def ^:private time-range-schema
[:map
[:start [:ref ::timestamp]]
[:stop [:ref ::timestamp]]])
(def ^:private link-schema
[:map
[:url [:or
[:cat [:= "File"] :string]
[:cat [:= "Search"] :string]
[:cat [:= "Complex"] [:map
[:protocol :string]
[:link :string]]]
[:cat [:= "Page_ref"] :string]
[:cat [:= "Block_ref"] :string]
[:cat [:= "Embed_data"] :string]]]
[:label [:sequential [:ref ::inline]]]
(field-optional-and-maybe-nil :title :string)
[:full_text :string]
[:metadata :string]])
(def latex-fragment-schema
[:or
[:tuple [:= "Inline"] :string]
[:tuple [:= "Displayed"] :string]])
(def inline-ast-schema
[:schema {:registry {::timestamp timestamp-schema
::time-range time-range-schema
::link link-schema
::inline
[:or
[:tuple [:= "Emphasis"]
[:tuple
[:tuple [:enum "Italic" "Bold" "Underline" "Strike_through" "Highlight"]]
[:sequential [:ref ::inline]]]]
[:tuple [:= "Break_Line"]]
[:tuple [:= "Hard_Break_Line"]]
[:tuple [:= "Verbatim"] :string]
[:tuple [:= "Code"] :string]
[:tuple [:= "Tag"] [:sequential [:ref ::inline]]]
[:tuple [:= "Spaces"] :string]
[:tuple [:= "Plain"] :string]
[:tuple [:= "Link"] [:ref ::link]]
[:tuple [:= "Nested_link"] nested-link-schema]
[:tuple [:= "Target"] :string]
[:tuple [:= "Subscript"] [:sequential [:ref ::inline]]]
[:tuple [:= "Superscript"] [:sequential [:ref ::inline]]]
[:tuple [:= "Footnote_Reference"] [:map
[:id :int]
[:name :string]
(field-optional-and-maybe-nil
:definition [:sequential [:ref ::inline]])]]
[:tuple [:= "Cookie"] [:or
[:tuple [:= "Percent"] :int]
[:catn [:label [:= "Absolute"]] [:current :int] [:total :int]]]]
[:tuple [:= "Latex_Fragment"] latex-fragment-schema]
[:tuple [:= "Macro"] [:map
[:name :string]
[:arguments [:sequential :string]]]]
[:tuple [:= "Entity"] [:map
[:name :string]
[:latex :string]
[:latex_mathp :boolean]
[:html :string]
[:ascii :string]
[:unicode :string]]]
[:tuple [:= "Timestamp"] [:or
[:tuple [:= "Scheduled"] [:ref ::timestamp]]
[:tuple [:= "Deadline"] [:ref ::timestamp]]
[:tuple [:= "Date"] [:ref ::timestamp]]
[:tuple [:= "Closed"] [:ref ::timestamp]]
[:tuple [:= "Clock"] [:or
[:tuple [:= "Started"] [:ref ::timestamp]]
[:tuple [:= "Stopped"] [:ref ::time-range]]]]
[:tuple [:= "Range"] [:ref ::time-range]]]]
[:tuple [:= "Radio_Target"] :string]
[:tuple [:= "Export_Snippet"] :string :string]
[:tuple [:= "Inline_Source_Block"] [:map
[:language :string]
[:options :string]
[:code :string]]]
[:tuple [:= "Email"] [:map
[:local_part :string]
[:domain :string]]]
[:tuple [:= "Inline_Hiccup"] :string]
[:tuple [:= "Inline_Html"] :string]]}}
::inline])
(def ^:private list-item-schema
[:map
[:content [:sequential [:ref ::block]]]
[:items [:sequential [:ref ::list-item]]]
(field-optional-and-maybe-nil
:number :int)
[:name [:sequential [:ref ::inline]]]
(field-optional-and-maybe-nil
:checkbox :boolean)
[:indent :int]
[:ordered :boolean]])
(def ^:private heading-schema
[:map
[:title [:sequential [:ref ::inline]]]
[:tags [:sequential :string]]
(field-optional-and-maybe-nil
:marker :string)
[:level :int]
(field-optional-and-maybe-nil
:numbering [:sequential :int])
(field-optional-and-maybe-nil
:priority :string)
[:anchor :string]
[:meta :map]
(field-optional-and-maybe-nil
:size :int)])
(def block-ast-schema
[:schema {:registry {::inline inline-ast-schema
::list-item list-item-schema
::block
[:or
[:tuple [:= "Paragraph"] [:sequential [:ref ::inline]]]
[:tuple [:= "Paragraph_Sep"] :int]
[:tuple [:= "Heading"] heading-schema]
[:tuple [:= "List"] [:sequential [:ref ::list-item]]]
[:tuple [:= "Directive"] :string :string]
[:tuple [:= "Results"]]
[:tuple [:= "Example"] [:sequential :string]]
[:tuple [:= "Src"] [:map
[:lines [:sequential :string]]
(field-optional-and-maybe-nil
:language :string)
(field-optional-and-maybe-nil
:options [:sequential :string])
[:pos_meta pos-schema]]]
[:tuple [:= "Quote"] [:sequential [:ref ::block]]]
[:catn
[:label [:= "Export"]]
[:type :string]
[:options [:maybe [:sequential :string]]]
[:content :string]]
[:tuple [:= "CommentBlock"] [:sequential :string]]
[:catn
[:label [:= "Custom"]]
[:type :string]
[:options [:maybe :string]]
[:result [:sequential [:ref ::block]]]
[:content :string]]
[:tuple [:= "Latex_Fragment"] latex-fragment-schema]
[:catn
[:label [:= "Latex_Environment"]]
[:name :string]
[:options [:maybe :string]]
[:content :string]]
[:tuple [:= "Displayed_Math"] :string]
[:tuple [:= "Drawer"] :string [:sequential :string]]
[:tuple [:= "Property_Drawer"]
[:sequential
[:catn [:k :string] [:v :string] [:other-info [:sequential [:ref ::inline]]]]]]
[:tuple [:= "Footnote_Definition"] :string [:sequential [:ref ::inline]]]
[:tuple [:= "Horizontal_Rule"]]
[:tuple [:= "Table"]
[:map
(field-optional-and-maybe-nil
:header [:sequential [:sequential [:ref ::inline]]])
[:groups [:sequential [:sequential [:sequential [:sequential [:ref ::inline]]]]]]
[:col_groups [:sequential :int]]]]
[:tuple [:= "Comment"] :string]
[:tuple [:= "Raw_Html"] :string]
[:tuple [:= "Hiccup"] :string]
;; this block type is not from mldoc,
;; but from `logseq.graph-parser.mldoc/collect-page-properties`
[:tuple [:= "Properties"] [:sequential :any]]]}}
::block])
(def block-ast-with-pos-coll-schema
[:sequential [:cat block-ast-schema [:maybe pos-schema]]])

171
deps/graph-parser/src/logseq/graph_parser/text.cljs vendored Normal file
View File

@@ -0,0 +1,171 @@
(ns logseq.graph-parser.text
"Miscellaneous text util fns for the parser. Used by file and DB graphs"
(:require ["path" :as node-path]
[clojure.set :as set]
[clojure.string :as string]
[goog.string :as gstring]
[logseq.common.util :as common-util]
[logseq.common.util.namespace :as ns-util]
[logseq.common.util.page-ref :as page-ref]
[logseq.graph-parser.mldoc :as gp-mldoc]
[logseq.graph-parser.property :as gp-property]))
(def get-file-basename page-ref/get-file-basename)
(defn- get-file-rootname
"Returns the rootname of a file path. e.g. /a/b/c.md -> c"
[path]
(when-not (string/blank? path)
(.-name (node-path/parse (string/replace path "+" "/")))))
(defn get-page-name
"Similar to page-ref/get-page-name but handles format-specific page-refs e.g. org/md"
[s]
(and (string? s)
(or (when-let [[_ label _path] (re-matches page-ref/markdown-page-ref-re s)]
(string/trim label))
(when-let [[_ path _label] (re-matches #"\[\[(file:.*)\]\[.+?\]\]" s)]
(some-> (get-file-rootname path)
(string/replace "." "/")))
(-> (re-matches page-ref/page-ref-any-re s)
second))))
(def page-ref-un-brackets! page-ref/page-ref-un-brackets!)
(defn get-nested-page-name
[page-name]
(when-let [first-match (re-find page-ref/page-ref-without-nested-re page-name)]
(second first-match)))
(defn- remove-level-space-aux!
[text pattern space? trim-left?]
(let [pattern (gstring/format
(if space?
"^[%s]+\\s+"
"^[%s]+\\s?")
pattern)
text (if trim-left? (string/triml text) text)]
(string/replace-first text (re-pattern pattern) "")))
(defn remove-level-spaces
([text format block-pattern]
(remove-level-spaces text format block-pattern false true))
([text format block-pattern space?]
(remove-level-spaces text format block-pattern space? true))
([text format block-pattern space? trim-left?]
(when format
(cond
(string/blank? text)
""
(and (= "markdown" (name format))
(string/starts-with? text "---"))
text
:else
(remove-level-space-aux! text block-pattern space? trim-left?)))))
(defn parse-non-string-property-value
"Return parsed non-string property value or nil if none is found"
[v]
(cond
(= v "true")
true
(= v "false")
false
(re-find #"^\d+$" v)
(parse-long v)))
(defn- get-ref-from-ast
[[typ data]]
(case typ
"Link"
(case (first (:url data))
"Page_ref"
(second (:url data))
"Search"
(second (:url data))
nil)
"Nested_link"
(get-page-name (:content data))
"Tag"
(if (= "Plain" (ffirst data))
(second (first data))
(get-ref-from-ast (first data)))
nil))
(defn extract-refs-from-mldoc-ast
[v]
(into #{}
(comp
(remove gp-mldoc/ast-link?)
(keep get-ref-from-ast)
(map string/trim))
v))
(defn- sep-by-comma
[s]
{:pre (string? s)}
(->>
(string/split s #"[\,，]{1}")
(map string/trim)
(remove string/blank?)
(set)))
(defn separated-by-commas?
[config-state k]
(let [k' (if (keyword? k) k (keyword k))]
(contains? (set/union gp-property/editable-linkable-built-in-properties
(set (get config-state :property/separated-by-commas)))
k')))
(defn- extract-refs-by-commas
[v format]
(let [plains (->> (map first (gp-mldoc/->edn v (gp-mldoc/default-config format)))
first
second
(filter #(and (vector? %) (= "Plain" (first %))))
(map second))]
(set (mapcat sep-by-comma plains))))
(defn- parse-property-refs [k v mldoc-references-ast config-state]
(let [refs (extract-refs-from-mldoc-ast mldoc-references-ast)
property-separated-by-commas? (separated-by-commas? config-state k)]
(if property-separated-by-commas?
(->> (extract-refs-by-commas v (get config-state :format :markdown))
(set/union refs))
refs)))
(defn parse-property
"Property value parsing that takes into account built-in properties, format
and user config"
[k v mldoc-references-ast config-state]
(let [v' (string/trim (str v))]
(cond
(contains? (set/union
(set (map name (gp-property/unparsed-built-in-properties)))
(get config-state :ignored-page-references-keywords))
(name k))
v'
(common-util/wrapped-by-quotes? v')
v'
;; parse property value as needed
:else
(let [refs (parse-property-refs k v' mldoc-references-ast config-state)]
(if (seq refs)
refs
(if-some [new-val (parse-non-string-property-value v')]
new-val
v'))))))
(def namespace-page? ns-util/namespace-page?)
(def get-namespace-last-part ns-util/get-last-part)

24
deps/graph-parser/src/logseq/graph_parser/utf8.cljs vendored Normal file
View File

@@ -0,0 +1,24 @@
(ns logseq.graph-parser.utf8
"Utf8 utilities used by the parser")
(defonce encoder
(js/TextEncoder. "utf-8"))
(defonce decoder
(js/TextDecoder. "utf-8"))
(defn encode
[s]
(.encode encoder s))
(defn decode
[arr]
(.decode decoder arr))
(defn substring
([arr start]
(decode (.subarray arr start)))
([arr start end]
(if end
(decode (.subarray arr start end))
(decode (.subarray arr start)))))