Initial import of LogDarioSeq baseline
Establishes version control for the personal Logseq DB fork before starting the server-client architecture migration and domain features described in docs/agent-guide and the approved implementation plan. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
24
deps/db/.carve/config.edn
vendored
Normal file
24
deps/db/.carve/config.edn
vendored
Normal file
@@ -0,0 +1,24 @@
|
||||
{:paths ["src"]
|
||||
:api-namespaces [logseq.db.common.sqlite
|
||||
logseq.db.sqlite.util
|
||||
logseq.db.common.sqlite-cli
|
||||
logseq.db.frontend.property
|
||||
logseq.db.frontend.property.build
|
||||
logseq.db.frontend.content
|
||||
logseq.db.common.order
|
||||
logseq.db.sqlite.create-graph
|
||||
logseq.db.frontend.malli-schema
|
||||
logseq.db.frontend.asset
|
||||
;; Used outside deps/db by frontend worker and electron modules.
|
||||
logseq.db.sqlite.backup
|
||||
;; Used outside deps/db by frontend handler and worker search.
|
||||
logseq.db.frontend.block-title
|
||||
;; Used outside deps/db by frontend worker query handlers.
|
||||
logseq.db.frontend.datalog
|
||||
;; Some fns are used by frontend but not worth moving over yet
|
||||
logseq.db.frontend.schema
|
||||
logseq.db.frontend.validate
|
||||
logseq.db.test.helper
|
||||
logseq.db
|
||||
logseq.db.frontend.property.type]
|
||||
:report {:format :ignore}}
|
||||
67
deps/db/.carve/ignore
vendored
Normal file
67
deps/db/.carve/ignore
vendored
Normal file
@@ -0,0 +1,67 @@
|
||||
;; API
|
||||
logseq.db.frontend.rules/db-query-dsl-rules
|
||||
;; API
|
||||
logseq.db.frontend.rules/extract-rules
|
||||
;; API
|
||||
logseq.db.frontend.rules/rules
|
||||
;; API
|
||||
logseq.db.frontend.rules/rules-dependencies
|
||||
;; API
|
||||
logseq.db.frontend.query-dsl/pre-transform
|
||||
;; API
|
||||
logseq.db.frontend.query-dsl/simplify-query
|
||||
;; API
|
||||
logseq.db.frontend.query-dsl/get-timestamp-property
|
||||
;; API
|
||||
logseq.db.frontend.query-dsl/custom-readers
|
||||
;; API
|
||||
logseq.db.frontend.inputs/resolve-input
|
||||
;; API
|
||||
logseq.db.frontend.class/build-new-class
|
||||
;; API
|
||||
logseq.db.frontend.class/logseq-class?
|
||||
;; API
|
||||
logseq.db.frontend.class/disallowed-inline-tags
|
||||
;; API
|
||||
logseq.db.frontend.db-ident/ensure-unique-db-ident
|
||||
;; API
|
||||
logseq.db.frontend.db-ident/replace-db-ident-random-suffix
|
||||
;; API
|
||||
logseq.db.sqlite.build/create-blocks
|
||||
;; API
|
||||
logseq.db.sqlite.export/build-export
|
||||
;; API
|
||||
logseq.db.sqlite.export/validate-export
|
||||
;; API
|
||||
logseq.db.sqlite.export/build-import
|
||||
;; API
|
||||
logseq.db.sqlite.export/diff-exports
|
||||
;; API
|
||||
logseq.db.common.view/get-property-values
|
||||
;; API
|
||||
logseq.db.common.view/get-view-data
|
||||
;; API
|
||||
logseq.db.common.initial-data/with-parent
|
||||
;; API
|
||||
logseq.db.common.initial-data/get-block-refs
|
||||
;; API
|
||||
logseq.db.common.initial-data/get-block-and-children
|
||||
;; API
|
||||
logseq.db.common.initial-data/get-initial-data
|
||||
;; API
|
||||
logseq.db.sqlite.debug/find-missing-addresses
|
||||
;; API
|
||||
logseq.db.sqlite.debug/find-missing-addresses-node-version
|
||||
;; API
|
||||
logseq.db.sqlite.gc/gc-kvs-table!
|
||||
;; API
|
||||
logseq.db.sqlite.gc/gc-kvs-table-node-version!
|
||||
;; API
|
||||
logseq.db.sqlite.gc/ensure-no-garbage
|
||||
;; API
|
||||
logseq.db.frontend.entity-util/entity->map
|
||||
;; documenting keywords
|
||||
logseq.db.frontend.kv-entity/kv-entities
|
||||
;; API
|
||||
logseq.db.common.normalize/replace-attr-retract-with-retract-entity-v2
|
||||
logseq.db.common.normalize/normalize-tx-data
|
||||
34
deps/db/.clj-kondo/config.edn
vendored
Normal file
34
deps/db/.clj-kondo/config.edn
vendored
Normal file
@@ -0,0 +1,34 @@
|
||||
{:linters
|
||||
{:aliased-namespace-symbol {:level :warning}
|
||||
:namespace-name-mismatch {:level :warning}
|
||||
:used-underscored-binding {:level :warning}
|
||||
:shadowed-var {:level :warning
|
||||
;; FIXME: Remove these as shadowing core fns isn't a good practice
|
||||
:exclude [val key]}
|
||||
|
||||
:consistent-alias
|
||||
{:aliases {clojure.string string
|
||||
logseq.db ldb
|
||||
logseq.db.common.entity-plus entity-plus
|
||||
logseq.db.common.order db-order
|
||||
logseq.db.common.sqlite common-sqlite
|
||||
logseq.db.common.view db-view
|
||||
logseq.db.frontend.content db-content
|
||||
logseq.db.frontend.class db-class
|
||||
logseq.db.frontend.db db-db
|
||||
logseq.db.frontend.db-ident db-ident
|
||||
logseq.db.frontend.inputs db-inputs
|
||||
logseq.db.frontend.property db-property
|
||||
logseq.db.frontend.property.build db-property-build
|
||||
logseq.db.frontend.property.type db-property-type
|
||||
logseq.db.frontend.rules rules
|
||||
logseq.db.frontend.schema db-schema
|
||||
logseq.db.frontend.validate db-validate
|
||||
logseq.db.sqlite.build sqlite-build
|
||||
logseq.db.common.initial-data common-initial-data
|
||||
logseq.db.common.sqlite-cli sqlite-cli
|
||||
logseq.db.sqlite.create-graph sqlite-create-graph
|
||||
logseq.db.sqlite.export sqlite-export
|
||||
logseq.db.sqlite.util sqlite-util}}}
|
||||
:skip-comments true
|
||||
:output {:progress true}}
|
||||
1
deps/db/.gitignore
vendored
Normal file
1
deps/db/.gitignore
vendored
Normal file
@@ -0,0 +1 @@
|
||||
/.clj-kondo/.cache
|
||||
71
deps/db/README.md
vendored
Normal file
71
deps/db/README.md
vendored
Normal file
@@ -0,0 +1,71 @@
|
||||
## Description
|
||||
|
||||
This library provides an API to the
|
||||
frontend([datascript](https://github.com/tonsky/datascript)) and
|
||||
backend([SQLite](https://www.sqlite.org/index.html)) databases from the Logseq
|
||||
app and the CLI. This library defines the core schema and fns for DB graphs.
|
||||
There are a few namespaces that also support file graphs for the graph-parser.
|
||||
This library is compatible with ClojureScript and with
|
||||
[nbb-logseq](https://github.com/logseq/nbb-logseq) to respectively provide
|
||||
frontend and commandline functionality.
|
||||
|
||||
## API
|
||||
|
||||
This library is under the parent namespace `logseq.db`. It provides the following namespaces:
|
||||
* `logseq.db` - main public ns. Most often used by frontend
|
||||
* `logseq.db.frontend.*` - frontend namespaces for DB graphs
|
||||
* `logseq.db.sqlite.*` - backend/sqlite namespaces for DB graphs
|
||||
* `logseq.db.common.*` - frontend and backend namespaces for DB graphs
|
||||
|
||||
The following namespaces are used with file graphs via the graph-parser: `logseq.db.common.order, logseq.db.frontend.entity-util and logseq.db.common.entity-plus`.
|
||||
|
||||
## Usage
|
||||
|
||||
See the frontend for example usage.
|
||||
|
||||
## Dev
|
||||
|
||||
This follows the practices that [the Logseq frontend
|
||||
follows](/docs/dev-practices.md). Most of the same linters are used, with
|
||||
configurations that are specific to this library. See [this library's CI
|
||||
file](/.github/workflows/deps-db.yml) for linting examples.
|
||||
|
||||
### Setup
|
||||
|
||||
To run linters and tests, you'll want to install pnpm dependencies once:
|
||||
```
|
||||
pnpm install
|
||||
```
|
||||
|
||||
This step is not needed if you're just running the application.
|
||||
|
||||
### Testing
|
||||
|
||||
Testing is done with nbb-logseq and
|
||||
[nbb-test-runner](https://github.com/nextjournal/nbb-test-runner). Some basic
|
||||
usage:
|
||||
|
||||
```
|
||||
# Run all tests
|
||||
$ pnpm test
|
||||
# List available options
|
||||
$ pnpm test -H
|
||||
# Run tests with :focus metadata flag
|
||||
$ pnpm test -i focus
|
||||
```
|
||||
### Datalog linting
|
||||
|
||||
Datalog rules for the client are linted through a script that also uses the datalog-parser. To run this linter:
|
||||
```
|
||||
bb lint:rules
|
||||
```
|
||||
|
||||
### Managing dependencies
|
||||
|
||||
The package.json dependencies are just for testing and should be updated if there is
|
||||
new behavior to test.
|
||||
|
||||
The deps.edn dependencies are used by both ClojureScript and nbb-logseq. Their
|
||||
versions should be backwards compatible with each other with priority given to
|
||||
the frontend. _No new dependency_ should be introduced to this library without
|
||||
an understanding of the tradeoffs of adding this to nbb-logseq.
|
||||
43
deps/db/bb.edn
vendored
Normal file
43
deps/db/bb.edn
vendored
Normal file
@@ -0,0 +1,43 @@
|
||||
{:paths ["src"]
|
||||
:min-bb-version "0.8.156"
|
||||
:deps
|
||||
{logseq/bb-tasks
|
||||
#_{:local/root "../../../bb-tasks"}
|
||||
{:git/url "https://github.com/logseq/bb-tasks"
|
||||
:git/sha "7cc218286c03f093996539b8195de56a0ec5df6a"}}
|
||||
|
||||
:pods
|
||||
{clj-kondo/clj-kondo {:version "2026.04.15"}}
|
||||
|
||||
:tasks
|
||||
{test:load-all-namespaces-with-nbb
|
||||
logseq.bb-tasks.nbb.test/load-all-namespaces
|
||||
|
||||
lint:large-vars
|
||||
logseq.bb-tasks.lint.large-vars/-main
|
||||
|
||||
lint:carve
|
||||
logseq.bb-tasks.lint.carve/-main
|
||||
|
||||
lint:ns-docstrings
|
||||
logseq.bb-tasks.lint.ns-docstrings/-main
|
||||
|
||||
lint:rules
|
||||
{:requires ([logseq.bb-tasks.lint.datalog :as datalog]
|
||||
[logseq.db.frontend.rules :as rules])
|
||||
:doc "Lint datalog rules for parsability and unbound variables"
|
||||
:task (datalog/lint-rules
|
||||
(set
|
||||
(concat (mapcat val (dissoc rules/rules :self-ref))
|
||||
;; TODO: Update linter to handle false positive on :task, :priority, :*property* rules
|
||||
(rules/extract-rules (dissoc rules/db-query-dsl-rules
|
||||
:task :priority
|
||||
:property :simple-query-property :private-property
|
||||
:property-scalar-default-value
|
||||
:property-missing-value
|
||||
:has-property-or-object-property)))))}}
|
||||
|
||||
:tasks/config
|
||||
{:large-vars
|
||||
{:max-lines-count 50
|
||||
:metadata-exceptions #{:large-vars/doc-var :large-vars/cleanup-todo}}}}
|
||||
24
deps/db/deps.edn
vendored
Normal file
24
deps/db/deps.edn
vendored
Normal file
@@ -0,0 +1,24 @@
|
||||
{:deps
|
||||
;; These nbb-logseq deps are kept in sync with https://github.com/logseq/nbb-logseq/blob/main/bb.edn
|
||||
{datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork
|
||||
:sha "3f141af97b70e1f14c65eaa119acd822ebece37e"}
|
||||
;; datascript/datascript {:local/root "../../../../datascript"}
|
||||
datascript-transit/datascript-transit {:mvn/version "0.3.0"
|
||||
:exclusions [datascript/datascript]}
|
||||
cljs-bean/cljs-bean {:mvn/version "1.9.0"}
|
||||
com.cognitect/transit-cljs {:mvn/version "0.8.280"}
|
||||
org.flatland/ordered {:mvn/version "1.15.12"}
|
||||
|
||||
;; Any other deps should be added here and to nbb.edn
|
||||
logseq/common {:local/root "../common"}
|
||||
logseq/clj-fractional-indexing {:git/url "https://github.com/logseq/clj-fractional-indexing"
|
||||
:sha "1087f0fb18aa8e25ee3bbbb0db983b7a29bce270"}
|
||||
borkdude/rewrite-edn {:mvn/version "0.5.9"}
|
||||
metosin/malli {:git/url "https://github.com/metosin/malli"
|
||||
:sha "52ea58a36ff5172b38dfc526ca638afa7226a4a0"}
|
||||
medley/medley {:mvn/version "1.4.0"}}
|
||||
|
||||
:aliases
|
||||
{:clj-kondo
|
||||
{:replace-deps {clj-kondo/clj-kondo {:mvn/version "2026.04.15"}}
|
||||
:main-opts ["-m" "clj-kondo.main"]}}}
|
||||
14
deps/db/nbb.edn
vendored
Normal file
14
deps/db/nbb.edn
vendored
Normal file
@@ -0,0 +1,14 @@
|
||||
{:paths ["src"]
|
||||
:deps
|
||||
{logseq/common
|
||||
{:local/root "../common"}
|
||||
medley/medley {:mvn/version "1.4.0"}
|
||||
metosin/malli {:git/url "https://github.com/metosin/malli"
|
||||
:sha "52ea58a36ff5172b38dfc526ca638afa7226a4a0"}
|
||||
;; Used by db scripts with outliner.cli
|
||||
borkdude/rewrite-edn {:mvn/version "0.5.9"}
|
||||
logseq/clj-fractional-indexing {:git/url "https://github.com/logseq/clj-fractional-indexing"
|
||||
:sha "1087f0fb18aa8e25ee3bbbb0db983b7a29bce270"}
|
||||
io.github.nextjournal/nbb-test-runner
|
||||
{:git/sha "b379325cfa5a3306180649da5de3bf5166414e71"}
|
||||
io.github.pez/baldr {:mvn/version "1.0.9"}}}
|
||||
22
deps/db/package.json
vendored
Normal file
22
deps/db/package.json
vendored
Normal file
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"name": "@logseq/db",
|
||||
"version": "1.0.0",
|
||||
"packageManager": "pnpm@10.33.0",
|
||||
"private": true,
|
||||
"devDependencies": {
|
||||
"@logseq/nbb-logseq": "git+https://github.com/logseq/nbb-logseq.git#feat-db-v34",
|
||||
"fs-extra": "^11.3.4"
|
||||
},
|
||||
"dependencies": {
|
||||
"better-sqlite3": "^12.8.0"
|
||||
},
|
||||
"scripts": {
|
||||
"test": "pnpm exec nbb-logseq -cp test -m nextjournal.test-runner",
|
||||
"test-v": "pnpm exec nbb-logseq -cp test -m logseq.db.test-runner"
|
||||
},
|
||||
"pnpm": {
|
||||
"onlyBuiltDependencies": [
|
||||
"better-sqlite3"
|
||||
]
|
||||
}
|
||||
}
|
||||
339
deps/db/pnpm-lock.yaml
generated
vendored
Normal file
339
deps/db/pnpm-lock.yaml
generated
vendored
Normal file
@@ -0,0 +1,339 @@
|
||||
lockfileVersion: '9.0'
|
||||
|
||||
settings:
|
||||
autoInstallPeers: true
|
||||
excludeLinksFromLockfile: false
|
||||
|
||||
importers:
|
||||
|
||||
.:
|
||||
dependencies:
|
||||
better-sqlite3:
|
||||
specifier: ^12.8.0
|
||||
version: 12.8.0
|
||||
devDependencies:
|
||||
'@logseq/nbb-logseq':
|
||||
specifier: git+https://github.com/logseq/nbb-logseq.git#feat-db-v34
|
||||
version: https://codeload.github.com/logseq/nbb-logseq/tar.gz/4d9f1382ce2cec6f39bf12df49ba425f1bd5f3fb
|
||||
fs-extra:
|
||||
specifier: ^11.3.4
|
||||
version: 11.3.4
|
||||
|
||||
packages:
|
||||
|
||||
'@logseq/nbb-logseq@https://codeload.github.com/logseq/nbb-logseq/tar.gz/4d9f1382ce2cec6f39bf12df49ba425f1bd5f3fb':
|
||||
resolution: {tarball: https://codeload.github.com/logseq/nbb-logseq/tar.gz/4d9f1382ce2cec6f39bf12df49ba425f1bd5f3fb}
|
||||
version: 1.2.173-feat-db-v34
|
||||
engines: {node: '>=20.10.0'}
|
||||
hasBin: true
|
||||
|
||||
base64-js@1.5.1:
|
||||
resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==}
|
||||
|
||||
better-sqlite3@12.8.0:
|
||||
resolution: {integrity: sha512-RxD2Vd96sQDjQr20kdP+F+dK/1OUNiVOl200vKBZY8u0vTwysfolF6Hq+3ZK2+h8My9YvZhHsF+RSGZW2VYrPQ==}
|
||||
engines: {node: 20.x || 22.x || 23.x || 24.x || 25.x}
|
||||
|
||||
bindings@1.5.0:
|
||||
resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==}
|
||||
|
||||
bl@4.1.0:
|
||||
resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==}
|
||||
|
||||
buffer@5.7.1:
|
||||
resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==}
|
||||
|
||||
chownr@1.1.4:
|
||||
resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==}
|
||||
|
||||
decompress-response@6.0.0:
|
||||
resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==}
|
||||
engines: {node: '>=10'}
|
||||
|
||||
deep-extend@0.6.0:
|
||||
resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==}
|
||||
engines: {node: '>=4.0.0'}
|
||||
|
||||
detect-libc@2.0.3:
|
||||
resolution: {integrity: sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==}
|
||||
engines: {node: '>=8'}
|
||||
|
||||
end-of-stream@1.4.4:
|
||||
resolution: {integrity: sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==}
|
||||
|
||||
expand-template@2.0.3:
|
||||
resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==}
|
||||
engines: {node: '>=6'}
|
||||
|
||||
file-uri-to-path@1.0.0:
|
||||
resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==}
|
||||
|
||||
fs-constants@1.0.0:
|
||||
resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==}
|
||||
|
||||
fs-extra@11.3.4:
|
||||
resolution: {integrity: sha512-CTXd6rk/M3/ULNQj8FBqBWHYBVYybQ3VPBw0xGKFe3tuH7ytT6ACnvzpIQ3UZtB8yvUKC2cXn1a+x+5EVQLovA==}
|
||||
engines: {node: '>=14.14'}
|
||||
|
||||
github-from-package@0.0.0:
|
||||
resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==}
|
||||
|
||||
graceful-fs@4.2.11:
|
||||
resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==}
|
||||
|
||||
ieee754@1.2.1:
|
||||
resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==}
|
||||
|
||||
import-meta-resolve@4.1.0:
|
||||
resolution: {integrity: sha512-I6fiaX09Xivtk+THaMfAwnA3MVA5Big1WHF1Dfx9hFuvNIWpXnorlkzhcQf6ehrqQiiZECRt1poOAkPmer3ruw==}
|
||||
|
||||
inherits@2.0.4:
|
||||
resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==}
|
||||
|
||||
ini@1.3.8:
|
||||
resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==}
|
||||
|
||||
jsonfile@6.2.0:
|
||||
resolution: {integrity: sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg==}
|
||||
|
||||
mimic-response@3.1.0:
|
||||
resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==}
|
||||
engines: {node: '>=10'}
|
||||
|
||||
minimist@1.2.8:
|
||||
resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==}
|
||||
|
||||
mkdirp-classic@0.5.3:
|
||||
resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==}
|
||||
|
||||
napi-build-utils@1.0.2:
|
||||
resolution: {integrity: sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==}
|
||||
|
||||
node-abi@3.71.0:
|
||||
resolution: {integrity: sha512-SZ40vRiy/+wRTf21hxkkEjPJZpARzUMVcJoQse2EF8qkUWbbO2z7vd5oA/H6bVH6SZQ5STGcu0KRDS7biNRfxw==}
|
||||
engines: {node: '>=10'}
|
||||
|
||||
once@1.4.0:
|
||||
resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==}
|
||||
|
||||
prebuild-install@7.1.2:
|
||||
resolution: {integrity: sha512-UnNke3IQb6sgarcZIDU3gbMeTp/9SSU1DAIkil7PrqG1vZlBtY5msYccSKSHDqa3hNg436IXK+SNImReuA1wEQ==}
|
||||
engines: {node: '>=10'}
|
||||
deprecated: No longer maintained. Please contact the author of the relevant native addon; alternatives are available.
|
||||
hasBin: true
|
||||
|
||||
pump@3.0.2:
|
||||
resolution: {integrity: sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==}
|
||||
|
||||
rc@1.2.8:
|
||||
resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==}
|
||||
hasBin: true
|
||||
|
||||
readable-stream@3.6.2:
|
||||
resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==}
|
||||
engines: {node: '>= 6'}
|
||||
|
||||
safe-buffer@5.2.1:
|
||||
resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==}
|
||||
|
||||
semver@7.6.3:
|
||||
resolution: {integrity: sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==}
|
||||
engines: {node: '>=10'}
|
||||
hasBin: true
|
||||
|
||||
simple-concat@1.0.1:
|
||||
resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==}
|
||||
|
||||
simple-get@4.0.1:
|
||||
resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==}
|
||||
|
||||
string_decoder@1.3.0:
|
||||
resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==}
|
||||
|
||||
strip-json-comments@2.0.1:
|
||||
resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==}
|
||||
engines: {node: '>=0.10.0'}
|
||||
|
||||
tar-fs@2.1.1:
|
||||
resolution: {integrity: sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==}
|
||||
|
||||
tar-stream@2.2.0:
|
||||
resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==}
|
||||
engines: {node: '>=6'}
|
||||
|
||||
tunnel-agent@0.6.0:
|
||||
resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==}
|
||||
|
||||
universalify@2.0.1:
|
||||
resolution: {integrity: sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==}
|
||||
engines: {node: '>= 10.0.0'}
|
||||
|
||||
util-deprecate@1.0.2:
|
||||
resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==}
|
||||
|
||||
wrappy@1.0.2:
|
||||
resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==}
|
||||
|
||||
snapshots:
|
||||
|
||||
'@logseq/nbb-logseq@https://codeload.github.com/logseq/nbb-logseq/tar.gz/4d9f1382ce2cec6f39bf12df49ba425f1bd5f3fb':
|
||||
dependencies:
|
||||
import-meta-resolve: 4.1.0
|
||||
|
||||
base64-js@1.5.1: {}
|
||||
|
||||
better-sqlite3@12.8.0:
|
||||
dependencies:
|
||||
bindings: 1.5.0
|
||||
prebuild-install: 7.1.2
|
||||
|
||||
bindings@1.5.0:
|
||||
dependencies:
|
||||
file-uri-to-path: 1.0.0
|
||||
|
||||
bl@4.1.0:
|
||||
dependencies:
|
||||
buffer: 5.7.1
|
||||
inherits: 2.0.4
|
||||
readable-stream: 3.6.2
|
||||
|
||||
buffer@5.7.1:
|
||||
dependencies:
|
||||
base64-js: 1.5.1
|
||||
ieee754: 1.2.1
|
||||
|
||||
chownr@1.1.4: {}
|
||||
|
||||
decompress-response@6.0.0:
|
||||
dependencies:
|
||||
mimic-response: 3.1.0
|
||||
|
||||
deep-extend@0.6.0: {}
|
||||
|
||||
detect-libc@2.0.3: {}
|
||||
|
||||
end-of-stream@1.4.4:
|
||||
dependencies:
|
||||
once: 1.4.0
|
||||
|
||||
expand-template@2.0.3: {}
|
||||
|
||||
file-uri-to-path@1.0.0: {}
|
||||
|
||||
fs-constants@1.0.0: {}
|
||||
|
||||
fs-extra@11.3.4:
|
||||
dependencies:
|
||||
graceful-fs: 4.2.11
|
||||
jsonfile: 6.2.0
|
||||
universalify: 2.0.1
|
||||
|
||||
github-from-package@0.0.0: {}
|
||||
|
||||
graceful-fs@4.2.11: {}
|
||||
|
||||
ieee754@1.2.1: {}
|
||||
|
||||
import-meta-resolve@4.1.0: {}
|
||||
|
||||
inherits@2.0.4: {}
|
||||
|
||||
ini@1.3.8: {}
|
||||
|
||||
jsonfile@6.2.0:
|
||||
dependencies:
|
||||
universalify: 2.0.1
|
||||
optionalDependencies:
|
||||
graceful-fs: 4.2.11
|
||||
|
||||
mimic-response@3.1.0: {}
|
||||
|
||||
minimist@1.2.8: {}
|
||||
|
||||
mkdirp-classic@0.5.3: {}
|
||||
|
||||
napi-build-utils@1.0.2: {}
|
||||
|
||||
node-abi@3.71.0:
|
||||
dependencies:
|
||||
semver: 7.6.3
|
||||
|
||||
once@1.4.0:
|
||||
dependencies:
|
||||
wrappy: 1.0.2
|
||||
|
||||
prebuild-install@7.1.2:
|
||||
dependencies:
|
||||
detect-libc: 2.0.3
|
||||
expand-template: 2.0.3
|
||||
github-from-package: 0.0.0
|
||||
minimist: 1.2.8
|
||||
mkdirp-classic: 0.5.3
|
||||
napi-build-utils: 1.0.2
|
||||
node-abi: 3.71.0
|
||||
pump: 3.0.2
|
||||
rc: 1.2.8
|
||||
simple-get: 4.0.1
|
||||
tar-fs: 2.1.1
|
||||
tunnel-agent: 0.6.0
|
||||
|
||||
pump@3.0.2:
|
||||
dependencies:
|
||||
end-of-stream: 1.4.4
|
||||
once: 1.4.0
|
||||
|
||||
rc@1.2.8:
|
||||
dependencies:
|
||||
deep-extend: 0.6.0
|
||||
ini: 1.3.8
|
||||
minimist: 1.2.8
|
||||
strip-json-comments: 2.0.1
|
||||
|
||||
readable-stream@3.6.2:
|
||||
dependencies:
|
||||
inherits: 2.0.4
|
||||
string_decoder: 1.3.0
|
||||
util-deprecate: 1.0.2
|
||||
|
||||
safe-buffer@5.2.1: {}
|
||||
|
||||
semver@7.6.3: {}
|
||||
|
||||
simple-concat@1.0.1: {}
|
||||
|
||||
simple-get@4.0.1:
|
||||
dependencies:
|
||||
decompress-response: 6.0.0
|
||||
once: 1.4.0
|
||||
simple-concat: 1.0.1
|
||||
|
||||
string_decoder@1.3.0:
|
||||
dependencies:
|
||||
safe-buffer: 5.2.1
|
||||
|
||||
strip-json-comments@2.0.1: {}
|
||||
|
||||
tar-fs@2.1.1:
|
||||
dependencies:
|
||||
chownr: 1.1.4
|
||||
mkdirp-classic: 0.5.3
|
||||
pump: 3.0.2
|
||||
tar-stream: 2.2.0
|
||||
|
||||
tar-stream@2.2.0:
|
||||
dependencies:
|
||||
bl: 4.1.0
|
||||
end-of-stream: 1.4.4
|
||||
fs-constants: 1.0.0
|
||||
inherits: 2.0.4
|
||||
readable-stream: 3.6.2
|
||||
|
||||
tunnel-agent@0.6.0:
|
||||
dependencies:
|
||||
safe-buffer: 5.2.1
|
||||
|
||||
universalify@2.0.1: {}
|
||||
|
||||
util-deprecate@1.0.2: {}
|
||||
|
||||
wrappy@1.0.2: {}
|
||||
76
deps/db/script/create_graph.cljs
vendored
Normal file
76
deps/db/script/create_graph.cljs
vendored
Normal file
@@ -0,0 +1,76 @@
|
||||
(ns create-graph
|
||||
"A script that creates or updates a DB graph given a sqlite.build EDN file.
|
||||
If the given graph already exists, the EDN file updates the graph."
|
||||
(:require ["fs" :as fs]
|
||||
["path" :as node-path]
|
||||
[babashka.cli :as cli]
|
||||
[clojure.edn :as edn]
|
||||
[logseq.db :as ldb]
|
||||
[logseq.db.common.sqlite-cli :as sqlite-cli]
|
||||
[logseq.db.frontend.validate :as db-validate]
|
||||
[logseq.db.sqlite.export :as sqlite-export]
|
||||
[logseq.outliner.cli :as outliner-cli]
|
||||
[clojure.pprint :as pprint]
|
||||
[nbb.classpath :as cp]
|
||||
[nbb.core :as nbb]))
|
||||
|
||||
(defn- resolve-path
|
||||
"If relative path, resolve with $ORIGINAL_PWD"
|
||||
[path]
|
||||
(if (node-path/isAbsolute path)
|
||||
path
|
||||
(node-path/join (or js/process.env.ORIGINAL_PWD ".") path)))
|
||||
|
||||
(defn- validate-db [db db-name options]
|
||||
(if-let [errors (:errors
|
||||
(db-validate/validate-local-db!
|
||||
db
|
||||
(merge options {:db-name db-name :verbose true})))]
|
||||
(do
|
||||
(println "Found" (count errors)
|
||||
(if (= 1 (count errors)) "entity" "entities")
|
||||
"with errors:")
|
||||
(pprint/pprint errors)
|
||||
(js/process.exit 1))
|
||||
(println "Valid!")))
|
||||
|
||||
(def spec
|
||||
"Options spec"
|
||||
{:help {:alias :h
|
||||
:desc "Print help"}
|
||||
:validate {:alias :v
|
||||
:desc "Validate db after creation"}
|
||||
:import {:alias :i
|
||||
:desc "Import edn file using sqlite-export"}})
|
||||
|
||||
(defn -main [args]
|
||||
(let [{options :opts args' :args} (cli/parse-args args {:spec spec})
|
||||
[graph-dir edn-path] args'
|
||||
_ (when (or (nil? graph-dir) (nil? edn-path) (:help options))
|
||||
(println (str "Usage: $0 GRAPH-NAME EDN-PATH [OPTIONS]\nOptions:\n"
|
||||
(cli/format-opts {:spec spec})))
|
||||
(js/process.exit 1))
|
||||
init-conn-args (conj (sqlite-cli/->open-db-args graph-dir))
|
||||
sqlite-build-edn (merge (if (:import options) {} {:auto-create-ontology? true})
|
||||
(-> (resolve-path edn-path) fs/readFileSync str edn/read-string))
|
||||
graph-exists? (fs/existsSync (apply node-path/join init-conn-args))
|
||||
db-name (if (= 1 (count init-conn-args)) (first init-conn-args) (second init-conn-args))
|
||||
conn (apply outliner-cli/init-conn
|
||||
(conj init-conn-args {:classpath (cp/get-classpath) :import-type :cli/create-graph}))
|
||||
{:keys [init-tx block-props-tx misc-tx] :as _txs}
|
||||
(if (:import options)
|
||||
(sqlite-export/build-import sqlite-build-edn @conn {})
|
||||
(outliner-cli/build-blocks-tx sqlite-build-edn))]
|
||||
(println "Generating" (count (filter :block/name init-tx)) "pages and"
|
||||
(count (filter :block/title init-tx)) "blocks ...")
|
||||
;; (fs/writeFileSync "txs.edn" (with-out-str (cljs.pprint/pprint _txs)))
|
||||
;; (cljs.pprint/pprint _txs)
|
||||
(ldb/transact! conn init-tx)
|
||||
(when (seq block-props-tx) (ldb/transact! conn block-props-tx))
|
||||
(when (seq misc-tx) (ldb/transact! conn misc-tx))
|
||||
(println (if graph-exists? "Updated graph" "Created graph") (str db-name "!"))
|
||||
(when (:validate options)
|
||||
(validate-db @conn db-name {}))))
|
||||
|
||||
(when (= nbb/*file* (nbb/invoked-file))
|
||||
(-main *command-line-args*))
|
||||
23
deps/db/script/create_graph/inferred.edn
vendored
Normal file
23
deps/db/script/create_graph/inferred.edn
vendored
Normal file
@@ -0,0 +1,23 @@
|
||||
;; Script that generates classes and properties for a demo of inferring properties.
|
||||
;; To generate this graph:
|
||||
;; bb dev:create inferred deps/db/create_graph/inferred.edn
|
||||
;;
|
||||
;; To try the demo in the UI, in any page type:
|
||||
;; - Good Will Hunting #Movie #Ben-Affleck
|
||||
;; or
|
||||
;; - DB 3 #Meeting #Tienson
|
||||
{:auto-create-ontology? true
|
||||
:classes {:Movie {:build/class-properties [:actor :comment]}
|
||||
:Meeting {:build/class-properties [:attendee :duration]}}
|
||||
:properties
|
||||
{:actor {:logseq.property/type :node
|
||||
:db/cardinality :many
|
||||
:build/property-classes [:Person]}
|
||||
:attendee {:logseq.property/type :node
|
||||
:db/cardinality :many
|
||||
:build/property-classes [:Person]}}
|
||||
:pages-and-blocks
|
||||
[{:page {:block/title "Matt-Damon" :build/tags [:Person]}}
|
||||
{:page {:block/title "Ben-Affleck" :build/tags [:Person]}}
|
||||
{:page {:block/title "Tienson" :build/tags [:Person]}}
|
||||
{:page {:block/title "Zhiyuan" :build/tags [:Person]}}]}
|
||||
25
deps/db/script/dump_datoms.cljs
vendored
Normal file
25
deps/db/script/dump_datoms.cljs
vendored
Normal file
@@ -0,0 +1,25 @@
|
||||
(ns dump-datoms
|
||||
"A script that dumps all eavt datoms to a specified edn file
|
||||
|
||||
$ pnpm exec nbb-logseq script/dump_datoms.cljs db-name datoms.edn"
|
||||
(:require ["fs" :as fs]
|
||||
["path" :as node-path]
|
||||
[clojure.pprint :as pprint]
|
||||
[datascript.core :as d]
|
||||
[logseq.db.common.sqlite-cli :as sqlite-cli]
|
||||
[nbb.core :as nbb]))
|
||||
|
||||
(defn -main [args]
|
||||
(when (< (count args) 2)
|
||||
(println "Usage: $0 GRAPH FILE")
|
||||
(js/process.exit 1))
|
||||
(let [[graph-name file*] args
|
||||
conn (apply sqlite-cli/open-db! (sqlite-cli/->open-db-args graph-name))
|
||||
datoms (mapv #(vec %) (d/datoms @conn :eavt))
|
||||
parent-dir (or js/process.env.ORIGINAL_PWD ".")
|
||||
file (node-path/join parent-dir file*)]
|
||||
(println "Writing" (count datoms) "datoms to" file)
|
||||
(fs/writeFileSync file (with-out-str (pprint/pprint datoms)))))
|
||||
|
||||
(when (= nbb/*file* (nbb/invoked-file))
|
||||
(-main *command-line-args*))
|
||||
93
deps/db/script/export_client_ops_ops.cljs
vendored
Normal file
93
deps/db/script/export_client_ops_ops.cljs
vendored
Normal file
@@ -0,0 +1,93 @@
|
||||
(ns export-client-ops-ops
|
||||
"Export client-ops tx entities to an EDN file sorted by :db-sync/created-at asc.
|
||||
|
||||
Examples:
|
||||
$ pnpm exec nbb-logseq script/export_client_ops_ops.cljs /path/to/client-ops.sqlite
|
||||
$ pnpm exec nbb-logseq script/export_client_ops_ops.cljs /path/to/client-ops.sqlite --out client-ops-ops.edn"
|
||||
(:require ["fs" :as fs]
|
||||
["path" :as node-path]
|
||||
[babashka.cli :as cli]
|
||||
[clojure.pprint :as pprint]
|
||||
[datascript.core :as d]
|
||||
[logseq.db.common.sqlite-cli :as sqlite-cli]
|
||||
[nbb.core :as nbb]))
|
||||
|
||||
(def spec
|
||||
{:help {:alias :h
|
||||
:desc "Show help"}
|
||||
:out {:alias :o
|
||||
:desc "Output EDN file path"
|
||||
:coerce :string}})
|
||||
|
||||
(def op-keys
|
||||
[:db-sync/tx-id
|
||||
:db-sync/created-at
|
||||
:db-sync/pending?
|
||||
:db-sync/failed?
|
||||
:db-sync/outliner-op
|
||||
:db-sync/forward-outliner-ops
|
||||
:db-sync/inverse-outliner-ops
|
||||
:db-sync/inferred-outliner-ops?
|
||||
:db-sync/normalized-tx-data
|
||||
:db-sync/reversed-tx-data])
|
||||
|
||||
(defn usage []
|
||||
(str "Usage: pnpm exec nbb-logseq script/export_client_ops_ops.cljs <client-ops-db> [--out <file.edn>]\n"
|
||||
"Options:\n"
|
||||
(cli/format-opts {:spec spec})))
|
||||
|
||||
(defn resolve-path
|
||||
[path*]
|
||||
(if (node-path/isAbsolute path*)
|
||||
path*
|
||||
(node-path/join (or js/process.env.ORIGINAL_PWD ".") path*)))
|
||||
|
||||
(defn default-output-path
|
||||
[input-path]
|
||||
(let [resolved (resolve-path input-path)
|
||||
dir (node-path/dirname resolved)
|
||||
base (node-path/basename resolved)
|
||||
name (node-path/parse base)
|
||||
stem (or (.-name name) base)]
|
||||
(node-path/join dir (str stem "-ops.edn"))))
|
||||
|
||||
(defn entity->op
|
||||
[db eid]
|
||||
(let [ent (d/entity db eid)]
|
||||
(merge
|
||||
{:db/id (:db/id ent)}
|
||||
(select-keys (into {} ent) op-keys))))
|
||||
|
||||
(defn read-ops
|
||||
[conn]
|
||||
(let [db @conn]
|
||||
(->> (d/datoms db :avet :db-sync/created-at)
|
||||
(map (fn [datom] (entity->op db (:e datom))))
|
||||
(sort-by (juxt :db-sync/created-at :db/id))
|
||||
vec)))
|
||||
|
||||
(defn -main
|
||||
[argv]
|
||||
(let [{:keys [opts args]} (cli/parse-args argv {:spec spec})
|
||||
[input-path] args]
|
||||
(when (or (:help opts) (nil? input-path))
|
||||
(println (usage))
|
||||
(js/process.exit (if (:help opts) 0 1)))
|
||||
(let [db-path (resolve-path input-path)
|
||||
_ (when-not (fs/existsSync db-path)
|
||||
(println "Client ops db file does not exist:" db-path)
|
||||
(js/process.exit 1))
|
||||
out-path (resolve-path (or (:out opts) (default-output-path db-path)))
|
||||
open-args (sqlite-cli/->open-db-args db-path)
|
||||
{:keys [conn sqlite]} (apply sqlite-cli/open-sqlite-datascript! open-args)]
|
||||
(try
|
||||
(let [ops (read-ops conn)]
|
||||
(println "Read" (count ops) "ops from" db-path)
|
||||
(println "Writing ops EDN to" out-path)
|
||||
(fs/writeFileSync out-path (with-out-str (pprint/pprint ops))))
|
||||
(finally
|
||||
(when sqlite
|
||||
(.close sqlite)))))))
|
||||
|
||||
(when (= nbb/*file* (nbb/invoked-file))
|
||||
(-main *command-line-args*))
|
||||
70
deps/db/script/query.cljs
vendored
Normal file
70
deps/db/script/query.cljs
vendored
Normal file
@@ -0,0 +1,70 @@
|
||||
(ns query
|
||||
"A script that queries any db graph from the commandline e.g.
|
||||
|
||||
$ pnpm exec nbb-logseq script/query.cljs db-name '[:find (pull ?b [:block/name :block/title]) :where [?b :block/created-at]]'"
|
||||
(:require ["child_process" :as child-process]
|
||||
[babashka.cli :as cli]
|
||||
[clojure.edn :as edn]
|
||||
[clojure.pprint :as pprint]
|
||||
[datascript.core :as d]
|
||||
[logseq.db.common.sqlite-cli :as sqlite-cli]
|
||||
[logseq.db.frontend.rules :as rules]
|
||||
[nbb.core :as nbb]))
|
||||
|
||||
(defn- sh
|
||||
"Run shell cmd synchronously and print to inherited streams by default. Aims
|
||||
to be similar to babashka.tasks/shell"
|
||||
[cmd opts]
|
||||
(child-process/spawnSync (first cmd)
|
||||
(clj->js (rest cmd))
|
||||
(clj->js (merge {:stdio "inherit"} opts))))
|
||||
|
||||
(def spec
|
||||
"Options spec"
|
||||
{:help {:alias :h
|
||||
:desc "Print help"}
|
||||
:verbose {:alias :v
|
||||
:desc "Print more info"}
|
||||
:raw {:alias :r
|
||||
:desc "Print results plainly. Useful when piped to bb"}
|
||||
:additional-graphs {:alias :a
|
||||
:coerce []
|
||||
:desc "Additional graphs to query"}
|
||||
:entity {:alias :e
|
||||
:coerce []
|
||||
:desc "Lookup entities instead of query"}})
|
||||
|
||||
(defn query-graph [graph args'' options]
|
||||
(let [conn (apply sqlite-cli/open-db! (sqlite-cli/->open-db-args graph))
|
||||
results (if (:entity options)
|
||||
(map #(when-let [ent (d/entity @conn
|
||||
(if (string? %) (edn/read-string %) %))]
|
||||
(into {:db/id (:db/id ent)} ent))
|
||||
(:entity options))
|
||||
;; assumes no :in are in queries
|
||||
(let [query (into (edn/read-string (first args'')) [:in '$ '%])
|
||||
res (d/q query @conn (rules/extract-rules rules/db-query-dsl-rules))]
|
||||
;; Remove nesting for most queries which just have one :find binding
|
||||
(if (= 1 (count (first res))) (mapv first res) res)))]
|
||||
(when (:verbose options) (println "DB contains" (count (d/datoms @conn :eavt)) "datoms"))
|
||||
(if (:raw options)
|
||||
(prn results)
|
||||
(if (zero? (.-status (child-process/spawnSync "which" #js ["puget"])))
|
||||
(sh ["puget"] {:input (pr-str results) :stdio ["pipe" "inherit" "inherit"]})
|
||||
(pprint/pprint results)))))
|
||||
|
||||
(defn -main [args]
|
||||
(let [{options :opts args' :args} (cli/parse-args args {:spec spec})
|
||||
[graph-dir & args''] args'
|
||||
_ (when (or (nil? graph-dir) (:help options))
|
||||
(println (str "Usage: $0 GRAPH-NAME [& ARGS] [OPTIONS]\nOptions:\n"
|
||||
(cli/format-opts {:spec spec})))
|
||||
(js/process.exit 1))
|
||||
graph-dirs (cond-> [graph-dir]
|
||||
(:additional-graphs options)
|
||||
(into (:additional-graphs options)))]
|
||||
(doseq [graph-dir graph-dirs]
|
||||
(query-graph graph-dir args'' options))))
|
||||
|
||||
(when (= nbb/*file* (nbb/invoked-file))
|
||||
(-main *command-line-args*))
|
||||
208
deps/db/script/replay_sync_artifact.cljs
vendored
Normal file
208
deps/db/script/replay_sync_artifact.cljs
vendored
Normal file
@@ -0,0 +1,208 @@
|
||||
(ns replay-sync-artifact
|
||||
"Replay sync simulation artifacts with DataScript + checksum logic in nbb."
|
||||
(:require ["node:fs" :as fs]
|
||||
["node:path" :as node-path]
|
||||
[babashka.cli :as cli]
|
||||
[clojure.string :as string]
|
||||
[datascript.core :as d]
|
||||
[logseq.db-sync.checksum :as sync-checksum]
|
||||
[logseq.db.frontend.schema :as db-schema]
|
||||
[nbb.core :as nbb]))
|
||||
|
||||
(def cli-spec
|
||||
{:help {:alias :h
|
||||
:desc "Show help"}
|
||||
:artifact {:alias :a
|
||||
:desc "Path to artifact.json"
|
||||
:coerce :string}
|
||||
:client {:alias :c
|
||||
:desc "Only replay one client instance index"
|
||||
:coerce :long}
|
||||
:round {:alias :r
|
||||
:desc "Only replay one round index"
|
||||
:coerce :long}
|
||||
:pretty {:desc "Pretty-print JSON output"}})
|
||||
|
||||
(def checksum-attr-map
|
||||
{"uuid" :block/uuid
|
||||
"title" :block/title
|
||||
"name" :block/name
|
||||
"parent" :block/parent
|
||||
"page" :block/page
|
||||
"order" :block/order
|
||||
":block/uuid" :block/uuid
|
||||
":block/title" :block/title
|
||||
":block/name" :block/name
|
||||
":block/parent" :block/parent
|
||||
":block/page" :block/page
|
||||
":block/order" :block/order
|
||||
"block/uuid" :block/uuid
|
||||
"block/title" :block/title
|
||||
"block/name" :block/name
|
||||
"block/parent" :block/parent
|
||||
"block/page" :block/page
|
||||
"block/order" :block/order})
|
||||
|
||||
(def checksum-attrs (set (vals checksum-attr-map)))
|
||||
|
||||
(defn usage []
|
||||
(str "Usage: pnpm exec nbb-logseq -cp src:script:../db-sync/src script/replay_sync_artifact.cljs --artifact <path> [--client <n>] [--round <n>] [--pretty]\n"
|
||||
"Options:\n"
|
||||
(cli/format-opts {:spec cli-spec})))
|
||||
|
||||
(defn parse-uuid
|
||||
[value]
|
||||
(when (string? value)
|
||||
(try
|
||||
(uuid value)
|
||||
(catch :default _
|
||||
nil))))
|
||||
|
||||
(defn parse-int
|
||||
[value]
|
||||
(when-not (nil? value)
|
||||
(let [n (js/Number value)]
|
||||
(when (js/Number.isInteger n)
|
||||
(int n)))))
|
||||
|
||||
(defn parse-attr
|
||||
[value]
|
||||
(cond
|
||||
(keyword? value)
|
||||
value
|
||||
|
||||
(string? value)
|
||||
(let [trimmed (string/trim value)]
|
||||
(or (get checksum-attr-map trimmed)
|
||||
(get checksum-attr-map (string/lower-case trimmed))
|
||||
(when (string/starts-with? trimmed ":")
|
||||
(get checksum-attr-map (subs trimmed 1)))))
|
||||
|
||||
:else
|
||||
nil))
|
||||
|
||||
(defn snapshot-block->entity
|
||||
[block]
|
||||
(let [id (parse-int (get block "id"))
|
||||
uuid-value (parse-uuid (get block "uuid"))]
|
||||
(when (and id uuid-value)
|
||||
(cond-> {:db/id id
|
||||
:block/uuid uuid-value}
|
||||
(string? (get block "title"))
|
||||
(assoc :block/title (get block "title"))
|
||||
|
||||
(string? (get block "name"))
|
||||
(assoc :block/name (get block "name"))
|
||||
|
||||
(string? (get block "order"))
|
||||
(assoc :block/order (get block "order"))
|
||||
|
||||
(parse-int (get block "parentId"))
|
||||
(assoc :block/parent (parse-int (get block "parentId")))
|
||||
|
||||
(parse-int (get block "pageId"))
|
||||
(assoc :block/page (parse-int (get block "pageId")))))))
|
||||
|
||||
(defn datom->tx
|
||||
[datom]
|
||||
(let [e (parse-int (get datom "e"))
|
||||
attr (parse-attr (get datom "a"))
|
||||
added? (not= false (get datom "added"))
|
||||
value (get datom "v")]
|
||||
(when (and e (contains? checksum-attrs attr))
|
||||
(let [value' (case attr
|
||||
:block/uuid (or (parse-uuid value) value)
|
||||
:block/parent (or (parse-int value) value)
|
||||
:block/page (or (parse-int value) value)
|
||||
value)]
|
||||
[(if added? :db/add :db/retract) e attr value']))))
|
||||
|
||||
(defn replay-round
|
||||
[client round]
|
||||
(let [initial-db (get round "initialDb")
|
||||
blocks (if (map? initial-db) (get initial-db "blocks") nil)
|
||||
tx-log (-> (get round "txCapture") (get "txLog"))
|
||||
initial-entities (if (sequential? blocks)
|
||||
(keep snapshot-block->entity blocks)
|
||||
[])
|
||||
db0 (d/db-with (d/empty-db db-schema/schema) initial-entities)
|
||||
checksum0 (sync-checksum/recompute-checksum db0)
|
||||
replay-result
|
||||
(reduce
|
||||
(fn [{:keys [db checksum mismatches tx-count] :as acc} tx-entry]
|
||||
(let [datoms (if (map? tx-entry) (get tx-entry "datoms") nil)
|
||||
tx-data (if (sequential? datoms) (keep datom->tx datoms) [])
|
||||
op-index (when (map? tx-entry) (parse-int (get tx-entry "opIndex")))]
|
||||
(if (empty? tx-data)
|
||||
(assoc acc :tx-count (inc tx-count))
|
||||
(let [report (d/with db tx-data)
|
||||
db' (:db-after report)
|
||||
incremental (sync-checksum/update-checksum checksum report)
|
||||
full (sync-checksum/recompute-checksum db')]
|
||||
{:db db'
|
||||
:checksum incremental
|
||||
:tx-count (inc tx-count)
|
||||
:mismatches
|
||||
(cond-> mismatches
|
||||
(not= incremental full)
|
||||
(conj {:txIndex tx-count
|
||||
:opIndex op-index
|
||||
:incremental incremental
|
||||
:full full
|
||||
:txSize (count tx-data)}))}))))
|
||||
{:db db0
|
||||
:checksum checksum0
|
||||
:mismatches []
|
||||
:tx-count 0}
|
||||
(if (sequential? tx-log) tx-log []))]
|
||||
{:instanceIndex (parse-int (get client "instanceIndex"))
|
||||
:round (parse-int (get round "round"))
|
||||
:initialBlockCount (count initial-entities)
|
||||
:txCount (:tx-count replay-result)
|
||||
:mismatchCount (count (:mismatches replay-result))
|
||||
:firstMismatch (first (:mismatches replay-result))
|
||||
:finalChecksum (:checksum replay-result)}))
|
||||
|
||||
(defn select-rounds
|
||||
[client opts]
|
||||
(let [rounds (if (sequential? (get client "rounds")) (get client "rounds") [])]
|
||||
(if-let [round-index (:round opts)]
|
||||
(filter #(= round-index (parse-int (get % "round"))) rounds)
|
||||
rounds)))
|
||||
|
||||
(defn select-clients
|
||||
[artifact opts]
|
||||
(let [clients (if (sequential? (get artifact "clients")) (get artifact "clients") [])]
|
||||
(if-let [instance-index (:client opts)]
|
||||
(filter #(= instance-index (parse-int (get % "instanceIndex"))) clients)
|
||||
clients)))
|
||||
|
||||
(defn -main
|
||||
[argv]
|
||||
(let [{:keys [opts]} (cli/parse-args argv {:spec cli-spec})
|
||||
artifact-path (:artifact opts)]
|
||||
(when (or (:help opts) (string/blank? artifact-path))
|
||||
(println (usage))
|
||||
(js/process.exit (if (:help opts) 0 1)))
|
||||
(let [resolved-path (.resolve node-path artifact-path)
|
||||
content (.readFileSync fs resolved-path "utf8")
|
||||
artifact (js->clj (js/JSON.parse content))
|
||||
selected-clients (vec (select-clients artifact opts))
|
||||
results (->> selected-clients
|
||||
(mapcat (fn [client]
|
||||
(map #(replay-round client %)
|
||||
(select-rounds client opts))))
|
||||
(vec))
|
||||
payload {:artifactPath resolved-path
|
||||
:runId (get artifact "runId")
|
||||
:createdAt (get artifact "createdAt")
|
||||
:selectedClient (or (:client opts) nil)
|
||||
:selectedRound (or (:round opts) nil)
|
||||
:resultCount (count results)
|
||||
:results results}]
|
||||
(if (:pretty opts)
|
||||
(println (js/JSON.stringify (clj->js payload) nil 2))
|
||||
(println (js/JSON.stringify (clj->js payload)))))))
|
||||
|
||||
(when (= nbb/*file* (nbb/invoked-file))
|
||||
(-main *command-line-args*))
|
||||
1032
deps/db/script/replay_sync_sqlite.cljs
vendored
Normal file
1032
deps/db/script/replay_sync_sqlite.cljs
vendored
Normal file
File diff suppressed because it is too large
Load Diff
939
deps/db/src/logseq/db.cljs
vendored
Normal file
939
deps/db/src/logseq/db.cljs
vendored
Normal file
@@ -0,0 +1,939 @@
|
||||
(ns logseq.db
|
||||
"Main namespace for db fns. All fns are only for DB graphs"
|
||||
(:require [clojure.set :as set]
|
||||
[clojure.string :as string]
|
||||
[clojure.walk :as walk]
|
||||
[datascript.conn :as dc]
|
||||
[datascript.core :as d]
|
||||
[datascript.storage :as storage]
|
||||
[datascript.impl.entity :as de]
|
||||
[logseq.common.config :as common-config]
|
||||
[logseq.common.plural :as common-plural]
|
||||
[logseq.common.util :as common-util]
|
||||
[logseq.common.uuid :as common-uuid]
|
||||
[logseq.db.common.delete-blocks :as delete-blocks] ;; Load entity extensions
|
||||
[logseq.db.common.entity-plus :as entity-plus]
|
||||
[logseq.db.common.initial-data :as common-initial-data]
|
||||
[logseq.db.frontend.class :as db-class]
|
||||
[logseq.db.frontend.db :as db-db]
|
||||
[logseq.db.frontend.entity-util :as entity-util]
|
||||
[logseq.db.frontend.property :as db-property]
|
||||
[logseq.db.frontend.validate :as db-validate]
|
||||
[logseq.db.sqlite.util :as sqlite-util]
|
||||
[logseq.common.log :as log])
|
||||
(:refer-clojure :exclude [object?]))
|
||||
|
||||
(def built-in? entity-util/built-in?)
|
||||
(def built-in-class-property? db-db/built-in-class-property?)
|
||||
(def private-built-in-page? db-db/private-built-in-page?)
|
||||
|
||||
(def write-transit-str sqlite-util/write-transit-str)
|
||||
(def read-transit-str sqlite-util/read-transit-str)
|
||||
(def build-favorite-tx db-db/build-favorite-tx)
|
||||
|
||||
(defonce *transact-fn (atom nil))
|
||||
(defonce *transact-invalid-callback (atom nil))
|
||||
(defonce *transact-pipeline-fn (atom nil))
|
||||
(defonce *debounce-fn (atom nil))
|
||||
(def ^:dynamic *batch-tx-report?* false)
|
||||
|
||||
(defn register-transact-fn!
|
||||
[f]
|
||||
(when f (reset! *transact-fn f)))
|
||||
(defn register-transact-invalid-callback-fn!
|
||||
[f]
|
||||
(when f (reset! *transact-invalid-callback f)))
|
||||
(defn register-transact-pipeline-fn!
|
||||
[f]
|
||||
(when f (reset! *transact-pipeline-fn f)))
|
||||
(defn register-debounce-fn!
|
||||
[f]
|
||||
(when f (reset! *debounce-fn f)))
|
||||
|
||||
(defn- remove-temp-block-data
|
||||
[tx-data]
|
||||
(let [remove-block-temp-f (fn [m]
|
||||
(->> (remove (fn [[k _v]]
|
||||
(= "block.temp" (namespace k))) m)
|
||||
(into {})))]
|
||||
(keep (fn [data]
|
||||
(cond
|
||||
(map? data)
|
||||
(cond->
|
||||
(remove-block-temp-f data)
|
||||
(seq (:block/refs data))
|
||||
(update :block/refs (fn [refs]
|
||||
(map (fn [ref]
|
||||
(if (map? ref) (remove-block-temp-f ref) ref)) refs))))
|
||||
(and (vector? data)
|
||||
(contains? #{:db/add :db/retract} (first data))
|
||||
(> (count data) 2)
|
||||
(keyword? (nth data 2))
|
||||
(= "block.temp" (namespace (nth data 2))))
|
||||
nil
|
||||
:else
|
||||
data))
|
||||
tx-data)))
|
||||
|
||||
(defn entity->db-id
|
||||
[tx-data]
|
||||
(walk/prewalk
|
||||
(fn [f]
|
||||
(if (de/entity? f)
|
||||
(if-let [id (:db/id f)]
|
||||
id
|
||||
(throw (ex-info "ldb/transact! doesn't support Entity"
|
||||
{:entity f
|
||||
:tx-data tx-data})))
|
||||
f))
|
||||
tx-data))
|
||||
|
||||
(defn- throw-if-page-has-block-parent!
|
||||
[db tx-data]
|
||||
(when (some (fn [d] (and (:added d)
|
||||
(= :block/parent (:a d))
|
||||
(let [entity (d/entity db (:e d))
|
||||
parent (:block/parent entity)]
|
||||
(and (entity-util/page? entity)
|
||||
(= (:v d) (:db/id parent))
|
||||
(not (entity-util/page? (d/entity db (:v d)))))))) tx-data)
|
||||
(throw (ex-info "Page can't have block as parent"
|
||||
{:tx-data tx-data}))))
|
||||
|
||||
(comment
|
||||
(defn debounced-store-db
|
||||
[conn]
|
||||
(when-some [_storage (storage/storage @conn)]
|
||||
(when-not (:batch-tx? @conn)
|
||||
(let [f (if (exists? js/process)
|
||||
d/store
|
||||
(or @*debounce-fn d/store))]
|
||||
(f @conn))))))
|
||||
|
||||
(defn- should-run-transact-pipeline?
|
||||
[conn db tx-meta]
|
||||
(and (entity-plus/db-based-graph? db)
|
||||
(not
|
||||
(or (:rtc-download-graph? tx-meta)
|
||||
(:reset-conn! tx-meta)
|
||||
(:initial-db? tx-meta)
|
||||
(:skip-validate-db? tx-meta false)
|
||||
;; used by `batch-transact-with-temp-conn!`
|
||||
(:skip-validate-db? @conn)
|
||||
(:logseq.graph-parser.exporter/new-graph? tx-meta)))))
|
||||
|
||||
(defn- should-validate-pipeline-result?
|
||||
[tx-meta]
|
||||
(not (:fix-db? tx-meta)))
|
||||
|
||||
(defn- tx-report-pipeline-data
|
||||
[tx-report]
|
||||
(map (fn [[e a v t]]
|
||||
[e a v t])
|
||||
(:tx-data tx-report)))
|
||||
|
||||
(defn- invalid-tx-debug-data
|
||||
[tx-meta tx-data errors tx-report]
|
||||
{:tx-meta tx-meta
|
||||
:tx-data tx-data
|
||||
:errors errors
|
||||
:pipeline-tx-data (tx-report-pipeline-data tx-report)})
|
||||
|
||||
(defn- throw-invalid-tx!
|
||||
[tx-meta tx-data errors tx-report]
|
||||
;; notify ui
|
||||
(when-let [f @*transact-invalid-callback]
|
||||
(f tx-report errors))
|
||||
(let [debug-data (invalid-tx-debug-data tx-meta tx-data errors tx-report)]
|
||||
(prn :debug :invalid-data debug-data)
|
||||
(prn :debug :errors errors)
|
||||
(throw (ex-info "DB write failed with invalid data" debug-data))))
|
||||
|
||||
(defn- transact-sync
|
||||
[conn tx-data tx-meta]
|
||||
(try
|
||||
(loop []
|
||||
(let [db @conn
|
||||
run-pipeline? (should-run-transact-pipeline? conn db tx-meta)]
|
||||
(if run-pipeline?
|
||||
(let [tx-report* (d/with db tx-data tx-meta)
|
||||
pipeline-f @*transact-pipeline-fn
|
||||
tx-report (if pipeline-f (pipeline-f tx-report*) tx-report*)
|
||||
_ (throw-if-page-has-block-parent! (:db-after tx-report) (:tx-data tx-report))
|
||||
[validate-result errors] (if (should-validate-pipeline-result? tx-meta)
|
||||
(db-validate/validate-tx-report tx-report nil)
|
||||
[true nil])]
|
||||
(cond
|
||||
(not validate-result)
|
||||
(if (identical? db @conn)
|
||||
(throw-invalid-tx! tx-meta tx-data errors tx-report)
|
||||
(recur))
|
||||
|
||||
(and tx-report (seq (:tx-data tx-report)))
|
||||
(if (compare-and-set! conn db (:db-after tx-report))
|
||||
(do
|
||||
(dc/store-after-transact! conn tx-report)
|
||||
(dc/run-callbacks conn tx-report)
|
||||
tx-report)
|
||||
(recur))
|
||||
|
||||
:else
|
||||
tx-report))
|
||||
(d/transact! conn tx-data tx-meta))))
|
||||
(catch :default e
|
||||
(when-not (or (:db-sync/suppress-transact-failed-log? tx-meta)
|
||||
(and (:db-sync/suppress-stale-rebase-transact-failed-log? tx-meta)
|
||||
(= :entity-id/missing (:error (ex-data e)))))
|
||||
(prn :debug :transact-failed
|
||||
:tx-meta tx-meta
|
||||
:tx-data tx-data
|
||||
:error (str e))
|
||||
(js/console.error e))
|
||||
(throw e))))
|
||||
|
||||
(defn transact!
|
||||
"`repo-or-conn`: repo for UI thread and conn for worker/node"
|
||||
([repo-or-conn tx-data]
|
||||
(transact! repo-or-conn tx-data nil))
|
||||
([repo-or-conn tx-data tx-meta]
|
||||
(let [tx-data (->> tx-data
|
||||
entity->db-id
|
||||
(map (fn [m]
|
||||
(if (map? m)
|
||||
(cond->
|
||||
(dissoc m :block/children :block/meta :block/top? :block/bottom? :block/anchor
|
||||
:block/level :block/container :db/other-tx
|
||||
:block/unordered)
|
||||
(not @*transact-fn)
|
||||
(dissoc :block.temp/load-status))
|
||||
m)))
|
||||
(remove-temp-block-data)
|
||||
(remove (fn [m] (and (map? m) (= (:db/ident m) :block/path-refs))))
|
||||
(common-util/fast-remove-nils)
|
||||
(remove (fn [m] (or ;; db/id
|
||||
(integer? m)
|
||||
(empty? m)))))
|
||||
tx-data (if-not (string? repo-or-conn)
|
||||
(delete-blocks/expand-delete-blocks-tx @repo-or-conn tx-data tx-meta)
|
||||
tx-data)
|
||||
delete-blocks-tx (when-not (string? repo-or-conn)
|
||||
(delete-blocks/update-refs-history @repo-or-conn tx-data tx-meta))
|
||||
tx-data (concat tx-data delete-blocks-tx)]
|
||||
|
||||
;; Ensure worker can handle the request sequentially (one by one)
|
||||
;; Because UI assumes that the in-memory db has all the data except the last one transaction
|
||||
(when (seq tx-data)
|
||||
|
||||
;; (prn :debug :transact :sync? (= d/transact! (or @*transact-fn d/transact!)) :tx-meta tx-meta)
|
||||
;; (cljs.pprint/pprint tx-data)
|
||||
;; (js/console.trace)
|
||||
|
||||
(let [tx-meta (cond-> tx-meta
|
||||
*batch-tx-report?*
|
||||
(assoc :batch-tx-report? true))]
|
||||
(if-let [transact-fn @*transact-fn]
|
||||
(transact-fn repo-or-conn tx-data tx-meta)
|
||||
(transact-sync repo-or-conn tx-data tx-meta)))))))
|
||||
|
||||
(defn- make-conn [db opts]
|
||||
;; `datascript.conn/->Conn` is not exposed in nbb runtime.
|
||||
;; Start from a fresh conn and merge the desired internal state.
|
||||
(let [conn (d/create-conn (:schema db))]
|
||||
(swap! (:atom conn) merge opts)
|
||||
conn))
|
||||
|
||||
(defn- conn-from-db
|
||||
"Forked conn-from-db to not invoke `d/store`, it's unsafe to store during nested batch tx."
|
||||
[db]
|
||||
(if-some [_storage (storage/storage db)]
|
||||
(make-conn
|
||||
db
|
||||
{:db db
|
||||
:tx-tail []
|
||||
:db-last-stored db})
|
||||
(make-conn db {:db db})))
|
||||
|
||||
(defn temp-conn-from-db
|
||||
"Create an isolated in-memory conn from `db` that can read storage-backed data without storing writes."
|
||||
[db]
|
||||
(doto (conn-from-db db)
|
||||
(swap! assoc
|
||||
:skip-store? true
|
||||
:skip-validate-db? true)))
|
||||
|
||||
(defn batch-transact-with-temp-conn!
|
||||
"Run batched tx work against a temporary conn, then apply all collected tx-data
|
||||
to `conn` with a single final `transact!`.
|
||||
|
||||
Semantics:
|
||||
- Uses `d/conn-from-db` so batch work runs on an isolated in-memory conn.
|
||||
- Temp conn can still read from storage-backed db state, but cannot write to disk
|
||||
(`:skip-store? true`).
|
||||
- Defers db validation during inner batch ops (`:skip-validate-db? true`), then
|
||||
validates on the final `transact!` to `conn`.
|
||||
- Supports nested usage.
|
||||
|
||||
Notes:
|
||||
- `batch-tx-fn` is called as `(batch-tx-fn temp-conn *batch-tx-data)`.
|
||||
- `listen-db` (if provided) receives each intermediate tx-report from temp conn.
|
||||
- Do not rely on returned tx-report shape for undo/redo behavior."
|
||||
[conn tx-meta batch-tx-fn & {:keys [listen-db before-commit]}]
|
||||
(let [temp-conn (temp-conn-from-db @conn)
|
||||
*batch-tx-data (volatile! [])
|
||||
*complete? (volatile! false)]
|
||||
;; can read from disk, write is disallowed
|
||||
(swap! temp-conn assoc
|
||||
:batch-tx? true
|
||||
:skip-validate-db? true)
|
||||
(d/listen! temp-conn ::temp-conn-batch-tx
|
||||
(fn [{:keys [tx-data] :as tx-report}]
|
||||
(vswap! *batch-tx-data into tx-data)
|
||||
(when (fn? listen-db)
|
||||
(listen-db tx-report))))
|
||||
(try
|
||||
(batch-tx-fn temp-conn *batch-tx-data)
|
||||
(when (fn? before-commit)
|
||||
(before-commit))
|
||||
(vreset! *complete? true)
|
||||
(let [tx-data @*batch-tx-data]
|
||||
(when (and @*complete? (seq tx-data))
|
||||
;; transact tx-data to `conn` and validate db
|
||||
(transact! conn tx-data tx-meta)))
|
||||
(finally
|
||||
(d/unlisten! temp-conn ::temp-conn-batch-tx)
|
||||
(reset! temp-conn nil)
|
||||
(vreset! *batch-tx-data nil)))))
|
||||
|
||||
(defn- batch-transact-listen!
|
||||
[conn *tx-data listen-db]
|
||||
(d/listen! conn ::batch-tx
|
||||
(fn [tx-report]
|
||||
(swap! *tx-data into (:tx-data tx-report))
|
||||
(when (fn? listen-db)
|
||||
(listen-db tx-report)))))
|
||||
|
||||
(defn- batch-transact-cleanup!
|
||||
[conn]
|
||||
(d/unlisten! conn ::batch-tx)
|
||||
(swap! conn dissoc :skip-store? :batch-tx?))
|
||||
|
||||
(defn batch-transact!
|
||||
"Run batched tx work on `conn` and persist once at the end.
|
||||
Not nestable; throws when called inside another `:batch-tx?`."
|
||||
[conn tx-meta batch-tx-fn & {:keys [listen-db]}]
|
||||
(let [db-before @conn
|
||||
*tx-data (atom [])]
|
||||
(try
|
||||
(when (:batch-tx? @conn)
|
||||
(throw (ex-info "batch-transact! can't be nested called" {:tx-meta tx-meta})))
|
||||
(batch-transact-listen! conn *tx-data listen-db)
|
||||
(swap! conn assoc :skip-store? true :batch-tx? true)
|
||||
(binding [*batch-tx-report?* true]
|
||||
(batch-tx-fn conn))
|
||||
(batch-transact-cleanup! conn)
|
||||
(when-some [_storage (storage/storage @conn)]
|
||||
(d/store @conn)
|
||||
;; batch-transact! bypasses conn/store-after-transact!, so keep tail bookkeeping in sync
|
||||
;; with the just-persisted root snapshot.
|
||||
(swap! (:atom conn) assoc
|
||||
:tx-tail []
|
||||
:db-last-stored @conn))
|
||||
(let [batch-tx-data @*tx-data
|
||||
_ (reset! *tx-data nil)
|
||||
tx-report {:db-before db-before
|
||||
:db-after @conn
|
||||
:tx-meta (assoc tx-meta :batch-final-tx-report? true)
|
||||
:tx-data batch-tx-data}]
|
||||
(dc/run-callbacks conn tx-report)
|
||||
tx-report)
|
||||
(catch :default e
|
||||
(log/error e)
|
||||
(reset! conn db-before)
|
||||
(throw e))
|
||||
(finally
|
||||
(batch-transact-cleanup! conn)
|
||||
(reset! *tx-data nil)))))
|
||||
|
||||
(def page? entity-util/page?)
|
||||
(def internal-page? entity-util/internal-page?)
|
||||
(def class? entity-util/class?)
|
||||
(def property? entity-util/property?)
|
||||
(def closed-value? entity-util/closed-value?)
|
||||
(def journal? entity-util/journal?)
|
||||
(def hidden? entity-util/hidden?)
|
||||
(def recycled? entity-util/recycled?)
|
||||
(def object? entity-util/object?)
|
||||
(def asset? entity-util/asset?)
|
||||
(def public-built-in-property? db-property/public-built-in-property?)
|
||||
(def get-entity-types entity-util/get-entity-types)
|
||||
(def internal-tags db-class/internal-tags)
|
||||
(def private-tags db-class/private-tags)
|
||||
(def extends-hidden-tags db-class/extends-hidden-tags)
|
||||
(def hidden-tags db-class/hidden-tags)
|
||||
|
||||
(defn sort-by-order
|
||||
[blocks]
|
||||
(sort-by :block/order blocks))
|
||||
|
||||
(defn- get-block-and-children-aux
|
||||
[entity & {:keys [include-property-block?]
|
||||
:or {include-property-block? false}
|
||||
:as opts}]
|
||||
(if-let [children (sort-by-order
|
||||
(if include-property-block?
|
||||
(let [children (entity-plus/lookup-kv-then-entity entity :block/_raw-parent)]
|
||||
(concat children
|
||||
(keep (fn [c] (:logseq.property/query c)) children)))
|
||||
(entity-plus/lookup-kv-then-entity entity :block/_parent)))]
|
||||
(cons entity (mapcat #(get-block-and-children-aux % opts) children))
|
||||
[entity]))
|
||||
|
||||
(defn get-block-and-children
|
||||
[db block-uuid & {:as opts}]
|
||||
(when-let [e (d/entity db [:block/uuid block-uuid])]
|
||||
(get-block-and-children-aux e opts)))
|
||||
|
||||
(defn get-page-blocks
|
||||
"Return blocks of the designated page, without using cache.
|
||||
page-id - eid"
|
||||
[db page-id & {:keys [pull-keys]
|
||||
:or {pull-keys '[*]}}]
|
||||
(when page-id
|
||||
(let [datoms (d/datoms db :avet :block/page page-id)
|
||||
block-eids (mapv :e datoms)]
|
||||
(d/pull-many db pull-keys block-eids))))
|
||||
|
||||
(defn get-page-blocks-count
|
||||
[db page-id]
|
||||
(count (d/datoms db :avet :block/page page-id)))
|
||||
|
||||
(defn- get-block-children-or-property-children
|
||||
[block parent]
|
||||
(let [from-property (:logseq.property/created-from-property block)
|
||||
closed-property (:block/closed-value-property block)]
|
||||
(sort-by-order (cond
|
||||
closed-property
|
||||
(:block/_closed-value-property closed-property)
|
||||
|
||||
from-property
|
||||
(filter (fn [e]
|
||||
(= (:db/id (:logseq.property/created-from-property e))
|
||||
(:db/id from-property)))
|
||||
(:block/_raw-parent parent))
|
||||
|
||||
:else
|
||||
(:block/_parent parent)))))
|
||||
|
||||
(defn- get-right-sibling-for-property-children
|
||||
[block parent]
|
||||
(assert (or (de/entity? block) (nil? block)))
|
||||
(let [children (get-block-children-or-property-children block parent)
|
||||
right (some (fn [child] (when (> (compare (:block/order child) (:block/order block)) 0) child)) children)]
|
||||
(when (not= (:db/id right) (:db/id block))
|
||||
right)))
|
||||
|
||||
(defn- get-ordinary-sibling
|
||||
[block direction]
|
||||
(let [db (.-db block)
|
||||
parent-id (:db/id (:block/parent block))
|
||||
block-order (:block/order block)
|
||||
[eligible? closer?] (case direction
|
||||
:left [neg? pos?]
|
||||
:right [pos? neg?])
|
||||
[sibling-id _]
|
||||
(reduce (fn [[_ best-order :as best] datom]
|
||||
(let [child-id (:e datom)
|
||||
child-order (some-> (d/datoms db :eavt child-id :block/order) first :v)]
|
||||
(if (and child-order
|
||||
(eligible? (compare child-order block-order))
|
||||
(not (seq (d/datoms db :avet :logseq.property/created-from-property child-id)))
|
||||
(not (seq (d/datoms db :avet :block/closed-value-property child-id)))
|
||||
(or (nil? best-order)
|
||||
(closer? (compare child-order best-order))))
|
||||
[child-id child-order]
|
||||
best)))
|
||||
[nil nil]
|
||||
(d/datoms db :avet :block/parent parent-id))]
|
||||
(when sibling-id
|
||||
(d/entity db sibling-id))))
|
||||
|
||||
(defn get-right-sibling
|
||||
[block]
|
||||
(assert (or (de/entity? block) (nil? block)))
|
||||
(when-let [parent (:block/parent block)]
|
||||
(cond
|
||||
(:block/closed-value-property block)
|
||||
(get-right-sibling-for-property-children block parent)
|
||||
|
||||
(:logseq.property/created-from-property block)
|
||||
(get-right-sibling-for-property-children block parent)
|
||||
|
||||
:else
|
||||
(get-ordinary-sibling block :right))))
|
||||
|
||||
(defn- get-left-sibling-for-property-children
|
||||
[block parent]
|
||||
(assert (or (de/entity? block) (nil? block)))
|
||||
(let [children (reverse (get-block-children-or-property-children block parent))
|
||||
left (some (fn [child] (when (< (compare (:block/order child) (:block/order block)) 0) child)) children)]
|
||||
(when (not= (:db/id left) (:db/id block))
|
||||
left)))
|
||||
|
||||
(defn get-left-sibling
|
||||
[block]
|
||||
(assert (or (de/entity? block) (nil? block)))
|
||||
(when-let [parent (:block/parent block)]
|
||||
(cond
|
||||
(:block/closed-value-property block)
|
||||
(get-left-sibling-for-property-children block parent)
|
||||
|
||||
(:logseq.property/created-from-property block)
|
||||
(get-left-sibling-for-property-children block parent)
|
||||
|
||||
:else
|
||||
(get-ordinary-sibling block :left))))
|
||||
|
||||
(defn get-down
|
||||
[block]
|
||||
(assert (or (de/entity? block) (nil? block)))
|
||||
(first (sort-by-order (:block/_parent block))))
|
||||
|
||||
(defn get-pages
|
||||
[db]
|
||||
(->> (d/q
|
||||
'[:find ?page-title
|
||||
:where
|
||||
[?page :block/name ?page-name]
|
||||
[(get-else $ ?page :block/title ?page-name) ?page-title]]
|
||||
db)
|
||||
(map first)
|
||||
(remove hidden?)))
|
||||
|
||||
(def get-first-page-by-name common-initial-data/get-first-page-by-name)
|
||||
|
||||
(defn page-exists?
|
||||
"Returns all page db ids that given title and one of the given `tags`."
|
||||
[db page-name tags]
|
||||
(when page-name
|
||||
(let [tags' (if (coll? tags) (set tags) #{tags})]
|
||||
;; Classes and properties are case sensitive and can be looked up
|
||||
;; as such in case-sensitive contexts e.g. no Page
|
||||
(if (and (seq tags') (every? #{:logseq.class/Tag :logseq.class/Property} tags'))
|
||||
(seq
|
||||
(d/q
|
||||
'[:find [?p ...]
|
||||
:in $ ?name [?tag-ident ...]
|
||||
:where
|
||||
[?p :block/title ?name]
|
||||
[?p :block/tags ?tag]
|
||||
[?tag :db/ident ?tag-ident]]
|
||||
db
|
||||
page-name
|
||||
tags'))
|
||||
;; TODO: Decouple db graphs from file specific :block/name
|
||||
(seq
|
||||
(d/q
|
||||
'[:find [?p ...]
|
||||
:in $ ?name [?tag-ident ...]
|
||||
:where
|
||||
[?p :block/name ?name]
|
||||
[?p :block/tags ?tag]
|
||||
[?tag :db/ident ?tag-ident]]
|
||||
db
|
||||
(common-util/page-name-sanity-lc page-name)
|
||||
tags'))))))
|
||||
|
||||
(defn get-page
|
||||
"Get a page given its unsanitized name or uuid"
|
||||
[db page-id-name-or-uuid]
|
||||
(when db
|
||||
(if (number? page-id-name-or-uuid)
|
||||
(d/entity db page-id-name-or-uuid)
|
||||
(if-let [id (if (uuid? page-id-name-or-uuid) page-id-name-or-uuid
|
||||
(parse-uuid page-id-name-or-uuid))]
|
||||
(d/entity db [:block/uuid id])
|
||||
(d/entity db (get-first-page-by-name db (name page-id-name-or-uuid)))))))
|
||||
|
||||
(defn get-journal-page
|
||||
"Get a journal page given its unsanitized name.
|
||||
This will be useful for DB graphs later as we can switch to a different lookup
|
||||
approach for journals e.g. like get-built-in-page"
|
||||
[db page-name]
|
||||
(when db
|
||||
(d/entity db (get-first-page-by-name db page-name))))
|
||||
|
||||
(defn get-journal-page-by-day
|
||||
"Get a journal page given its :block/journal-day value."
|
||||
[db journal-day]
|
||||
(when (and db journal-day)
|
||||
(when-let [eid (some-> (first (d/datoms db :avet :block/journal-day journal-day))
|
||||
:e)]
|
||||
(d/entity db eid))))
|
||||
|
||||
(def get-built-in-page db-db/get-built-in-page)
|
||||
|
||||
(def library? db-db/library?)
|
||||
(defn get-library-page
|
||||
[db]
|
||||
(get-built-in-page db common-config/library-page-name))
|
||||
|
||||
(defn get-case-page
|
||||
"Case sensitive version of get-page. For use with DB graphs"
|
||||
[db page-name-or-uuid]
|
||||
(when db
|
||||
(if-let [id (if (uuid? page-name-or-uuid) page-name-or-uuid
|
||||
(parse-uuid page-name-or-uuid))]
|
||||
(d/entity db [:block/uuid id])
|
||||
(d/entity db (common-initial-data/get-first-page-by-title db page-name-or-uuid)))))
|
||||
|
||||
(defn page-empty?
|
||||
"Whether a page is empty. Does it has a non-page block?
|
||||
`page-id` could be either a string or a db/id."
|
||||
[db page-id]
|
||||
(let [page-id (if (string? page-id)
|
||||
(get-first-page-by-name db page-id)
|
||||
page-id)
|
||||
page (d/entity db page-id)]
|
||||
(empty? (:block/_parent page))))
|
||||
|
||||
(defn get-first-child
|
||||
[db id]
|
||||
(first (sort-by-order (:block/_parent (d/entity db id)))))
|
||||
|
||||
(defn get-orphaned-pages
|
||||
[db {:keys [pages empty-ref-f built-in-pages-names]
|
||||
:or {empty-ref-f (fn [page] (zero? (count (:block/_refs page))))
|
||||
built-in-pages-names #{}}}]
|
||||
(let [pages (->> (or pages (get-pages db))
|
||||
(remove nil?))
|
||||
built-in-pages (set (map string/lower-case built-in-pages-names))
|
||||
orphaned-pages (->>
|
||||
(map
|
||||
(fn [page]
|
||||
(when-let [page (get-page db page)]
|
||||
(let [name' (:block/name page)]
|
||||
(and
|
||||
(empty-ref-f page)
|
||||
(or
|
||||
(page-empty? db (:db/id page))
|
||||
(let [first-child (get-first-child db (:db/id page))
|
||||
children (:block/_page page)]
|
||||
(and
|
||||
first-child
|
||||
(= 1 (count children))
|
||||
(contains? #{"" "-" "*"} (string/trim (:block/title first-child))))))
|
||||
(not (contains? built-in-pages name'))
|
||||
(not (property? page))
|
||||
;; a/b/c might be deleted but a/b/c/d still exists (for backward compatibility)
|
||||
(not (and (string/includes? name' "/")
|
||||
(not (journal? page))))
|
||||
(not (:block/properties page))
|
||||
page))))
|
||||
pages)
|
||||
(remove false?)
|
||||
(remove nil?)
|
||||
(remove hidden?))]
|
||||
orphaned-pages))
|
||||
|
||||
(defn has-children?
|
||||
[db block-id]
|
||||
(let [eid (if (uuid? block-id) [:block/uuid block-id] block-id)]
|
||||
(some? (:block/_parent (d/entity db eid)))))
|
||||
|
||||
(defn- collapsed-and-has-children?
|
||||
[db block]
|
||||
(and (:block/collapsed? block) (has-children? db (:block/uuid block))))
|
||||
|
||||
(defn get-block-last-direct-child-id
|
||||
"Notice: if `not-collapsed?` is true, will skip searching for any collapsed block."
|
||||
([db db-id]
|
||||
(get-block-last-direct-child-id db db-id false))
|
||||
([db db-id not-collapsed?]
|
||||
(when-let [block (d/entity db db-id)]
|
||||
(when (if not-collapsed?
|
||||
(not (collapsed-and-has-children? db block))
|
||||
true)
|
||||
(let [children (sort-by :block/order (:block/_parent block))]
|
||||
(:db/id (last children)))))))
|
||||
|
||||
(defn get-children
|
||||
"Doesn't include nested children."
|
||||
([block-entity]
|
||||
(get-children nil block-entity))
|
||||
([db block-entity-or-eid]
|
||||
(when-let [parent (cond
|
||||
(number? block-entity-or-eid)
|
||||
(d/entity db block-entity-or-eid)
|
||||
(uuid? block-entity-or-eid)
|
||||
(d/entity db [:block/uuid block-entity-or-eid])
|
||||
:else
|
||||
block-entity-or-eid)]
|
||||
(sort-by-order (:block/_parent parent)))))
|
||||
|
||||
(defn get-block-parents
|
||||
"Returns parents entities"
|
||||
[db block-id & {:keys [depth] :or {depth 100}}]
|
||||
(loop [block-id block-id
|
||||
parents' (list)
|
||||
d 1]
|
||||
(if (> d depth)
|
||||
parents'
|
||||
(if-let [parent (:block/parent (d/entity db [:block/uuid block-id]))]
|
||||
(recur (:block/uuid parent) (conj parents' parent) (inc d))
|
||||
parents'))))
|
||||
|
||||
(def get-block-children-ids common-initial-data/get-block-children-ids)
|
||||
(def get-block-full-children-ids common-initial-data/get-block-full-children-ids)
|
||||
|
||||
(defn- get-sorted-page-block-ids
|
||||
[db page-id]
|
||||
(let [root (d/entity db page-id)]
|
||||
(loop [result []
|
||||
children (sort-by-order (:block/_parent root))]
|
||||
(if (seq children)
|
||||
(let [child (first children)]
|
||||
(recur (conj result (:db/id child))
|
||||
(concat
|
||||
(sort-by-order (:block/_parent child))
|
||||
(rest children))))
|
||||
result))))
|
||||
|
||||
(defn sort-page-random-blocks
|
||||
"Blocks could be non consecutive."
|
||||
[db blocks]
|
||||
(assert (every? #(= (:block/page %) (:block/page (first blocks))) blocks) "Blocks must to be in a same page.")
|
||||
(let [page-id (:db/id (:block/page (first blocks)))
|
||||
;; TODO: there's no need to sort all the blocks
|
||||
sorted-ids (get-sorted-page-block-ids db page-id)
|
||||
blocks-map (zipmap (map :db/id blocks) blocks)]
|
||||
(keep blocks-map sorted-ids)))
|
||||
|
||||
(defn last-child-block?
|
||||
"The child block could be collapsed."
|
||||
[db parent-id child-id]
|
||||
(when-let [child (d/entity db child-id)]
|
||||
(cond
|
||||
(= parent-id child-id)
|
||||
true
|
||||
|
||||
(get-right-sibling child)
|
||||
false
|
||||
|
||||
:else
|
||||
(last-child-block? db parent-id (:db/id (:block/parent child))))))
|
||||
|
||||
(defn- consecutive-block?
|
||||
[db block-1 block-2]
|
||||
(let [aux-fn (fn [block-1 block-2]
|
||||
(and (= (:block/page block-1) (:block/page block-2))
|
||||
(or
|
||||
;; sibling or child
|
||||
(= (:db/id (get-left-sibling block-2)) (:db/id block-1))
|
||||
(when-let [prev-sibling (get-left-sibling (d/entity db (:db/id block-2)))]
|
||||
(last-child-block? db (:db/id prev-sibling) (:db/id block-1))))))]
|
||||
(or (aux-fn block-1 block-2) (aux-fn block-2 block-1))))
|
||||
|
||||
(defn get-non-consecutive-blocks
|
||||
[db blocks]
|
||||
(vec
|
||||
(keep-indexed
|
||||
(fn [i _block]
|
||||
(when (< (inc i) (count blocks))
|
||||
(when-not (consecutive-block? db (nth blocks i) (nth blocks (inc i)))
|
||||
(nth blocks i))))
|
||||
blocks)))
|
||||
|
||||
(defn new-block-id
|
||||
[]
|
||||
(common-uuid/gen-uuid))
|
||||
|
||||
(defn get-alias-source-page
|
||||
"return the source page of an alias"
|
||||
[db alias-id]
|
||||
(when alias-id
|
||||
(first (:block/_alias (d/entity db alias-id)))))
|
||||
|
||||
(def get-block-alias common-initial-data/get-block-alias)
|
||||
|
||||
(defn page-alias-set
|
||||
[db page-id]
|
||||
(->>
|
||||
(get-block-alias db page-id)
|
||||
(set)
|
||||
(set/union #{page-id})))
|
||||
|
||||
(def get-block-refs-count common-initial-data/get-block-refs-count)
|
||||
|
||||
(defn hidden-or-internal-tag?
|
||||
[e]
|
||||
(or (entity-util/hidden? e) (db-class/internal-tags (:db/ident e))))
|
||||
|
||||
(defn get-all-pages
|
||||
[db]
|
||||
(->>
|
||||
(d/datoms db :avet :block/name)
|
||||
(keep (fn [d]
|
||||
(let [e (d/entity db (:e d))]
|
||||
(when-not (hidden-or-internal-tag? e)
|
||||
e))))))
|
||||
|
||||
(defn get-key-value
|
||||
[db key-ident]
|
||||
(assert (= "logseq.kv" (namespace key-ident)) key-ident)
|
||||
(:kv/value (d/entity db key-ident)))
|
||||
|
||||
(def kv sqlite-util/kv)
|
||||
|
||||
(defn get-graph-rtc-uuid
|
||||
[db]
|
||||
(when db (get-key-value db :logseq.kv/graph-uuid)))
|
||||
|
||||
(defn get-graph-local-uuid
|
||||
[db]
|
||||
(when db (get-key-value db :logseq.kv/local-graph-uuid)))
|
||||
|
||||
(defn get-graph-schema-version
|
||||
[db]
|
||||
(when db (get-key-value db :logseq.kv/schema-version)))
|
||||
|
||||
(defn get-graph-remote-schema-version
|
||||
[db]
|
||||
(when db (get-key-value db :logseq.kv/remote-schema-version)))
|
||||
|
||||
(defn get-graph-rtc-e2ee?
|
||||
[db]
|
||||
(when db (get-key-value db :logseq.kv/graph-rtc-e2ee?)))
|
||||
|
||||
(def get-all-properties db-db/get-all-properties)
|
||||
(def get-class-extends db-class/get-class-extends)
|
||||
(def get-classes-parents db-db/get-classes-parents)
|
||||
(def get-page-parents db-db/get-page-parents)
|
||||
(def get-title-with-parents db-db/get-title-with-parents)
|
||||
(def class-instance? db-db/class-instance?)
|
||||
(def inline-tag? db-db/inline-tag?)
|
||||
(def node-display-type-classes db-db/node-display-type-classes)
|
||||
(def get-class-ident-by-display-type db-db/get-class-ident-by-display-type)
|
||||
(def get-display-type-by-class-ident db-db/get-display-type-by-class-ident)
|
||||
|
||||
(def get-recent-updated-pages common-initial-data/get-recent-updated-pages)
|
||||
|
||||
(def get-latest-journals common-initial-data/get-latest-journals)
|
||||
|
||||
(defn get-pages-relation
|
||||
[db with-journal?]
|
||||
(let [q (if with-journal?
|
||||
'[:find ?p ?ref-page
|
||||
:where
|
||||
[?block :block/page ?p]
|
||||
[?block :block/refs ?ref-page]]
|
||||
'[:find ?p ?ref-page
|
||||
:where
|
||||
[?block :block/page ?p]
|
||||
[?p :block/tags]
|
||||
(not [?p :block/tags :logseq.class/Journal])
|
||||
[?block :block/refs ?ref-page]])]
|
||||
(d/q q db)))
|
||||
|
||||
(defn get-all-tagged-pages
|
||||
[db]
|
||||
(d/q '[:find ?page ?tag
|
||||
:where
|
||||
[?page :block/tags ?tag]]
|
||||
db))
|
||||
|
||||
(defn page-in-library?
|
||||
"Check whether a `page` exists on the Library page"
|
||||
[db page]
|
||||
(when (page? page)
|
||||
(when-let [library-page (get-built-in-page db common-config/library-page-name)]
|
||||
(loop [parent (:block/parent page)]
|
||||
(cond
|
||||
(nil? parent)
|
||||
false
|
||||
(= (:db/id parent) (:db/id library-page))
|
||||
true
|
||||
:else
|
||||
(recur (:block/parent parent)))))))
|
||||
|
||||
(def get-class-title-with-extends db-db/get-class-title-with-extends)
|
||||
|
||||
(defn- bidirectional-property-attr?
|
||||
[db attr]
|
||||
(when (qualified-keyword? attr)
|
||||
(let [attr-ns (namespace attr)]
|
||||
(and (or (db-property/user-property-namespace? attr-ns)
|
||||
(db-property/plugin-property? attr))
|
||||
(when-let [property (d/entity db attr)]
|
||||
(and (= :db.type/ref (:db/valueType property))
|
||||
(seq (:logseq.property/classes property))))))))
|
||||
|
||||
(defn- get-bidirectional-property-attrs
|
||||
[db]
|
||||
(->> (d/q '[:find [?a ...]
|
||||
:where
|
||||
[?property :db/ident ?a]
|
||||
[?property :db/valueType :db.type/ref]
|
||||
[?property :logseq.property/classes ?class]]
|
||||
db)
|
||||
(filter (partial bidirectional-property-attr? db))))
|
||||
|
||||
(defn- get-ea-by-v
|
||||
[db attrs v]
|
||||
(->> attrs
|
||||
(mapcat (fn [attr]
|
||||
(map (fn [datom]
|
||||
[(:e datom) attr])
|
||||
(d/datoms db :avet attr v))))))
|
||||
|
||||
(defn- add-entity
|
||||
[acc class-id entity]
|
||||
(if class-id
|
||||
(update acc class-id (fnil conj #{}) entity)
|
||||
acc))
|
||||
|
||||
(defn- build-bidirectional-property-group
|
||||
[db [class-id entities]]
|
||||
(let [class (d/entity db class-id)]
|
||||
(when (true? (:logseq.property.class/enable-bidirectional? class))
|
||||
(let [custom-title (when-let [custom (:logseq.property.class/bidirectional-property-title class)]
|
||||
(if (string? custom)
|
||||
custom
|
||||
(db-property/property-value-content custom)))
|
||||
title (if (string/blank? custom-title)
|
||||
(common-plural/plural (:block/title class))
|
||||
custom-title)]
|
||||
{:title title
|
||||
:class (-> (into {} class)
|
||||
(assoc :db/id (:db/id class)))
|
||||
:entities (->> entities
|
||||
(sort-by :block/created-at))}))))
|
||||
|
||||
(defn get-bidirectional-properties
|
||||
"Given a target entity id, returns a seq of maps with:
|
||||
* :class - class entity
|
||||
* :title - pluralized class title
|
||||
* :entities - node entities that reference the target via ref properties"
|
||||
[db target-id]
|
||||
(when (and db target-id
|
||||
(not (:logseq.property/created-from-property (d/entity db target-id))))
|
||||
(let [bidirectional-attrs (get-bidirectional-property-attrs db)]
|
||||
(->> (get-ea-by-v db bidirectional-attrs target-id)
|
||||
(keep (fn [[e _a]]
|
||||
(when-let [entity (d/entity db e)]
|
||||
(when (and (not= (:db/id entity) target-id)
|
||||
(not (entity-util/recycled? entity))
|
||||
(not (entity-util/class? entity))
|
||||
(not (entity-util/property? entity)))
|
||||
(let [classes (filter entity-util/class? (:block/tags entity))]
|
||||
(when (seq classes)
|
||||
(keep (fn [class-ent]
|
||||
(when (and (not (built-in? class-ent))
|
||||
(not (entity-util/recycled? class-ent)))
|
||||
[(:db/id class-ent) entity]))
|
||||
classes)))))))
|
||||
(mapcat identity)
|
||||
(reduce (fn [acc [class-ent entity]]
|
||||
(add-entity acc class-ent entity))
|
||||
{})
|
||||
(keep (partial build-bidirectional-property-group db))
|
||||
(sort-by (comp :block/created-at :class))))))
|
||||
229
deps/db/src/logseq/db/common/delete_blocks.cljs
vendored
Normal file
229
deps/db/src/logseq/db/common/delete_blocks.cljs
vendored
Normal file
@@ -0,0 +1,229 @@
|
||||
(ns logseq.db.common.delete-blocks
|
||||
"Provides fn to handle any deletion to occur per ldb/transact!"
|
||||
(:require [clojure.string :as string]
|
||||
[datascript.core :as d]
|
||||
[logseq.common.util :as common-util]
|
||||
[logseq.common.util.block-ref :as block-ref]
|
||||
[logseq.common.util.page-ref :as page-ref]
|
||||
[logseq.db.common.entity-plus :as entity-plus]
|
||||
[logseq.db.frontend.entity-util :as entity-util]))
|
||||
|
||||
(defn- replace-ref-with-deleted-block-title
|
||||
[block ref-raw-title]
|
||||
(let [block-content (if (entity-util/asset? block)
|
||||
""
|
||||
(:block/title block))]
|
||||
(some-> ref-raw-title
|
||||
(string/replace (re-pattern (common-util/format "(?i){{embed \\(\\(%s\\)\\)\\s?}}" (str (:block/uuid block))))
|
||||
block-content)
|
||||
|
||||
(string/replace (block-ref/->block-ref (str (:block/uuid block)))
|
||||
block-content)
|
||||
(string/replace (page-ref/->page-ref (str (:block/uuid block)))
|
||||
block-content))))
|
||||
|
||||
(defn- build-retracted-tx
|
||||
([retracted-blocks]
|
||||
(build-retracted-tx retracted-blocks #{}))
|
||||
([retracted-blocks extra-retract-ids]
|
||||
(let [refs (->> (mapcat (fn [block] (:block/_refs block)) retracted-blocks)
|
||||
(common-util/distinct-by :db/id))
|
||||
retract-ids (into (set (map :db/id retracted-blocks)) extra-retract-ids)]
|
||||
(mapcat
|
||||
(fn [ref]
|
||||
(let [id (:db/id ref)
|
||||
replaced-title (when-not (contains? retract-ids id)
|
||||
(when-let [raw-title (:block/raw-title ref)]
|
||||
(reduce
|
||||
(fn [raw-title block]
|
||||
(replace-ref-with-deleted-block-title block raw-title))
|
||||
raw-title
|
||||
retracted-blocks)))
|
||||
tx (cond->
|
||||
(mapcat
|
||||
(fn [block]
|
||||
[[:db/retract (:db/id ref) :block/refs (:db/id block)]]) retracted-blocks)
|
||||
replaced-title
|
||||
(conj [:db/add id :block/title replaced-title]))]
|
||||
tx))
|
||||
refs))))
|
||||
|
||||
(defn- block-entity?
|
||||
[entity]
|
||||
(and (:block/uuid entity)
|
||||
(:block/page entity)
|
||||
(not (entity-util/page? entity))))
|
||||
|
||||
(defn- retracted-entities
|
||||
[db txs]
|
||||
(->> txs
|
||||
(keep (fn [tx]
|
||||
(when (and (vector? tx)
|
||||
(contains? #{:db.fn/retractEntity :db/retractEntity} (first tx)))
|
||||
(d/entity db (second tx)))))
|
||||
(common-util/distinct-by :db/id)))
|
||||
|
||||
(def ^:private property-history-ref-attrs
|
||||
#{:logseq.property.history/block
|
||||
:logseq.property.history/property
|
||||
:logseq.property.history/ref-value})
|
||||
|
||||
(defn- property-history-entity?
|
||||
[entity]
|
||||
(boolean
|
||||
(and entity
|
||||
(or (:logseq.property.history/block entity)
|
||||
(:logseq.property.history/property entity)
|
||||
(:logseq.property.history/ref-value entity)
|
||||
(contains? entity :logseq.property.history/scalar-value)))))
|
||||
|
||||
(defn- property-history-ref-retracted-entities
|
||||
[db txs]
|
||||
(->> txs
|
||||
(keep (fn [tx]
|
||||
(when (and (vector? tx)
|
||||
(= :db/retract (first tx))
|
||||
(contains? property-history-ref-attrs (nth tx 2 nil)))
|
||||
(let [entity (d/entity db (second tx))]
|
||||
(when (property-history-entity? entity)
|
||||
entity)))))
|
||||
(common-util/distinct-by :db/id)))
|
||||
|
||||
(defn- tx-entity-id
|
||||
[db eid]
|
||||
(cond
|
||||
(integer? eid)
|
||||
eid
|
||||
|
||||
(vector? eid)
|
||||
(some-> (d/entity db eid) :db/id)))
|
||||
|
||||
(defn- vector-adds-by-eid
|
||||
[txs]
|
||||
(reduce (fn [result tx]
|
||||
(if (and (vector? tx)
|
||||
(= :db/add (first tx))
|
||||
(integer? (second tx)))
|
||||
(update result (second tx) assoc (nth tx 2) (nth tx 3))
|
||||
result))
|
||||
{}
|
||||
txs))
|
||||
|
||||
(defn- property-history-ref-retracted-ids
|
||||
[txs]
|
||||
(->> txs
|
||||
(keep (fn [tx]
|
||||
(when (and (vector? tx)
|
||||
(= :db/retract (first tx))
|
||||
(integer? (second tx))
|
||||
(contains? property-history-ref-attrs (nth tx 2 nil)))
|
||||
(second tx))))
|
||||
set))
|
||||
|
||||
(defn- new-property-history-retract-tx
|
||||
[db txs retracted-ids]
|
||||
(let [referencing-retracted-entity?
|
||||
(fn [history]
|
||||
(or (contains? retracted-ids
|
||||
(tx-entity-id db (:logseq.property.history/block history)))
|
||||
(contains? retracted-ids
|
||||
(tx-entity-id db (:logseq.property.history/property history)))
|
||||
(contains? retracted-ids
|
||||
(tx-entity-id db (:logseq.property.history/ref-value history)))))
|
||||
map-retract-tx
|
||||
(keep (fn [tx]
|
||||
(when (and (map? tx)
|
||||
(:block/uuid tx)
|
||||
(property-history-entity? tx)
|
||||
(referencing-retracted-entity? tx))
|
||||
[:db/retractEntity [:block/uuid (:block/uuid tx)]]))
|
||||
txs)
|
||||
retracted-history-ref-ids (property-history-ref-retracted-ids txs)
|
||||
vector-retract-tx
|
||||
(->> (vector-adds-by-eid txs)
|
||||
(keep (fn [[eid tx-entity]]
|
||||
(when (and (:block/uuid tx-entity)
|
||||
(property-history-entity? tx-entity)
|
||||
(or (contains? retracted-history-ref-ids eid)
|
||||
(referencing-retracted-entity? tx-entity)))
|
||||
[:db/retractEntity eid]))))]
|
||||
(distinct (concat map-retract-tx vector-retract-tx))))
|
||||
|
||||
(defn- block-subtree-entities
|
||||
[root]
|
||||
(loop [pending [root]
|
||||
seen-ids #{}
|
||||
result []]
|
||||
(if-let [entity (first pending)]
|
||||
(if (or (nil? (:db/id entity))
|
||||
(contains? seen-ids (:db/id entity)))
|
||||
(recur (rest pending) seen-ids result)
|
||||
(recur (concat (rest pending)
|
||||
(filter block-entity?
|
||||
(entity-plus/lookup-kv-then-entity entity :block/_raw-parent)))
|
||||
(conj seen-ids (:db/id entity))
|
||||
(conj result entity)))
|
||||
result)))
|
||||
|
||||
(defn expand-delete-blocks-tx
|
||||
"Expands delete-blocks retracts to the current DB subtree before transacting."
|
||||
[db txs {:keys [outliner-op]}]
|
||||
(if (= :delete-blocks outliner-op)
|
||||
(let [subtree-tx (->> (retracted-entities db txs)
|
||||
(filter block-entity?)
|
||||
(mapcat block-subtree-entities)
|
||||
(map (fn [entity] [:db/retractEntity (:db/id entity)])))]
|
||||
(distinct (concat txs subtree-tx)))
|
||||
txs))
|
||||
|
||||
(defn- direct-cleanup-tx
|
||||
[entities]
|
||||
(let [retracted-blocks (filter block-entity? entities)
|
||||
retracted-history-self-entities (filter property-history-entity? entities)
|
||||
retracted-history-self-tx (map (fn [history] [:db/retractEntity (:db/id history)])
|
||||
retracted-history-self-entities)
|
||||
reaction-entities (->> entities
|
||||
(mapcat :logseq.property.reaction/_target)
|
||||
(common-util/distinct-by :db/id))
|
||||
retract-reactions-tx (map (fn [reaction] [:db/retractEntity (:db/id reaction)])
|
||||
reaction-entities)
|
||||
view-entities (->> entities
|
||||
(mapcat :logseq.property/_view-for)
|
||||
(common-util/distinct-by :db/id))
|
||||
history-entities (->> entities
|
||||
(mapcat (fn [entity]
|
||||
(concat (:logseq.property.history/_block entity)
|
||||
(:logseq.property.history/_property entity)
|
||||
(:logseq.property.history/_ref-value entity))))
|
||||
(common-util/distinct-by :db/id))
|
||||
retract-history-tx (map (fn [history] [:db/retractEntity (:db/id history)])
|
||||
history-entities)
|
||||
cleanup-retract-ids (->> (concat retracted-history-self-entities
|
||||
reaction-entities
|
||||
view-entities
|
||||
history-entities)
|
||||
(keep :db/id)
|
||||
set)
|
||||
retracted-tx (build-retracted-tx retracted-blocks cleanup-retract-ids)
|
||||
delete-views (map (fn [view] [:db/retractEntity (:db/id view)]) view-entities)]
|
||||
(vec (concat retracted-tx delete-views retracted-history-self-tx retract-history-tx retract-reactions-tx))))
|
||||
|
||||
(defn- build-cleanup-tx
|
||||
[db txs]
|
||||
(let [initial-entities (concat (retracted-entities db txs)
|
||||
(property-history-ref-retracted-entities db txs))]
|
||||
(loop [pending-entities initial-entities
|
||||
seen-ids #{}
|
||||
cleanup-tx (new-property-history-retract-tx db txs (set (map :db/id initial-entities)))]
|
||||
(if-let [entities (seq (remove #(contains? seen-ids (:db/id %))
|
||||
pending-entities))]
|
||||
(let [seen-ids' (into seen-ids (map :db/id) entities)
|
||||
next-tx (direct-cleanup-tx entities)]
|
||||
(recur (retracted-entities db next-tx) seen-ids' (into cleanup-tx next-tx)))
|
||||
(distinct cleanup-tx)))))
|
||||
|
||||
(defn update-refs-history
|
||||
"When an entity is deleted, related property history, views and reactions
|
||||
are deleted"
|
||||
[db txs _opts]
|
||||
(seq (build-cleanup-tx db txs)))
|
||||
265
deps/db/src/logseq/db/common/entity_plus.cljc
vendored
Normal file
265
deps/db/src/logseq/db/common/entity_plus.cljc
vendored
Normal file
@@ -0,0 +1,265 @@
|
||||
(ns logseq.db.common.entity-plus
|
||||
"Add map ops such as assoc/dissoc to datascript Entity.
|
||||
The db-based-graph? checks are still in this ns because it's unclear which of these
|
||||
lower level fns are still used by the graph-parser.
|
||||
|
||||
NOTE: This doesn't work for nbb/sci yet because of https://github.com/babashka/sci/issues/639"
|
||||
;; Disable clj linters since we don't support clj
|
||||
#?(:clj {:clj-kondo/config {:linters {:unresolved-namespace {:level :off}
|
||||
:unresolved-symbol {:level :off}}}})
|
||||
(:require #?(:org.babashka/nbb [datascript.db])
|
||||
[cljs.core]
|
||||
[clojure.data :as data]
|
||||
[datascript.core :as d]
|
||||
[datascript.impl.entity :as entity :refer [Entity]]
|
||||
[logseq.common.util.date-time :as date-time-util]
|
||||
[logseq.db.frontend.content :as db-content]
|
||||
[logseq.db.frontend.entity-util :as entity-util]
|
||||
[logseq.db.frontend.property :as db-property]))
|
||||
|
||||
(def nil-db-ident-entities
|
||||
"No such entities with these :db/ident, but `(d/entity <db> <ident>)` has been called somewhere."
|
||||
#{:block/tx-id :block/uuid :block/journal-day :block/_refs :block/level :block/heading-level
|
||||
:block/warning :block/name
|
||||
:block.temp/ast-title :block.temp/load-status :block.temp/has-children? :block.temp/ast-body
|
||||
|
||||
:db/valueType :db/cardinality :db/ident :db/index
|
||||
|
||||
:logseq.property/_query})
|
||||
|
||||
(def immutable-db-ident-entities
|
||||
"These db-ident entities are immutable,
|
||||
it means `(db/entity :block/title)` always return same result"
|
||||
#{:block/link :block/updated-at :block/refs :block/closed-value-property
|
||||
:block/created-at :block/collapsed? :block/tags :block/title
|
||||
:block/parent :block/order :block/page
|
||||
|
||||
:logseq.property/created-from-property
|
||||
:logseq.property/icon
|
||||
:logseq.property.asset/type
|
||||
:logseq.property.asset/checksum
|
||||
:logseq.property.node/display-type
|
||||
|
||||
:logseq.kv/db-type})
|
||||
|
||||
(assert (empty? (last (data/diff immutable-db-ident-entities nil-db-ident-entities))))
|
||||
|
||||
(def lookup-entity @#'entity/lookup-entity)
|
||||
|
||||
(def ^:private *seen-immutable-entities (volatile! {}))
|
||||
|
||||
(defn reset-immutable-entities-cache!
|
||||
[]
|
||||
(vreset! *seen-immutable-entities {}))
|
||||
|
||||
(defonce *reset-cache-background-task-running-f (atom nil))
|
||||
|
||||
(defn entity-memoized
|
||||
[db eid]
|
||||
(if (and (qualified-keyword? eid) (not (exists? js/process))) ; don't memoize on node
|
||||
(when-not (contains? nil-db-ident-entities eid) ;fast return nil
|
||||
(let [f @*reset-cache-background-task-running-f]
|
||||
(if (and (fn? f)
|
||||
(f :logseq.db.common.entity-plus/reset-immutable-entities-cache!)
|
||||
(contains? immutable-db-ident-entities eid)) ;return cache entity if possible which isn't nil
|
||||
(or (get @*seen-immutable-entities eid)
|
||||
(let [r (d/entity db eid)]
|
||||
(when r (vswap! *seen-immutable-entities assoc eid r))
|
||||
r))
|
||||
(d/entity db eid))))
|
||||
(d/entity db eid)))
|
||||
|
||||
(defn unsafe->Entity
|
||||
"Faster version of d/entity without checking e exists.
|
||||
Only use it in performance-critical areas and where the existence of 'e' is confirmed."
|
||||
[db e]
|
||||
{:pre [(pos-int? e)]}
|
||||
(Entity. db e (volatile! false) (volatile! {})))
|
||||
|
||||
(defn db-based-graph?
|
||||
"Whether the current graph is db-only. This should only be used in contexts
|
||||
where both DB and file graph databases are possible e.g. graph-parser"
|
||||
[db]
|
||||
(when db
|
||||
(identical? "db" (:kv/value (entity-memoized db :logseq.kv/db-type)))))
|
||||
|
||||
(defn- get-journal-title
|
||||
[db e]
|
||||
(date-time-util/int->journal-title (:block/journal-day e)
|
||||
(:logseq.property.journal/title-format (entity-memoized db :logseq.class/Journal))))
|
||||
|
||||
(defn- get-block-title
|
||||
[^Entity e k default-value]
|
||||
(let [db (.-db e)
|
||||
db-based? (db-based-graph? db)]
|
||||
(if (and db-based? (entity-util/journal? e))
|
||||
(get-journal-title db e)
|
||||
(or
|
||||
(get (.-kv e) k)
|
||||
(if db-based?
|
||||
(let [result (lookup-entity e k default-value)
|
||||
;; Replace title for pages only, otherwise it'll recursively
|
||||
;; replace block id refs if there're cycle references of blocks
|
||||
refs (:block/refs e)
|
||||
result' (if (and (string? result) refs)
|
||||
(db-content/id-ref->title-ref result refs
|
||||
{:db db
|
||||
:replace-pages-with-same-name? false})
|
||||
result)]
|
||||
(or result' default-value))
|
||||
(lookup-entity e k default-value))))))
|
||||
|
||||
(defn- lookup-kv-with-default-value
|
||||
[db ^Entity e k default-value]
|
||||
(or
|
||||
;; from kv
|
||||
(get (.-kv e) k)
|
||||
;; from db
|
||||
(let [result (lookup-entity e k default-value)]
|
||||
(if (some? result)
|
||||
result
|
||||
;; property default value
|
||||
(when (qualified-keyword? k)
|
||||
(when-let [property (entity-memoized db k)]
|
||||
(let [property-type (lookup-entity property :logseq.property/type nil)]
|
||||
(if (keyword-identical? :checkbox property-type)
|
||||
(lookup-entity property :logseq.property/scalar-default-value nil)
|
||||
(lookup-entity property :logseq.property/default-value nil)))))))))
|
||||
|
||||
(defn- get-property-keys
|
||||
[^Entity e]
|
||||
(let [db (.-db e)]
|
||||
(if (db-based-graph? db)
|
||||
(->> (map :a (d/datoms db :eavt (.-eid e)))
|
||||
distinct
|
||||
(filter db-property/property?))
|
||||
(keys (lookup-entity e :block/properties nil)))))
|
||||
|
||||
(defn- get-properties
|
||||
[^Entity e]
|
||||
(let [db (.-db e)]
|
||||
(if (db-based-graph? db)
|
||||
(lookup-entity e :block/properties
|
||||
(->> (into {} e)
|
||||
(filter (fn [[k _]] (db-property/property? k)))
|
||||
(into {})))
|
||||
(lookup-entity e :block/properties nil))))
|
||||
|
||||
;; (defonce *id->k-frequency (atom {}))
|
||||
(defn lookup-kv-then-entity
|
||||
([e k] (lookup-kv-then-entity e k nil))
|
||||
([^Entity e k default-value]
|
||||
(try
|
||||
(when k
|
||||
;; (swap! *id->k-frequency update-in [(.-eid e) k] inc)
|
||||
(let [db (.-db e)]
|
||||
(case k
|
||||
:block/raw-title
|
||||
(if (and (db-based-graph? db) (entity-util/journal? e))
|
||||
(get-journal-title db e)
|
||||
(lookup-entity e :block/title default-value))
|
||||
|
||||
:block/properties
|
||||
(get-properties e)
|
||||
|
||||
:block.temp/property-keys
|
||||
(get-property-keys e)
|
||||
|
||||
:block/title
|
||||
(get-block-title e k default-value)
|
||||
|
||||
:block/_parent
|
||||
(->> (lookup-entity e k default-value)
|
||||
(remove (fn [e] (or (:logseq.property/created-from-property e)
|
||||
(:block/closed-value-property e))))
|
||||
seq)
|
||||
|
||||
:block/_raw-parent
|
||||
(lookup-entity e :block/_parent default-value)
|
||||
|
||||
:property/closed-values
|
||||
(some->> (lookup-entity e :block/_closed-value-property default-value)
|
||||
(remove entity-util/recycled?)
|
||||
(sort-by :block/order))
|
||||
|
||||
(lookup-kv-with-default-value db e k default-value))))
|
||||
(catch :default e
|
||||
(js/console.error e)))))
|
||||
|
||||
(defn- cache-with-kv
|
||||
[^js this]
|
||||
(let [v @(.-cache this)]
|
||||
(concat (seq v)
|
||||
(seq (.-kv this)))))
|
||||
|
||||
(defn- entity-ish? [x]
|
||||
(instance? Entity x))
|
||||
|
||||
(defn- ->printable
|
||||
"Convert values so printing won't recurse forever:
|
||||
- Entity => {:db/id eid}
|
||||
- coll of Entity => coll of {:db/id ...}
|
||||
- maps are walked (rare but safe)"
|
||||
[x]
|
||||
(cond
|
||||
(entity-ish? x)
|
||||
{:db/id (.-eid ^Entity x)}
|
||||
|
||||
(map? x)
|
||||
(reduce-kv (fn [m k v] (assoc m k (->printable v))) {} x)
|
||||
|
||||
(sequential? x)
|
||||
(map ->printable x)
|
||||
|
||||
(set? x)
|
||||
(into #{} (map ->printable x))
|
||||
|
||||
:else x))
|
||||
|
||||
#?(:org.babashka/nbb
|
||||
nil
|
||||
:default
|
||||
(extend-type Entity
|
||||
cljs.core/IEncodeJS
|
||||
(-clj->js [_this] nil) ; avoid `clj->js` overhead when entity is passed to React components
|
||||
(-key->js [_this] nil)
|
||||
|
||||
IAssociative
|
||||
(-assoc [this k v]
|
||||
(assert (keyword? k) "attribute must be keyword")
|
||||
(set! (.-kv this) (assoc (.-kv this) k v))
|
||||
this)
|
||||
(-contains-key? [e k] (not= ::nf (lookup-kv-then-entity e k ::nf)))
|
||||
|
||||
IMap
|
||||
(-dissoc [this k]
|
||||
(assert (keyword? k) (str "attribute must be keyword: " k))
|
||||
(set! (.-kv this) (dissoc (.-kv this) k))
|
||||
this)
|
||||
|
||||
ISeqable
|
||||
(-seq [this]
|
||||
(entity/touch this)
|
||||
(cache-with-kv this))
|
||||
|
||||
IPrintWithWriter
|
||||
(-pr-writer [this writer opts]
|
||||
;; Touch ONLY this entity, to materialize its forward attrs
|
||||
(entity/touch this)
|
||||
(let [m0 (into {} (cache-with-kv this))
|
||||
m (-> (reduce-kv (fn [m k v] (assoc m k (->printable v))) {} m0)
|
||||
(assoc :db/id (.-eid this)))]
|
||||
(-pr-writer m writer opts)))
|
||||
|
||||
ICollection
|
||||
(-conj [this entry]
|
||||
(if (vector? entry)
|
||||
(let [[k v] entry]
|
||||
(-assoc this k v))
|
||||
(reduce (fn [this [k v]]
|
||||
(-assoc this k v)) this entry)))
|
||||
|
||||
ILookup
|
||||
(-lookup
|
||||
([this attr] (lookup-kv-then-entity this attr))
|
||||
([this attr not-found] (lookup-kv-then-entity this attr not-found)))))
|
||||
454
deps/db/src/logseq/db/common/initial_data.cljs
vendored
Normal file
454
deps/db/src/logseq/db/common/initial_data.cljs
vendored
Normal file
@@ -0,0 +1,454 @@
|
||||
(ns logseq.db.common.initial-data
|
||||
"Provides db helper fns for graph initialization and lazy loading entities"
|
||||
(:require [clojure.string :as string]
|
||||
[datascript.core :as d]
|
||||
[datascript.impl.entity :as de]
|
||||
[logseq.common.config :as common-config]
|
||||
[logseq.common.util :as common-util]
|
||||
[logseq.common.util.date-time :as date-time-util]
|
||||
[logseq.db.common.order :as db-order]
|
||||
[logseq.db.frontend.class :as db-class]
|
||||
[logseq.db.frontend.db :as db-db]
|
||||
[logseq.db.frontend.entity-util :as entity-util]
|
||||
[logseq.db.frontend.rules :as rules]))
|
||||
|
||||
;; FIXME: For DB graph built-in pages, look up by name -> uuid like
|
||||
;; get-built-in-page instead of this approach which is more error prone
|
||||
(defn get-first-page-by-name
|
||||
"Return the oldest page's db id for :block/name"
|
||||
[db page-name]
|
||||
(when (and db (string? page-name))
|
||||
(first (sort (map :e (entity-util/get-pages-by-name db page-name))))))
|
||||
|
||||
(defn get-first-page-by-title
|
||||
"Return the oldest page's db id for :block/title"
|
||||
[db page-name]
|
||||
{:pre [(string? page-name)]}
|
||||
(->> (d/datoms db :avet :block/title page-name)
|
||||
(filter (fn [d]
|
||||
(let [e (d/entity db (:e d))]
|
||||
(entity-util/page? e))))
|
||||
(map :e)
|
||||
sort
|
||||
first))
|
||||
|
||||
(defn get-block-alias
|
||||
[db eid]
|
||||
(->>
|
||||
(d/q
|
||||
'[:find [?e ...]
|
||||
:in $ ?eid %
|
||||
:where
|
||||
(alias ?eid ?e)]
|
||||
db
|
||||
eid
|
||||
(:alias rules/rules))
|
||||
distinct))
|
||||
|
||||
(defn- datom-v
|
||||
[db eid attr]
|
||||
(some-> (first (d/datoms db :eavt eid attr)) :v))
|
||||
|
||||
(defn- datom-vs
|
||||
[db eid attr]
|
||||
(map :v (d/datoms db :eavt eid attr)))
|
||||
|
||||
(defn- get-block-alias-ids
|
||||
[db eid]
|
||||
(distinct
|
||||
(concat
|
||||
(datom-vs db eid :block/alias)
|
||||
(map :e (d/datoms db :avet :block/alias eid)))))
|
||||
|
||||
(defn- hidden-eid-pred
|
||||
[db]
|
||||
(let [*cache (volatile! {})]
|
||||
(letfn [(hidden? [eid seen]
|
||||
(cond
|
||||
(nil? eid)
|
||||
false
|
||||
|
||||
(contains? @*cache eid)
|
||||
(get @*cache eid)
|
||||
|
||||
(contains? seen eid)
|
||||
false
|
||||
|
||||
:else
|
||||
(let [result (boolean
|
||||
(or (datom-v db eid :logseq.property/hide?)
|
||||
(datom-v db eid :logseq.property/deleted-at)
|
||||
(hidden? (datom-v db eid :block/parent) (conj seen eid))))]
|
||||
(vswap! *cache assoc eid result)
|
||||
result)))]
|
||||
(fn [eid]
|
||||
(hidden? eid #{})))))
|
||||
|
||||
(defn- hidden-ref-id-pred
|
||||
([db id]
|
||||
(hidden-ref-id-pred db id (hidden-eid-pred db)))
|
||||
([db id hidden-eid?]
|
||||
(let [entity (d/entity db id)
|
||||
entity-ident (:db/ident entity)
|
||||
class-ids (when (entity-util/class? entity)
|
||||
(let [children (db-class/get-structured-children db id)]
|
||||
(set (conj children id))))]
|
||||
(fn [ref-eid]
|
||||
(or
|
||||
(= ref-eid id)
|
||||
(= id (datom-v db ref-eid :block/page))
|
||||
(= id (datom-v db ref-eid :logseq.property/view-for))
|
||||
(hidden-eid? (datom-v db ref-eid :block/page))
|
||||
(hidden-eid? ref-eid)
|
||||
(and class-ids
|
||||
(some class-ids (datom-vs db ref-eid :block/tags)))
|
||||
(and entity-ident
|
||||
(seq (d/datoms db :eavt ref-eid entity-ident))))))))
|
||||
|
||||
(comment
|
||||
(defn- get-built-in-files
|
||||
[db]
|
||||
(let [files ["logseq/config.edn"
|
||||
"logseq/custom.css"
|
||||
"logseq/custom.js"]]
|
||||
(map #(d/pull db '[*] [:file/path %]) files))))
|
||||
|
||||
(defn- get-all-files
|
||||
[db]
|
||||
(->> (d/datoms db :avet :file/path)
|
||||
(mapcat (fn [e] (d/datoms db :eavt (:e e))))))
|
||||
|
||||
(defn- with-block-refs
|
||||
[db block]
|
||||
(update block :block/refs (fn [refs] (map (fn [ref] (d/pull db '[*] (:db/id ref))) refs))))
|
||||
|
||||
(defn with-parent
|
||||
[db block]
|
||||
(cond
|
||||
(:block/page block)
|
||||
(let [parent (when-let [e (d/entity db (:db/id (:block/parent block)))]
|
||||
(select-keys e [:db/id :block/uuid]))]
|
||||
(->>
|
||||
(assoc block :block/parent parent)
|
||||
(common-util/remove-nils-non-nested)
|
||||
(with-block-refs db)))
|
||||
|
||||
:else
|
||||
block))
|
||||
|
||||
(comment
|
||||
(defn- with-block-link
|
||||
[db block]
|
||||
(if (:block/link block)
|
||||
(update block :block/link (fn [link] (d/pull db '[*] (:db/id link))))
|
||||
block)))
|
||||
|
||||
(defn get-block-children-ids
|
||||
"Returns children ids, notice the result doesn't include property value children ids."
|
||||
[db block-eid & {:keys [include-collapsed-children?]
|
||||
:or {include-collapsed-children? true}}]
|
||||
(when-let [eid (:db/id (d/entity db block-eid))]
|
||||
(let [seen (volatile! #{})]
|
||||
(loop [eids-to-expand [eid]]
|
||||
(when (seq eids-to-expand)
|
||||
(let [children
|
||||
(mapcat (fn [eid]
|
||||
(let [e (d/entity db eid)]
|
||||
(when (or include-collapsed-children?
|
||||
(not (:block/collapsed? e))
|
||||
(entity-util/page? e))
|
||||
(:block/_parent e)))) eids-to-expand)
|
||||
ids-to-add (keep :db/id children)]
|
||||
(vswap! seen (partial apply conj) ids-to-add)
|
||||
(recur (keep :db/id children)))))
|
||||
@seen)))
|
||||
|
||||
(defn get-block-children
|
||||
"Including nested children, notice the result doesn't include property values."
|
||||
{:arglists '([db eid & {:keys [include-collapsed-children?]}])}
|
||||
[db eid & {:as opts}]
|
||||
(let [ids (get-block-children-ids db eid opts)]
|
||||
(when (seq ids)
|
||||
(map (fn [id] (d/entity db id)) ids))))
|
||||
|
||||
(defn get-block-full-children-ids
|
||||
"Including nested, collapsed and property value children."
|
||||
{:arglists '([db block-eid])}
|
||||
[db block-eid]
|
||||
(d/q
|
||||
'[:find [?c ...]
|
||||
:in $ ?id %
|
||||
:where
|
||||
(parent ?id ?c)]
|
||||
db
|
||||
block-eid
|
||||
(:parent rules/rules)))
|
||||
|
||||
(defn- with-raw-title
|
||||
[m entity]
|
||||
(if-let [raw-title (:block/raw-title entity)]
|
||||
(assoc m :block/title raw-title)
|
||||
m))
|
||||
|
||||
(defn- entity->map
|
||||
[entity & {:keys [level properties]
|
||||
:or {level 0}}]
|
||||
(let [opts {:level (inc level)}
|
||||
f (if (> level 0)
|
||||
identity
|
||||
(fn [e]
|
||||
(keep (fn [[k v]]
|
||||
(when (or (empty? properties) (properties k))
|
||||
(let [v' (cond
|
||||
(= k :block/parent)
|
||||
(:db/id v)
|
||||
(= k :block/tags)
|
||||
(map #(select-keys % [:db/id]) v)
|
||||
(= k :logseq.property/created-by-ref)
|
||||
(:db/id v)
|
||||
(de/entity? v)
|
||||
(entity->map v opts)
|
||||
(and (coll? v) (every? de/entity? v))
|
||||
(map #(entity->map % opts) v)
|
||||
:else
|
||||
v)]
|
||||
[k v'])))
|
||||
e)))
|
||||
m (->> (f entity)
|
||||
(into {}))]
|
||||
(-> m
|
||||
(with-raw-title entity)
|
||||
(assoc :db/id (:db/id entity)))))
|
||||
|
||||
(defn hidden-ref-pred
|
||||
"Build a predicate that determines if a ref block should be hidden for `id`.
|
||||
Reuses cached class/tag context so callers can check many refs efficiently."
|
||||
[db id]
|
||||
(let [entity (d/entity db id)
|
||||
entity-ident (:db/ident entity)
|
||||
class-ids (when (entity-util/class? entity)
|
||||
(let [children (db-class/get-structured-children db id)]
|
||||
(set (conj children id))))]
|
||||
(fn [ref-block]
|
||||
(or
|
||||
(= (:db/id ref-block) id)
|
||||
(= id (:db/id (:block/page ref-block)))
|
||||
(= id (:db/id (:logseq.property/view-for ref-block)))
|
||||
(entity-util/hidden? (:block/page ref-block))
|
||||
(entity-util/hidden? ref-block)
|
||||
(and class-ids
|
||||
(some class-ids (map :db/id (:block/tags ref-block))))
|
||||
(and entity-ident
|
||||
(some? (get ref-block entity-ident)))))))
|
||||
|
||||
(defn get-block-refs
|
||||
[db id]
|
||||
(let [with-alias (->> (get-block-alias db id)
|
||||
(cons id)
|
||||
distinct)
|
||||
hidden-ref?* (hidden-ref-pred db id)]
|
||||
(some->> with-alias
|
||||
(map #(d/entity db %))
|
||||
(mapcat :block/_refs)
|
||||
(remove hidden-ref?*))))
|
||||
|
||||
(defn get-block-refs-count
|
||||
[db id]
|
||||
(let [with-alias (->> (get-block-alias-ids db id)
|
||||
(cons id)
|
||||
distinct)
|
||||
hidden-ref?* (hidden-ref-id-pred db id)]
|
||||
(reduce
|
||||
(fn [total alias-id]
|
||||
(+ total
|
||||
(reduce (fn [n datom]
|
||||
(if (hidden-ref?* (:e datom))
|
||||
n
|
||||
(inc n)))
|
||||
0
|
||||
(d/datoms db :avet :block/refs alias-id))))
|
||||
0
|
||||
with-alias)))
|
||||
|
||||
(defn ^:large-vars/cleanup-todo get-block-and-children
|
||||
[db id-or-page-name {:keys [children? properties include-collapsed-children?]
|
||||
:or {include-collapsed-children? false}}]
|
||||
(let [block (let [eid (cond (uuid? id-or-page-name)
|
||||
[:block/uuid id-or-page-name]
|
||||
(integer? id-or-page-name)
|
||||
id-or-page-name
|
||||
:else nil)]
|
||||
(cond
|
||||
eid
|
||||
(d/entity db eid)
|
||||
(string? id-or-page-name)
|
||||
(d/entity db (get-first-page-by-name db (name id-or-page-name)))
|
||||
:else
|
||||
nil))
|
||||
block-refs-count? (some #{:block.temp/refs-count} properties)]
|
||||
(when block
|
||||
;; (prn :debug :get-block (:db/id block) (:block/title block) :children? children?
|
||||
;; :include-collapsed-children? include-collapsed-children?)
|
||||
(let [children (when children?
|
||||
(let [children-blocks (get-block-children db (:db/id block) {:include-collapsed-children? include-collapsed-children?})
|
||||
large-page? (>= (count children-blocks) 100)
|
||||
children (let [children' (if large-page?
|
||||
(:block/_parent block)
|
||||
children-blocks)]
|
||||
(->> children'
|
||||
(remove (fn [e] (:block/closed-value-property e)))))
|
||||
children-ids (set (map :db/id children))]
|
||||
(map
|
||||
(fn [block]
|
||||
(let [collapsed? (:block/collapsed? block)]
|
||||
(-> block
|
||||
(entity->map)
|
||||
(assoc :block.temp/has-children? (some? (:block/_parent block))
|
||||
:block.temp/load-status (if (and (not collapsed?)
|
||||
(or (and large-page?
|
||||
(every? children-ids (map :db/id (:block/_parent block))))
|
||||
(not large-page?)))
|
||||
:full
|
||||
:self)))))
|
||||
children)))
|
||||
block' (cond-> (entity->map block {:properties (set properties)})
|
||||
block-refs-count?
|
||||
(assoc :block.temp/refs-count (get-block-refs-count db (:db/id block)))
|
||||
true
|
||||
(assoc :block.temp/load-status (cond
|
||||
(and children? include-collapsed-children? (empty? properties))
|
||||
:full
|
||||
(and children? (empty? properties))
|
||||
:children
|
||||
:else
|
||||
:self)
|
||||
:block.temp/has-children? (some? (first (d/datoms db :avet :block/parent (:db/id block))))))]
|
||||
(cond->
|
||||
{:block block'}
|
||||
children?
|
||||
(assoc :children children))))))
|
||||
|
||||
(defn get-latest-journals
|
||||
[db]
|
||||
(let [today (date-time-util/date->int (js/Date.))]
|
||||
(->> (d/datoms db :avet :block/journal-day)
|
||||
vec
|
||||
rseq
|
||||
(keep (fn [d]
|
||||
(when (<= (:v d) today)
|
||||
(let [e (d/entity db (:e d))]
|
||||
(when (and (entity-util/journal? e)
|
||||
(:db/id e)
|
||||
(not (entity-util/recycled? e)))
|
||||
e)))))
|
||||
(common-util/distinct-by :db/id))))
|
||||
|
||||
(defn- get-structured-datoms
|
||||
[db]
|
||||
(let [class-property-id (:db/id (d/entity db :logseq.class/Property))]
|
||||
(->> (concat
|
||||
(d/datoms db :avet :block/tags :logseq.class/Tag)
|
||||
(d/datoms db :avet :block/tags :logseq.class/Property)
|
||||
(d/datoms db :avet :block/closed-value-property))
|
||||
(mapcat (fn [d]
|
||||
(let [block-datoms (d/datoms db :eavt (:e d))
|
||||
property-description-datoms
|
||||
(when (= (:v d) class-property-id)
|
||||
(when-let [desc (:logseq.property/description (d/entity db (:e d)))]
|
||||
(d/datoms db :eavt (:db/id desc))))]
|
||||
(if (seq property-description-datoms)
|
||||
(concat block-datoms property-description-datoms)
|
||||
block-datoms)))))))
|
||||
|
||||
(defn- get-favorites
|
||||
"Favorites page and its blocks"
|
||||
[db]
|
||||
(let [page-id (get-first-page-by-name db common-config/favorites-page-name)
|
||||
block (d/entity db page-id)
|
||||
children (:block/_page block)]
|
||||
(when block
|
||||
(concat (d/datoms db :eavt (:db/id block))
|
||||
(->> (keep :block/link children)
|
||||
(mapcat (fn [l]
|
||||
(d/datoms db :eavt (:db/id l)))))
|
||||
(mapcat (fn [child]
|
||||
(d/datoms db :eavt (:db/id child)))
|
||||
children)))))
|
||||
|
||||
(defn get-recent-updated-pages
|
||||
[db]
|
||||
(when db
|
||||
(some->>
|
||||
(d/datoms db :avet :block/updated-at)
|
||||
rseq
|
||||
(keep (fn [datom]
|
||||
(let [page (first (d/datoms db :eavt (:e datom) :block/page))]
|
||||
(when-not (or page
|
||||
(let [title (:v (first (d/datoms db :eavt (:e datom) :block/title)))]
|
||||
(string/blank? title)))
|
||||
(let [e (d/entity db (:e datom))]
|
||||
(when (and
|
||||
(entity-util/page? e)
|
||||
(not (entity-util/hidden? e)))
|
||||
e))))))
|
||||
(take 15))))
|
||||
|
||||
(defn- get-all-user-datoms
|
||||
[db]
|
||||
(when (d/entity db :logseq.property.user/email)
|
||||
(->> (d/datoms db :avet :logseq.property.user/email)
|
||||
(mapcat
|
||||
(fn [d] (d/datoms db :eavt (:e d)))))))
|
||||
|
||||
(defn- get-list-style-values
|
||||
[db]
|
||||
(->> (d/datoms db :avet :logseq.property/order-list-type)
|
||||
(map :v)
|
||||
(distinct)
|
||||
(mapcat (fn [v] (d/datoms db :eavt v)))))
|
||||
|
||||
(defn get-initial-data
|
||||
"Returns current database schema and initial data"
|
||||
[db]
|
||||
(let [_ (reset! db-order/*max-key (db-order/get-max-order db))
|
||||
schema (:schema db)
|
||||
idents (mapcat (fn [id]
|
||||
(when-let [e (d/entity db id)]
|
||||
(d/datoms db :eavt (:db/id e))))
|
||||
[:logseq.kv/db-type
|
||||
:logseq.kv/schema-version
|
||||
:logseq.kv/graph-uuid
|
||||
:logseq.kv/local-graph-uuid
|
||||
:logseq.kv/graph-rtc-e2ee?
|
||||
:logseq.kv/graph-remote?
|
||||
:logseq.kv/latest-code-lang
|
||||
:logseq.kv/graph-backup-folder
|
||||
:logseq.property/empty-placeholder])
|
||||
favorites (get-favorites db)
|
||||
recent-updated-pages (let [pages (get-recent-updated-pages db)]
|
||||
(mapcat (fn [p] (d/datoms db :eavt (:db/id p))) pages))
|
||||
all-files (get-all-files db)
|
||||
structured-datoms (get-structured-datoms db)
|
||||
user-datoms (get-all-user-datoms db)
|
||||
list-style-datoms (get-list-style-values db)
|
||||
pages-datoms (let [contents-id (get-first-page-by-title db "Contents")
|
||||
capture-page-id (:db/id (db-db/get-built-in-page db common-config/quick-add-page-name))
|
||||
views-id (get-first-page-by-title db common-config/views-page-name)
|
||||
recycle-id (get-first-page-by-title db "Recycle")]
|
||||
(mapcat #(d/datoms db :eavt %)
|
||||
(remove nil? [contents-id capture-page-id views-id recycle-id])))
|
||||
data (->> (concat idents
|
||||
structured-datoms
|
||||
user-datoms
|
||||
list-style-datoms
|
||||
favorites
|
||||
recent-updated-pages
|
||||
all-files
|
||||
pages-datoms)
|
||||
distinct
|
||||
(remove (fn [d]
|
||||
(contains? #{:block/created-at :block/updated-at
|
||||
:block/tx-id :logseq.property/created-by-ref}
|
||||
(:a d)))))]
|
||||
{:schema schema
|
||||
:initial-data data}))
|
||||
196
deps/db/src/logseq/db/common/normalize.cljs
vendored
Normal file
196
deps/db/src/logseq/db/common/normalize.cljs
vendored
Normal file
@@ -0,0 +1,196 @@
|
||||
(ns logseq.db.common.normalize
|
||||
"Normalize && denormalize eid for sync"
|
||||
(:require [datascript.core :as d]))
|
||||
|
||||
(defn remove-retract-entity-ref
|
||||
[db tx-data]
|
||||
(let [retracted (-> (keep (fn [[op value]]
|
||||
(when (and (= op :db/retractEntity)
|
||||
(nil? (d/entity db value)))
|
||||
value)) tx-data)
|
||||
set)
|
||||
retract-uuids (set (map second retracted))
|
||||
retracted-ids (into retract-uuids
|
||||
;; handle temp ids
|
||||
(map str retract-uuids))]
|
||||
(if (seq retracted)
|
||||
(remove (fn [item]
|
||||
(and (= 5 (count item))
|
||||
(or
|
||||
(contains? retracted (second item))
|
||||
(contains? retracted-ids (second item))
|
||||
(contains? retracted (nth item 3))
|
||||
(contains? retracted-ids (nth item 3))))) tx-data)
|
||||
tx-data)))
|
||||
|
||||
(defn replace-attr-retract-with-retract-entity-v2
|
||||
[db normalized-tx-data]
|
||||
(->> normalized-tx-data
|
||||
(map (fn [[op eid a v t]]
|
||||
(cond
|
||||
(and (= op :db/retract) (= a :block/uuid))
|
||||
[:db/retractEntity eid]
|
||||
(and a (some? v))
|
||||
[op eid a v t]
|
||||
:else
|
||||
[op eid])))
|
||||
(remove-retract-entity-ref db)))
|
||||
|
||||
(defn replace-attr-retract-with-retract-entity
|
||||
[db-after tx-data]
|
||||
(let [retract-eids-by-entity
|
||||
(into {}
|
||||
(keep (fn [d]
|
||||
(when (and (= :block/uuid (:a d))
|
||||
(false? (:added d)))
|
||||
(let [entity (d/entity db-after [:block/uuid (:v d)])]
|
||||
(when (not= (:db/id entity) (:e d)) ; eid changed
|
||||
[(:e d) (if entity
|
||||
(:e d)
|
||||
[:block/uuid (:v d)])])))))
|
||||
tx-data)]
|
||||
(loop [result []
|
||||
seen #{}
|
||||
[d & more] tx-data]
|
||||
(if-not d
|
||||
result
|
||||
(if-let [eid (get retract-eids-by-entity (:e d))]
|
||||
(if (contains? seen (:e d))
|
||||
(recur result seen more)
|
||||
(recur (conj result [:db/retractEntity eid])
|
||||
(conj seen (:e d))
|
||||
more))
|
||||
(recur (conj result d) seen more))))))
|
||||
|
||||
(defn eid->lookup
|
||||
[db e]
|
||||
(when-let [entity (d/entity db e)]
|
||||
(if-let [id (:block/uuid entity)]
|
||||
[:block/uuid id]
|
||||
(:db/ident entity))))
|
||||
|
||||
(defn eid->tempid
|
||||
[db e]
|
||||
(when-let [entity (d/entity db e)]
|
||||
(when-let [id (if-let [id (:block/uuid entity)]
|
||||
id
|
||||
(:db/ident entity))]
|
||||
(str id))))
|
||||
|
||||
(defn- sort-datoms
|
||||
"Properties first"
|
||||
[datoms]
|
||||
(sort-by
|
||||
(fn [item]
|
||||
(case (nth item 1)
|
||||
:db/ident
|
||||
0
|
||||
:db/valueType
|
||||
1
|
||||
:db/cardinality
|
||||
2
|
||||
3))
|
||||
datoms))
|
||||
|
||||
(defn remove-conflict-datoms
|
||||
[datoms]
|
||||
(->> datoms
|
||||
(group-by (fn [d] (take 4 d))) ; group by '(e a v tx)
|
||||
(keep (fn [[_eavt same-eavt-datoms]]
|
||||
(first (rseq same-eavt-datoms))))
|
||||
;; sort by :tx, use nth to make this fn works on both vector and datom
|
||||
(sort-by #(nth % 3))))
|
||||
|
||||
(defn- retract-entity-op?
|
||||
[item]
|
||||
(and (= 2 (count item))
|
||||
(= :db/retractEntity (first item))))
|
||||
|
||||
(defn- retract-entity-match-keys
|
||||
[e]
|
||||
(if (and (vector? e) (= :block/uuid (first e)))
|
||||
(let [block-uuid (second e)]
|
||||
#{e block-uuid (str block-uuid)})
|
||||
#{e}))
|
||||
|
||||
(defn reorder-retract-entity
|
||||
[tx-data]
|
||||
(let [retract-ops (filter retract-entity-op? tx-data)
|
||||
{recreated-block-retract-ops true
|
||||
end-retract-ops false} (->> retract-ops
|
||||
(group-by (fn [[_ [_ id]]]
|
||||
(boolean
|
||||
(some (fn [x]
|
||||
(and
|
||||
(vector? x)
|
||||
(>= (count x) 4)
|
||||
(= (first x) :db/add)
|
||||
(= (nth x 2) :block/uuid)
|
||||
(= (nth x 3) id))) tx-data)))))
|
||||
retract-keys (->> retract-ops
|
||||
(map second)
|
||||
(mapcat retract-entity-match-keys)
|
||||
set)
|
||||
datom-for-retracted-eid?
|
||||
(fn [item]
|
||||
(and (vector? item)
|
||||
(>= (count item) 4)
|
||||
(contains? retract-keys (second item))))
|
||||
datoms-for-retracted-eids (filter datom-for-retracted-eid? tx-data)
|
||||
others (remove (fn [item]
|
||||
(or (retract-entity-op? item)
|
||||
(datom-for-retracted-eid? item)))
|
||||
tx-data)]
|
||||
(concat recreated-block-retract-ops datoms-for-retracted-eids others end-retract-ops)))
|
||||
|
||||
(defn- resolve-eid
|
||||
[db-before db-after retract? e]
|
||||
(if retract?
|
||||
(eid->lookup db-before e)
|
||||
(or (eid->lookup db-before e)
|
||||
(eid->tempid db-after e))))
|
||||
|
||||
(defn- ref-value-type?
|
||||
[db-after db-before attr]
|
||||
(or (= :db.type/ref (:db/valueType (d/entity db-after attr)))
|
||||
(= :db.type/ref (:db/valueType (d/entity db-before attr)))))
|
||||
|
||||
(defn normalize-datom
|
||||
[db-after db-before [e a v t added]]
|
||||
(let [retract? (not added)
|
||||
e' (resolve-eid db-before db-after retract? e)
|
||||
v' (if (and (integer? v)
|
||||
(pos? v)
|
||||
(ref-value-type? db-after db-before a))
|
||||
(resolve-eid db-before db-after retract? v)
|
||||
v)]
|
||||
(when (and (some? e') (some? v'))
|
||||
(if added
|
||||
[:db/add e' a v' t]
|
||||
[:db/retract e' a v' t]))))
|
||||
|
||||
(defn- normalize-retract-entity-item
|
||||
[db-before d]
|
||||
(when-let [[op e] (and (= 2 (count d))
|
||||
(= :db/retractEntity (first d))
|
||||
d)]
|
||||
(when-let [e' (eid->lookup db-before e)]
|
||||
[op e'])))
|
||||
|
||||
(defn- normalize-tx-item
|
||||
[db-after db-before d]
|
||||
(case (count d)
|
||||
5 (normalize-datom db-after db-before d)
|
||||
2 (normalize-retract-entity-item db-before d)
|
||||
nil))
|
||||
|
||||
(defn normalize-tx-data
|
||||
[db-after db-before tx-data]
|
||||
(->> tx-data
|
||||
remove-conflict-datoms
|
||||
(replace-attr-retract-with-retract-entity db-after)
|
||||
sort-datoms
|
||||
(keep #(normalize-tx-item db-after db-before %))
|
||||
(remove-retract-entity-ref db-after)
|
||||
reorder-retract-entity
|
||||
distinct))
|
||||
78
deps/db/src/logseq/db/common/order.cljs
vendored
Normal file
78
deps/db/src/logseq/db/common/order.cljs
vendored
Normal file
@@ -0,0 +1,78 @@
|
||||
(ns logseq.db.common.order
|
||||
"Use fractional-indexing order for blocks/properties/closed values/etc.
|
||||
Used by DB and file graphs"
|
||||
(:require [logseq.clj-fractional-indexing :as index]
|
||||
[datascript.core :as d]))
|
||||
|
||||
(defonce *max-key (atom nil))
|
||||
|
||||
(defn reset-max-key!
|
||||
([key]
|
||||
(reset-max-key! *max-key key))
|
||||
([max-key-atom key]
|
||||
(when (and key (or (nil? @max-key-atom)
|
||||
(> (compare key @max-key-atom) 0)))
|
||||
(reset! max-key-atom key))))
|
||||
|
||||
(defn gen-key
|
||||
([]
|
||||
(gen-key @*max-key nil))
|
||||
([end]
|
||||
(gen-key @*max-key end))
|
||||
([start end & {:keys [max-key-atom]
|
||||
:or {max-key-atom *max-key}}]
|
||||
(let [k (index/generate-key-between start end)]
|
||||
(reset-max-key! max-key-atom k)
|
||||
k)))
|
||||
|
||||
(defn get-max-order
|
||||
[db]
|
||||
(:v (first (d/rseek-datoms db :avet :block/order))))
|
||||
|
||||
(defn gen-n-keys
|
||||
[n start end & {:keys [max-key-atom]
|
||||
:or {max-key-atom *max-key}}]
|
||||
(let [ks (index/generate-n-keys-between start end n)]
|
||||
(reset-max-key! max-key-atom (last ks))
|
||||
ks))
|
||||
|
||||
(defn validate-order-key?
|
||||
[key]
|
||||
(index/validate-order-key key index/base-62-digits)
|
||||
true)
|
||||
|
||||
(defn get-prev-order
|
||||
[db property value-id]
|
||||
(let [value (d/entity db value-id)]
|
||||
(if property
|
||||
(let [values (->> (:property/closed-values property)
|
||||
reverse)]
|
||||
(some (fn [e]
|
||||
(when (and (< (compare (:block/order e) (:block/order value)) 0)
|
||||
(not= (:db/id e) (:db/id value)))
|
||||
(:block/order e))) values))
|
||||
(let [properties (->> (d/datoms db :avet :block/tags :logseq.class/Property)
|
||||
(map (fn [d] (d/entity db (:e d))))
|
||||
(sort-by :block/order)
|
||||
reverse)]
|
||||
(some (fn [property]
|
||||
(when (and (< (compare (:block/order property) (:block/order value)) 0)
|
||||
(not= (:db/id property) (:db/id value)))
|
||||
(:block/order property))) properties)))))
|
||||
|
||||
(defn get-next-order
|
||||
[db property value-id]
|
||||
(let [value (d/entity db value-id)]
|
||||
(if property
|
||||
(let [values (:property/closed-values property)]
|
||||
(some (fn [e]
|
||||
(when (and (> (compare (:block/order e) (:block/order value)) 0)
|
||||
(not= (:db/id e) (:db/id value)))
|
||||
(:block/order e))) values))
|
||||
(let [properties (->> (d/datoms db :avet :block/tags :logseq.class/Property)
|
||||
(map (fn [d] (d/entity db (:e d))))
|
||||
(sort-by :block/order))]
|
||||
(some (fn [property]
|
||||
(when (and (> (compare (:block/order property) (:block/order value)) 0)
|
||||
(not= (:db/id property) (:db/id value)))
|
||||
(:block/order property))) properties)))))
|
||||
326
deps/db/src/logseq/db/common/reference.cljs
vendored
Normal file
326
deps/db/src/logseq/db/common/reference.cljs
vendored
Normal file
@@ -0,0 +1,326 @@
|
||||
(ns logseq.db.common.reference
|
||||
"References"
|
||||
(:require [clojure.set :as set]
|
||||
[clojure.string :as string]
|
||||
[datascript.core :as d]
|
||||
[logseq.db :as ldb]
|
||||
[logseq.db.common.initial-data :as common-initial-data]
|
||||
[logseq.db.frontend.class :as db-class]
|
||||
[logseq.db.frontend.entity-util :as entity-util]))
|
||||
|
||||
;; -----------------------------------------------------------------------------
|
||||
;; Helpers (fast datoms access)
|
||||
;; -----------------------------------------------------------------------------
|
||||
|
||||
(defn- has-datom? [db e a v]
|
||||
(boolean (seq (d/datoms db :eavt e a v))))
|
||||
|
||||
(defn- entid
|
||||
"Normalize a datom value into an entity id.
|
||||
In DataScript, ref values are often raw entids (numbers), not maps."
|
||||
[v]
|
||||
(cond
|
||||
(nil? v) nil
|
||||
(number? v) v
|
||||
(and (map? v) (contains? v :db/id)) (:db/id v)
|
||||
:else v))
|
||||
|
||||
(defn- datom-v
|
||||
"Return the first value for attr a on entity e, normalized to entid."
|
||||
[db e a]
|
||||
(some-> (first (d/datoms db :eavt e a))
|
||||
:v
|
||||
entid))
|
||||
|
||||
(defn- datom-vs
|
||||
"Return all values for attr a on entity e, normalized to entids, as a set."
|
||||
[db e a]
|
||||
(into #{}
|
||||
(comp (map :v) (map entid))
|
||||
(d/datoms db :eavt e a)))
|
||||
|
||||
(defn- get-path-refs
|
||||
[db entity]
|
||||
(let [refs (mapcat :block/refs (ldb/get-block-parents db (:block/uuid entity)))
|
||||
block-page (:block/page entity)]
|
||||
(->> (cond->> refs (some? block-page) (cons block-page))
|
||||
distinct)))
|
||||
|
||||
(defn- get-ref-pages-count
|
||||
[db id ref-blocks children-ids]
|
||||
(when (seq ref-blocks)
|
||||
(let [children (->> children-ids
|
||||
(map (fn [id] (d/entity db id))))
|
||||
hidden-ref?* (common-initial-data/hidden-ref-pred db id)]
|
||||
(->> (concat (mapcat #(get-path-refs db %) ref-blocks)
|
||||
(mapcat :block/refs (concat ref-blocks children)))
|
||||
frequencies
|
||||
(keep (fn [[ref size]]
|
||||
(when (and (ldb/page? ref)
|
||||
(not= (:db/id ref) id)
|
||||
(not= :block/tags (:db/ident ref))
|
||||
(not (hidden-ref?* ref)))
|
||||
[(:block/title ref) size])))
|
||||
(sort-by second #(> %1 %2))))))
|
||||
|
||||
(defn- child-ids
|
||||
"Direct children via AVET on :block/parent."
|
||||
[db parent-eid]
|
||||
(into #{} (map :e) (d/datoms db :avet :block/parent parent-eid)))
|
||||
|
||||
(defn- own-refs
|
||||
"Refs contributed by this block itself:
|
||||
- direct :block/refs
|
||||
- implicit :block/page as a ref"
|
||||
[db eid]
|
||||
(let [direct (datom-vs db eid :block/refs)
|
||||
page (datom-v db eid :block/page)]
|
||||
(cond-> direct
|
||||
page (conj page))))
|
||||
|
||||
(defn- effective-refs-fn
|
||||
"effective-refs(eid) = own-refs(eid) ∪ effective-refs(parent(eid))"
|
||||
[db]
|
||||
(let [memo (volatile! {})]
|
||||
(letfn [(eff [eid]
|
||||
(if (contains? @memo eid)
|
||||
(get @memo eid)
|
||||
(let [own (own-refs db eid)
|
||||
res (if-let [p (datom-v db eid :block/parent)]
|
||||
(into own (eff p))
|
||||
own)]
|
||||
(vswap! memo assoc eid res)
|
||||
res)))]
|
||||
eff)))
|
||||
|
||||
(defn- allowed-subtree-refs-fn
|
||||
"Like subtree-refs-fn but PRUNES branches that are under an excluded ref.
|
||||
This implements:
|
||||
- parent should not be excluded just because a child has an excluded ref
|
||||
- BUT refs (e.g. [[bar]]) that appear under an excluded block (e.g. [[baz]])
|
||||
should NOT satisfy includes for ancestors.
|
||||
|
||||
Concretely: if (effective-refs node) contains any excludes, this node and its
|
||||
descendants contribute NOTHING to the include reachability set."
|
||||
[db eff excludes]
|
||||
(let [memo (volatile! {})]
|
||||
(letfn [(blocked? [eid]
|
||||
(and (seq excludes)
|
||||
(some #(contains? (eff eid) %) excludes)))
|
||||
(sub [eid]
|
||||
(if (contains? @memo eid)
|
||||
(get @memo eid)
|
||||
(let [res (if (blocked? eid)
|
||||
#{}
|
||||
(reduce (fn [acc c] (into acc (sub c)))
|
||||
(own-refs db eid)
|
||||
(child-ids db eid)))]
|
||||
(vswap! memo assoc eid res)
|
||||
res)))]
|
||||
sub)))
|
||||
|
||||
(defn- matches-filters?
|
||||
"Include semantics: AND (must all be present) against include-set.
|
||||
Exclude semantics: NONE (must not contain any) against exclude-set."
|
||||
[include-set exclude-set includes excludes]
|
||||
(and
|
||||
(or (empty? includes)
|
||||
(every? #(contains? include-set %) includes))
|
||||
(or (empty? excludes)
|
||||
(not (some #(contains? exclude-set %) excludes)))))
|
||||
|
||||
(defn- filter-matched-ref-blocks
|
||||
[db top-ref-block-ids includes excludes {:keys [eff class-ok? can-satisfy-includes? allowed-subrefs]}]
|
||||
(loop [stack (vec top-ref-block-ids)
|
||||
visited #{}
|
||||
out #{}]
|
||||
(if (empty? stack)
|
||||
out
|
||||
(let [eid (peek stack)
|
||||
stack (pop stack)]
|
||||
(if (contains? visited eid)
|
||||
(recur stack visited out)
|
||||
(let [visited (conj visited eid)
|
||||
eff-refs (eff eid)]
|
||||
(cond
|
||||
;; don't match this node, but still traverse
|
||||
(not (class-ok? eid))
|
||||
(recur (into stack (child-ids db eid)) visited out)
|
||||
|
||||
;; prune: AND includes cannot be satisfied anywhere below
|
||||
(not (can-satisfy-includes? eff-refs eid))
|
||||
(recur stack visited out)
|
||||
|
||||
:else
|
||||
(let [include-set (into eff-refs (allowed-subrefs eid))
|
||||
exclude-set eff-refs
|
||||
out' (if (matches-filters? include-set exclude-set includes excludes)
|
||||
(conj out eid)
|
||||
out)]
|
||||
(recur (into stack (child-ids db eid)) visited out')))))))))
|
||||
|
||||
(defn- matched-ref-block-ids-under-top
|
||||
"Return the set of block ids (top or child) that match filters.
|
||||
|
||||
Semantics (matches your example):
|
||||
- Includes (AND) may be satisfied by descendants, BUT only from descendants
|
||||
that are NOT under an excluded ref.
|
||||
Example:
|
||||
- [[foo]]
|
||||
- [[baz]]
|
||||
- [[bar]]
|
||||
With include=bar, exclude=baz => foo should NOT match because bar is under baz.
|
||||
|
||||
- Excludes are checked ONLY against effective-refs(node) (self + parents),
|
||||
so a parent isn't excluded just because a child mentions an excluded page.
|
||||
|
||||
Pruning:
|
||||
- If includes cannot be satisfied anywhere below (effective + allowed-subtree),
|
||||
prune subtree."
|
||||
[db top-ref-block-ids includes excludes class-ids]
|
||||
(let [eff (effective-refs-fn db)
|
||||
;; allowed subtree refs for includes (prunes excluded branches)
|
||||
allowed-subrefs (allowed-subtree-refs-fn db eff (->> excludes (remove nil?) vec))
|
||||
includes (->> includes (remove nil?) vec)
|
||||
excludes (->> excludes (remove nil?) vec)
|
||||
class-ids (when (seq class-ids) (->> class-ids (remove nil?) vec))
|
||||
|
||||
can-satisfy-includes? (fn [eff-refs node]
|
||||
(or (empty? includes)
|
||||
(let [possible (into eff-refs (allowed-subrefs node))]
|
||||
(every? #(contains? possible %) includes))))
|
||||
|
||||
class-ok? (fn [eid]
|
||||
(or (empty? class-ids)
|
||||
(not (some #(has-datom? db eid :block/tags %) class-ids))))]
|
||||
|
||||
(filter-matched-ref-blocks db top-ref-block-ids includes excludes
|
||||
{:eff eff
|
||||
:class-ok? class-ok?
|
||||
:can-satisfy-includes? can-satisfy-includes?
|
||||
:allowed-subrefs allowed-subrefs})))
|
||||
|
||||
;; -----------------------------------------------------------------------------
|
||||
;; Expand matched refs up to top refs
|
||||
;; -----------------------------------------------------------------------------
|
||||
|
||||
(defn- expand-to-top-refs
|
||||
"Given matched refs, add ancestors until reaching a top ref in `top-ref-ids`.
|
||||
Uses cached parent lookups (datoms only)."
|
||||
[db top-ref-ids matched-ref-ids]
|
||||
(let [parent-cache (volatile! {}) ;; eid -> parent-eid|nil
|
||||
result (volatile! #{})
|
||||
top? (fn [eid] (contains? top-ref-ids eid))
|
||||
parent-of (fn [eid]
|
||||
(if (contains? @parent-cache eid)
|
||||
(get @parent-cache eid)
|
||||
(let [p (some-> (first (d/datoms db :eavt eid :block/parent))
|
||||
:v
|
||||
entid)]
|
||||
(vswap! parent-cache assoc eid p)
|
||||
p)))]
|
||||
(doseq [start matched-ref-ids]
|
||||
(loop [eid start]
|
||||
(when eid
|
||||
(cond
|
||||
(contains? @result eid)
|
||||
nil
|
||||
(top? eid)
|
||||
(vswap! result conj eid)
|
||||
:else
|
||||
(do
|
||||
(vswap! result conj eid)
|
||||
(recur (parent-of eid)))))))
|
||||
@result))
|
||||
|
||||
;; -----------------------------------------------------------------------------
|
||||
;; Public API
|
||||
;; -----------------------------------------------------------------------------
|
||||
|
||||
(defn get-filters
|
||||
[page]
|
||||
(let [included-pages (:logseq.property.linked-references/includes page)
|
||||
excluded-pages (:logseq.property.linked-references/excludes page)]
|
||||
(when (or (seq included-pages) (seq excluded-pages))
|
||||
{:included included-pages
|
||||
:excluded excluded-pages})))
|
||||
|
||||
(defn- linked-reference-top-block-ids
|
||||
[db ids class-ids]
|
||||
(->> ids
|
||||
(mapcat (fn [pid] (:block/_refs (d/entity db pid))))
|
||||
(remove (fn [ref]
|
||||
(or
|
||||
(when class-ids
|
||||
(some class-ids (map :db/id (:block/tags ref))))
|
||||
(entity-util/hidden? ref)
|
||||
(entity-util/hidden? (:block/page ref)))))
|
||||
(map :db/id)
|
||||
set))
|
||||
|
||||
(defn- linked-reference-result
|
||||
[db id include-ref-pages-count? has-filters? ref-blocks children-ids]
|
||||
(cond->
|
||||
{:ref-blocks ref-blocks
|
||||
:ref-matched-children-ids (when has-filters? children-ids)}
|
||||
include-ref-pages-count?
|
||||
(assoc :ref-pages-count (get-ref-pages-count db id ref-blocks children-ids))))
|
||||
|
||||
(defn get-linked-references
|
||||
([db id]
|
||||
(get-linked-references db id {:include-ref-pages-count? true}))
|
||||
([db id {:keys [include-ref-pages-count?]
|
||||
:or {include-ref-pages-count? true}}]
|
||||
(let [entity (d/entity db id)
|
||||
ids (set (cons id (ldb/get-block-alias db id)))
|
||||
page-filters (get-filters entity)
|
||||
excludes (map :db/id (:excluded page-filters))
|
||||
includes (map :db/id (:included page-filters))
|
||||
has-filters? (or (seq excludes) (seq includes))
|
||||
|
||||
class-ids (when (ldb/class? entity)
|
||||
(let [class-children (db-class/get-structured-children db id)]
|
||||
(set (conj class-children id))))
|
||||
;; Collect all top ref blocks that directly reference the page (or any alias).
|
||||
full-ref-block-ids (linked-reference-top-block-ids db ids class-ids)
|
||||
;; matched can be top or child ids
|
||||
matched-ref-block-ids
|
||||
(when has-filters?
|
||||
(matched-ref-block-ids-under-top db full-ref-block-ids includes excludes class-ids))
|
||||
;; Expand matches up to top refs so we can show parent chains and matched children.
|
||||
matched-refs-with-children-ids
|
||||
(when has-filters?
|
||||
(expand-to-top-refs db full-ref-block-ids matched-ref-block-ids))
|
||||
final-ref-ids
|
||||
(if has-filters?
|
||||
(set/intersection full-ref-block-ids matched-refs-with-children-ids)
|
||||
full-ref-block-ids)
|
||||
;; Materialize only at the end.
|
||||
ref-blocks (map #(d/entity db %) final-ref-ids)
|
||||
children-ids
|
||||
(if has-filters?
|
||||
(set (remove full-ref-block-ids matched-refs-with-children-ids))
|
||||
(when include-ref-pages-count?
|
||||
(->> ref-blocks
|
||||
(mapcat (fn [ref] (ldb/get-block-children-ids db (:db/id ref))))
|
||||
set)))]
|
||||
(linked-reference-result db id include-ref-pages-count? has-filters? ref-blocks children-ids))))
|
||||
|
||||
(defn get-unlinked-references
|
||||
[db id]
|
||||
(let [entity (d/entity db id)
|
||||
title (string/lower-case (:block/title entity))]
|
||||
(when-not (string/blank? title)
|
||||
(let [ids (->> (d/datoms db :avet :block/title)
|
||||
(keep (fn [d]
|
||||
(when (and (not= id (:e d))
|
||||
(string/includes? (string/lower-case (:v d)) title))
|
||||
(:e d)))))]
|
||||
(keep
|
||||
(fn [eid]
|
||||
(let [e (d/entity db eid)]
|
||||
(when-not (or (some #(= id %) (map :db/id (:block/refs e)))
|
||||
(:block/link e)
|
||||
(ldb/built-in? e))
|
||||
e)))
|
||||
ids)))))
|
||||
38
deps/db/src/logseq/db/common/sqlite.cljs
vendored
Normal file
38
deps/db/src/logseq/db/common/sqlite.cljs
vendored
Normal file
@@ -0,0 +1,38 @@
|
||||
(ns logseq.db.common.sqlite
|
||||
"Provides common sqlite util fns that work on browser and node"
|
||||
(:require ["path" :as node-path]
|
||||
[clojure.string :as string]
|
||||
[datascript.core :as d]
|
||||
[logseq.common.graph-dir :as graph-dir]
|
||||
[logseq.common.path :as path]
|
||||
[logseq.db.sqlite.util :as sqlite-util]))
|
||||
|
||||
(defn create-kvs-table!
|
||||
"Creates a sqlite table for use with datascript.storage if one doesn't exist"
|
||||
[sqlite-db]
|
||||
(.exec sqlite-db "create table if not exists kvs (addr INTEGER primary key, content TEXT, addresses JSON)"))
|
||||
|
||||
(defn get-storage-conn
|
||||
"Given a datascript storage, returns a datascript connection for it"
|
||||
[storage schema]
|
||||
(or (d/restore-conn storage)
|
||||
(d/create-conn schema {:storage storage})))
|
||||
|
||||
(defn sanitize-db-name
|
||||
[db-name]
|
||||
(-> db-name
|
||||
(string/replace sqlite-util/db-version-prefix "")
|
||||
(string/replace "/" "_")
|
||||
(string/replace "\\" "_")
|
||||
(string/replace ":" "_")));; windows
|
||||
|
||||
(defn get-db-full-path
|
||||
[graphs-dir db-name]
|
||||
(let [graph-dir-name (graph-dir/repo->encoded-graph-dir-name db-name)
|
||||
graph-dir (node-path/join graphs-dir graph-dir-name)]
|
||||
[graph-dir-name (path/path-join graph-dir "db.sqlite")]))
|
||||
|
||||
(defn get-db-backups-path
|
||||
[graphs-dir db-name]
|
||||
(let [graph-dir-name (graph-dir/repo->encoded-graph-dir-name db-name)]
|
||||
(path/path-join graphs-dir graph-dir-name "backups")))
|
||||
103
deps/db/src/logseq/db/common/sqlite_cli.cljs
vendored
Normal file
103
deps/db/src/logseq/db/common/sqlite_cli.cljs
vendored
Normal file
@@ -0,0 +1,103 @@
|
||||
(ns ^:node-only logseq.db.common.sqlite-cli
|
||||
"Primary ns to interact with DB files with node.js based CLIs"
|
||||
(:require ["better-sqlite3" :as sqlite3]
|
||||
["path" :as node-path]
|
||||
[cljs-bean.core :as bean]
|
||||
[clojure.string :as string]
|
||||
[datascript.storage :refer [IStorage]]
|
||||
[logseq.common.graph :as common-graph]
|
||||
[logseq.db.common.sqlite :as common-sqlite]
|
||||
[logseq.db.frontend.schema :as db-schema]
|
||||
[logseq.db.sqlite.util :as sqlite-util]))
|
||||
|
||||
;; Reference same sqlite default class in cljs + nbb without needing .cljc
|
||||
(def sqlite (if (find-ns 'nbb.core) (aget sqlite3 "default") sqlite3))
|
||||
|
||||
(defn query
|
||||
"Run a sql query against the given better-sqlite3 db"
|
||||
[^js db sql]
|
||||
(let [stmt (.prepare db sql)]
|
||||
(.all ^object stmt)))
|
||||
|
||||
(defn- upsert-addr-content!
|
||||
"Upsert addr+data-seq. Should be functionally equivalent to db-worker/upsert-addr-content!"
|
||||
[^js db data]
|
||||
(assert db ::upsert-addr-content!)
|
||||
(let [insert (.prepare db "INSERT INTO kvs (addr, content, addresses) values ($addr, $content, $addresses) on conflict(addr) do update set content = $content, addresses = $addresses")
|
||||
insert-many (.transaction ^object db
|
||||
(fn [data]
|
||||
(doseq [item data]
|
||||
(.run ^object insert item))))]
|
||||
(insert-many data)))
|
||||
|
||||
(defn- restore-data-from-addr
|
||||
"Should be functionally equivalent to db-worker/restore-data-from-addr"
|
||||
[db addr]
|
||||
(when-let [result (-> (query db (str "select content, addresses from kvs where addr = " addr))
|
||||
first)]
|
||||
(let [{:keys [content addresses]} (bean/->clj result)
|
||||
addresses (when addresses
|
||||
(js/JSON.parse addresses))
|
||||
data (sqlite-util/read-transit-str content)]
|
||||
(if (and addresses (map? data))
|
||||
(assoc data :addresses addresses)
|
||||
data))))
|
||||
|
||||
(defn new-sqlite-storage
|
||||
"Creates a datascript storage for sqlite. Should be functionally equivalent to db-worker/new-sqlite-storage"
|
||||
[db]
|
||||
(reify IStorage
|
||||
(-store [_ addr+data-seq _delete-addrs]
|
||||
;; Only difference from db-worker impl is that js data maps don't start with '$' e.g. :$addr -> :addr
|
||||
(let [data (map
|
||||
(fn [[addr data]]
|
||||
(let [data' (if (map? data) (dissoc data :addresses) data)
|
||||
addresses (when (map? data)
|
||||
(when-let [addresses (:addresses data)]
|
||||
(js/JSON.stringify (bean/->js addresses))))]
|
||||
#js {:addr addr
|
||||
:content (sqlite-util/write-transit-str data')
|
||||
:addresses addresses}))
|
||||
addr+data-seq)]
|
||||
(upsert-addr-content! db data)))
|
||||
(-restore [_ addr]
|
||||
(restore-data-from-addr db addr))))
|
||||
|
||||
(defn open-sqlite-datascript!
|
||||
"Returns a map including `conn` for datascript connection and `sqlite` for sqlite connection"
|
||||
([db-full-path]
|
||||
(open-sqlite-datascript! nil db-full-path))
|
||||
([graphs-dir db-name]
|
||||
(let [db-full-path
|
||||
(if (nil? graphs-dir)
|
||||
db-name
|
||||
(second (common-sqlite/get-db-full-path graphs-dir db-name)))
|
||||
db (new sqlite db-full-path nil)]
|
||||
(common-sqlite/create-kvs-table! db)
|
||||
(let [storage (new-sqlite-storage db)
|
||||
conn (common-sqlite/get-storage-conn storage db-schema/schema)]
|
||||
{:sqlite db
|
||||
:conn conn}))))
|
||||
|
||||
(defn open-db!
|
||||
"For a given database name, opens a sqlite db connection for it, creates
|
||||
needed sqlite tables if not created and returns a datascript connection that's
|
||||
connected to the sqlite db"
|
||||
([db-full-path]
|
||||
(open-db! nil db-full-path))
|
||||
([graphs-dir db-name]
|
||||
(:conn (open-sqlite-datascript! graphs-dir db-name))))
|
||||
|
||||
(defn ->open-db-args
|
||||
"Creates args for open-db from a graph arg. Works for relative and absolute paths and
|
||||
defaults to ~/logseq/graphs/ when no '/' present in name"
|
||||
[graph-dir-or-path]
|
||||
;; Pass full path directly to allow for paths that don't have standard graph naming convention
|
||||
(if (node-path/isAbsolute graph-dir-or-path)
|
||||
[graph-dir-or-path]
|
||||
(if (string/includes? graph-dir-or-path "/")
|
||||
(let [resolve-path' #(if (node-path/isAbsolute %) %
|
||||
;; $ORIGINAL_PWD used by bb tasks to correct current dir
|
||||
(node-path/join (or js/process.env.ORIGINAL_PWD ".") %))]
|
||||
((juxt node-path/dirname node-path/basename) (resolve-path' graph-dir-or-path)))
|
||||
[(common-graph/expand-home (common-graph/get-default-graphs-dir)) graph-dir-or-path])))
|
||||
724
deps/db/src/logseq/db/common/view.cljs
vendored
Normal file
724
deps/db/src/logseq/db/common/view.cljs
vendored
Normal file
@@ -0,0 +1,724 @@
|
||||
(ns logseq.db.common.view
|
||||
"Main namespace for view fns."
|
||||
(:require [clojure.set :as set]
|
||||
[clojure.string :as string]
|
||||
[datascript.core :as d]
|
||||
[datascript.impl.entity :as de]
|
||||
[logseq.common.util :as common-util]
|
||||
[logseq.db :as ldb]
|
||||
[logseq.db.common.entity-plus :as entity-plus]
|
||||
[logseq.db.common.initial-data :as common-initial-data]
|
||||
[logseq.db.common.reference :as db-reference]
|
||||
[logseq.db.frontend.class :as db-class]
|
||||
[logseq.db.frontend.entity-util :as entity-util]
|
||||
[logseq.db.frontend.property :as db-property]
|
||||
[logseq.db.frontend.property.type :as db-property-type]
|
||||
[logseq.db.frontend.rules :as rules]))
|
||||
|
||||
(def valid-type-for-sort? (some-fn number? string? boolean?))
|
||||
|
||||
(defn get-property-value-for-search
|
||||
[block property]
|
||||
(let [v (get block (:db/ident property))]
|
||||
(if (valid-type-for-sort? v) ;fast path
|
||||
v
|
||||
(let [typ (:logseq.property/type property)
|
||||
many? (keyword-identical? :db.cardinality/many (get property :db/cardinality))
|
||||
number-type? (or (keyword-identical? :number typ)
|
||||
(keyword-identical? :datetime typ))]
|
||||
(if many?
|
||||
(let [col (->> (if (db-property-type/all-ref-property-types typ) (map db-property/property-value-content v) v)
|
||||
(remove nil?))]
|
||||
(if number-type?
|
||||
(reduce + (filter number? col))
|
||||
(string/join ", " col)))
|
||||
(let [v' (if (db-property-type/all-ref-property-types typ) (db-property/property-value-content v) v)]
|
||||
(cond
|
||||
(and number-type? (number? v')) v'
|
||||
:else v')))))))
|
||||
|
||||
(defn- get-value-for-sort
|
||||
[property]
|
||||
(let [db-ident (or (:db/ident property) (:id property))
|
||||
closed-values (seq (:property/closed-values property))
|
||||
closed-value->sort-number (when closed-values
|
||||
(->> (zipmap (map :db/id closed-values)
|
||||
(if (every? :block/order closed-values)
|
||||
(map :block/order closed-values)
|
||||
(range 0 (count closed-values))))
|
||||
(into {})))
|
||||
get-property-value-fn (fn [entity]
|
||||
(if (de/entity? property)
|
||||
(if (= :date (:logseq.property/type property))
|
||||
(:block/journal-day (get entity db-ident))
|
||||
(get-property-value-for-search entity property))
|
||||
(get entity db-ident)))]
|
||||
(fn [entity]
|
||||
(cond
|
||||
closed-values
|
||||
(closed-value->sort-number (:db/id (get entity db-ident)))
|
||||
:else
|
||||
(let [v (get-property-value-fn entity)]
|
||||
(when (valid-type-for-sort? v)
|
||||
v))))))
|
||||
|
||||
(defn- by-one-sorting
|
||||
[{:keys [asc? get-value]}]
|
||||
(let [cmp (if asc? compare #(compare %2 %1))]
|
||||
(fn [a b]
|
||||
(cmp (get-value a) (get-value b)))))
|
||||
|
||||
(defn- sort-ref-entities-by-single-property
|
||||
"get all entities sorted by `major-sorting`"
|
||||
[entities {:keys [_id asc?]} get-value-fn]
|
||||
(let [sorting {:asc? asc?
|
||||
:get-value get-value-fn}
|
||||
sort-cmp (by-one-sorting sorting)]
|
||||
(sort sort-cmp entities)))
|
||||
|
||||
(defn- sort-by-single-property
|
||||
[db {:keys [id asc?] :as sorting} entities partition?]
|
||||
(let [property (or (d/entity db id) {:db/ident id})
|
||||
get-value-fn (memoize (get-value-for-sort property))
|
||||
entities' (if (vector? entities) entities (vec entities))
|
||||
datom-sort-supported? (contains? #{:block/updated-at :block/created-at :block/title}
|
||||
(:db/ident property))
|
||||
use-datom-sort? (and datom-sort-supported?
|
||||
(not= :db.type/ref (:db/valueType property))
|
||||
(> (count entities') 10000))
|
||||
sorted-entities (->>
|
||||
(cond
|
||||
(= id :block.temp/refs-count)
|
||||
(cond-> (sort-by :block.temp/refs-count entities')
|
||||
(not asc?)
|
||||
reverse)
|
||||
|
||||
use-datom-sort?
|
||||
(let [datoms (cond->
|
||||
(->> (d/datoms db :avet id)
|
||||
(common-util/distinct-by :e)
|
||||
vec)
|
||||
(not asc?)
|
||||
rseq)
|
||||
row-ids (set (map :db/id entities'))
|
||||
id->row (zipmap (map :db/id entities') entities')]
|
||||
(keep
|
||||
(fn [d]
|
||||
(when (row-ids (:e d))
|
||||
(id->row (:e d))))
|
||||
datoms))
|
||||
|
||||
:else
|
||||
(sort-ref-entities-by-single-property entities' sorting get-value-fn))
|
||||
|
||||
distinct)]
|
||||
(if partition?
|
||||
(partition-by get-value-fn sorted-entities)
|
||||
sorted-entities)))
|
||||
|
||||
(defn- sort-entities-by-minor-sorting
|
||||
"minor-sorting - [{:keys [id asc?]} ...]"
|
||||
[db partitioned-entities-by-major-sorting minor-sorting]
|
||||
(let [sorting
|
||||
(map (fn [{:keys [id asc?]}]
|
||||
(let [property (d/entity db id)]
|
||||
{:asc? asc?
|
||||
:get-value (memoize (get-value-for-sort property))}))
|
||||
minor-sorting)
|
||||
sort-cmp (common-util/by-sorting sorting)]
|
||||
(mapcat (fn [entities] (sort sort-cmp entities)) partitioned-entities-by-major-sorting)))
|
||||
|
||||
(defn sort-entities
|
||||
[db sorting entities]
|
||||
(let [major-sorting (or (first sorting)
|
||||
{:id :block/updated-at :asc? false})
|
||||
minor-sorting (seq (rest sorting))
|
||||
major-sorted-entities
|
||||
(sort-by-single-property db major-sorting entities (not-empty minor-sorting))]
|
||||
(if minor-sorting
|
||||
(sort-entities-by-minor-sorting db major-sorted-entities minor-sorting)
|
||||
major-sorted-entities)))
|
||||
|
||||
(defn get-property-value-content
|
||||
[db value]
|
||||
(when value
|
||||
(cond
|
||||
(uuid? value)
|
||||
(db-property/property-value-content (d/entity db [:block/uuid value]))
|
||||
(de/entity? value)
|
||||
(db-property/property-value-content value)
|
||||
(keyword? value)
|
||||
(str value)
|
||||
:else
|
||||
value)))
|
||||
|
||||
(defn- match-property-value-as-entity?
|
||||
"Determines if the property value entity should be treated as an entity. For some property types
|
||||
like :default, we want match on the entity's content as that is what the user sees and interacts with"
|
||||
[property-value-entity property-entity]
|
||||
;; Allow pvalue entities with :db/ident e.g. closed values like status OR for any type
|
||||
;; that aren't text types
|
||||
(or (:db/ident property-value-entity)
|
||||
(not (contains? db-property-type/closed-value-property-types (:logseq.property/type property-entity)))))
|
||||
|
||||
(defn- empty-value?
|
||||
[v]
|
||||
(or (nil? v)
|
||||
(= :logseq.property/empty-placeholder v)
|
||||
(and (string? v) (string/blank? v))
|
||||
(and (coll? v) (empty? v))))
|
||||
|
||||
(defn- ^:large-vars/cleanup-todo row-matched?
|
||||
[db row filters input]
|
||||
(let [or? (:or? filters)
|
||||
check-f (if or? some every?)]
|
||||
(and
|
||||
(if (string/blank? input)
|
||||
true
|
||||
(string/includes? (string/lower-case (:block/title row)) (string/lower-case input)))
|
||||
(check-f
|
||||
(fn [[property-ident operator match]]
|
||||
(if (nil? match)
|
||||
true
|
||||
(boolean
|
||||
(let [value (get row property-ident)
|
||||
value' (cond
|
||||
(set? value) value
|
||||
(nil? value) nil
|
||||
:else #{value})
|
||||
entity? (de/entity? (first value'))
|
||||
result
|
||||
(case operator
|
||||
:is
|
||||
(cond
|
||||
(boolean? match)
|
||||
(= (boolean (get-property-value-content db (get row property-ident))) match)
|
||||
(= :empty match)
|
||||
(empty-value? value)
|
||||
(empty? match)
|
||||
true
|
||||
(and (empty? match) (empty? value'))
|
||||
true
|
||||
:else
|
||||
(if entity?
|
||||
(let [property (d/entity db property-ident)]
|
||||
(if (match-property-value-as-entity? (first value') property)
|
||||
(boolean (some match (map :block/uuid value')))
|
||||
(boolean (seq (set/intersection (set (map db-property/property-value-content value'))
|
||||
(set (map (comp db-property/property-value-content #(d/entity db [:block/uuid %]))
|
||||
match)))))))
|
||||
(boolean (some match value'))))
|
||||
|
||||
:is-not
|
||||
(cond
|
||||
(boolean? match)
|
||||
(not= (boolean (get-property-value-content db (get row property-ident))) match)
|
||||
(= :empty match)
|
||||
(not (empty-value? value))
|
||||
(and (empty? match) (seq value'))
|
||||
true
|
||||
(and (seq match) (empty? value'))
|
||||
true
|
||||
:else
|
||||
(if entity?
|
||||
(let [property (d/entity db property-ident)]
|
||||
(if (match-property-value-as-entity? (first value') property)
|
||||
(not (some match (map :block/uuid value')))
|
||||
(empty? (set/intersection (set (map db-property/property-value-content value'))
|
||||
(set (map (comp db-property/property-value-content #(d/entity db [:block/uuid %]))
|
||||
match))))))
|
||||
(not (some match value'))))
|
||||
|
||||
:text-contains
|
||||
(some (fn [v]
|
||||
(if-let [property-value (get-property-value-content db v)]
|
||||
(string/includes? (string/lower-case property-value) (string/lower-case match))
|
||||
false))
|
||||
value')
|
||||
|
||||
:text-not-contains
|
||||
(not-any? #(string/includes? (str (get-property-value-content db %)) match) value')
|
||||
|
||||
:number-gt
|
||||
(when value
|
||||
(if match (some #(> (get-property-value-content db %) match) value') true))
|
||||
:number-gte
|
||||
(when value
|
||||
(if match (some #(>= (get-property-value-content db %) match) value') true))
|
||||
:number-lt
|
||||
(when value
|
||||
(if match (some #(< (get-property-value-content db %) match) value') true))
|
||||
:number-lte
|
||||
(when value
|
||||
(if match (some #(<= (get-property-value-content db %) match) value') true))
|
||||
|
||||
:between
|
||||
(if (seq match)
|
||||
(some (fn [value-entity]
|
||||
(let [[start end] match
|
||||
value (get-property-value-content db value-entity)
|
||||
conditions [(if start (<= start value) true)
|
||||
(if end (<= value end) true)]]
|
||||
(if (seq match) (every? true? conditions) true))) value')
|
||||
true)
|
||||
|
||||
:date-before
|
||||
(when value
|
||||
(if match (some #(< (:block/journal-day %) (:block/journal-day match)) value') true))
|
||||
|
||||
:date-after
|
||||
(when value
|
||||
(if match (some #(> (:block/journal-day %) (:block/journal-day match)) value') true))
|
||||
|
||||
:before
|
||||
(when value
|
||||
(let [search-value (common-util/get-timestamp match)]
|
||||
(if search-value (<= value search-value) true)))
|
||||
|
||||
:after
|
||||
(when value
|
||||
(let [search-value (common-util/get-timestamp match)]
|
||||
(if search-value (>= value search-value) true)))
|
||||
|
||||
true)]
|
||||
result))))
|
||||
(:filters filters)))))
|
||||
|
||||
(defn- ->filter-match-id
|
||||
[db v]
|
||||
(cond
|
||||
(nil? v) nil
|
||||
(number? v) v
|
||||
(uuid? v) (some-> (d/entity db [:block/uuid v]) :db/id)
|
||||
(de/entity? v) (:db/id v)
|
||||
(and (map? v) (contains? v :db/id)) (:db/id v)
|
||||
:else nil))
|
||||
|
||||
(defn- build-fast-filter-pred
|
||||
"Build a faster matcher for common filter shapes while preserving semantics.
|
||||
Currently optimized for a single ref property filter with :is/:is-not."
|
||||
[db filters input]
|
||||
(when (and (string/blank? input)
|
||||
(map? filters)
|
||||
(not (:or? filters)))
|
||||
(let [clauses (:filters filters)]
|
||||
(when (= 1 (count clauses))
|
||||
(let [[property-ident operator match] (first clauses)
|
||||
property (d/entity db property-ident)
|
||||
ref-property? (= :db.type/ref (:db/valueType property))]
|
||||
(when (and ref-property?
|
||||
(#{:is :is-not} operator)
|
||||
(set? match)
|
||||
(seq match)
|
||||
(not (contains? match :empty)))
|
||||
(let [match-ids (set (keep #(->filter-match-id db %) match))]
|
||||
(when (seq match-ids)
|
||||
(fn [row]
|
||||
(let [v (get row property-ident)
|
||||
value-col (cond
|
||||
(set? v) v
|
||||
(nil? v) nil
|
||||
:else #{v})
|
||||
hit? (boolean (some match-ids (keep #(->filter-match-id db %) value-col)))]
|
||||
(if (= operator :is) hit? (not hit?))))))))))))
|
||||
|
||||
(defn- get-exclude-page-ids
|
||||
[db]
|
||||
(let [property-tag-id (:db/id (d/entity db :logseq.class/Property))]
|
||||
(persistent!
|
||||
(reduce (fn [result d]
|
||||
(conj! result (:e d)))
|
||||
(transient #{})
|
||||
(concat
|
||||
(d/datoms db :avet :logseq.property/hide? true)
|
||||
(d/datoms db :avet :logseq.property/deleted-at)
|
||||
(d/datoms db :avet :logseq.property/built-in? true)
|
||||
(d/datoms db :avet :block/tags property-tag-id))))))
|
||||
|
||||
(defn- get-entities-for-all-pages [db sorting property-ident]
|
||||
(let [refs-count? (and (coll? sorting) (some (fn [m] (= (:id m) :block.temp/refs-count)) sorting))
|
||||
exclude-ids (get-exclude-page-ids db)]
|
||||
(persistent!
|
||||
(reduce (fn [result d]
|
||||
(let [eid (:e d)]
|
||||
(if (contains? exclude-ids eid)
|
||||
result
|
||||
(let [e (entity-plus/unsafe->Entity db eid)]
|
||||
(if (entity-util/hidden? e)
|
||||
result
|
||||
(conj! result
|
||||
(cond-> e
|
||||
refs-count?
|
||||
(assoc :block.temp/refs-count (common-initial-data/get-block-refs-count db eid)))))))))
|
||||
(transient [])
|
||||
(d/datoms db :avet property-ident)))))
|
||||
|
||||
(def ^:private fast-all-pages-sort-ids
|
||||
"Sort ids supported by a datom-order fast path"
|
||||
#{:block/updated-at :block/created-at :block/title :block/name})
|
||||
|
||||
(defn- get-all-page-ids-fast
|
||||
"Fast path for all-pages where only sorted page ids are needed. Avoids
|
||||
hydrating every page entity by deriving ordering from indexed datoms."
|
||||
[db sorting]
|
||||
(let [major-sorting (or (first sorting)
|
||||
{:id :block/updated-at :asc? false})
|
||||
minor-sorting (seq (rest sorting))]
|
||||
(when (and (empty? minor-sorting)
|
||||
(contains? fast-all-pages-sort-ids (:id major-sorting)))
|
||||
(let [exclude-ids (get-exclude-page-ids db)
|
||||
page-ids (persistent!
|
||||
(reduce (fn [result datom]
|
||||
(let [eid (:e datom)]
|
||||
(if (contains? exclude-ids eid)
|
||||
result
|
||||
(conj! result eid))))
|
||||
(transient [])
|
||||
(d/datoms db :avet :block/name)))
|
||||
sort-id (:id major-sorting)
|
||||
asc? (:asc? major-sorting)
|
||||
get-sort-value (memoize
|
||||
(fn [eid]
|
||||
(get (entity-plus/unsafe->Entity db eid) sort-id)))
|
||||
cmp (fn [eid-a eid-b]
|
||||
(let [va (get-sort-value eid-a)
|
||||
vb (get-sort-value eid-b)
|
||||
c (cond
|
||||
(and (nil? va) (nil? vb)) 0
|
||||
(nil? va) 1
|
||||
(nil? vb) -1
|
||||
:else (compare va vb))]
|
||||
(if asc? c (- c))))]
|
||||
(sort cmp page-ids)))))
|
||||
|
||||
(defn- get-entities
|
||||
[db view feat-type property-ident view-for-id* sorting {:keys [include-ref-pages-count?]
|
||||
:or {include-ref-pages-count? true}}]
|
||||
(let [view-for (:logseq.property/view-for view)
|
||||
view-for-id (or (:db/id view-for) view-for-id*)
|
||||
non-hidden-e (fn [id] (let [e (d/entity db id)]
|
||||
(when-not (entity-util/hidden? e)
|
||||
e)))]
|
||||
(case feat-type
|
||||
:all-pages
|
||||
(get-entities-for-all-pages db sorting property-ident)
|
||||
|
||||
:class-objects
|
||||
(db-class/get-class-objects db view-for-id)
|
||||
|
||||
:property-objects
|
||||
(->>
|
||||
(d/q
|
||||
'[:find [?b ...]
|
||||
:in $ % ?prop
|
||||
:where
|
||||
(has-property-or-object-property? ?b ?prop)]
|
||||
db
|
||||
(rules/extract-rules rules/db-query-dsl-rules [:has-property-or-object-property]
|
||||
{:deps rules/rules-dependencies})
|
||||
property-ident)
|
||||
(keep (fn [id] (non-hidden-e id))))
|
||||
|
||||
:linked-references
|
||||
(db-reference/get-linked-references db view-for-id
|
||||
{:include-ref-pages-count?
|
||||
(not (false? include-ref-pages-count?))})
|
||||
|
||||
:unlinked-references
|
||||
(db-reference/get-unlinked-references db view-for-id)
|
||||
|
||||
:query-result
|
||||
nil
|
||||
|
||||
nil)))
|
||||
|
||||
(defn- get-view-entities
|
||||
[db view-id & {:keys [view-for-id view-feature-type sorting include-ref-pages-count?]}]
|
||||
(let [view (d/entity db view-id)
|
||||
feat-type (or view-feature-type (:logseq.property.view/feature-type view))
|
||||
sorting (or sorting (:logseq.property.table/sorting view))
|
||||
index-attr (case feat-type
|
||||
:all-pages
|
||||
:block/name
|
||||
:class-objects
|
||||
:block/tags
|
||||
:property-objects
|
||||
(let [view-for (:logseq.property/view-for view)]
|
||||
(:db/ident view-for))
|
||||
nil)]
|
||||
(get-entities db view feat-type index-attr view-for-id sorting
|
||||
(cond-> {}
|
||||
(some? include-ref-pages-count?)
|
||||
(assoc :include-ref-pages-count? include-ref-pages-count?)))))
|
||||
|
||||
(defn- get-view-property-values
|
||||
[db property-ident {:keys [view-id query-entity-ids]}]
|
||||
(let [empty-id (:db/id (d/entity db :logseq.property/empty-placeholder))
|
||||
entities-result (get-view-entities db view-id)
|
||||
entities (cond
|
||||
query-entity-ids
|
||||
(keep #(d/entity db %) query-entity-ids)
|
||||
(map? entities-result)
|
||||
(:ref-blocks entities-result)
|
||||
:else
|
||||
entities-result)]
|
||||
(->> (mapcat (fn [entity]
|
||||
(let [v (get entity property-ident)]
|
||||
(if (set? v) v #{v})))
|
||||
entities)
|
||||
(remove nil?)
|
||||
(keep (fn [e]
|
||||
(when-not (and (de/entity? e) (entity-util/recycled? e))
|
||||
(when-let [label (get-property-value-content db e)]
|
||||
(when-not (or (string/blank? (str label))
|
||||
(= empty-id (:db/id e)))
|
||||
{:label (str label)
|
||||
:value (if (de/entity? e)
|
||||
(select-keys e [:db/id :block/uuid])
|
||||
e)})))))
|
||||
(common-util/distinct-by :label))))
|
||||
|
||||
(defn ^:api get-property-values
|
||||
[db property-ident {:keys [view-id _query-entity-ids] :as option}]
|
||||
(let [property (d/entity db property-ident)
|
||||
default-value (:logseq.property/default-value property)
|
||||
ref-type? (= :db.type/ref (:db/valueType property))
|
||||
values (if view-id
|
||||
(get-view-property-values db property-ident option)
|
||||
;; get all values
|
||||
(->> (d/datoms db :avet property-ident)
|
||||
(map (fn [d]
|
||||
(:v d)))
|
||||
distinct
|
||||
(keep (fn [v]
|
||||
(let [e (when ref-type? (d/entity db v))]
|
||||
(when-not (and ref-type? (entity-util/recycled? e))
|
||||
(let [[label value] (cond ref-type?
|
||||
[(db-property/property-value-content e)
|
||||
(select-keys e [:db/id :block/uuid])]
|
||||
;; FIXME: Move query concerns out of :label as UI labels are usually strings
|
||||
;; All non-string values need to be passed to the query builder since non-ref prop values use the actual value
|
||||
;; This check is less fragile than listing all the property types to support e.g. :datetime, :checkbox, :keyword, :any
|
||||
(not (string? v))
|
||||
[v v]
|
||||
:else
|
||||
[(str v) v])]
|
||||
{:label label
|
||||
:value value})))))))]
|
||||
(->>
|
||||
(if (and default-value (not (entity-util/recycled? default-value)))
|
||||
(cons {:label (get-property-value-content db default-value)
|
||||
:value (select-keys default-value [:db/id :block/uuid])}
|
||||
values)
|
||||
values)
|
||||
(common-util/distinct-by :label))))
|
||||
|
||||
(defn- get-query-properties
|
||||
[query entities]
|
||||
(let [properties (when (and (coll? query) (= :find (first query)))
|
||||
(let [expr (second query)]
|
||||
(when (= 'pull (first expr))
|
||||
(last expr))))]
|
||||
(if (and (seq properties) (not= properties ['*]))
|
||||
properties
|
||||
(distinct (mapcat keys entities)))))
|
||||
|
||||
(defn- linked-references-page-list-view-data
|
||||
[view entities-result entities]
|
||||
(let [groups-sort-by-property-ident (or (:db/ident (:logseq.property.view/sort-groups-by-property view))
|
||||
:block/journal-day)
|
||||
desc? (:logseq.property.view/sort-groups-desc? view)
|
||||
page-sort-value (fn [page]
|
||||
(let [v (get page groups-sort-by-property-ident)]
|
||||
(if (and (= groups-sort-by-property-ident :block/journal-day)
|
||||
(not desc?)
|
||||
(nil? (:block/journal-day page)))
|
||||
js/Number.MAX_SAFE_INTEGER
|
||||
v)))
|
||||
page-sorters (cond->
|
||||
[{:get-value (fn [[page _blocks]] (page-sort-value page))
|
||||
:asc? (not desc?)}]
|
||||
(not= groups-sort-by-property-ident :block/title)
|
||||
(conj {:get-value (fn [[page _blocks]] (:block/title page))
|
||||
:asc? (not desc?)}))
|
||||
sorted-page-groups (sort (common-util/by-sorting page-sorters)
|
||||
(group-by #(or (:block/page %) %) entities))
|
||||
block-row (fn [block]
|
||||
{:db/id (:db/id block)
|
||||
:block/parent (:block/uuid (:block/parent block))})
|
||||
nested-group (fn [[_parent blocks]]
|
||||
[(:block/uuid (first blocks))
|
||||
(map block-row (ldb/sort-by-order blocks))])
|
||||
data (map (fn [[page blocks]]
|
||||
[(select-keys page [:db/id :db/ident :block/uuid :block/title :block/name
|
||||
:logseq.property/value :logseq.property/icon :block/tags])
|
||||
(->> blocks
|
||||
(group-by :block/parent)
|
||||
(sort-by (fn [[parent _]] (:block/order parent)))
|
||||
(map nested-group))])
|
||||
sorted-page-groups)]
|
||||
(merge
|
||||
{:count (count entities)
|
||||
:data data}
|
||||
(select-keys entities-result [:ref-pages-count :ref-matched-children-ids]))))
|
||||
|
||||
(defn ^:api ^:large-vars/cleanup-todo get-view-data
|
||||
[db view-id {:keys [journals? _view-for-id view-feature-type group-by-property-ident input query-entity-ids query filters sorting]
|
||||
:as opts}]
|
||||
;; TODO: create a view for journals maybe?
|
||||
(cond
|
||||
journals?
|
||||
(let [journals (vec (ldb/get-latest-journals db))
|
||||
index (mapv #(select-keys % [:db/id :block/journal-day]) journals)]
|
||||
{:count (count index)
|
||||
:data index})
|
||||
:else
|
||||
(let [view (d/entity db view-id)
|
||||
group-by-property (:logseq.property.view/group-by-property view)
|
||||
list-view? (= :logseq.property.view/type.list (:db/ident (:logseq.property.view/type view)))
|
||||
group-by-property-ident (or (:db/ident group-by-property) group-by-property-ident)
|
||||
group-by-closed-values? (some? (:property/closed-values group-by-property))
|
||||
ref-property? (= (:db/valueType group-by-property) :db.type/ref)
|
||||
filters (or (:logseq.property.table/filters view) filters)
|
||||
feat-type (or view-feature-type (:logseq.property.view/feature-type view))
|
||||
query? (= feat-type :query-result)
|
||||
query-entity-ids (when (seq query-entity-ids) (set query-entity-ids))
|
||||
sorting (let [sorting* (:logseq.property.table/sorting view)]
|
||||
(if (or (= sorting* :logseq.property/empty-placeholder) (empty? sorting*))
|
||||
(or sorting [{:id :block/updated-at :asc? false}])
|
||||
sorting*))
|
||||
fast-all-pages-ids (when (and (= feat-type :all-pages)
|
||||
(not query?)
|
||||
(nil? group-by-property-ident)
|
||||
(empty? filters)
|
||||
(string/blank? input))
|
||||
(get-all-page-ids-fast db sorting))]
|
||||
(if fast-all-pages-ids
|
||||
{:count (count fast-all-pages-ids)
|
||||
:data fast-all-pages-ids}
|
||||
(let [entities-result (if query?
|
||||
(keep (fn [id]
|
||||
(let [e (d/entity db id)]
|
||||
(when-not (= :logseq.property/query (:db/ident (:logseq.property/created-from-property e)))
|
||||
e)))
|
||||
query-entity-ids)
|
||||
(get-view-entities db view-id opts))
|
||||
entities (if (= feat-type :linked-references)
|
||||
(:ref-blocks entities-result)
|
||||
entities-result)
|
||||
filtered-entities (if (or (seq filters) (not (string/blank? input)))
|
||||
(let [filter-pred (or (build-fast-filter-pred db filters input)
|
||||
(fn [row] (row-matched? db row filters input)))]
|
||||
(into [] (filter filter-pred) entities))
|
||||
entities)
|
||||
nested-list-view? (and list-view?
|
||||
(some :block/page filtered-entities))
|
||||
group-by-page? (= group-by-property-ident :block/page)
|
||||
linked-references-page-list-fast-path?
|
||||
(and (= feat-type :linked-references)
|
||||
group-by-page?
|
||||
list-view?
|
||||
(empty? filters)
|
||||
(string/blank? input))
|
||||
group-values
|
||||
(fn group-values [ent]
|
||||
(let [pvalue (get ent group-by-property-ident)
|
||||
values (if (and (not (de/entity? pvalue))
|
||||
(coll? pvalue)
|
||||
(not (map? pvalue)))
|
||||
(seq pvalue)
|
||||
[pvalue])]
|
||||
(or
|
||||
(seq
|
||||
(map
|
||||
(fn [value]
|
||||
(if (de/entity? value)
|
||||
(if (match-property-value-as-entity? value group-by-property)
|
||||
value
|
||||
(db-property/property-value-content value))
|
||||
value))
|
||||
values))
|
||||
[nil])))
|
||||
result (if linked-references-page-list-fast-path?
|
||||
nil
|
||||
(if group-by-property-ident
|
||||
(let [groups-sort-by-property-ident (or (:db/ident (:logseq.property.view/sort-groups-by-property view))
|
||||
:block/journal-day)
|
||||
desc? (:logseq.property.view/sort-groups-desc? view)
|
||||
result (->> filtered-entities
|
||||
(reduce (fn [groups ent]
|
||||
(reduce
|
||||
(fn [groups value]
|
||||
(update groups value (fnil conj []) ent))
|
||||
groups
|
||||
(group-values ent)))
|
||||
{})
|
||||
(seq))
|
||||
keyfn (fn [groups-sort-by-property-ident]
|
||||
(fn [[by-value _]]
|
||||
(cond
|
||||
group-by-page?
|
||||
(let [v (get by-value groups-sort-by-property-ident)]
|
||||
(if (and (= groups-sort-by-property-ident :block/journal-day) (not desc?)
|
||||
(nil? (:block/journal-day by-value)))
|
||||
;; Use MAX_SAFE_INTEGER so non-journal pages (without :block/journal-day) are sorted
|
||||
;; after all journal pages when sorting by journal date.
|
||||
js/Number.MAX_SAFE_INTEGER
|
||||
v))
|
||||
group-by-closed-values?
|
||||
(:block/order by-value)
|
||||
ref-property?
|
||||
;; For value-ref types (e.g. :number), group-values has already
|
||||
;; extracted the scalar content, so by-value is no longer an entity.
|
||||
;; Only re-extract for entity group keys (e.g. :node/:class).
|
||||
(if (de/entity? by-value)
|
||||
(db-property/property-value-content by-value)
|
||||
by-value)
|
||||
:else
|
||||
by-value)))]
|
||||
(sort (common-util/by-sorting
|
||||
(cond->
|
||||
[{:get-value (keyfn groups-sort-by-property-ident)
|
||||
:asc? (not desc?)}]
|
||||
(not= groups-sort-by-property-ident :block/title)
|
||||
(conj {:get-value (keyfn :block/title)
|
||||
:asc? (not desc?)})))
|
||||
result))
|
||||
(sort-entities db sorting filtered-entities)))
|
||||
data' (if linked-references-page-list-fast-path?
|
||||
nil
|
||||
(if group-by-property-ident
|
||||
(map
|
||||
(fn [[by-value entities]]
|
||||
(let [by-value' (if (de/entity? by-value)
|
||||
(select-keys by-value [:db/id :db/ident :block/uuid :block/title :block/name :logseq.property/value :logseq.property/icon :block/tags])
|
||||
by-value)
|
||||
group (if nested-list-view?
|
||||
(let [parent-groups (->> entities
|
||||
(group-by :block/parent)
|
||||
(sort-by (fn [[parent _]] (:block/order parent))))]
|
||||
(map
|
||||
(fn [[_parent blocks]]
|
||||
[(:block/uuid (first blocks))
|
||||
(map (fn [b]
|
||||
{:db/id (:db/id b)
|
||||
:block/parent (:block/uuid (:block/parent b))})
|
||||
(ldb/sort-by-order blocks))])
|
||||
parent-groups))
|
||||
(->> (sort-entities db sorting entities)
|
||||
(map :db/id)))]
|
||||
[by-value' group]))
|
||||
result)
|
||||
(map :db/id result)))
|
||||
dedupe-data? (or (= feat-type :property-objects) query?)]
|
||||
(if linked-references-page-list-fast-path?
|
||||
(linked-references-page-list-view-data view entities-result entities)
|
||||
(cond->
|
||||
{:count (count filtered-entities)
|
||||
:data (if dedupe-data?
|
||||
(distinct data')
|
||||
data')}
|
||||
(= feat-type :linked-references)
|
||||
(merge (select-keys entities-result [:ref-pages-count :ref-matched-children-ids]))
|
||||
query?
|
||||
(assoc :properties (get-query-properties query entities-result)))))))))
|
||||
29
deps/db/src/logseq/db/frontend/asset.cljs
vendored
Normal file
29
deps/db/src/logseq/db/frontend/asset.cljs
vendored
Normal file
@@ -0,0 +1,29 @@
|
||||
(ns logseq.db.frontend.asset
|
||||
"Asset fns used in node and browser contexts"
|
||||
(:require ["path" :as node-path]
|
||||
[clojure.string :as string]))
|
||||
|
||||
(defn- decode-digest
|
||||
[^js/Uint8Array digest]
|
||||
(.. (js/Array.from digest)
|
||||
(map (fn [s] (.. s (toString 16) (padStart 2 "0"))))
|
||||
(join "")))
|
||||
|
||||
(defn <get-file-array-buffer-checksum
|
||||
"Given a file's ArrayBuffer or String, returns its checksum in a promise"
|
||||
[s]
|
||||
(let [array-buffer (if (string? s)
|
||||
(.encode (js/TextEncoder.) s) s)]
|
||||
(-> (js/crypto.subtle.digest "SHA-256" array-buffer)
|
||||
(.then (fn [dig] (js/Uint8Array. dig)))
|
||||
(.then decode-digest))))
|
||||
|
||||
(defn asset-path->type
|
||||
"Create asset type given asset path"
|
||||
[path]
|
||||
(string/lower-case (.substr (node-path/extname path) 1)))
|
||||
|
||||
(defn asset-name->title
|
||||
"Create asset title given asset path's basename"
|
||||
[path-basename]
|
||||
(.-name (node-path/parse path-basename)))
|
||||
118
deps/db/src/logseq/db/frontend/block_title.cljs
vendored
Normal file
118
deps/db/src/logseq/db/frontend/block_title.cljs
vendored
Normal file
@@ -0,0 +1,118 @@
|
||||
(ns logseq.db.frontend.block-title
|
||||
"Shared block title formatting for DB graph entities."
|
||||
(:require [clojure.string :as string]
|
||||
[datascript.core :as d]
|
||||
[datascript.impl.entity :as de]
|
||||
[logseq.common.util.namespace :as ns-util]
|
||||
[logseq.db.frontend.class :as db-class]
|
||||
[logseq.db.frontend.db :as db-db]
|
||||
[logseq.db.frontend.entity-util :as entity-util]))
|
||||
|
||||
(defn- resolve-entity
|
||||
[db lookup-ref]
|
||||
(when db
|
||||
(d/entity db lookup-ref)))
|
||||
|
||||
(defn- needs-resolved-block?
|
||||
[block]
|
||||
(or (nil? (:block/title block))
|
||||
(entity-util/class? block)
|
||||
(some number? (:block/tags block))))
|
||||
|
||||
(defn- resolve-block
|
||||
[db block]
|
||||
(cond
|
||||
(de/entity? block)
|
||||
block
|
||||
|
||||
(and (needs-resolved-block? block)
|
||||
(number? (:db/id block)))
|
||||
(or (resolve-entity db (:db/id block)) block)
|
||||
|
||||
(and (needs-resolved-block? block)
|
||||
(uuid? (:block/uuid block)))
|
||||
(or (resolve-entity db [:block/uuid (:block/uuid block)]) block)
|
||||
|
||||
:else
|
||||
block))
|
||||
|
||||
(defn- class-title-conflicts?
|
||||
[db class-entity]
|
||||
(let [class-title (:block/title class-entity)
|
||||
class-id (:db/id class-entity)]
|
||||
(when (and db class-title class-id)
|
||||
(boolean
|
||||
(d/q '[:find ?other .
|
||||
:in $ ?class-title ?class-id
|
||||
:where
|
||||
[?other :block/title ?class-title]
|
||||
[?other :block/tags :logseq.class/Tag]
|
||||
[(not= ?other ?class-id)]
|
||||
(not [?other :logseq.property/deleted-at])]
|
||||
db class-title class-id)))))
|
||||
|
||||
(defn- class-title-with-extends
|
||||
[class-entity title]
|
||||
(let [extends (some->> (:logseq.property.class/extends class-entity)
|
||||
(remove (fn [extend]
|
||||
(or (:logseq.property/built-in? extend)
|
||||
(= (:block/title class-entity) (:block/title extend)))))
|
||||
vec)]
|
||||
(if (seq extends)
|
||||
(str (if (= 1 (count extends))
|
||||
(:block/title (first extends))
|
||||
(->> (take 2 extends)
|
||||
(map :block/title)
|
||||
(string/join " | ")))
|
||||
ns-util/parent-char
|
||||
title)
|
||||
title)))
|
||||
|
||||
(defn- resolve-tag
|
||||
[db tag]
|
||||
(if (number? tag)
|
||||
(resolve-entity db tag)
|
||||
tag))
|
||||
|
||||
(defn block-unique-title
|
||||
"Return a display title that disambiguates duplicate class titles and appends
|
||||
user-visible tags.
|
||||
|
||||
`title` may be supplied when a caller has already prepared a display title,
|
||||
such as a search snippet with highlight markers."
|
||||
[db block {:keys [with-tags? alias truncate? title]
|
||||
:or {with-tags? true
|
||||
truncate? true}}]
|
||||
(let [block-e (resolve-block db block)]
|
||||
(if (entity-util/built-in? block-e)
|
||||
(:block/title block-e)
|
||||
(let [class? (entity-util/class? block-e)
|
||||
tags (when (and with-tags? (not class?))
|
||||
(remove (fn [tag]
|
||||
(or (some-> (:block/raw-title block-e) (db-db/inline-tag? tag))
|
||||
(db-class/private-tags (:db/ident tag))))
|
||||
(map #(resolve-tag db %) (or (:block/tags block)
|
||||
(:block/tags block-e)))))
|
||||
base-title (if class?
|
||||
(let [display-title (or title (:block/title block-e))]
|
||||
(if (class-title-conflicts? db block-e)
|
||||
(class-title-with-extends block-e display-title)
|
||||
display-title))
|
||||
(or title (:block/title block-e)))
|
||||
trunc-title (if (and truncate? base-title (> (count base-title) 256))
|
||||
(subs base-title 0 256)
|
||||
base-title)
|
||||
title (if (seq tags)
|
||||
(str (or trunc-title "")
|
||||
" "
|
||||
(string/join
|
||||
", "
|
||||
(keep (fn [tag]
|
||||
(when-let [title (:block/title tag)]
|
||||
(str "#" title)))
|
||||
tags)))
|
||||
trunc-title)]
|
||||
(when title
|
||||
(str title
|
||||
(when alias
|
||||
(str " -> alias: " alias))))))))
|
||||
234
deps/db/src/logseq/db/frontend/class.cljs
vendored
Normal file
234
deps/db/src/logseq/db/frontend/class.cljs
vendored
Normal file
@@ -0,0 +1,234 @@
|
||||
(ns logseq.db.frontend.class
|
||||
"Class related fns for DB graphs and frontend/datascript usage"
|
||||
(:require [clojure.set :as set]
|
||||
[clojure.string :as string]
|
||||
[datascript.core :as d]
|
||||
[datascript.impl.entity :as de]
|
||||
[flatland.ordered.map :refer [ordered-map]]
|
||||
[logseq.common.defkeywords :refer [defkeywords]]
|
||||
[logseq.db.frontend.db-ident :as db-ident]
|
||||
[logseq.db.frontend.entity-util :as entity-util]
|
||||
[logseq.db.frontend.rules :as rules]
|
||||
[logseq.db.sqlite.util :as sqlite-util]))
|
||||
|
||||
;; Main class vars
|
||||
;; ===============
|
||||
|
||||
(def ^:large-vars/data-var built-in-classes
|
||||
"Map of built-in classes for db graphs with their :db/ident as keys"
|
||||
(apply
|
||||
ordered-map
|
||||
(defkeywords
|
||||
:logseq.class/Root {:title "Root Tag"}
|
||||
|
||||
:logseq.class/Tag {:title "Tag"}
|
||||
|
||||
:logseq.class/Property {:title "Property"}
|
||||
|
||||
:logseq.class/Page {:title "Page"}
|
||||
|
||||
:logseq.class/Journal
|
||||
{:title "Journal"
|
||||
:properties {:logseq.property.class/extends :logseq.class/Page
|
||||
:logseq.property.journal/title-format "MMM do, yyyy"}}
|
||||
|
||||
;; TODO: Remove deprecated
|
||||
:logseq.class/Whiteboard
|
||||
{:title "Whiteboard"
|
||||
:properties {:logseq.property.class/extends :logseq.class/Page}}
|
||||
|
||||
:logseq.class/Task
|
||||
{:title "Task"
|
||||
:schema {:properties [:logseq.property/status :logseq.property/priority :logseq.property/deadline :logseq.property/scheduled]}}
|
||||
|
||||
:logseq.class/Comments
|
||||
{:title "Comments"
|
||||
:properties {:logseq.property.class/hide-from-node true
|
||||
:logseq.property/icon {:type :tabler-icon, :id "message-circle"}}
|
||||
:schema {:properties [:logseq.property.comments/blocks]}}
|
||||
|
||||
:logseq.class/Comment
|
||||
{:title "Comment"
|
||||
:properties {:logseq.property.class/hide-from-node true}}
|
||||
|
||||
:logseq.class/Query
|
||||
{:title "Query"
|
||||
:properties {:logseq.property/icon {:type :tabler-icon :id "search"}}
|
||||
:schema {:properties [:logseq.property/query]}}
|
||||
|
||||
:logseq.class/Card
|
||||
{:title "Card"
|
||||
:schema {:properties [:logseq.property.fsrs/state :logseq.property.fsrs/due]}}
|
||||
|
||||
:logseq.class/Cards
|
||||
{:title "Cards"
|
||||
:properties {:logseq.property/icon {:type :tabler-icon :id "search"}
|
||||
:logseq.property.class/extends :logseq.class/Query}}
|
||||
|
||||
:logseq.class/Asset
|
||||
{:title "Asset"
|
||||
:properties {;; :logseq.property/icon {:type :tabler-icon :id "file"}
|
||||
:logseq.property.class/hide-from-node true
|
||||
:logseq.property.view/type :logseq.property.view/type.gallery}
|
||||
:schema {:properties [:logseq.property.asset/type :logseq.property.asset/size :logseq.property.asset/checksum]
|
||||
:required-properties [:logseq.property.asset/type :logseq.property.asset/size :logseq.property.asset/checksum]}}
|
||||
|
||||
:logseq.class/Code-block
|
||||
{:title "Code"
|
||||
:properties {:logseq.property.class/hide-from-node true}
|
||||
:schema {:properties [:logseq.property.node/display-type :logseq.property.code/lang]}}
|
||||
|
||||
:logseq.class/Quote-block
|
||||
{:title "Quote"
|
||||
:properties {:logseq.property.class/hide-from-node true}
|
||||
:schema {:properties [:logseq.property.node/display-type]}}
|
||||
|
||||
:logseq.class/Math-block
|
||||
{:title "Math"
|
||||
:properties {:logseq.property.class/hide-from-node true}
|
||||
:schema {:properties [:logseq.property.node/display-type]}}
|
||||
|
||||
:logseq.class/Pdf-annotation
|
||||
{:title "PDF Annotation"
|
||||
:properties {:logseq.property.class/hide-from-node true}
|
||||
:schema {:properties [:logseq.property/ls-type :logseq.property.pdf/hl-color :logseq.property/asset
|
||||
:logseq.property.pdf/hl-page :logseq.property.pdf/hl-value
|
||||
:logseq.property.pdf/hl-type :logseq.property.pdf/hl-image]
|
||||
:required-properties [:logseq.property/ls-type :logseq.property.pdf/hl-color :logseq.property/asset
|
||||
:logseq.property.pdf/hl-page :logseq.property.pdf/hl-value]}}
|
||||
|
||||
:logseq.class/Template
|
||||
{:title "Template"
|
||||
:schema {:properties [:logseq.property/template-applied-to]}}
|
||||
|
||||
;; TODO: Add more classes such as :book, :paper, :movie, :music, :project)
|
||||
)))
|
||||
|
||||
(def page-children-classes
|
||||
"Children of :logseq.class/Page"
|
||||
(set
|
||||
(keep (fn [[class-ident m]]
|
||||
(when (= (get-in m [:properties :logseq.property.class/extends]) :logseq.class/Page) class-ident))
|
||||
built-in-classes)))
|
||||
|
||||
(def page-classes
|
||||
"Built-in classes that behave like a page. Classes should match entity-util/page?"
|
||||
(into #{:logseq.class/Page :logseq.class/Tag :logseq.class/Property}
|
||||
page-children-classes))
|
||||
|
||||
(def internal-tags
|
||||
"Built-in classes that are hidden on a node and all pages view"
|
||||
#{:logseq.class/Page :logseq.class/Property :logseq.class/Tag :logseq.class/Root
|
||||
:logseq.class/Asset})
|
||||
|
||||
(def private-tags
|
||||
"Built-in classes that are private and should not be used by a user directly."
|
||||
(set/union (disj internal-tags :logseq.class/Root)
|
||||
#{:logseq.class/Journal :logseq.class/Whiteboard
|
||||
:logseq.class/Pdf-annotation}))
|
||||
|
||||
(def block-kind-tags
|
||||
#{:logseq.class/Cards :logseq.class/Code-block
|
||||
:logseq.class/Math-block :logseq.class/Quote-block
|
||||
:logseq.class/Query :logseq.class/Pdf-annotation
|
||||
:logseq.class/Template})
|
||||
|
||||
(def disallowed-inline-tags
|
||||
"Classes that should be removed from inline tags"
|
||||
(set/union page-classes
|
||||
private-tags
|
||||
block-kind-tags))
|
||||
|
||||
(def extends-hidden-tags
|
||||
"Built-in classes that are hidden when choosing extends"
|
||||
(set/union
|
||||
private-tags
|
||||
block-kind-tags))
|
||||
|
||||
(def hidden-tags
|
||||
"Built-in classes that are hidden in a few contexts like property values"
|
||||
#{:logseq.class/Page :logseq.class/Root :logseq.class/Asset})
|
||||
|
||||
;; Helper fns
|
||||
;; ==========
|
||||
(defn get-structured-children
|
||||
"Returns all children of a class"
|
||||
[db eid]
|
||||
(->>
|
||||
(d/q '[:find [?c ...]
|
||||
:in $ ?p %
|
||||
:where
|
||||
(class-extends ?p ?c)]
|
||||
db
|
||||
eid
|
||||
(:class-extends rules/rules))
|
||||
(remove #{eid})))
|
||||
|
||||
(defn get-class-extends
|
||||
"Returns all extends of a class"
|
||||
[class]
|
||||
(assert (de/entity? class) "get-class-extends `class` should be an entity")
|
||||
(loop [extends (:logseq.property.class/extends class)
|
||||
result []]
|
||||
(if (seq extends)
|
||||
(recur (mapcat :logseq.property.class/extends extends)
|
||||
(into result extends))
|
||||
(reverse (distinct result)))))
|
||||
|
||||
(defn create-user-class-ident-from-name
|
||||
"Creates a class :db/ident for a default user namespace.
|
||||
NOTE: Only use this when creating a db-ident for a new class."
|
||||
[db class-name & {:keys [ident-namespace]}]
|
||||
(let [ident-ns (or ident-namespace "user.class")
|
||||
db-ident (db-ident/create-db-ident-from-name ident-ns class-name)]
|
||||
(if db
|
||||
(db-ident/ensure-unique-db-ident db db-ident)
|
||||
db-ident)))
|
||||
|
||||
(defn build-new-class
|
||||
"Builds a new class with a unique :db/ident. Also throws exception for user
|
||||
facing messages when name is invalid"
|
||||
[db page-m & {:as option}]
|
||||
{:pre [(string? (:block/title page-m))]}
|
||||
(let [db-ident (create-user-class-ident-from-name db (:block/title page-m) option)]
|
||||
(sqlite-util/build-new-class (assoc page-m :db/ident db-ident))))
|
||||
|
||||
(defonce logseq-class "logseq.class")
|
||||
|
||||
(defn logseq-class?
|
||||
"Determines if keyword is a logseq class"
|
||||
[kw]
|
||||
(= logseq-class (namespace kw)))
|
||||
|
||||
(defn user-class-namespace?
|
||||
"Determines if namespace string is a user class"
|
||||
[s]
|
||||
(string/includes? s ".class"))
|
||||
|
||||
(defn- hidden-class-object?
|
||||
[e]
|
||||
(if (entity-util/property? e)
|
||||
(or (:logseq.property/deleted-at e)
|
||||
(and (entity-util/built-in? e)
|
||||
(not (:logseq.property/public? e))))
|
||||
(entity-util/hidden? e)))
|
||||
|
||||
(defn get-class-objects
|
||||
"Get class objects including children classes'"
|
||||
[db class-id]
|
||||
(let [class-children (get-structured-children db class-id)
|
||||
class-ids (distinct (conj class-children class-id))]
|
||||
(->> class-ids
|
||||
(mapcat (fn [id] (d/datoms db :avet :block/tags id)))
|
||||
(reduce (fn [[seen result] d]
|
||||
(let [eid (:e d)]
|
||||
(if (contains? seen eid)
|
||||
[seen result]
|
||||
(let [e (d/entity db eid)
|
||||
seen' (conj seen eid)]
|
||||
(if (hidden-class-object? e)
|
||||
[seen' result]
|
||||
[seen' (conj! result e)])))))
|
||||
[#{} (transient [])])
|
||||
second
|
||||
persistent!)))
|
||||
265
deps/db/src/logseq/db/frontend/content.cljs
vendored
Normal file
265
deps/db/src/logseq/db/frontend/content.cljs
vendored
Normal file
@@ -0,0 +1,265 @@
|
||||
(ns logseq.db.frontend.content
|
||||
"Fns to handle block content e.g. internal ids"
|
||||
(:require [clojure.string :as string]
|
||||
[datascript.core :as d]
|
||||
[logseq.common.util :as common-util]
|
||||
[logseq.common.util.page-ref :as page-ref]
|
||||
[logseq.db.frontend.entity-util :as entity-util]))
|
||||
|
||||
;; [[uuid]]
|
||||
(def id-ref-pattern
|
||||
(re-pattern
|
||||
(str
|
||||
"\\[\\["
|
||||
"("
|
||||
common-util/uuid-pattern
|
||||
")"
|
||||
"\\]\\]")))
|
||||
|
||||
(def ^:private id-or-tag-ref-pattern
|
||||
(re-pattern
|
||||
(str
|
||||
"(#?)"
|
||||
"\\[\\["
|
||||
"("
|
||||
common-util/uuid-pattern
|
||||
")"
|
||||
"\\]\\]")))
|
||||
|
||||
(defn content-id-ref->page
|
||||
"Convert id ref backs to page name using refs."
|
||||
[content refs]
|
||||
(reduce
|
||||
(fn [content ref]
|
||||
(if (:block/title ref)
|
||||
(string/replace content (page-ref/->page-ref (:block/uuid ref)) (:block/title ref))
|
||||
content))
|
||||
content
|
||||
refs))
|
||||
|
||||
(defn- sort-refs
|
||||
"Nested pages first"
|
||||
[refs]
|
||||
(sort-by
|
||||
(fn [ref]
|
||||
(when-let [title (and (map? ref) (:block/title ref))]
|
||||
[(boolean (re-find page-ref/page-ref-without-nested-re (:block/title ref)))
|
||||
title]))
|
||||
>
|
||||
refs))
|
||||
|
||||
(defn id-ref->title-ref
|
||||
"Convert id ref backs to page name refs using refs."
|
||||
[content* refs* & {:keys [db replace-block-id? replace-pages-with-same-name?]
|
||||
:or {replace-block-id? false
|
||||
replace-pages-with-same-name? true}}]
|
||||
(when (string? content*)
|
||||
(let [refs (if replace-block-id?
|
||||
;; The caller need to handle situations including
|
||||
;; mutual references and circle references.
|
||||
refs*
|
||||
(cond->> (filter entity-util/page? refs*)
|
||||
(and db (false? replace-pages-with-same-name?))
|
||||
(remove (fn [e]
|
||||
(> (count (entity-util/get-pages-by-name db (:block/title e))) 1)))))
|
||||
content (str content*)]
|
||||
(if (re-find id-ref-pattern content)
|
||||
(reduce
|
||||
(fn [content ref]
|
||||
(if (:block/title ref)
|
||||
(let [content' (if (not (string/includes? (:block/title ref) " "))
|
||||
(string/replace content
|
||||
(str "#" (page-ref/->page-ref (:block/uuid ref)))
|
||||
(str "#" (:block/title ref)))
|
||||
content)]
|
||||
(string/replace content' (page-ref/->page-ref (:block/uuid ref))
|
||||
(page-ref/->page-ref (:block/title ref))))
|
||||
content))
|
||||
content
|
||||
(sort-refs refs))
|
||||
content))))
|
||||
|
||||
(defn get-matched-ids
|
||||
[content]
|
||||
(->> (re-seq id-ref-pattern content)
|
||||
(distinct)
|
||||
(map second)
|
||||
(map uuid)))
|
||||
|
||||
(defn- replace-tag-ref
|
||||
[content page-name id]
|
||||
(let [page (if (string/includes? page-name " ") (page-ref/->page-ref page-name) page-name)
|
||||
wrapped-id (page-ref/->page-ref id)
|
||||
page-name (common-util/format "#%s" page)
|
||||
r (common-util/format "#%s" wrapped-id)]
|
||||
;; hash tag parsing rules https://github.com/logseq/mldoc/blob/701243eaf9b4157348f235670718f6ad19ebe7f8/test/test_markdown.ml#L631
|
||||
;; Safari doesn't support look behind, don't use
|
||||
;; TODO: parse via mldoc
|
||||
(string/replace content
|
||||
(re-pattern (str "(?i)(^|\\s|\\()(" (common-util/escape-regex-chars page-name) ")(?=[,\\.\\)]*($|\\s|\\)))"))
|
||||
;; case_insense^ ^lhs ^_grp2 look_ahead^ ^_grp3
|
||||
(fn [[_match lhs _grp2 _grp3]]
|
||||
(str lhs r)))))
|
||||
|
||||
(defn- replace-page-ref
|
||||
[content page-name id]
|
||||
(let [[page wrapped-id] (map page-ref/->page-ref [page-name id])]
|
||||
(string/replace content
|
||||
;; Don't replace #[[]] as that is a tag and is handled separately in replace-tag-ref
|
||||
(re-pattern (str "(?i)" "(^|[^#])"
|
||||
(common-util/escape-regex-chars page))) (str "$1" wrapped-id))))
|
||||
|
||||
(defn- replace-page-ref-with-id
|
||||
[content page-name id replace-tag?]
|
||||
(let [page-name (-> (str page-name)
|
||||
(string/replace "HashTag-" "#"))
|
||||
id (str id)
|
||||
content' (replace-page-ref content page-name id)]
|
||||
(if replace-tag?
|
||||
(replace-tag-ref content' page-name id)
|
||||
content')))
|
||||
|
||||
(defn- ref-replacement-title
|
||||
[{:block/keys [title] original-page-name :block.temp/original-page-name}]
|
||||
(or (when (and (string? original-page-name)
|
||||
(not (common-util/uuid-string? original-page-name)))
|
||||
original-page-name)
|
||||
title))
|
||||
|
||||
(defn title-ref->id-ref
|
||||
"Convert ref to id refs e.g. `[[page name]] -> [[uuid]]."
|
||||
[title refs & {:keys [replace-tag?]
|
||||
:or {replace-tag? true}}]
|
||||
(assert (string? title))
|
||||
(let [refs' (->> refs
|
||||
(map (fn [ref]
|
||||
;; remove uuid references since they're introduced to detect multiple pages
|
||||
;; that have the same name
|
||||
(if (and (map? ref) (some-> (:block.temp/original-page-name ref) common-util/uuid-string?))
|
||||
(dissoc ref :block.temp/original-page-name)
|
||||
ref)))
|
||||
(map
|
||||
(fn [ref]
|
||||
(if (and (vector? ref) (= :block/uuid (first ref)))
|
||||
{:block/uuid (second ref)
|
||||
:block/title (str (first ref))}
|
||||
ref)))
|
||||
sort-refs)]
|
||||
(reduce
|
||||
(fn [content {uuid' :block/uuid :as ref}]
|
||||
(replace-page-ref-with-id content (ref-replacement-title ref) uuid' replace-tag?))
|
||||
title
|
||||
(filter #(and (:block/title %) (:block/uuid %)) refs'))))
|
||||
|
||||
(defn update-block-content
|
||||
"Replace `[[internal-id]]` with `[[page name]]`"
|
||||
[db item eid]
|
||||
(if-let [content (:block/title item)]
|
||||
(let [refs (:block/refs (d/entity db eid))]
|
||||
(assoc item :block/title (id-ref->title-ref content refs)))
|
||||
item))
|
||||
|
||||
(defn replace-tags-with-id-refs
|
||||
"Replace tag names in content with page-ref ids e.g. #TAG -> [[UUID]].
|
||||
Ignore case because tags in content can have any case and still have a valid ref"
|
||||
[content tags]
|
||||
(->>
|
||||
(reduce
|
||||
(fn [content tag]
|
||||
(let [id-ref (page-ref/->page-ref (:block/uuid tag))]
|
||||
(-> content
|
||||
;; #[[favorite book]]
|
||||
(common-util/replace-ignore-case
|
||||
(str "#" (page-ref/->page-ref (:block/title tag)))
|
||||
id-ref)
|
||||
;; #book
|
||||
(common-util/replace-ignore-case (str "#" (:block/title tag)) id-ref))))
|
||||
content
|
||||
(sort-refs tags))
|
||||
(string/trim)))
|
||||
|
||||
(defn replace-tag-refs-with-page-refs
|
||||
"Replace tag refs in content with page refs e.g. #[[UUID]] -> [[UUID]]"
|
||||
[content tags]
|
||||
(->>
|
||||
(reduce
|
||||
(fn [content tag]
|
||||
(let [id-ref (page-ref/->page-ref (:block/uuid tag))]
|
||||
(-> content
|
||||
;; #[[favorite book]]
|
||||
(common-util/replace-ignore-case
|
||||
(str "#" id-ref)
|
||||
id-ref)
|
||||
;; #book
|
||||
(common-util/replace-ignore-case (str "#" id-ref) id-ref))))
|
||||
content
|
||||
(sort-refs tags))
|
||||
(string/trim)))
|
||||
|
||||
(defn- title-ref-replacement
|
||||
[id->title matched hash-prefix id]
|
||||
(if-let [ref-title (get id->title id)]
|
||||
(if (and (= "#" hash-prefix)
|
||||
(not (string/includes? ref-title " ")))
|
||||
(str "#" ref-title)
|
||||
(str hash-prefix (page-ref/->page-ref ref-title)))
|
||||
matched))
|
||||
|
||||
(defn- replace-title-refs-once
|
||||
[content id->title]
|
||||
(string/replace content id-or-tag-ref-pattern
|
||||
#(apply title-ref-replacement id->title %)))
|
||||
|
||||
(defn- ref->title-entry
|
||||
[replace-block-refs? {:block/keys [title]
|
||||
block-uuid :block/uuid
|
||||
:as ref}]
|
||||
(when (and block-uuid
|
||||
(string? title)
|
||||
(or replace-block-refs?
|
||||
(entity-util/page? ref)))
|
||||
[(str block-uuid) title]))
|
||||
|
||||
(defn- block-ref-id->title
|
||||
[ent max-depth replace-block-refs?]
|
||||
(loop [frontier (set (:block/refs ent))
|
||||
seen-ids #{}
|
||||
id->title {}
|
||||
depth 0]
|
||||
(if (or (>= depth max-depth)
|
||||
(empty? frontier))
|
||||
id->title
|
||||
(let [refs (filter map? frontier)
|
||||
new-refs (remove (fn [ref]
|
||||
(contains? seen-ids (:block/uuid ref)))
|
||||
refs)
|
||||
seen-ids' (into seen-ids (keep :block/uuid) new-refs)
|
||||
id->title' (into id->title
|
||||
(keep #(ref->title-entry replace-block-refs? %))
|
||||
new-refs)
|
||||
next-frontier (->> new-refs
|
||||
(mapcat :block/refs)
|
||||
(filter map?)
|
||||
set)]
|
||||
(recur next-frontier seen-ids' id->title' (inc depth))))))
|
||||
|
||||
(defn recur-replace-uuid-in-block-title
|
||||
"Convert id ref (recursively) backs to page name refs, returns replaced title"
|
||||
([ent]
|
||||
(recur-replace-uuid-in-block-title ent 10))
|
||||
([ent max-depth]
|
||||
(recur-replace-uuid-in-block-title ent max-depth {}))
|
||||
([ent max-depth {:keys [replace-block-refs?]
|
||||
:or {replace-block-refs? true}}]
|
||||
(let [title (:block/title ent)]
|
||||
(if (some->> title (re-find id-ref-pattern))
|
||||
(let [id->title (block-ref-id->title ent max-depth replace-block-refs?)]
|
||||
(loop [result title depth 0]
|
||||
(if (or (>= depth max-depth)
|
||||
(not (re-find id-ref-pattern result)))
|
||||
result
|
||||
(let [next-result (replace-title-refs-once result id->title)]
|
||||
(if (= result next-result)
|
||||
result
|
||||
(recur next-result (inc depth)))))))
|
||||
title))))
|
||||
35
deps/db/src/logseq/db/frontend/datalog.cljc
vendored
Normal file
35
deps/db/src/logseq/db/frontend/datalog.cljc
vendored
Normal file
@@ -0,0 +1,35 @@
|
||||
(ns logseq.db.frontend.datalog
|
||||
"Utility fns related to datalog queries and rules")
|
||||
|
||||
(defn find-rules-in-where
|
||||
"Given where clauses and a set of valid rules, returns rules found in where
|
||||
clause as keywords. A more advanced version of this would use a datalog parser
|
||||
and not require valid-rules"
|
||||
[where valid-rules]
|
||||
(->> where
|
||||
flatten
|
||||
distinct
|
||||
(filter #(and (symbol? %) (contains? valid-rules (keyword %))))
|
||||
(map keyword)))
|
||||
|
||||
(defn query-vec->map
|
||||
"Converts query vec to query map. Modified version of
|
||||
datascript.parser/query->map which preserves insertion order in case map is
|
||||
converted back to vec"
|
||||
[query-vec]
|
||||
(loop [parsed (array-map) key nil qs query-vec]
|
||||
(if-let [q (first qs)]
|
||||
(if (keyword? q)
|
||||
(recur parsed q (next qs))
|
||||
(recur (update-in parsed [key] (fnil conj []) q) key (next qs)))
|
||||
parsed)))
|
||||
|
||||
(defn add-to-end-of-query-section
|
||||
"Adds vec of elements to end of a query section e.g. :find or :in"
|
||||
[query-vec query-kw elems]
|
||||
(let [query-map (query-vec->map query-vec)]
|
||||
(vec
|
||||
(reduce (fn [acc [k v]]
|
||||
(concat acc [k] v (when (= k query-kw) elems)))
|
||||
'()
|
||||
query-map))))
|
||||
141
deps/db/src/logseq/db/frontend/db.cljs
vendored
Normal file
141
deps/db/src/logseq/db/frontend/db.cljs
vendored
Normal file
@@ -0,0 +1,141 @@
|
||||
(ns logseq.db.frontend.db
|
||||
"DB graph fns commonly used outside db dep"
|
||||
(:require [clojure.set :as set]
|
||||
[clojure.string :as string]
|
||||
[datascript.core :as d]
|
||||
[logseq.common.config :as common-config]
|
||||
[logseq.common.util.namespace :as ns-util]
|
||||
[logseq.common.util.page-ref :as page-ref]
|
||||
[logseq.common.uuid :as common-uuid]
|
||||
[logseq.db.frontend.class :as db-class]
|
||||
[logseq.db.frontend.entity-util :as entity-util]
|
||||
[logseq.db.frontend.property :as db-property]))
|
||||
|
||||
(defn built-in-class-property?
|
||||
"Whether property a built-in property for the specific class"
|
||||
[class-entity property-entity]
|
||||
(and (entity-util/built-in? class-entity)
|
||||
(entity-util/class? class-entity)
|
||||
(entity-util/built-in? property-entity)
|
||||
(contains? (set (get-in (db-class/built-in-classes (:db/ident class-entity)) [:schema :properties]))
|
||||
(:db/ident property-entity))))
|
||||
|
||||
(defn private-built-in-page?
|
||||
"Private built-in pages should not be navigable, searchable or editable by users. Later it
|
||||
could be useful to use this for the All Pages view"
|
||||
[page]
|
||||
(cond (entity-util/property? page)
|
||||
(not (db-property/public-built-in-property? page))
|
||||
(or (entity-util/class? page) (entity-util/internal-page? page))
|
||||
false
|
||||
;; Default to true for closed value and future internal types.
|
||||
;; Other types are not considered because they aren't built-in
|
||||
:else
|
||||
true))
|
||||
|
||||
(defn build-favorite-tx
|
||||
"Builds tx for a favorite block in favorite page"
|
||||
[favorite-uuid]
|
||||
{:block/link [:block/uuid favorite-uuid]
|
||||
:block/title ""})
|
||||
|
||||
(defn get-all-properties
|
||||
[db]
|
||||
(->> (d/datoms db :avet :block/tags :logseq.class/Property)
|
||||
(map (fn [d] (d/entity db (:e d))))))
|
||||
|
||||
(defn get-page-parents
|
||||
[node]
|
||||
(when-let [parent (:block/parent node)]
|
||||
(loop [current-parent parent
|
||||
parents' []]
|
||||
(if (and current-parent
|
||||
(not (contains? parents' current-parent)))
|
||||
(recur (:block/parent current-parent)
|
||||
(conj parents' current-parent))
|
||||
(vec (reverse parents'))))))
|
||||
|
||||
(defn get-class-title-with-extends
|
||||
[entity]
|
||||
(let [extends (some->> (:logseq.property.class/extends entity)
|
||||
(remove (fn [extend]
|
||||
(or (:logseq.property/built-in? extend)
|
||||
(= (:block/title entity) (:block/title extend)))))
|
||||
vec)]
|
||||
(if (seq extends)
|
||||
(str (if (= 1 (count extends))
|
||||
(:block/title (first extends))
|
||||
(->> (take 2 extends)
|
||||
(map :block/title)
|
||||
(string/join " | ")))
|
||||
ns-util/parent-char (:block/title entity))
|
||||
(:block/title entity))))
|
||||
|
||||
(defn get-title-with-parents
|
||||
[entity]
|
||||
(cond
|
||||
(entity-util/class? entity)
|
||||
(get-class-title-with-extends entity)
|
||||
|
||||
(entity-util/page? entity)
|
||||
(let [parents' (->> (get-page-parents entity)
|
||||
(remove (fn [e]
|
||||
(and (:logseq.property/built-in? e) (= common-config/library-page-name (:block/title e))))))]
|
||||
(string/join
|
||||
ns-util/parent-char
|
||||
(map :block/title (conj (vec parents') entity))))
|
||||
|
||||
:else
|
||||
(:block/title entity)))
|
||||
|
||||
(defn get-classes-parents
|
||||
"Returns all parents of all classes. Like get-class-extends but for multiple classes"
|
||||
[tags]
|
||||
(let [tags' (filter entity-util/class? tags)
|
||||
result (mapcat db-class/get-class-extends tags')]
|
||||
(set result)))
|
||||
|
||||
(defn class-instance?
|
||||
"Whether `object` is an instance of `class`"
|
||||
[class object]
|
||||
(let [tags (:block/tags object)
|
||||
tags-ids (set (map :db/id tags))]
|
||||
(or
|
||||
(contains? tags-ids (:db/id class))
|
||||
(let [class-parent-ids (set (map :db/id (get-classes-parents tags)))]
|
||||
(contains? (set/union class-parent-ids tags-ids) (:db/id class))))))
|
||||
|
||||
(defn inline-tag?
|
||||
[block-raw-title tag]
|
||||
(assert (string? block-raw-title) "block-raw-title should be a string")
|
||||
(string/includes? block-raw-title (str "#" (page-ref/->page-ref (:block/uuid tag)))))
|
||||
|
||||
(defonce node-display-type-classes
|
||||
#{:logseq.class/Code-block :logseq.class/Math-block :logseq.class/Quote-block})
|
||||
|
||||
(defn get-class-ident-by-display-type
|
||||
[display-type]
|
||||
(case display-type
|
||||
:code :logseq.class/Code-block
|
||||
:math :logseq.class/Math-block
|
||||
:quote :logseq.class/Quote-block
|
||||
nil))
|
||||
|
||||
(defn get-display-type-by-class-ident
|
||||
[class-ident]
|
||||
(case class-ident
|
||||
:logseq.class/Code-block :code
|
||||
:logseq.class/Math-block :math
|
||||
:logseq.class/Quote-block :quote
|
||||
nil))
|
||||
|
||||
(defn get-built-in-page
|
||||
[db title]
|
||||
(when db
|
||||
(let [id (common-uuid/gen-uuid :builtin-block-uuid title)]
|
||||
(d/entity db [:block/uuid id]))))
|
||||
|
||||
(defn library?
|
||||
[page]
|
||||
(and (entity-util/built-in? page)
|
||||
(= common-config/library-page-name (:block/title page))))
|
||||
97
deps/db/src/logseq/db/frontend/db_ident.cljc
vendored
Normal file
97
deps/db/src/logseq/db/frontend/db_ident.cljc
vendored
Normal file
@@ -0,0 +1,97 @@
|
||||
(ns logseq.db.frontend.db-ident
|
||||
"Helper fns for class and property :db/ident"
|
||||
(:require [clojure.string :as string]
|
||||
[datascript.core :as d]))
|
||||
|
||||
(defn ensure-unique-db-ident
|
||||
"Ensures the given db-ident is unique. If a db-ident conflicts, it is made
|
||||
unique by adding a suffix with a unique number e.g. :db-ident-1 :db-ident-2"
|
||||
[db db-ident]
|
||||
(if (d/entity db db-ident)
|
||||
(let [existing-idents
|
||||
(d/q '[:find [?ident ...]
|
||||
:in $ ?ident-name
|
||||
:where
|
||||
[?b :db/ident ?ident]
|
||||
[(str ?ident) ?str-ident]
|
||||
[(clojure.string/starts-with? ?str-ident ?ident-name)]]
|
||||
db
|
||||
(str db-ident "-"))
|
||||
new-ident (if-let [max-num (->> existing-idents
|
||||
(keep #(parse-long (string/replace-first (str %) (str db-ident "-") "")))
|
||||
(apply max))]
|
||||
(keyword (namespace db-ident) (str (name db-ident) "-" (inc max-num)))
|
||||
(keyword (namespace db-ident) (str (name db-ident) "-1")))]
|
||||
new-ident)
|
||||
db-ident))
|
||||
|
||||
(def ^:private non-int-char-range "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
|
||||
|
||||
(def alphabet
|
||||
(mapv str "_-0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"))
|
||||
|
||||
#_:clj-kondo/ignore
|
||||
(defn- random-bytes
|
||||
[size]
|
||||
#?(:org.babashka/nbb
|
||||
nil
|
||||
:default
|
||||
(let [seed (js/Uint8Array. size)]
|
||||
(.getRandomValues js/crypto seed)
|
||||
(array-seq seed))))
|
||||
|
||||
(defn nano-id
|
||||
"Random id generator"
|
||||
([]
|
||||
(nano-id 21))
|
||||
([size]
|
||||
(let [mask' 0x3f]
|
||||
(loop [bs (random-bytes size)
|
||||
id ""]
|
||||
(if bs
|
||||
(recur (next bs)
|
||||
(->> (first bs)
|
||||
(bit-and mask')
|
||||
alphabet
|
||||
(str id)))
|
||||
id)))))
|
||||
|
||||
(defn normalize-ident-name-part
|
||||
[name-string]
|
||||
(->> (string/replace-first name-string #"^(\d)" "NUM-$1")
|
||||
;; '-' must go last in char class
|
||||
(filter #(re-find #"[0-9a-zA-Z*+!_'?<>=-]{1}" %))
|
||||
(apply str)))
|
||||
|
||||
(defn create-db-ident-from-name
|
||||
"Creates a :db/ident for a class or property by sanitizing the given name.
|
||||
The created ident must obey clojure's rules for keywords i.e.
|
||||
be a valid symbol per https://clojure.org/reference/reader#_symbols
|
||||
|
||||
NOTE: Only use this when creating a db-ident for a new class/property. Using
|
||||
this in read-only contexts like querying can result in db-ident conflicts"
|
||||
[user-namespace name-string]
|
||||
{:pre [(or (keyword? user-namespace) (string? user-namespace)) (string? name-string)]}
|
||||
(assert (not (re-find #"^(logseq|block)(\.|$)" (name user-namespace)))
|
||||
"New ident is not allowed to use an internal namespace")
|
||||
(if #?(:org.babashka/nbb true
|
||||
;; Use $LOGSEQ_STABLE_IDENTS when we want stable idents e.g. tests
|
||||
:cljs (and (exists? js/process) js/process.env.LOGSEQ_STABLE_IDENTS)
|
||||
:default false)
|
||||
(keyword user-namespace (normalize-ident-name-part name-string))
|
||||
(let [plugin? (string/starts-with? user-namespace "plugin.class.")
|
||||
suffix (str "-"
|
||||
(rand-nth non-int-char-range)
|
||||
(nano-id 7))]
|
||||
(keyword user-namespace
|
||||
(str
|
||||
(normalize-ident-name-part name-string)
|
||||
(when-not plugin?
|
||||
suffix))))))
|
||||
|
||||
(defn replace-db-ident-random-suffix
|
||||
[db-ident-kw new-suffix]
|
||||
(assert (keyword? db-ident-kw))
|
||||
(assert (and (string? new-suffix) (= 8 (count new-suffix))))
|
||||
(keyword (namespace db-ident-kw)
|
||||
(string/replace-first (name db-ident-kw) #"-.{8}$" (str "-" new-suffix))))
|
||||
115
deps/db/src/logseq/db/frontend/entity_util.cljs
vendored
Normal file
115
deps/db/src/logseq/db/frontend/entity_util.cljs
vendored
Normal file
@@ -0,0 +1,115 @@
|
||||
(ns logseq.db.frontend.entity-util
|
||||
"Lower level entity util fns for DB graphs"
|
||||
(:require [clojure.string :as string]
|
||||
[datascript.core :as d]
|
||||
[datascript.db]
|
||||
[datascript.impl.entity :as de]
|
||||
[logseq.common.util :as common-util])
|
||||
(:refer-clojure :exclude [object?]))
|
||||
|
||||
(defn- has-tag?
|
||||
[entity tag-ident]
|
||||
(when (or (map? entity) (de/entity? entity))
|
||||
(some (fn [t]
|
||||
(or (keyword-identical? (:db/ident t) tag-ident)
|
||||
(keyword-identical? t tag-ident)))
|
||||
(:block/tags entity))))
|
||||
|
||||
(comment
|
||||
(require '[logseq.common.profile :as c.p])
|
||||
(do (vreset! c.p/*key->call-count {})
|
||||
(vreset! c.p/*key->time-sum {}))
|
||||
(c.p/profile-fn! has-tag? :print-on-call? false))
|
||||
|
||||
(defn internal-page?
|
||||
[entity]
|
||||
(has-tag? entity :logseq.class/Page))
|
||||
|
||||
(defn class?
|
||||
[entity]
|
||||
(has-tag? entity :logseq.class/Tag))
|
||||
|
||||
(defn property?
|
||||
[entity]
|
||||
(has-tag? entity :logseq.class/Property))
|
||||
|
||||
(defn closed-value?
|
||||
[entity]
|
||||
(some? (:block/closed-value-property entity)))
|
||||
|
||||
(defn journal?
|
||||
"Given a page entity or map, check if it is a journal page"
|
||||
[entity]
|
||||
(has-tag? entity :logseq.class/Journal))
|
||||
|
||||
(defn page?
|
||||
[entity]
|
||||
(or (internal-page? entity)
|
||||
(journal? entity)
|
||||
(class? entity)
|
||||
(property? entity)))
|
||||
|
||||
(defn asset?
|
||||
"Given an entity or map, check if it is an asset block"
|
||||
[entity]
|
||||
;; Can't use :block/tags because this is used in some perf sensitive fns like ldb/transact!
|
||||
(some? (:logseq.property.asset/type entity)))
|
||||
|
||||
(defn hidden?
|
||||
[page]
|
||||
(letfn [(hidden-parent? [entity seen]
|
||||
(when (and entity
|
||||
(:db/id entity)
|
||||
(not (contains? seen (:db/id entity))))
|
||||
(or (:logseq.property/hide? entity)
|
||||
(:logseq.property/deleted-at entity)
|
||||
(hidden-parent? (:block/parent entity) (conj seen (:db/id entity))))))]
|
||||
(boolean
|
||||
(when page
|
||||
(if (string? page)
|
||||
(string/starts-with? page "$$$")
|
||||
(when (or (map? page) (de/entity? page))
|
||||
(or (:logseq.property/hide? page)
|
||||
(:logseq.property/deleted-at page)
|
||||
(hidden-parent? (:block/parent page) #{}))))))))
|
||||
|
||||
(defn recycled?
|
||||
[entity]
|
||||
(letfn [(recycled-parent? [parent seen]
|
||||
(when (and parent
|
||||
(:db/id parent)
|
||||
(not (contains? seen (:db/id parent))))
|
||||
(or (:logseq.property/deleted-at parent)
|
||||
(recycled-parent? (:block/parent parent) (conj seen (:db/id parent))))))]
|
||||
(boolean
|
||||
(when (or (map? entity) (de/entity? entity))
|
||||
(or (:logseq.property/deleted-at entity)
|
||||
(recycled-parent? (:block/parent entity) #{}))))))
|
||||
|
||||
(defn object?
|
||||
[node]
|
||||
(seq (:block/tags node)))
|
||||
|
||||
(defn get-entity-types
|
||||
"Get entity types from :block/tags"
|
||||
[entity]
|
||||
(let [ident->type {:logseq.class/Tag :class
|
||||
:logseq.class/Property :property
|
||||
:logseq.class/Journal :journal
|
||||
:logseq.class/Page :page}]
|
||||
(set (map #(ident->type (:db/ident %)) (:block/tags entity)))))
|
||||
|
||||
(defn built-in?
|
||||
"Built-in page or block"
|
||||
[entity]
|
||||
(:logseq.property/built-in? entity))
|
||||
|
||||
(defn get-pages-by-name
|
||||
[db page-name]
|
||||
(d/datoms db :avet :block/name (common-util/page-name-sanity-lc page-name)))
|
||||
|
||||
(defn entity->map
|
||||
"Convert a db Entity to a map"
|
||||
[e]
|
||||
(assert (de/entity? e))
|
||||
(assoc (into {} e) :db/id (:db/id e)))
|
||||
162
deps/db/src/logseq/db/frontend/inputs.cljs
vendored
Normal file
162
deps/db/src/logseq/db/frontend/inputs.cljs
vendored
Normal file
@@ -0,0 +1,162 @@
|
||||
(ns logseq.db.frontend.inputs
|
||||
"Handles :inputs in queries"
|
||||
(:require [cljs-time.core :as t]
|
||||
[logseq.common.util.date-time :as date-time-util]
|
||||
[logseq.common.util :as common-util]
|
||||
[logseq.common.util.page-ref :as page-ref]
|
||||
[datascript.core :as d]
|
||||
[clojure.string :as string]))
|
||||
|
||||
(defn date-at-local-ms
|
||||
"Returns the milliseconds representation of the provided time, in the local timezone.
|
||||
For example, if you run this function at 10pm EDT in the EDT timezone on May 31st,
|
||||
it will return 1622433600000, which is equivalent to Mon May 31 2021 00 :00:00."
|
||||
([hours mins secs millisecs]
|
||||
(date-at-local-ms (.now js/Date) hours mins secs millisecs))
|
||||
([date hours mins secs millisecs]
|
||||
(.setHours (js/Date. date) hours mins secs millisecs)))
|
||||
|
||||
(defn- old->new-relative-date-format [input]
|
||||
(let [count' (re-find #"^\d+" (name input))
|
||||
plus-minus (if (re-find #"after" (name input)) "+" "-")
|
||||
ms? (string/ends-with? (name input) "-ms")]
|
||||
(keyword :today (str plus-minus count' "d" (if ms? "-ms" "")))))
|
||||
|
||||
(comment
|
||||
(old->new-relative-date-format "1d")
|
||||
(old->new-relative-date-format "1d-before")
|
||||
(old->new-relative-date-format "1d-after")
|
||||
(old->new-relative-date-format "1d-before-ms")
|
||||
(old->new-relative-date-format "1d-after-ms")
|
||||
(old->new-relative-date-format "1w-after-ms"))
|
||||
|
||||
(defn- get-relative-date [input]
|
||||
(case (or (namespace input) "today")
|
||||
"today" (t/today)))
|
||||
|
||||
(defn- get-offset-date [relative-date direction amount unit]
|
||||
(let [offset-fn (case direction "+" t/plus "-" t/minus)
|
||||
offset-amount (parse-long amount)
|
||||
offset-unit-fn (case unit
|
||||
"d" t/days
|
||||
"w" t/weeks
|
||||
"m" t/months
|
||||
"y" t/years)]
|
||||
(offset-fn (offset-fn relative-date (offset-unit-fn offset-amount)))))
|
||||
|
||||
(defn- get-ts-units
|
||||
"There are currently several time suffixes being used in inputs:
|
||||
- ms: milliseconds, will return a time relative to the direction the date is being adjusted
|
||||
- start: will return the time at the start of the day [00:00:00.000]
|
||||
- end: will return the time at the end of the day [23:59:59.999]
|
||||
- HHMM: will return the specified time at the turn of the minute [HH:MM:00.000]
|
||||
- HHMMSS: will return the specified time at the turm of the second [HH:MM:SS.000]
|
||||
- HHMMSSmmm: will return the specified time at the turn of the millisecond [HH:MM:SS.mmm]
|
||||
|
||||
The latter three will be capped to the maximum allowed for each unit so they will always be valid times"
|
||||
[offset-direction offset-time]
|
||||
(case offset-time
|
||||
"ms" (if (= offset-direction "+") [23 59 59 999] [0 0 0 0])
|
||||
"start" [0 0 0 0]
|
||||
"end" [23 59 59 999]
|
||||
;; if it's not a matching string, then assume it is HHMM
|
||||
(let [[h1 h2 m1 m2 s1 s2 ms1 ms2 ms3] (str offset-time "000000000")]
|
||||
[(min 23 (parse-long (str h1 h2)))
|
||||
(min 59 (parse-long (str m1 m2)))
|
||||
(min 59 (parse-long (str s1 s2)))
|
||||
(min 999 (parse-long (str ms1 ms2 ms3)))])))
|
||||
|
||||
(defn- keyword-input-dispatch [input]
|
||||
(cond
|
||||
(#{:current-page :query-page :current-block :parent-block :today :yesterday :tomorrow :right-now-ms} input) input
|
||||
|
||||
(re-find #"^[+-]\d+[dwmy]?$" (name input)) :relative-date
|
||||
(re-find #"^[+-]\d+[dwmy]-(ms|start|end|\d{2}|\d{4}|\d{6}|\d{9})?$" (name input)) :relative-date-time
|
||||
|
||||
(= :start-of-today-ms input) :today-time
|
||||
(= :end-of-today-ms input) :today-time
|
||||
(re-find #"^today-(start|end|\d{2}|\d{4}|\d{6}|\d{9})$" (name input)) :today-time
|
||||
|
||||
(re-find #"^\d+d(-before|-after|-before-ms|-after-ms)?$" (name input)) :DEPRECATED-relative-date))
|
||||
|
||||
(defmulti resolve-keyword-input (fn [_db input _opts] (keyword-input-dispatch input)))
|
||||
|
||||
(defmethod resolve-keyword-input :current-page [_ _ {:keys [current-page-fn]}]
|
||||
(when current-page-fn
|
||||
(some-> (current-page-fn) string/lower-case)))
|
||||
|
||||
(defmethod resolve-keyword-input :query-page [db _ {:keys [current-block-uuid]}]
|
||||
(when-let [current-block (and current-block-uuid (d/entity db [:block/uuid current-block-uuid]))]
|
||||
(get-in current-block [:block/page :block/name])))
|
||||
|
||||
(defmethod resolve-keyword-input :current-block [db _ {:keys [current-block-uuid]}]
|
||||
(when current-block-uuid
|
||||
(:db/id (d/entity db [:block/uuid current-block-uuid]))))
|
||||
|
||||
(defmethod resolve-keyword-input :parent-block [db _ {:keys [current-block-uuid]}]
|
||||
(when current-block-uuid
|
||||
(:db/id (:block/parent (d/entity db [:block/uuid current-block-uuid])))))
|
||||
|
||||
(defmethod resolve-keyword-input :today [_ _ _]
|
||||
(date-time-util/date->int (t/today)))
|
||||
|
||||
(defmethod resolve-keyword-input :yesterday [_ _ _]
|
||||
(date-time-util/date->int (t/minus (t/today) (t/days 1))))
|
||||
|
||||
(defmethod resolve-keyword-input :tomorrow [_ _ _]
|
||||
(date-time-util/date->int (t/plus (t/today) (t/days 1))))
|
||||
|
||||
(defmethod resolve-keyword-input :right-now-ms [_ _ _]
|
||||
(common-util/time-ms))
|
||||
|
||||
;; today-time returns an epoch int
|
||||
(defmethod resolve-keyword-input :today-time [_db input _opts]
|
||||
(let [[hh mm ss ms] (case input
|
||||
:start-of-today-ms [0 0 0 0]
|
||||
:end-of-today-ms [23 59 59 999]
|
||||
(get-ts-units nil (subs (name input) 6)))]
|
||||
(date-at-local-ms (t/today) hh mm ss ms)))
|
||||
|
||||
;; relative-date returns a YYYMMDD string
|
||||
(defmethod resolve-keyword-input :relative-date [_ input _]
|
||||
(let [relative-to (get-relative-date input)
|
||||
[_ offset-direction offset offset-unit] (re-find #"^([+-])(\d+)([dwmy])$" (name input))
|
||||
offset-date (get-offset-date relative-to offset-direction offset offset-unit)]
|
||||
(date-time-util/date->int offset-date)))
|
||||
|
||||
;; relative-date-time returns an epoch int
|
||||
(defmethod resolve-keyword-input :relative-date-time [_ input _]
|
||||
(let [relative-to (get-relative-date input)
|
||||
[_ offset-direction offset offset-unit ts] (re-find #"^([+-])(\d+)([dwmy])-(ms|start|end|\d{2,9})$" (name input))
|
||||
offset-date (get-offset-date relative-to offset-direction offset offset-unit)
|
||||
[hh mm ss ms] (get-ts-units offset-direction ts)]
|
||||
(date-at-local-ms offset-date hh mm ss ms)))
|
||||
|
||||
(defmethod resolve-keyword-input :DEPRECATED-relative-date [db input opts]
|
||||
;; This handles all of the cases covered by the following:
|
||||
;; :Xd, :Xd-before, :Xd-before-ms, :Xd-after, :Xd-after-ms
|
||||
(resolve-keyword-input db (old->new-relative-date-format input) opts))
|
||||
|
||||
(defmethod resolve-keyword-input :default [_ _ _]
|
||||
nil)
|
||||
|
||||
(defn resolve-input
|
||||
"Main fn for resolving advanced query :inputs"
|
||||
[db input {:keys [current-block-uuid current-page-fn]
|
||||
:or {current-page-fn (constantly nil)}}]
|
||||
(cond
|
||||
(keyword? input)
|
||||
(or
|
||||
(resolve-keyword-input db input {:current-block-uuid current-block-uuid
|
||||
:current-page-fn current-page-fn})
|
||||
;; The input is returned back unresolved if a resolver communicates it
|
||||
;; can't resolve it by returning nil. We may want to error if this is too
|
||||
;; subtle for the user
|
||||
input)
|
||||
|
||||
(and (string? input) (page-ref/page-ref? input))
|
||||
(-> (page-ref/get-page-name input)
|
||||
(string/lower-case))
|
||||
|
||||
:else
|
||||
input))
|
||||
33
deps/db/src/logseq/db/frontend/kv_entity.cljs
vendored
Normal file
33
deps/db/src/logseq/db/frontend/kv_entity.cljs
vendored
Normal file
@@ -0,0 +1,33 @@
|
||||
(ns logseq.db.frontend.kv-entity
|
||||
"kv entities used by logseq db"
|
||||
(:require [logseq.common.defkeywords :refer [defkeywords]]))
|
||||
|
||||
(def kv-entities
|
||||
(apply
|
||||
hash-map
|
||||
(defkeywords
|
||||
:logseq.kv/db-type {:doc "Set to \"db\" if it's a db-graph"}
|
||||
:logseq.kv/graph-uuid {:doc "Store graph-uuid if it's a rtc enabled graph"
|
||||
:rtc {:rtc/ignore-entity-when-init-upload true}}
|
||||
:logseq.kv/local-graph-uuid {:doc "Store graph-uuid if it's a local graph"
|
||||
:rtc {:rtc/ignore-entity-when-init-upload true}}
|
||||
:logseq.kv/import-type {:doc "If graph is imported, identifies how a graph is imported including which UI or CLI import process. CLI scripts can set this to a custom value.
|
||||
UI values include :file-graph and :sqlite-db and CLI values start with :cli e.g. :cli/default."}
|
||||
:logseq.kv/imported-at {:doc "Time if graph is imported"}
|
||||
:logseq.kv/graph-local-tx {:doc "local rtc tx-id"
|
||||
:rtc {:rtc/ignore-entity-when-init-upload true}}
|
||||
:logseq.kv/schema-version {:doc "Graph's current schema version"}
|
||||
:logseq.kv/remote-schema-version {:doc "Graph's remote schema version.
|
||||
RTC won't start when major-schema-versions don't match"
|
||||
:rtc {:rtc/ignore-entity-when-init-upload true}}
|
||||
:logseq.kv/graph-created-at {:doc "Graph's created at time"}
|
||||
:logseq.kv/latest-code-lang {:doc "Latest lang used by a #Code-block"
|
||||
:rtc {:rtc/ignore-entity-when-init-upload true}}
|
||||
:logseq.kv/graph-backup-folder {:doc "Backup folder for automated backup feature"
|
||||
:rtc {:rtc/ignore-entity-when-init-upload true}}
|
||||
:logseq.kv/graph-initial-schema-version {:doc "Graph's schema version when created"}
|
||||
|
||||
:logseq.kv/graph-last-gc-at {:doc "Last time graph gc at"
|
||||
:rtc {:rtc/ignore-entity-when-init-upload true}}
|
||||
:logseq.kv/graph-rtc-e2ee? {:doc "true if it's a rtc graph with E2EE enabled"}
|
||||
:logseq.kv/graph-remote? {:doc "true if it's a remote graph"})))
|
||||
647
deps/db/src/logseq/db/frontend/malli_schema.cljs
vendored
Normal file
647
deps/db/src/logseq/db/frontend/malli_schema.cljs
vendored
Normal file
@@ -0,0 +1,647 @@
|
||||
(ns logseq.db.frontend.malli-schema
|
||||
"Malli schemas and fns for logseq.db.frontend.*"
|
||||
(:require [clojure.set :as set]
|
||||
[clojure.string :as string]
|
||||
[datascript.core :as d]
|
||||
[datascript.impl.entity :as de]
|
||||
[logseq.db.common.entity-plus :as entity-plus]
|
||||
[logseq.db.common.order :as db-order]
|
||||
[logseq.db.frontend.class :as db-class]
|
||||
[logseq.db.frontend.entity-util :as entity-util]
|
||||
[logseq.db.frontend.property :as db-property]
|
||||
[logseq.db.frontend.property.type :as db-property-type]
|
||||
[logseq.db.frontend.schema :as db-schema]))
|
||||
|
||||
;; :db/ident malli schemas
|
||||
;; =======================
|
||||
|
||||
(def db-attribute-ident
|
||||
(into [:enum] db-property/db-attribute-properties))
|
||||
|
||||
(def logseq-property-ident
|
||||
[:and :keyword [:fn
|
||||
{:error/message "should be a valid logseq property namespace"}
|
||||
db-property/logseq-property?]])
|
||||
|
||||
(def block-order
|
||||
[:and :string [:fn
|
||||
{:error/message "should be a valid fractional index"}
|
||||
db-order/validate-order-key?]])
|
||||
|
||||
(def internal-property-ident
|
||||
[:or logseq-property-ident db-attribute-ident])
|
||||
|
||||
(defn user-property?
|
||||
"Determines if keyword/ident is a user property"
|
||||
[kw]
|
||||
(db-property/user-property-namespace? (namespace kw)))
|
||||
|
||||
(def user-property-ident
|
||||
[:and :qualified-keyword [:fn
|
||||
{:error/message "should be a valid user property namespace"}
|
||||
user-property?]])
|
||||
|
||||
(def plugin-property-ident
|
||||
[:and :qualified-keyword [:fn
|
||||
{:error/message "should be a valid plugin property namespace"}
|
||||
db-property/plugin-property?]])
|
||||
|
||||
(def logseq-ident-namespaces
|
||||
"Set of all namespaces Logseq uses for :db/ident except for
|
||||
db-attribute-ident. It's important to grow this list purposefully and have it
|
||||
start with 'logseq' to allow for users and 3rd party plugins to provide their
|
||||
own namespaces to core concepts."
|
||||
(into db-property/logseq-property-namespaces #{db-class/logseq-class "logseq.kv"}))
|
||||
|
||||
(def logseq-ident
|
||||
[:and :keyword [:fn
|
||||
{:error/message "should be a valid :db/ident namespace"}
|
||||
(fn logseq-namespace? [k]
|
||||
(contains? logseq-ident-namespaces (namespace k)))]])
|
||||
|
||||
(defn class?
|
||||
"Determines if keyword/ident is a logseq or user class"
|
||||
[kw]
|
||||
(string/includes? (namespace kw) ".class"))
|
||||
|
||||
(def class-ident
|
||||
[:and :qualified-keyword [:fn
|
||||
{:error/message "should be a valid class namespace"}
|
||||
class?]])
|
||||
;; Helper fns
|
||||
;; ==========
|
||||
(defn- empty-placeholder-value? [db property property-val]
|
||||
(if (= :db.type/ref (:db/valueType property))
|
||||
(and (integer? property-val)
|
||||
(= :logseq.property/empty-placeholder (:db/ident (d/entity db property-val))))
|
||||
(= :logseq.property/empty-placeholder property-val)))
|
||||
|
||||
(defn internal-ident?
|
||||
"Determines if given ident is created by Logseq. All Logseq internal idents
|
||||
must start with 'block' or 'logseq' to keep Logseq internals from leaking
|
||||
across namespaces and to allow for users and 3rd party plugins to choose
|
||||
any other namespace"
|
||||
[ident]
|
||||
(or (contains? db-property/db-attribute-properties ident)
|
||||
(contains? logseq-ident-namespaces (namespace ident))))
|
||||
|
||||
(defn validate-property-value
|
||||
"Validates the property value in a property tuple. The property value is
|
||||
expected to be a coll if the property has a :many cardinality. validate-fn is
|
||||
a fn that is called directly on each value to return a truthy value.
|
||||
validate-fn varies by property type"
|
||||
[db validate-fn [property property-val] & {:keys [new-closed-value? :closed-values-validate? _skip-strict-url-validate?]
|
||||
:as validate-options}]
|
||||
;; For debugging
|
||||
;; (when (not (internal-ident? (:db/ident property))) (prn :validate-val (dissoc property :property/closed-values) property-val))
|
||||
(let [validate-fn' (if (db-property-type/property-types-with-db (:logseq.property/type property))
|
||||
(fn [value]
|
||||
(validate-fn db value validate-options))
|
||||
validate-fn)
|
||||
validate-fn'' (if (and closed-values-validate?
|
||||
(db-property-type/closed-value-property-types (:logseq.property/type property))
|
||||
;; new closed values aren't associated with the property yet
|
||||
(not new-closed-value?)
|
||||
(seq (:property/closed-values property)))
|
||||
(fn closed-value-valid? [val]
|
||||
(and (validate-fn' val)
|
||||
(let [ids (set (map :db/id (:property/closed-values property)))
|
||||
result (contains? ids val)]
|
||||
(when-not result
|
||||
(js/console.error (str "Error: not a closed value, id: " val ", existing choices: " ids ", property: " (:db/ident property))))
|
||||
result)))
|
||||
validate-fn')]
|
||||
(if (db-property/many? property)
|
||||
(or (every? validate-fn'' property-val)
|
||||
(empty-placeholder-value? db property (first property-val)))
|
||||
(or (validate-fn'' property-val)
|
||||
;; also valid if value is empty-placeholder
|
||||
(empty-placeholder-value? db property property-val)))))
|
||||
|
||||
(def required-properties
|
||||
"Set of properties required by a schema and that are validated directly in a schema instead
|
||||
of validate-property-value"
|
||||
(set/union
|
||||
(set (get-in db-class/built-in-classes [:logseq.class/Asset :schema :required-properties]))
|
||||
#{:logseq.property/created-from-property :logseq.property/value
|
||||
:logseq.property.history/scalar-value :logseq.property.history/block
|
||||
:logseq.property.history/property :logseq.property.history/ref-value
|
||||
:logseq.property.class/extends
|
||||
:logseq.property.reaction/emoji-id
|
||||
:logseq.property.reaction/target}))
|
||||
|
||||
(defn- property-entity->map
|
||||
"Provide the minimal number of property attributes to validate the property
|
||||
and to reduce noise in error messages. The resulting map should be the same as
|
||||
what the frontend property since they both call validate-property-value"
|
||||
[property]
|
||||
;; use explicit call to be nbb compatible
|
||||
(let [closed-values (entity-plus/lookup-kv-then-entity property :property/closed-values)]
|
||||
(cond-> (select-keys property [:db/ident :db/valueType :db/cardinality :logseq.property/type])
|
||||
(seq closed-values)
|
||||
(assoc :property/closed-values closed-values))))
|
||||
|
||||
(defn update-properties-in-ents
|
||||
"Prepares properties in entities to be validated by DB schema"
|
||||
[db ents]
|
||||
;; required-properties allows schemas like property-value-block to require
|
||||
;; properties in their schema that they depend on
|
||||
(let [exceptions-to-block-properties (-> required-properties
|
||||
(into db-property/schema-properties)
|
||||
(conj :block/tags))
|
||||
page-class-id (:db/id (d/entity db :logseq.class/Page))
|
||||
all-page-class-ids (set (map #(:db/id (d/entity db %)) db-class/page-classes))]
|
||||
(mapv
|
||||
(fn [ent]
|
||||
(reduce (fn [m [k v]]
|
||||
(if-let [property (and (db-property/property? k)
|
||||
(not (contains? exceptions-to-block-properties k))
|
||||
(d/entity db k))]
|
||||
(update m :block/properties (fnil conj [])
|
||||
[(property-entity->map property) v])
|
||||
(if (= :block/tags k)
|
||||
;; Provides additional options map to validation for data about current entity being tagged
|
||||
(let [property (d/entity db :block/tags)]
|
||||
(assoc m k [(property-entity->map property)
|
||||
v
|
||||
(merge (select-keys ent [:logseq.property/built-in?])
|
||||
{:page-class-id page-class-id
|
||||
:all-page-class-ids all-page-class-ids})]))
|
||||
(assoc m k v))))
|
||||
{}
|
||||
ent))
|
||||
ents)))
|
||||
|
||||
(defn datoms->entity-maps
|
||||
"Returns entity maps for given :eavt datoms indexed by db/id. Optional keys:
|
||||
* :entity-fn - Optional fn that given an entity id, returns entity. Defaults
|
||||
to just doing a lookup based on existing entity-maps to be as performant as possible"
|
||||
[datoms & {:keys [entity-fn]}]
|
||||
(let [ent-maps
|
||||
(reduce (fn [acc {:keys [a e v]}]
|
||||
(if (contains? db-schema/card-many-attributes a)
|
||||
(update acc e update a (fnil conj #{}) v)
|
||||
;; If there's already a val, don't clobber it and automatically start collecting it as a :many
|
||||
(if-let [existing-val (get-in acc [e a])]
|
||||
(if (set? existing-val)
|
||||
(update acc e assoc a (conj existing-val v))
|
||||
(update acc e assoc a #{existing-val v}))
|
||||
(update acc e assoc a v))))
|
||||
{}
|
||||
datoms)
|
||||
entity-fn' (or entity-fn
|
||||
(let [db-ident-maps (dissoc (into {} (map (juxt :db/ident identity) (vals ent-maps))) nil)]
|
||||
#(get db-ident-maps %)))]
|
||||
(-> ent-maps
|
||||
(update-vals
|
||||
(fn [m]
|
||||
(->> m
|
||||
(map (fn [[k v]]
|
||||
(if-let [property (and (db-property/property? k)
|
||||
(entity-fn' k))]
|
||||
(if (and (db-property/many? property)
|
||||
(not (set? v)))
|
||||
;; Fix :many property values that only had one value
|
||||
[k #{v}]
|
||||
[k v])
|
||||
[k v])))
|
||||
(into {})))))))
|
||||
|
||||
(defn datoms->entities
|
||||
"Returns a vec of entity maps given :eavt datoms"
|
||||
[datoms]
|
||||
(mapv (fn [[db-id m]] (assoc m :db/id db-id))
|
||||
(datoms->entity-maps datoms)))
|
||||
|
||||
(assert (every? #(re-find #"^(block|logseq\.)" (namespace %)) db-property/db-attribute-properties)
|
||||
"All db-attribute idents start with an internal namespace")
|
||||
(assert (every? #(re-find #"^logseq\." %) logseq-ident-namespaces)
|
||||
"All logseq idents start with an internal namespace")
|
||||
|
||||
;; Main malli schemas
|
||||
;; ==================
|
||||
;; These schemas should be data vars to remain as simple and reusable as possible
|
||||
|
||||
(def ^:dynamic *db-for-validate-fns*
|
||||
"Used by validate-fns which need db as input"
|
||||
nil)
|
||||
|
||||
(def ^:dynamic *skip-strict-url-validate?*
|
||||
"`true` allows updating a block's other property when it has invalid URL value"
|
||||
false)
|
||||
|
||||
(def ^:dynamic *closed-values-validate?*
|
||||
"By default this is false because we can't ensure this when merging updates from server.
|
||||
`true` allows for non RTC graphs to have higher data quality and avoid
|
||||
possible UX bugs related to closed values."
|
||||
false)
|
||||
|
||||
(def property-tuple
|
||||
"A tuple of a property map and a property value"
|
||||
(into
|
||||
[:multi {:dispatch #(-> % first :logseq.property/type)}]
|
||||
(map (fn [[prop-type value-schema]]
|
||||
[prop-type
|
||||
(let [schema-fn (if (vector? value-schema) (last value-schema) value-schema)
|
||||
error-message (when (vector? value-schema)
|
||||
(and (map? (second value-schema))
|
||||
(:error/message (second value-schema))))]
|
||||
[:fn
|
||||
(when error-message
|
||||
{:error/message error-message})
|
||||
(fn [tuple]
|
||||
(validate-property-value *db-for-validate-fns* schema-fn tuple
|
||||
{:skip-strict-url-validate? *skip-strict-url-validate?*
|
||||
:closed-values-validate? *closed-values-validate?*}))])])
|
||||
db-property-type/built-in-validation-schemas)))
|
||||
|
||||
(def block-properties
|
||||
"Validates a block's properties as property pairs. Properties are
|
||||
a vector of tuples instead of a map in order to validate each
|
||||
property with its property value that is valid for its type"
|
||||
[:sequential property-tuple])
|
||||
|
||||
(def block-tags
|
||||
[:and
|
||||
property-tuple
|
||||
;; Important to keep data integrity of built-in entities. Ensure UI doesn't accidentally modify them
|
||||
[:fn {:error/message "should only have one tag for a built-in entity"}
|
||||
(fn [[_k v opts]]
|
||||
(if (:logseq.property/built-in? opts)
|
||||
(= 1 (count v))
|
||||
true))]
|
||||
;; Ensure use of :logseq.class/Page is consistent and simple. Doing so reduces complexity elsewhere
|
||||
;; and allows for Page to exist as its own public concept later
|
||||
[:fn {:error/message "should not have other built-in page tags when tagged with #Page"}
|
||||
(fn [[_k v {:keys [page-class-id all-page-class-ids]}]]
|
||||
(if (contains? v page-class-id)
|
||||
(empty? (set/intersection (disj v page-class-id) all-page-class-ids))
|
||||
true))]])
|
||||
|
||||
(def page-or-block-attrs
|
||||
"Common attributes for page and normal blocks"
|
||||
[[:block/uuid :uuid]
|
||||
[:block/created-at :int]
|
||||
[:block/updated-at :int]
|
||||
[:logseq.property/deleted-at {:optional true} :int]
|
||||
;; Injected by update-properties-in-ents
|
||||
[:block/properties {:optional true} block-properties]
|
||||
[:block/tags {:optional true} block-tags]
|
||||
[:block/refs {:optional true} [:set :int]]
|
||||
[:block/tx-id {:optional true} :int]
|
||||
[:block/collapsed? {:optional true} :boolean]
|
||||
[:block/warning {:optional true} [:keyword]]
|
||||
[:logseq.property/created-by-ref {:optional true} :int]])
|
||||
|
||||
(def page-attrs
|
||||
"Common attributes for pages"
|
||||
[[:block/name :string]
|
||||
[:block/title :string]])
|
||||
|
||||
(def property-attrs
|
||||
"Common attributes for properties"
|
||||
[[:db/index {:optional true} :boolean]
|
||||
[:db/valueType {:optional true} [:enum :db.type/ref]]
|
||||
[:db/cardinality {:optional true} [:enum :db.cardinality/many :db.cardinality/one]]
|
||||
[:block/order {:optional true} block-order]
|
||||
[:logseq.property/classes {:optional true} [:set :int]]])
|
||||
|
||||
(def normal-page
|
||||
(vec
|
||||
(concat
|
||||
[:map {:error/path ["normal-page"]}
|
||||
;; journal-day is only set for journal pages
|
||||
[:block/journal-day {:optional true} :int]
|
||||
[:block/parent {:optional true} :int]
|
||||
[:block/order {:optional true} block-order]]
|
||||
page-attrs
|
||||
page-or-block-attrs)))
|
||||
|
||||
(def class-page
|
||||
[:or
|
||||
(vec
|
||||
(concat
|
||||
[:map {:error/path ["class-page"]}
|
||||
[:db/ident class-ident]
|
||||
[:logseq.property.class/extends [:set :int]]]
|
||||
page-attrs
|
||||
page-or-block-attrs))
|
||||
(vec
|
||||
(concat
|
||||
[:map {:error/path ["class-page"]}
|
||||
[:db/ident [:= :logseq.class/Root]]]
|
||||
page-attrs
|
||||
page-or-block-attrs))])
|
||||
|
||||
(def property-common-schema-attrs
|
||||
"Property :schema attributes common to all properties"
|
||||
[[:logseq.property/hide? {:optional true} :boolean]
|
||||
[:logseq.property/public? {:optional true} :boolean]
|
||||
[:logseq.property/ui-position {:optional true} [:enum :properties :block-left :block-right :block-below]]])
|
||||
|
||||
(def internal-property
|
||||
(vec
|
||||
(concat
|
||||
[:map {:error/path ["internal-property"]}
|
||||
[:db/ident internal-property-ident]
|
||||
[:logseq.property/type (apply vector :enum (into db-property-type/internal-built-in-property-types
|
||||
db-property-type/user-built-in-property-types))]
|
||||
[:logseq.property/view-context {:optional true} [:enum :page :block :class :property :never]]]
|
||||
property-common-schema-attrs
|
||||
property-attrs
|
||||
page-attrs
|
||||
page-or-block-attrs)))
|
||||
|
||||
(def user-property
|
||||
(vec
|
||||
(concat
|
||||
[:map {:error/path ["user-property"]}
|
||||
[:db/ident user-property-ident]
|
||||
[:logseq.property/type (apply vector :enum (into db-property-type/user-allowed-internal-property-types
|
||||
db-property-type/user-built-in-property-types))]]
|
||||
property-common-schema-attrs
|
||||
property-attrs
|
||||
page-attrs
|
||||
page-or-block-attrs)))
|
||||
|
||||
(def plugin-property
|
||||
(vec
|
||||
(concat
|
||||
[:map {:error/path ["plugin-property"]}
|
||||
[:db/ident plugin-property-ident]
|
||||
[:logseq.property/type (apply vector :enum (concat db-property-type/user-built-in-property-types [:json :string :page]))]]
|
||||
property-common-schema-attrs
|
||||
property-attrs
|
||||
page-attrs
|
||||
page-or-block-attrs)))
|
||||
|
||||
(def property-page
|
||||
[:multi {:dispatch (fn [m]
|
||||
(let [ident (:db/ident m)]
|
||||
|
||||
(cond
|
||||
(or (and ident (db-property/logseq-property? ident))
|
||||
(contains? db-property/db-attribute-properties (:db/ident m)))
|
||||
:internal
|
||||
|
||||
(and ident (db-property/plugin-property? ident))
|
||||
:plugin
|
||||
|
||||
:else
|
||||
:user)))}
|
||||
[:internal internal-property]
|
||||
[:plugin plugin-property]
|
||||
[:malli.core/default user-property]])
|
||||
|
||||
(def hidden-page
|
||||
(vec
|
||||
(concat
|
||||
[:map {:error/path ["hidden-page"]}
|
||||
;; pages from :default property uses this but closed-value pages don't
|
||||
[:block/order {:optional true} block-order]
|
||||
[:logseq.property/hide? [:enum true]]]
|
||||
page-attrs
|
||||
page-or-block-attrs)))
|
||||
|
||||
(def block-attrs
|
||||
"Common attributes for normal blocks"
|
||||
[[:block/title :string]
|
||||
[:block/parent :int]
|
||||
[:block/order block-order]
|
||||
;; refs
|
||||
[:block/page :int]
|
||||
[:block/link {:optional true} :int]
|
||||
[:logseq.property/created-from-property {:optional true} :int]])
|
||||
|
||||
;; TODO: Remove deprecated
|
||||
(def whiteboard-block
|
||||
"A (shape) block for whiteboard"
|
||||
(vec
|
||||
(concat
|
||||
[:map {:error/path ["whiteboard-block"]}]
|
||||
[[:block/title :string]
|
||||
[:block/parent :int]
|
||||
;; These blocks only associate with pages of type "whiteboard"
|
||||
[:block/page :int]]
|
||||
page-or-block-attrs)))
|
||||
|
||||
(def property-value-block
|
||||
"A common property value for user properties"
|
||||
(vec
|
||||
(concat
|
||||
[:map {:error/path "property-value-block"}]
|
||||
[[:logseq.property/value [:or :string :double :boolean]]
|
||||
[:logseq.property/created-from-property :int]]
|
||||
(remove #(#{:block/title :logseq.property/created-from-property} (first %)) block-attrs)
|
||||
page-or-block-attrs)))
|
||||
|
||||
(def reaction-entity
|
||||
"A reaction entity referencing a target node"
|
||||
(vec
|
||||
[:map {:error/path ["reaction-entity"]}
|
||||
[:block/uuid :uuid]
|
||||
[:logseq.property.reaction/emoji-id :string]
|
||||
[:logseq.property.reaction/target :int]
|
||||
[:block/created-at :int]
|
||||
[:block/tx-id {:optional true} :int]
|
||||
[:block/properties {:optional true} block-properties]
|
||||
[:block/refs {:optional true} [:set :int]]]))
|
||||
|
||||
(def property-history-block*
|
||||
[:map {:error/path ["property-history-block"]}
|
||||
[:block/uuid :uuid]
|
||||
[:block/created-at :int]
|
||||
[:block/updated-at {:optional true} :int]
|
||||
[:logseq.property.history/block :int]
|
||||
[:logseq.property.history/property :int]
|
||||
[:logseq.property.history/ref-value {:optional true} :int]
|
||||
[:logseq.property.history/scalar-value {:optional true} :any]
|
||||
[:block/properties {:optional true} block-properties]
|
||||
[:block/tx-id {:optional true} :int]])
|
||||
|
||||
(def property-history-block
|
||||
"A closed value for a property with closed/allowed values"
|
||||
[:and property-history-block*
|
||||
[:fn {:error/message ":logseq.property.history/ref-value or :logseq.property.history/scalar-value required"
|
||||
:error/path [:logseq.property.history/ref-value]}
|
||||
(fn [m]
|
||||
(or (:logseq.property.history/ref-value m)
|
||||
(some? (:logseq.property.history/scalar-value m))))]])
|
||||
|
||||
(def closed-value-block*
|
||||
(vec
|
||||
(concat
|
||||
[:map]
|
||||
[;; for built-in properties
|
||||
[:db/ident {:optional true} logseq-property-ident]
|
||||
[:block/title {:optional true} :string]
|
||||
[:logseq.property/value {:optional true} [:or :string :double]]
|
||||
[:logseq.property/created-from-property :int]
|
||||
[:block/closed-value-property {:optional true} [:set :int]]]
|
||||
(remove #(#{:block/title :logseq.property/created-from-property} (first %)) block-attrs)
|
||||
page-or-block-attrs)))
|
||||
|
||||
(def closed-value-block
|
||||
"A closed value for a property with closed/allowed values"
|
||||
[:and closed-value-block*
|
||||
[:fn {:error/message ":block/title or :logseq.property/value required"
|
||||
:error/path [:logseq.property/value]}
|
||||
(fn [m]
|
||||
(or (:block/title m) (:logseq.property/value m)))]])
|
||||
|
||||
(def normal-block
|
||||
"A block with content and no special type or tag behavior"
|
||||
(vec
|
||||
(concat
|
||||
[:map {:error/path ["normal-block"]}]
|
||||
block-attrs
|
||||
page-or-block-attrs)))
|
||||
|
||||
(def block
|
||||
"A block has content and a page"
|
||||
[:or
|
||||
normal-block
|
||||
whiteboard-block])
|
||||
|
||||
(def asset-block
|
||||
"A block tagged with #Asset"
|
||||
(vec
|
||||
(concat
|
||||
[:map {:error/path ["asset-block"]}]
|
||||
;; TODO: Derive required property types from existing schema in frontend.property
|
||||
[[:logseq.property.asset/type :string]
|
||||
[:logseq.property.asset/checksum :string]
|
||||
[:logseq.property.asset/size :int]
|
||||
[:logseq.property.asset/width {:optional true} :int]
|
||||
[:logseq.property.asset/height {:optional true} :int]
|
||||
[:logseq.property.asset/align {:optional true} :keyword]]
|
||||
block-attrs
|
||||
page-or-block-attrs)))
|
||||
|
||||
(def file-block
|
||||
[:map {:error/path ["file-block"]}
|
||||
[:block/uuid :uuid]
|
||||
[:block/tx-id {:optional true} :int]
|
||||
;; App doesn't use timestamps but migrations may
|
||||
[:block/created-at {:optional true} :int]
|
||||
[:block/updated-at {:optional true} :int]
|
||||
[:file/content :string]
|
||||
[:file/path :string]
|
||||
[:file/size {:optional true} :int]
|
||||
[:file/created-at inst?]
|
||||
[:file/last-modified-at inst?]])
|
||||
|
||||
(def db-ident-key-val
|
||||
"A key value map with :db/ident and :kv/value"
|
||||
[:map {:error/path ["db-ident-key-val"]}
|
||||
[:db/ident logseq-ident]
|
||||
[:kv/value :any]
|
||||
[:block/tx-id {:optional true} :int]])
|
||||
|
||||
(def property-value-placeholder
|
||||
[:map {:error/path ["property-value-placeholder"]}
|
||||
[:db/ident [:= :logseq.property/empty-placeholder]]
|
||||
[:block/uuid :uuid]
|
||||
[:block/tx-id {:optional true} :int]
|
||||
[:block/created-at {:optional true} :int]
|
||||
[:block/updated-at {:optional true} :int]])
|
||||
|
||||
(defn- whiteboard?
|
||||
[entity]
|
||||
(when (or (map? entity) (de/entity? entity))
|
||||
(some (fn [t]
|
||||
(or (keyword-identical? (:db/ident t) :logseq.class/Whiteboard)
|
||||
(keyword-identical? t :logseq.class/Whiteboard)))
|
||||
(:block/tags entity))))
|
||||
|
||||
(defn entity-dispatch-key [db ent]
|
||||
(let [d (if (:block/uuid ent) (d/entity db [:block/uuid (:block/uuid ent)]) ent)
|
||||
;; order matters as some block types are a subset of others e.g. :whiteboard
|
||||
dispatch-key (cond
|
||||
(:logseq.property.reaction/target d)
|
||||
:reaction-entity
|
||||
(entity-util/property? d)
|
||||
:property
|
||||
(entity-util/class? d)
|
||||
:class
|
||||
(and (entity-util/page? d)
|
||||
(true? (:logseq.property/hide? d)))
|
||||
:hidden
|
||||
;; TODO: Remove deprecated
|
||||
(whiteboard? d)
|
||||
:normal-page
|
||||
(entity-util/page? d)
|
||||
:normal-page
|
||||
(entity-util/asset? d)
|
||||
:asset-block
|
||||
(:file/path d)
|
||||
:file-block
|
||||
(:logseq.property.history/block d)
|
||||
:property-history-block
|
||||
(:block/closed-value-property d)
|
||||
:closed-value-block
|
||||
(and (:logseq.property/created-from-property d) (:logseq.property/value d))
|
||||
:property-value-block
|
||||
(= (:db/ident d) :logseq.property/empty-placeholder)
|
||||
:property-value-placeholder
|
||||
(:block/uuid d)
|
||||
:block
|
||||
(:db/ident d)
|
||||
:db-ident-key-value)]
|
||||
dispatch-key))
|
||||
|
||||
(def Data
|
||||
(into
|
||||
[:multi {:dispatch (fn [d] (entity-dispatch-key *db-for-validate-fns* d))}]
|
||||
{:property property-page
|
||||
:class class-page
|
||||
:hidden hidden-page
|
||||
:normal-page normal-page
|
||||
:reaction-entity reaction-entity
|
||||
:property-history-block property-history-block
|
||||
:closed-value-block closed-value-block
|
||||
:property-value-block property-value-block
|
||||
:block block
|
||||
:asset-block asset-block
|
||||
:file-block file-block
|
||||
:db-ident-key-value db-ident-key-val
|
||||
:property-value-placeholder property-value-placeholder}))
|
||||
|
||||
(def DB
|
||||
"Malli schema for entities from db-schema/schema. In order to
|
||||
thoroughly validate properties, the entities and this schema should be
|
||||
prepared with update-properties-in-ents and update-properties-in-schema
|
||||
respectively"
|
||||
[:sequential Data])
|
||||
|
||||
;; Keep malli schema in sync with db schema
|
||||
;; ========================================
|
||||
(let [malli-many-ref-attrs (->> (concat property-attrs page-attrs block-attrs page-or-block-attrs (rest closed-value-block*))
|
||||
(filter #(= (last %) [:set :int]))
|
||||
(map first)
|
||||
(into db-property/public-db-attribute-properties)
|
||||
set)]
|
||||
(when-let [undeclared-ref-attrs (seq (remove malli-many-ref-attrs db-schema/card-many-ref-type-attributes))]
|
||||
(throw (ex-info (str "The malli DB schema is missing the following cardinality-many ref attributes from datascript's schema: "
|
||||
(string/join ", " undeclared-ref-attrs))
|
||||
{}))))
|
||||
|
||||
(let [malli-one-ref-attrs (->> (concat property-attrs page-attrs block-attrs page-or-block-attrs (rest normal-page))
|
||||
(filter #(= (last %) :int))
|
||||
(map first)
|
||||
set)]
|
||||
(when-let [undeclared-ref-attrs (seq (remove malli-one-ref-attrs db-schema/card-one-ref-type-attributes))]
|
||||
(throw (ex-info (str "The malli DB schema is missing the following cardinality-one ref attributes from datascript's schema: "
|
||||
(string/join ", " undeclared-ref-attrs))
|
||||
{}))))
|
||||
|
||||
(let [malli-non-ref-attrs (->> (concat property-attrs page-attrs block-attrs page-or-block-attrs (rest normal-page))
|
||||
(concat (rest file-block) (rest property-value-block)
|
||||
(rest db-ident-key-val) (rest internal-property))
|
||||
(remove #(= (last %) [:set :int]))
|
||||
(map first)
|
||||
set)]
|
||||
(when-let [undeclared-attrs (seq (remove malli-non-ref-attrs db-schema/db-non-ref-attributes))]
|
||||
(throw (ex-info (str "The malli DB schema is missing the following non ref attributes from datascript's schema: "
|
||||
(string/join ", " undeclared-attrs))
|
||||
{}))))
|
||||
1024
deps/db/src/logseq/db/frontend/property.cljs
vendored
Normal file
1024
deps/db/src/logseq/db/frontend/property.cljs
vendored
Normal file
File diff suppressed because it is too large
Load Diff
163
deps/db/src/logseq/db/frontend/property/build.cljs
vendored
Normal file
163
deps/db/src/logseq/db/frontend/property/build.cljs
vendored
Normal file
@@ -0,0 +1,163 @@
|
||||
(ns logseq.db.frontend.property.build
|
||||
"Builds core property concepts"
|
||||
(:require [logseq.common.util :as common-util]
|
||||
[logseq.common.uuid :as common-uuid]
|
||||
[logseq.db.common.order :as db-order]
|
||||
[logseq.db.frontend.property :as db-property]
|
||||
[logseq.db.frontend.property.type :as db-property-type]
|
||||
[logseq.db.sqlite.util :as sqlite-util]))
|
||||
|
||||
(defn- closed-value-new-block
|
||||
[block-id block-type value property]
|
||||
(let [property-id (:db/ident property)]
|
||||
(merge {:block/uuid block-id
|
||||
:block/page property-id
|
||||
:block/closed-value-property property-id
|
||||
:logseq.property/created-from-property (if (= property-id :logseq.property/default-value)
|
||||
[:block/uuid block-id]
|
||||
property-id)
|
||||
:block/parent property-id}
|
||||
(if (db-property-type/property-value-content? block-type property)
|
||||
{:logseq.property/value value}
|
||||
{:block/title value}))))
|
||||
|
||||
(defn build-closed-value-block
|
||||
"Builds a closed value block to be transacted"
|
||||
[block-uuid block-type block-value property {:keys [db-ident icon]}]
|
||||
(assert block-uuid (uuid? block-uuid))
|
||||
(cond->
|
||||
(closed-value-new-block block-uuid block-type block-value property)
|
||||
(and db-ident (keyword? db-ident))
|
||||
(assoc :db/ident db-ident)
|
||||
|
||||
icon
|
||||
(assoc :logseq.property/icon icon)
|
||||
|
||||
true
|
||||
sqlite-util/block-with-timestamps))
|
||||
|
||||
(defn closed-values->blocks
|
||||
[property]
|
||||
(map (fn [{uuid' :uuid :keys [db-ident value icon schema properties]}]
|
||||
(cond->
|
||||
(build-closed-value-block
|
||||
uuid'
|
||||
(:type schema)
|
||||
value
|
||||
property
|
||||
{:db-ident db-ident :icon icon})
|
||||
(seq properties)
|
||||
(merge properties)
|
||||
true
|
||||
(assoc :block/order (db-order/gen-key))))
|
||||
(:closed-values property)))
|
||||
|
||||
(defn build-closed-values
|
||||
"Builds all the tx needed for property with closed values including
|
||||
the hidden page and closed value blocks as needed"
|
||||
[db-ident prop-name property {:keys [property-attributes properties]}]
|
||||
(let [property-schema (or (:schema property)
|
||||
(db-property/get-property-schema property))
|
||||
property-tx (merge (sqlite-util/build-new-property db-ident property-schema {:title prop-name
|
||||
:ref-type? true
|
||||
:properties properties})
|
||||
property-attributes)]
|
||||
(into [property-tx]
|
||||
(closed-values->blocks property))))
|
||||
|
||||
(defn build-property-value-block
|
||||
"Builds a property value entity given a block map/entity, a property entity or
|
||||
ident and its property value. Takes the following options:
|
||||
* :block-uuid - :block/uuid for property value entity
|
||||
* :properties - Additional properties and attributes to add to entity"
|
||||
[block property value & {:keys [block-uuid properties]}]
|
||||
(let [block-id (or (:db/id block) (:db/ident block))]
|
||||
(cond->
|
||||
(merge
|
||||
{:block/uuid (or block-uuid (common-uuid/gen-uuid))
|
||||
:block/page (if (:block/page block)
|
||||
(:db/id (:block/page block))
|
||||
;; page block
|
||||
block-id)
|
||||
:block/parent block-id
|
||||
:logseq.property/created-from-property (if (= (:db/ident property) :logseq.property/default-value)
|
||||
block-id
|
||||
(or (:db/id property) {:db/ident (:db/ident property)}))
|
||||
:block/order (db-order/gen-key)}
|
||||
(if (db-property-type/property-value-content? (:logseq.property/type property) property)
|
||||
{:logseq.property/value value}
|
||||
{:block/title value}))
|
||||
true
|
||||
common-util/block-with-timestamps
|
||||
properties
|
||||
(merge properties))))
|
||||
|
||||
(defn build-property-values-tx-m
|
||||
"Builds a map of property names to their property value blocks to be
|
||||
transacted, given a block and a properties map with raw property values. The
|
||||
properties map can have keys that are db-idents or they can be maps. If a map,
|
||||
it should have :original-property-id and :db/ident keys. See
|
||||
->property-value-tx-m for such an example. Options:
|
||||
|
||||
* :pure? - ensure this fn is a pure function
|
||||
* :pvalue-map? - When set, property value is passed as a map with keys :value and :attributes. This
|
||||
allows property values to have additional attributes"
|
||||
[block properties & {:keys [pure? pvalue-map?]}]
|
||||
;; Build :db/id out of uuid if block doesn't have one for tx purposes
|
||||
(let [block' (if (:db/id block) block (assoc block :db/id [:block/uuid (:block/uuid block)]))]
|
||||
(->> properties
|
||||
(map (fn [[k v*]]
|
||||
(let [property-map (if (map? k) k {:db/ident k})
|
||||
gen-uuid-value-prefix (when pure?
|
||||
(or (:db/ident block) (:block/uuid block)))
|
||||
->pvalue #(if pvalue-map? (:value %) %)
|
||||
v (if (set? v*)
|
||||
(set (map ->pvalue v*))
|
||||
(->pvalue v*))
|
||||
->value-block-opts
|
||||
(fn ->value-block-opts [v']
|
||||
(cond-> {}
|
||||
(and pvalue-map? (seq (:attributes v')))
|
||||
(assoc :properties (:attributes v'))
|
||||
pure?
|
||||
(assoc :block-uuid
|
||||
(common-uuid/gen-uuid :builtin-block-uuid
|
||||
(str gen-uuid-value-prefix "-" (->pvalue v'))))))]
|
||||
(assert (:db/ident property-map) "Key in map must have a :db/ident")
|
||||
(when pure? (assert (some? gen-uuid-value-prefix) block))
|
||||
[(or (:original-property-id property-map) (:db/ident property-map))
|
||||
(cond
|
||||
(and (set? v) (every? uuid? v))
|
||||
(set (map #(vector :block/uuid %) v))
|
||||
(set? v*)
|
||||
(set
|
||||
(map #(build-property-value-block block' property-map (->pvalue %) (->value-block-opts %))
|
||||
v*))
|
||||
(uuid? v)
|
||||
[:block/uuid v]
|
||||
:else
|
||||
(build-property-value-block block' property-map v (->value-block-opts v*)))])))
|
||||
(into {}))))
|
||||
|
||||
(defn- lookup-id?
|
||||
[v]
|
||||
(and (vector? v)
|
||||
(= 2 (count v))
|
||||
(= :block/uuid (first v))
|
||||
(uuid? (second v))))
|
||||
|
||||
(defn build-properties-with-ref-values
|
||||
"Given a properties map with property values to be transacted e.g. from
|
||||
build-property-values-tx-m, build a properties map to be transacted with the block"
|
||||
[prop-vals-tx-m]
|
||||
(update-vals prop-vals-tx-m
|
||||
(fn [v]
|
||||
(cond
|
||||
(and (set? v) (every? lookup-id? v))
|
||||
v
|
||||
(set? v)
|
||||
(set (map #(vector :block/uuid (:block/uuid %)) v))
|
||||
(lookup-id? v)
|
||||
v
|
||||
:else
|
||||
(vector :block/uuid (:block/uuid v))))))
|
||||
253
deps/db/src/logseq/db/frontend/property/type.cljs
vendored
Normal file
253
deps/db/src/logseq/db/frontend/property/type.cljs
vendored
Normal file
@@ -0,0 +1,253 @@
|
||||
(ns logseq.db.frontend.property.type
|
||||
"Provides property types and related helper fns e.g. property value validation
|
||||
fns and their allowed schema attributes"
|
||||
(:require [clojure.set :as set]
|
||||
[clojure.string :as string]
|
||||
[datascript.core :as d]
|
||||
[logseq.common.util.macro :as macro-util]
|
||||
[logseq.db.frontend.entity-util :as entity-util]))
|
||||
|
||||
;; Config vars
|
||||
;; ===========
|
||||
;; These vars enumerate all known property types and their associated behaviors
|
||||
;; except for validation which is in its own section
|
||||
|
||||
(def internal-built-in-property-types
|
||||
"Valid property types only for use by internal built-in-properties"
|
||||
#{:keyword :map :coll :any :entity :class :page :property :string :json :raw-number})
|
||||
|
||||
(def user-built-in-property-types
|
||||
"Valid property types for users in order they appear in the UI"
|
||||
[:default :number :date :datetime :checkbox :url :node :asset])
|
||||
|
||||
(def user-allowed-internal-property-types
|
||||
"Internal property types that users are allowed to store. These aren't available in the UI
|
||||
so these would normally be created via EDN or the API."
|
||||
#{:map :json :string})
|
||||
|
||||
(assert (set/subset? user-allowed-internal-property-types internal-built-in-property-types))
|
||||
|
||||
(def closed-value-property-types
|
||||
"Valid property :type for closed values"
|
||||
#{:default :number :url})
|
||||
|
||||
(def cardinality-property-types
|
||||
"Valid property types that can change cardinality"
|
||||
#{:default :number :url :date :node :asset})
|
||||
|
||||
(def default-value-ref-property-types
|
||||
"Valid ref property :type for default value support"
|
||||
#{:default :number :checkbox})
|
||||
|
||||
(def text-ref-property-types
|
||||
"Valid ref property :types that support text"
|
||||
#{:default :url :entity})
|
||||
|
||||
(assert (set/subset? cardinality-property-types (set user-built-in-property-types))
|
||||
"All closed value types are valid property types")
|
||||
|
||||
(assert (set/subset? closed-value-property-types (set user-built-in-property-types))
|
||||
"All closed value types are valid property types")
|
||||
|
||||
(def original-value-ref-property-types
|
||||
"Property value ref types where the refed entity stores its value in
|
||||
:logseq.property/value e.g. :number is stored as a number. new value-ref-property-types
|
||||
should default to this as it allows for more querying power"
|
||||
#{:number})
|
||||
|
||||
(def value-ref-property-types
|
||||
"Property value ref types where the refed entities either store their value in
|
||||
:logseq.property/value or :block/title (for :default)"
|
||||
(into #{:default :url} original-value-ref-property-types))
|
||||
|
||||
(def user-ref-property-types
|
||||
"User ref types. Property values that users see are stored in either
|
||||
:logseq.property/value or :block/title. :block/title is for all the page related types"
|
||||
(into #{:date :node :asset} value-ref-property-types))
|
||||
|
||||
(assert (set/subset? user-ref-property-types
|
||||
(set user-built-in-property-types))
|
||||
"All ref types are valid property types")
|
||||
|
||||
(def all-ref-property-types
|
||||
"All ref types - user and internal"
|
||||
(into #{:entity :class :page :property} user-ref-property-types))
|
||||
|
||||
(assert (set/subset? all-ref-property-types
|
||||
(set/union (set user-built-in-property-types) internal-built-in-property-types))
|
||||
"All ref types are valid property types")
|
||||
|
||||
;; Property value validation
|
||||
;; =========================
|
||||
;; TODO:
|
||||
;; Validate && list fixes for non-validated values when updating property schema
|
||||
|
||||
(defn url?
|
||||
"Test if it is a `protocol://`-style URL. Allows custom protocol such as `zotero`.
|
||||
Originally from common-util/url? but does not need to be the same"
|
||||
[s]
|
||||
(and (string? s)
|
||||
(try
|
||||
(not (contains? #{nil} (.-origin (js/URL. s))))
|
||||
(catch :default _e
|
||||
false))))
|
||||
|
||||
(defn macro-url?
|
||||
[s]
|
||||
;; TODO: Confirm that macro expanded value is url when it's easier to pass data into validations
|
||||
(macro-util/macro? s))
|
||||
|
||||
(defn- url-entity?
|
||||
"Empty string, url or macro url"
|
||||
[db val {:keys [new-closed-value? skip-strict-url-validate?]}]
|
||||
(if new-closed-value?
|
||||
(or (url? val) (macro-url? val))
|
||||
(when-let [ent (d/entity db val)]
|
||||
(let [title (:block/title ent)]
|
||||
(if skip-strict-url-validate?
|
||||
(string? title)
|
||||
(or (string/blank? title) (url? title) (macro-url? title)))))))
|
||||
|
||||
(defn- entity?
|
||||
[db id]
|
||||
(some? (d/entity db id)))
|
||||
|
||||
(defn- class-entity?
|
||||
[db id]
|
||||
(entity-util/class? (d/entity db id)))
|
||||
|
||||
(defn- property-entity?
|
||||
[db id]
|
||||
(entity-util/property? (d/entity db id)))
|
||||
|
||||
(defn- page-entity?
|
||||
[db id]
|
||||
(entity-util/page? (d/entity db id)))
|
||||
|
||||
(defn- number-entity?
|
||||
[db id-or-value {:keys [new-closed-value?]}]
|
||||
(if new-closed-value?
|
||||
(number? id-or-value)
|
||||
(when-let [entity (d/entity db id-or-value)]
|
||||
(number? (:logseq.property/value entity)))))
|
||||
|
||||
(defn- text-entity?
|
||||
[db s {:keys [new-closed-value?]}]
|
||||
(if new-closed-value?
|
||||
(string? s)
|
||||
(when-let [ent (d/entity db s)]
|
||||
(and (string? (:block/title ent))
|
||||
(some? (:block/page ent))))))
|
||||
|
||||
(defn- node-entity?
|
||||
[db val]
|
||||
(when-let [ent (d/entity db val)]
|
||||
(some? (:block/title ent))))
|
||||
|
||||
;; :logseq.class/Asset is in ldb/private-tags, so users can't type `#Asset`
|
||||
;; inline — we use the class programmatically here to scope :asset property
|
||||
;; values to uploaded asset blocks.
|
||||
(defn- asset-entity?
|
||||
[db val]
|
||||
(when-let [ent (d/entity db val)]
|
||||
(and (some? (:block/title ent))
|
||||
(contains? (set (map :db/ident (:block/tags ent)))
|
||||
:logseq.class/Asset))))
|
||||
|
||||
(defn- date?
|
||||
[db val]
|
||||
(when-let [ent (d/entity db val)]
|
||||
(and (some? (:block/title ent))
|
||||
(entity-util/journal? ent))))
|
||||
|
||||
;; Internal usage
|
||||
(def internal-validation-schemas
|
||||
{:string [:fn
|
||||
{:error/message "should be a string"}
|
||||
string?]
|
||||
:json [:fn
|
||||
{:error/message "should be JSON string"}
|
||||
string?]
|
||||
:raw-number [:fn
|
||||
{:error/message "should be a raw number"}
|
||||
number?]
|
||||
:entity [:fn
|
||||
{:error/message "should be an Entity"}
|
||||
entity?]
|
||||
:class [:fn
|
||||
{:error/message "should be a Class"}
|
||||
class-entity?]
|
||||
:property [:fn
|
||||
{:error/message "should be a Property"}
|
||||
property-entity?]
|
||||
:page [:fn
|
||||
{:error/message "should be a Page"}
|
||||
page-entity?]
|
||||
:keyword [:fn
|
||||
{:error/message "should be a Clojure keyword"}
|
||||
keyword?]
|
||||
:map [:fn
|
||||
{:error/message "should be a Clojure map"}
|
||||
map?]
|
||||
;; coll elements are ordered as it's saved as a vec
|
||||
:coll [:fn
|
||||
{:error/message "should be a collection"}
|
||||
coll?]
|
||||
:any some?})
|
||||
|
||||
(def built-in-validation-schemas
|
||||
"Map of types to malli validation schemas that validate a property value for that type"
|
||||
(into
|
||||
{:default [:fn
|
||||
{:error/message "should be a text block"}
|
||||
text-entity?]
|
||||
:number [:fn
|
||||
{:error/message "should be a number"}
|
||||
number-entity?]
|
||||
:date [:fn
|
||||
{:error/message "should be a journal date"}
|
||||
date?]
|
||||
:datetime [:fn
|
||||
{:error/message "should be a datetime"}
|
||||
number?]
|
||||
:checkbox [:fn
|
||||
{:error/message "should be a boolean"}
|
||||
boolean?]
|
||||
:url [:fn
|
||||
{:error/message "should be a URL"}
|
||||
url-entity?]
|
||||
:node [:fn
|
||||
{:error/message "should be a node with a title"}
|
||||
node-entity?]
|
||||
:asset [:fn
|
||||
{:error/message "should be an asset node"}
|
||||
asset-entity?]}
|
||||
internal-validation-schemas))
|
||||
|
||||
(assert (= (set (keys built-in-validation-schemas))
|
||||
(into internal-built-in-property-types
|
||||
user-built-in-property-types))
|
||||
"Built-in property types must be equal")
|
||||
|
||||
(def property-types-with-db
|
||||
"Property types whose validation fn requires a datascript db"
|
||||
#{:default :url :number :date :node :asset :entity :class :property :page})
|
||||
|
||||
;; Helper fns
|
||||
;; ==========
|
||||
(defn infer-property-type-from-value
|
||||
"Infers a user defined built-in :type from property value(s)"
|
||||
[val]
|
||||
(cond
|
||||
(number? val) :number
|
||||
(url? val) :url
|
||||
(contains? #{true false} val) :checkbox
|
||||
:else :default))
|
||||
|
||||
(defn property-value-content?
|
||||
"Whether property value should be stored in :logseq.property/value"
|
||||
[block-type property]
|
||||
(or
|
||||
(original-value-ref-property-types (:logseq.property/type property))
|
||||
(and (= (:db/ident property) :logseq.property/default-value)
|
||||
(original-value-ref-property-types block-type))))
|
||||
72
deps/db/src/logseq/db/frontend/query_dsl.cljs
vendored
Normal file
72
deps/db/src/logseq/db/frontend/query_dsl.cljs
vendored
Normal file
@@ -0,0 +1,72 @@
|
||||
(ns logseq.db.frontend.query-dsl
|
||||
"Pure parsing helpers shared by renderer and worker query DSL code."
|
||||
(:require [clojure.string :as string]
|
||||
[clojure.walk :as walk]
|
||||
[logseq.common.util :as common-util]
|
||||
[logseq.common.util.page-ref :as page-ref]
|
||||
[logseq.db.frontend.property :as db-property]))
|
||||
|
||||
(def ^:private tag-placeholder "~~~tag-placeholder~~~")
|
||||
|
||||
(defn pre-transform
|
||||
[s]
|
||||
(if (common-util/wrapped-by-quotes? s)
|
||||
s
|
||||
(let [quoted-page-ref
|
||||
(fn [matches]
|
||||
(let [match' (string/replace (second matches) "#" tag-placeholder)]
|
||||
(str "\"" page-ref/left-brackets match' page-ref/right-brackets "\"")))]
|
||||
(some-> s
|
||||
(string/replace #"\"?\[\[(.*?)\]\]\"?" quoted-page-ref)
|
||||
(string/replace #"\(between ([^\)]+)\)"
|
||||
(fn [[_ x]]
|
||||
(->> (string/split x #" ")
|
||||
(remove string/blank?)
|
||||
(map (fn [value]
|
||||
(if (or (contains? #{"+" "-"} (first value))
|
||||
(and (common-util/safe-re-find #"\d" (first value))
|
||||
(some #(string/ends-with? value %)
|
||||
["y" "m" "d" "h" "min"])))
|
||||
(keyword (name value))
|
||||
value)))
|
||||
(string/join " ")
|
||||
(common-util/format "(between %s)"))))
|
||||
(string/replace #"\"[^\"]+\""
|
||||
#(string/replace % "#" tag-placeholder))
|
||||
(string/replace " #" " #tag ")
|
||||
(string/replace #"^#" "#tag ")
|
||||
(string/replace tag-placeholder "#")))))
|
||||
|
||||
(defn simplify-query
|
||||
[query]
|
||||
(if (string? query)
|
||||
query
|
||||
(walk/postwalk
|
||||
(fn [form]
|
||||
(if (and (coll? form)
|
||||
(contains? #{'and 'or} (first form))
|
||||
(= 2 (count form)))
|
||||
(second form)
|
||||
form))
|
||||
query)))
|
||||
|
||||
(defn get-timestamp-property
|
||||
[form]
|
||||
(let [property-name (second form)]
|
||||
(when (or (keyword? property-name)
|
||||
(symbol? property-name)
|
||||
(string? property-name))
|
||||
(let [property (-> property-name
|
||||
name
|
||||
string/lower-case
|
||||
(string/replace "_" "-")
|
||||
keyword)]
|
||||
(if (db-property/property? property)
|
||||
property
|
||||
(case property
|
||||
:created-at :block/created-at
|
||||
:updated-at :block/updated-at
|
||||
nil))))))
|
||||
|
||||
(def custom-readers
|
||||
{:readers {'tag page-ref/->page-ref}})
|
||||
360
deps/db/src/logseq/db/frontend/rules.cljc
vendored
Normal file
360
deps/db/src/logseq/db/frontend/rules.cljc
vendored
Normal file
@@ -0,0 +1,360 @@
|
||||
(ns ^:bb-compatible logseq.db.frontend.rules
|
||||
"Datalog rules for DB graphs")
|
||||
|
||||
(def ^:large-vars/data-var rules
|
||||
"Rules used mainly in frontend.db.model"
|
||||
;; rule "parent" is optimized for parent node -> child node nesting queries
|
||||
{:parent
|
||||
'[[(parent ?p ?c)
|
||||
[?c :block/parent ?p]]
|
||||
[(parent ?p ?c)
|
||||
[?t :block/parent ?p]
|
||||
(parent ?t ?c)]]
|
||||
|
||||
:class-extends
|
||||
'[[(class-extends ?p ?c)
|
||||
[?c :logseq.property.class/extends ?p]]
|
||||
[(class-extends ?p ?c)
|
||||
[?t :logseq.property.class/extends ?p]
|
||||
(class-extends ?t ?c)]]
|
||||
|
||||
:alias
|
||||
'[[(alias ?e2 ?e1)
|
||||
[?e2 :block/alias ?e1]]
|
||||
[(alias ?e2 ?e1)
|
||||
[?e1 :block/alias ?e2]]]
|
||||
|
||||
:self-ref
|
||||
'[(self-ref ?b ?ref)
|
||||
[?b :block/refs ?ref]]
|
||||
|
||||
:has-ref
|
||||
'[[(has-ref ?b ?r)
|
||||
[?b :block/refs ?r]]
|
||||
[(has-ref ?b ?r)
|
||||
(parent ?p ?b)
|
||||
[?p :block/refs ?r]]]})
|
||||
|
||||
;; Rules writing advice
|
||||
;; ====================
|
||||
;; Select rules carefully, as it is critical for performance.
|
||||
;; The rules have different clause order and resolving directions.
|
||||
;; Clause order Reference:
|
||||
;; https://docs.datomic.com/on-prem/query/query-executing.html#clause-order
|
||||
;; Recursive optimization Reference:
|
||||
;; https://stackoverflow.com/questions/42457136/recursive-datalog-queries-for-datomic-really-slow
|
||||
;; Should optimize for query the descendents of a block
|
||||
;; Quote:
|
||||
;; My theory is that your rules are not written in a way that Datalog can optimize for this read pattern - probably resulting in a traversal of all the entities. I suggest to rewrite them as follows:
|
||||
;; [[(ubersymbol ?c ?p)
|
||||
;; (?c :ml/parent ?p)]
|
||||
;; [(ubersymbol ?c ?p)
|
||||
;; ;; we bind a child of the ancestor, instead of a parent of the descendant
|
||||
;; (?c1 :ml/parent ?p)
|
||||
;; (ubersymbol ?c ?c1)]]
|
||||
|
||||
;; This way of writing the ruleset is optimized to find the descendants of some node. The way you originally wrote it is optimized to find the anscestors of some node.
|
||||
|
||||
;; from https://stackoverflow.com/questions/43784258/find-entities-whose-ref-to-many-attribute-contains-all-elements-of-input
|
||||
;; Quote:
|
||||
;; You're tackling the general problem of 'dynamic conjunction' in Datomic's Datalog.
|
||||
;; Write a dynamic Datalog query which uses 2 negations and 1 disjunction or a recursive rule
|
||||
;; Datalog has no direct way of expressing dynamic conjunction (logical AND / 'for all ...' / set intersection).
|
||||
;; However, you can achieve it in pure Datalog by combining one disjunction
|
||||
;; (logical OR / 'exists ...' / set union) and two negations, i.e
|
||||
;; (For all ?g in ?Gs p(?e,?g)) <=> NOT(Exists ?g in ?Gs, such that NOT(p(?e, ?g)))
|
||||
|
||||
;; [(matches-all ?e ?a ?vs)
|
||||
;; [(first ?vs) ?v0]
|
||||
;; [?e ?a ?v0]
|
||||
;; (not-join [?e ?vs]
|
||||
;; [(identity ?vs) [?v ...]]
|
||||
;; (not-join [?e ?v]
|
||||
;; [?e ?a ?v]))]
|
||||
|
||||
(def ^:large-vars/data-var db-query-dsl-rules
|
||||
"Rules used by frontend.query.dsl for DB graphs"
|
||||
(merge
|
||||
rules
|
||||
|
||||
{:page-ref
|
||||
'[(page-ref ?b ?ref)
|
||||
(has-ref ?b ?ref)]
|
||||
|
||||
:block-content
|
||||
'[(block-content ?b ?query)
|
||||
[?b :block/title ?content]
|
||||
[(clojure.string/includes? ?content ?query)]]
|
||||
|
||||
:page
|
||||
'[(page ?b ?page-name)
|
||||
[?b :block/page ?bp]
|
||||
[?bp :block/name ?page-name]]
|
||||
|
||||
:between
|
||||
'[(between ?b ?start ?end)
|
||||
[?b :block/page ?p]
|
||||
[?p :block/tags :logseq.class/Journal]
|
||||
[?p :block/journal-day ?d]
|
||||
[(>= ?d ?start)]
|
||||
[(<= ?d ?end)]]
|
||||
|
||||
:ref->val
|
||||
'[[(ref->val ?pv ?val)
|
||||
[?pv :block/title ?val]]
|
||||
[(ref->val ?pv ?val)
|
||||
[?pv :logseq.property/value ?val]]]
|
||||
|
||||
:property-missing-value
|
||||
'[(property-missing-value ?b ?prop-e ?default-p ?default-v)
|
||||
[?t :logseq.property.class/properties ?prop-e]
|
||||
[?prop-e :db/ident ?prop]
|
||||
(object-has-class-property? ?b ?prop)
|
||||
;; Notice: `(missing? )` doesn't work here because `de/entity`
|
||||
;; returns the default value if there's no value yet.
|
||||
[(get-else $ ?b ?prop "N/A") ?prop-v]
|
||||
[(= ?prop-v "N/A")]
|
||||
[?prop-e ?default-p ?default-v]]
|
||||
|
||||
:scalar-property-value
|
||||
'[[(scalar-property-value ?b ?prop-e ?val)
|
||||
[?prop-e :db/ident ?prop]
|
||||
[?b ?prop ?val]]]
|
||||
|
||||
:scalar-property-value-with-default
|
||||
'[[(scalar-property-value-with-default ?b ?prop-e ?val)
|
||||
(scalar-property-value ?b ?prop-e ?val)]
|
||||
|
||||
[(scalar-property-value-with-default ?b ?prop-e ?val)
|
||||
(property-missing-value ?b ?prop-e :logseq.property/scalar-default-value ?val)]]
|
||||
|
||||
:ref-property-value
|
||||
'[[(ref-property-value ?b ?prop-e ?val)
|
||||
[?prop-e :db/ident ?prop]
|
||||
[?b ?prop ?pv]
|
||||
(ref->val ?pv ?val)]]
|
||||
|
||||
:ref-property-value-with-default
|
||||
'[[(ref-property-value-with-default ?b ?prop-e ?val)
|
||||
(ref-property-value ?b ?prop-e ?val)]
|
||||
[(ref-property-value-with-default ?b ?prop-e ?val)
|
||||
(property-missing-value ?b ?prop-e :logseq.property/default-value ?pv)
|
||||
(ref->val ?pv ?val)]]
|
||||
|
||||
:object-has-class-property
|
||||
'[(object-has-class-property? ?b ?prop)
|
||||
[?prop-e :db/ident ?prop]
|
||||
[?t :logseq.property.class/properties ?prop-e]
|
||||
[?b :block/tags ?tc]
|
||||
(or
|
||||
[(= ?t ?tc)]
|
||||
(class-extends ?t ?tc))]
|
||||
|
||||
:has-property-or-object-property
|
||||
'[(has-property-or-object-property? ?b ?prop)
|
||||
[?prop-e :db/ident ?prop]
|
||||
(or
|
||||
[?b ?prop _]
|
||||
(object-has-class-property? ?b ?prop))]
|
||||
|
||||
;; Checks if a property exists for simple queries. Supports default values
|
||||
:has-simple-query-property
|
||||
'[(has-simple-query-property ?b ?prop)
|
||||
[?prop-e :db/ident ?prop]
|
||||
[?prop-e :block/tags :logseq.class/Property]
|
||||
(has-property-or-object-property? ?b ?prop)
|
||||
(or
|
||||
[(missing? $ ?prop-e :logseq.property/public?)]
|
||||
[?prop-e :logseq.property/public? true])]
|
||||
|
||||
;; Same as has-simple-query-property except it returns public and private properties like :block/title
|
||||
:has-private-simple-query-property
|
||||
'[(has-private-simple-query-property ?b ?prop)
|
||||
[?prop-e :db/ident ?prop]
|
||||
[?prop-e :block/tags :logseq.class/Property]
|
||||
(has-property-or-object-property? ?b ?prop)]
|
||||
|
||||
;; Checks if a property exists for any features that are not simple queries
|
||||
:has-property
|
||||
'[(has-property ?b ?prop)
|
||||
[?b ?prop _]
|
||||
[?prop-e :db/ident ?prop]
|
||||
[?prop-e :block/tags :logseq.class/Property]
|
||||
(or
|
||||
[(missing? $ ?prop-e :logseq.property/public?)]
|
||||
[?prop-e :logseq.property/public? true])]
|
||||
|
||||
;; Checks if a property has a value for simple queries. Supports default values
|
||||
:scalar-property
|
||||
'[(scalar-property ?b ?prop ?val)
|
||||
[?prop-e :db/ident ?prop]
|
||||
(scalar-property-value ?b ?prop-e ?val)
|
||||
(or
|
||||
[(missing? $ ?prop-e :logseq.property/public?)]
|
||||
[?prop-e :logseq.property/public? true])]
|
||||
|
||||
:scalar-property-with-default
|
||||
'[(scalar-property-with-default ?b ?prop ?val)
|
||||
[?prop-e :db/ident ?prop]
|
||||
(scalar-property-value-with-default ?b ?prop-e ?val)
|
||||
(or
|
||||
[(missing? $ ?prop-e :logseq.property/public?)]
|
||||
[?prop-e :logseq.property/public? true])]
|
||||
|
||||
:ref-property
|
||||
'[(ref-property ?b ?prop ?val)
|
||||
[?prop-e :db/ident ?prop]
|
||||
(ref-property-value ?b ?prop-e ?val)
|
||||
(or
|
||||
[(missing? $ ?prop-e :logseq.property/public?)]
|
||||
[?prop-e :logseq.property/public? true])]
|
||||
|
||||
:ref-property-with-default
|
||||
'[(ref-property-with-default ?b ?prop ?val)
|
||||
[?prop-e :db/ident ?prop]
|
||||
(ref-property-value-with-default ?b ?prop-e ?val)
|
||||
(or
|
||||
[(missing? $ ?prop-e :logseq.property/public?)]
|
||||
[?prop-e :logseq.property/public? true])]
|
||||
|
||||
;; Same as ref-property/scalar-property except it returns public and private properties like :block/title
|
||||
:private-scalar-property
|
||||
'[(private-scalar-property ?b ?prop ?val)
|
||||
[?prop-e :db/ident ?prop]
|
||||
[?prop-e :block/tags :logseq.class/Property]
|
||||
(scalar-property-value ?b ?prop-e ?val)]
|
||||
|
||||
:private-scalar-property-with-default
|
||||
'[(private-scalar-property-with-default ?b ?prop ?val)
|
||||
[?prop-e :db/ident ?prop]
|
||||
[?prop-e :block/tags :logseq.class/Property]
|
||||
(scalar-property-value-with-default ?b ?prop-e ?val)]
|
||||
|
||||
:private-ref-property
|
||||
'[(private-ref-property ?b ?prop ?val)
|
||||
[?prop-e :db/ident ?prop]
|
||||
[?prop-e :block/tags :logseq.class/Property]
|
||||
(ref-property-value ?b ?prop-e ?val)]
|
||||
|
||||
:private-ref-property-with-default
|
||||
'[(private-ref-property-with-default ?b ?prop ?val)
|
||||
[?prop-e :db/ident ?prop]
|
||||
[?prop-e :block/tags :logseq.class/Property]
|
||||
(ref-property-value-with-default ?b ?prop-e ?val)]
|
||||
|
||||
;; `property` is slow so don't use it for user-facing queries.
|
||||
;; However this is still useful as it is the only self-contained way to query properties
|
||||
:property
|
||||
'[(property ?b ?prop ?val)
|
||||
[?prop-e :db/ident ?prop]
|
||||
[?prop-e :block/tags :logseq.class/Property]
|
||||
(or
|
||||
[(missing? $ ?prop-e :logseq.property/public?)]
|
||||
[?prop-e :logseq.property/public? true])
|
||||
[?b ?prop ?pv]
|
||||
(or
|
||||
;; non-ref value
|
||||
(and
|
||||
[(missing? $ ?prop-e :db/valueType)]
|
||||
[?b ?prop ?val])
|
||||
;; ref value
|
||||
(and
|
||||
[?prop-e :db/valueType :db.type/ref]
|
||||
(or [?pv :block/title ?val]
|
||||
[?pv :logseq.property/value ?val])))]
|
||||
|
||||
:tags
|
||||
'[;; Case 1: input is number entity id
|
||||
[(tag-spec->tag ?tag ?spec)
|
||||
[(number? ?spec)]
|
||||
[(identity ?spec) ?tag]]
|
||||
|
||||
;; Case 2: input is :block/title
|
||||
[(tag-spec->tag ?tag ?spec)
|
||||
[?tag :block/title ?spec]]
|
||||
|
||||
;; Case 3: input is db/ident
|
||||
[(tag-spec->tag ?tag ?spec)
|
||||
[?tag :db/ident ?spec]]
|
||||
|
||||
;; --- Main rule -----------------------------------------------------------
|
||||
|
||||
[(tags ?b ?tags)
|
||||
;; enumerate user input set
|
||||
[(identity ?tags) [?spec ...]]
|
||||
(tag-spec->tag ?tag ?spec)
|
||||
|
||||
;; tag/class attached to block
|
||||
[?b :block/tags ?tc]
|
||||
|
||||
;; direct or descendant
|
||||
(or
|
||||
[(= ?tag ?tc)]
|
||||
(class-extends ?tag ?tc))
|
||||
|
||||
[(missing? $ ?b :block/link)]]]
|
||||
|
||||
:task
|
||||
'[(task ?b ?statuses)
|
||||
(ref-property-with-default ?b :logseq.property/status ?val)
|
||||
[(contains? ?statuses ?val)]]
|
||||
|
||||
:priority
|
||||
'[(priority ?b ?priorities)
|
||||
(ref-property-with-default ?b :logseq.property/priority ?priority)
|
||||
[(contains? ?priorities ?priority)]]}))
|
||||
|
||||
(def rules-dependencies
|
||||
"For db graphs, a map of rule names and the rules they depend on. If this map
|
||||
becomes long or brittle, we could do scan rules for their deps with something
|
||||
like find-rules-in-where"
|
||||
{:has-ref #{:parent}
|
||||
:page-ref #{:has-ref}
|
||||
|
||||
;; simple query helpers
|
||||
:task #{:ref-property-with-default}
|
||||
:priority #{:ref-property-with-default}
|
||||
:tags #{:class-extends}
|
||||
|
||||
:has-property-or-object-property #{:object-has-class-property}
|
||||
:object-has-class-property #{:class-extends}
|
||||
:has-simple-query-property #{:has-property-or-object-property}
|
||||
:has-private-simple-query-property #{:has-property-or-object-property}
|
||||
:property-missing-value #{:object-has-class-property}
|
||||
:ref-property-value #{:ref->val}
|
||||
:scalar-property #{:scalar-property-value}
|
||||
:scalar-property-with-default #{:scalar-property-value-with-default}
|
||||
:scalar-property-value-with-default #{:scalar-property-value :property-missing-value}
|
||||
:ref-property #{:ref-property-value}
|
||||
:ref-property-value-with-default #{:ref-property-value :property-missing-value}
|
||||
:ref-property-with-default #{:ref-property-value-with-default}
|
||||
:private-scalar-property #{:scalar-property-value}
|
||||
:private-scalar-property-with-default #{:scalar-property-value-with-default}
|
||||
:private-ref-property #{:ref-property-value}
|
||||
:private-ref-property-with-default #{:ref-property-value-with-default}})
|
||||
|
||||
(defn- get-full-deps
|
||||
[deps rules-deps]
|
||||
(loop [deps' deps
|
||||
result #{}]
|
||||
(if (seq deps')
|
||||
(recur (mapcat rules-deps deps')
|
||||
(into result deps'))
|
||||
result)))
|
||||
|
||||
(defn extract-rules
|
||||
"Given a rules map and the rule names to extract, returns a vector of rules to
|
||||
be passed to datascript.core/q. Can handle rules with multiple or single clauses.
|
||||
Takes following options:
|
||||
* :deps - A map of rule names to their dependencies. Only one-level of dependencies are resolved.
|
||||
No dependencies are detected by default though we could add it later e.g. find-rules-in-where"
|
||||
([rules-m] (extract-rules rules-m (keys rules-m)))
|
||||
([rules-m rules' & {:keys [deps]}]
|
||||
(let [rules-with-deps (if (map? deps)
|
||||
(get-full-deps rules' deps)
|
||||
rules')]
|
||||
(vec
|
||||
(mapcat #(let [val (rules-m %)]
|
||||
;; if vector?, rule has multiple clauses
|
||||
(if (vector? (first val)) val [val]))
|
||||
rules-with-deps)))))
|
||||
142
deps/db/src/logseq/db/frontend/schema.cljs
vendored
Normal file
142
deps/db/src/logseq/db/frontend/schema.cljs
vendored
Normal file
@@ -0,0 +1,142 @@
|
||||
(ns logseq.db.frontend.schema
|
||||
"Schema related fns"
|
||||
(:require [clojure.set :as set]
|
||||
[clojure.string :as string]))
|
||||
|
||||
(def schema-version? (every-pred map? :major))
|
||||
|
||||
(defn parse-schema-version
|
||||
"Return schema-version({:major <num> :minor <num>}).
|
||||
supported input: 10, \"10.1\", [10 1]"
|
||||
[string-or-compatible-number]
|
||||
(cond
|
||||
(schema-version? string-or-compatible-number) string-or-compatible-number
|
||||
(and (sequential? string-or-compatible-number)
|
||||
(first string-or-compatible-number))
|
||||
{:major (first string-or-compatible-number)
|
||||
:minor (second string-or-compatible-number)}
|
||||
|
||||
(int? string-or-compatible-number) {:major string-or-compatible-number :minor nil}
|
||||
(string? string-or-compatible-number)
|
||||
(let [[major minor] (map parse-long (string/split string-or-compatible-number #"\."))]
|
||||
(assert (some? major))
|
||||
{:major major :minor minor})
|
||||
:else
|
||||
(throw (ex-info (str "Bad schema version: " string-or-compatible-number) {:data string-or-compatible-number}))))
|
||||
|
||||
(defn compare-schema-version
|
||||
[x y]
|
||||
(apply compare
|
||||
(map (juxt :major :minor)
|
||||
[(parse-schema-version x) (parse-schema-version y)])))
|
||||
|
||||
(def version (parse-schema-version "65.33"))
|
||||
|
||||
(defn major-version
|
||||
"Return a number.
|
||||
Compatible with current schema-version number.
|
||||
schema-version-old: 10, a number
|
||||
schema-version-new: \"12.34\", string, <major-num>.<minor-num>"
|
||||
[schema-version]
|
||||
(if (schema-version? schema-version)
|
||||
(:major schema-version)
|
||||
(:major (parse-schema-version schema-version))))
|
||||
|
||||
(defn schema-version->string
|
||||
[schema-version]
|
||||
(cond
|
||||
(string? schema-version) schema-version
|
||||
(int? schema-version) (str schema-version)
|
||||
(schema-version? schema-version)
|
||||
(if-let [minor (:minor schema-version)]
|
||||
(str (:major schema-version) "." minor)
|
||||
(str (:major schema-version)))
|
||||
:else (throw (ex-info "Not a schema-version" {:data schema-version}))))
|
||||
|
||||
(def ^:large-vars/data-var schema
|
||||
"Schema for DB graphs"
|
||||
{:db/ident {:db/unique :db.unique/identity}
|
||||
:kv/value {}
|
||||
|
||||
:block/uuid {:db/unique :db.unique/identity}
|
||||
:block/parent {:db/valueType :db.type/ref
|
||||
:db/index true}
|
||||
:block/order {:db/index true}
|
||||
:block/collapsed? {}
|
||||
|
||||
;; belongs to which page
|
||||
:block/page {:db/valueType :db.type/ref
|
||||
:db/index true}
|
||||
;; reference blocks
|
||||
:block/refs {:db/valueType :db.type/ref
|
||||
:db/cardinality :db.cardinality/many}
|
||||
:block/tags {:db/valueType :db.type/ref
|
||||
:db/cardinality :db.cardinality/many}
|
||||
|
||||
;; which block this block links to, used for tag, embeds
|
||||
:block/link {:db/valueType :db.type/ref
|
||||
:db/index true}
|
||||
|
||||
;; for pages
|
||||
:block/alias {:db/valueType :db.type/ref
|
||||
:db/cardinality :db.cardinality/many
|
||||
:db/index true}
|
||||
|
||||
:block/created-at {:db/index true}
|
||||
:block/updated-at {:db/index true}
|
||||
|
||||
;; page additional attributes
|
||||
;; page's name, lowercase
|
||||
:block/name {:db/index true} ; remove db/unique for :block/name
|
||||
|
||||
;; page's original name
|
||||
:block/title {:db/index true}
|
||||
|
||||
;; page's journal day
|
||||
:block/journal-day {:db/index true}
|
||||
|
||||
;; latest tx that affected the block
|
||||
:block/tx-id {}
|
||||
:block/closed-value-property {:db/valueType :db.type/ref
|
||||
:db/cardinality :db.cardinality/many}
|
||||
|
||||
;; file
|
||||
:file/path {:db/unique :db.unique/identity}
|
||||
:file/content {}
|
||||
:file/created-at {}
|
||||
:file/last-modified-at {}
|
||||
:file/size {}})
|
||||
|
||||
;; If only block/title changes
|
||||
(def retract-attributes
|
||||
"Retract attributes for DB graphs"
|
||||
#{:block/warning})
|
||||
|
||||
;; DB graph helpers
|
||||
;; ================
|
||||
(def ref-type-attributes
|
||||
(into #{}
|
||||
(keep (fn [[attr-name attr-body-map]]
|
||||
(when (= :db.type/ref (:db/valueType attr-body-map))
|
||||
attr-name)))
|
||||
schema))
|
||||
|
||||
(def card-many-attributes
|
||||
(into #{}
|
||||
(keep (fn [[attr-name attr-body-map]]
|
||||
(when (= :db.cardinality/many (:db/cardinality attr-body-map))
|
||||
attr-name)))
|
||||
schema))
|
||||
|
||||
(def card-many-ref-type-attributes
|
||||
(set/intersection card-many-attributes ref-type-attributes))
|
||||
|
||||
(def card-one-ref-type-attributes
|
||||
(set/difference ref-type-attributes card-many-attributes))
|
||||
|
||||
(def db-non-ref-attributes
|
||||
(->> schema
|
||||
(keep (fn [[k v]]
|
||||
(when (not (:db/valueType v))
|
||||
k)))
|
||||
set))
|
||||
153
deps/db/src/logseq/db/frontend/validate.cljs
vendored
Normal file
153
deps/db/src/logseq/db/frontend/validate.cljs
vendored
Normal file
@@ -0,0 +1,153 @@
|
||||
(ns logseq.db.frontend.validate
|
||||
"Validate frontend db for DB graphs"
|
||||
(:require [clojure.pprint :as pprint]
|
||||
[datascript.core :as d]
|
||||
[logseq.db.frontend.malli-schema :as db-malli-schema]
|
||||
[logseq.db.frontend.property :as db-property]
|
||||
[malli.core :as m]
|
||||
[malli.error :as me]
|
||||
[malli.util :as mu]))
|
||||
|
||||
(def ^:private db-schema-validator (m/validator db-malli-schema/DB))
|
||||
(def ^:private db-schema-explainer (m/explainer db-malli-schema/DB))
|
||||
(def ^:private closed-db-schema-validator (m/validator (mu/closed-schema db-malli-schema/DB)))
|
||||
(def ^:private closed-db-schema-explainer (m/explainer (mu/closed-schema db-malli-schema/DB)))
|
||||
|
||||
(defn get-schema-validator
|
||||
[closed-schema?]
|
||||
(if closed-schema? closed-db-schema-validator db-schema-validator))
|
||||
|
||||
(defn get-schema-explainer
|
||||
[closed-schema?]
|
||||
(if closed-schema? closed-db-schema-explainer db-schema-explainer))
|
||||
|
||||
(defn validate-tx-report
|
||||
"Validates the datascript tx-report for entities that have changed. Returns
|
||||
boolean indicating if db is valid"
|
||||
[{:keys [db-after tx-data tx-meta]} {:keys [closed-schema?]}]
|
||||
(binding [db-malli-schema/*skip-strict-url-validate?* true]
|
||||
(let [changed-ids (->> tx-data (keep :e) distinct)
|
||||
tx-datoms (mapcat (fn [id]
|
||||
(d/datoms db-after :eavt id))
|
||||
changed-ids)
|
||||
ent-maps* (map (fn [[db-id m]]
|
||||
;; Add :db/id for debugging
|
||||
(assoc m :db/id db-id))
|
||||
(db-malli-schema/datoms->entity-maps tx-datoms {:entity-fn #(d/entity db-after %)}))
|
||||
ent-maps (db-malli-schema/update-properties-in-ents db-after ent-maps*)
|
||||
validator (get-schema-validator closed-schema?)]
|
||||
(binding [db-malli-schema/*db-for-validate-fns* db-after]
|
||||
(let [invalid-ent-maps (remove
|
||||
;; remove :db/id as it adds needless declarations to schema
|
||||
#(validator [(dissoc % :db/id)])
|
||||
ent-maps)
|
||||
schema-errors
|
||||
(when (seq invalid-ent-maps)
|
||||
(prn "Invalid datascript entities detected amongst changed entity ids:" changed-ids :tx-meta tx-meta)
|
||||
(let [explainer (get-schema-explainer closed-schema?)]
|
||||
(doall
|
||||
(map
|
||||
(fn [m]
|
||||
(let [m' (update m :block/properties (fn [properties]
|
||||
(map (fn [[p v]]
|
||||
[(:db/ident p) v])
|
||||
properties)))
|
||||
data {:entity-map m'
|
||||
:errors (me/humanize (explainer [(dissoc m :db/id)]))}]
|
||||
(try
|
||||
(pprint/pprint data)
|
||||
(catch :default _e
|
||||
(prn data)))
|
||||
data))
|
||||
invalid-ent-maps))))
|
||||
errors (->> schema-errors
|
||||
(remove nil?)
|
||||
vec)]
|
||||
(if (seq errors)
|
||||
[false errors]
|
||||
[true nil]))))))
|
||||
|
||||
(defn- group-errors-by-entity
|
||||
"Groups malli errors by entities. db is used for providing more debugging info"
|
||||
[db ent-maps errors]
|
||||
(assert (vector? ent-maps) "Must be a vec for grouping to work")
|
||||
(->> errors
|
||||
(group-by #(-> % :in first))
|
||||
(map (fn [[idx errors']]
|
||||
(let [ent (get ent-maps idx)]
|
||||
{:entity (cond-> ent
|
||||
;; Provide additional page info for debugging
|
||||
(:block/page ent)
|
||||
(update :block/page
|
||||
(fn [id]
|
||||
(let [page-ent (d/entity db id)]
|
||||
(cond-> (select-keys page-ent [:block/name :db/id :block/created-at])
|
||||
(:block/tags page-ent)
|
||||
(assoc :block/tags (mapv :db/ident (:block/tags page-ent))))))))
|
||||
:dispatch-key (->> (dissoc ent :db/id) (db-malli-schema/entity-dispatch-key db))
|
||||
:errors errors'})))))
|
||||
|
||||
(defn validate-db
|
||||
"Validates all the entities of the given db using :eavt datoms. Returns a map
|
||||
with info about db being validated. If there are errors, they are placed on
|
||||
:errors and grouped by entity"
|
||||
[db]
|
||||
(let [datoms (d/datoms db :eavt)
|
||||
ent-maps* (db-malli-schema/datoms->entities datoms)
|
||||
ent-maps (mapv
|
||||
;; Remove some UI interactions adding this e.g. import
|
||||
#(dissoc % :block.temp/load-status :block.temp/has-children?)
|
||||
(db-malli-schema/update-properties-in-ents db ent-maps*))
|
||||
errors (binding [db-malli-schema/*db-for-validate-fns* db]
|
||||
(-> (map (fn [e]
|
||||
(dissoc e :db/id))
|
||||
ent-maps) closed-db-schema-explainer :errors))]
|
||||
(cond-> {:datom-count (count datoms)
|
||||
:entities ent-maps*}
|
||||
(some? errors)
|
||||
(assoc :errors (map #(update % :errors (fn [errs] (me/humanize {:errors errs})))
|
||||
(group-errors-by-entity db ent-maps errors))))))
|
||||
|
||||
(defn graph-counts
|
||||
"Calculates graph-wide counts given a graph's db and its entities from :eavt datoms"
|
||||
[db entities]
|
||||
(let [classes-count (count (d/datoms db :avet :block/tags :logseq.class/Tag))
|
||||
properties-count (count (d/datoms db :avet :block/tags :logseq.class/Property))]
|
||||
{:entities (count entities)
|
||||
:pages (count (filter :block/name entities))
|
||||
;; Nodes that aren't pages
|
||||
:blocks (count (filter :block/page entities))
|
||||
:classes classes-count
|
||||
:properties properties-count
|
||||
;; Objects that aren't classes or properties
|
||||
:objects (- (count (d/datoms db :avet :block/tags)) classes-count properties-count)
|
||||
:property-pairs (count (mapcat #(-> % db-property/properties (dissoc :block/tags)) entities))}))
|
||||
|
||||
(defn validate-local-db!
|
||||
"Validates a local (non-RTC) DB like validate-db! but with default behavior,
|
||||
options and logging specific to a local DB. Used by CLI, importer and tests"
|
||||
[db & {:keys [db-name open-schema verbose]}]
|
||||
(let [datoms (d/datoms db :eavt)
|
||||
ent-maps* (db-malli-schema/datoms->entities datoms)
|
||||
_ (when verbose
|
||||
(println "Read graph" (str db-name " with counts: "
|
||||
(pr-str (assoc (graph-counts db ent-maps*)
|
||||
:datoms (count datoms))))))
|
||||
ent-maps (mapv
|
||||
;; Remove some UI interactions adding this e.g. import
|
||||
#(dissoc % :block.temp/load-status :block.temp/has-children?)
|
||||
(db-malli-schema/update-properties-in-ents db ent-maps*))
|
||||
explainer (get-schema-explainer (not open-schema))]
|
||||
(when-let [explanation (binding [db-malli-schema/*db-for-validate-fns* db
|
||||
db-malli-schema/*closed-values-validate?* true]
|
||||
(->> (map (fn [e] (dissoc e :db/id)) ent-maps) explainer not-empty))]
|
||||
(let [ent-errors
|
||||
(->> (group-errors-by-entity db ent-maps (:errors explanation))
|
||||
(map #(update % :errors
|
||||
(fn [errs]
|
||||
;; errs looks like: {178 {:logseq.property/hide? ["disallowed key"]}}
|
||||
;; map is indexed by :in which is unused since all errors are for the same map
|
||||
(->> (me/humanize {:errors errs})
|
||||
vals
|
||||
(apply merge-with into))))))]
|
||||
{:errors ent-errors}))))
|
||||
58
deps/db/src/logseq/db/sqlite/backup.cljs
vendored
Normal file
58
deps/db/src/logseq/db/sqlite/backup.cljs
vendored
Normal file
@@ -0,0 +1,58 @@
|
||||
(ns logseq.db.sqlite.backup
|
||||
"Shared SQLite backup utilities for Node runtimes."
|
||||
(:require ["node:fs" :as fs]
|
||||
["node:sqlite" :as node-sqlite]
|
||||
[clojure.string :as string]
|
||||
[goog.object :as gobj]
|
||||
[promesa.core :as p]))
|
||||
|
||||
(defn- resolve-database-sync-ctor
|
||||
[]
|
||||
(or (gobj/get node-sqlite "DatabaseSync")
|
||||
(some-> (gobj/get node-sqlite "default")
|
||||
(gobj/get "DatabaseSync"))
|
||||
(let [default-export (gobj/get node-sqlite "default")]
|
||||
(when (fn? default-export)
|
||||
default-export))
|
||||
(throw (ex-info "node:sqlite DatabaseSync constructor missing"
|
||||
{:module-keys (js->clj (js/Object.keys node-sqlite))}))))
|
||||
|
||||
(defn- resolve-sqlite-backup-fn
|
||||
[]
|
||||
(or (gobj/get node-sqlite "backup")
|
||||
(some-> (gobj/get node-sqlite "default")
|
||||
(gobj/get "backup"))
|
||||
(throw (ex-info "node:sqlite backup function missing"
|
||||
{:module-keys (js->clj (js/Object.keys node-sqlite))}))))
|
||||
|
||||
(def ^:private sqlite-backup-fn
|
||||
(resolve-sqlite-backup-fn))
|
||||
|
||||
(defn- remove-file-if-exists!
|
||||
[path]
|
||||
(fs/rmSync path #js {:force true}))
|
||||
|
||||
(defn backup-connection!
|
||||
[^js db path]
|
||||
(try
|
||||
(-> (sqlite-backup-fn db path)
|
||||
(p/catch (fn [error]
|
||||
(remove-file-if-exists! path)
|
||||
(throw error))))
|
||||
(catch :default error
|
||||
(remove-file-if-exists! path)
|
||||
(p/rejected error))))
|
||||
|
||||
(defn backup-db-file!
|
||||
([src-path dst-path]
|
||||
(let [DatabaseSync (resolve-database-sync-ctor)
|
||||
db (new DatabaseSync src-path)]
|
||||
(-> (backup-connection! db dst-path)
|
||||
(p/finally (fn []
|
||||
(try
|
||||
(.close db)
|
||||
(catch :default error
|
||||
(when-not (string/includes? (str error) "database is not open")
|
||||
(throw error)))))))))
|
||||
([db _src-path dst-path]
|
||||
(backup-connection! db dst-path)))
|
||||
1067
deps/db/src/logseq/db/sqlite/build.cljs
vendored
Normal file
1067
deps/db/src/logseq/db/sqlite/build.cljs
vendored
Normal file
File diff suppressed because it is too large
Load Diff
275
deps/db/src/logseq/db/sqlite/create_graph.cljs
vendored
Normal file
275
deps/db/src/logseq/db/sqlite/create_graph.cljs
vendored
Normal file
@@ -0,0 +1,275 @@
|
||||
(ns logseq.db.sqlite.create-graph
|
||||
"Helper fns for creating a DB graph"
|
||||
(:require [clojure.string :as string]
|
||||
[logseq.common.config :as common-config]
|
||||
[logseq.common.util :as common-util]
|
||||
[logseq.common.uuid :as common-uuid]
|
||||
[logseq.db.frontend.class :as db-class]
|
||||
[logseq.db.frontend.entity-util :as entity-util]
|
||||
[logseq.db.frontend.malli-schema :as db-malli-schema]
|
||||
[logseq.db.frontend.property :as db-property]
|
||||
[logseq.db.frontend.property.build :as db-property-build]
|
||||
[logseq.db.frontend.property.type :as db-property-type]
|
||||
[logseq.db.frontend.schema :as db-schema]
|
||||
[logseq.db.sqlite.util :as sqlite-util]))
|
||||
|
||||
(defn mark-block-as-built-in [block]
|
||||
(assoc block :logseq.property/built-in? true))
|
||||
|
||||
(defn- schema->qualified-property-keyword
|
||||
[prop-schema]
|
||||
(reduce-kv
|
||||
(fn [r k v]
|
||||
(if-let [new-k (and (simple-keyword? k) (db-property/schema-properties-map k))]
|
||||
(assoc r new-k v)
|
||||
(assoc r k v)))
|
||||
{}
|
||||
prop-schema))
|
||||
|
||||
(defn- ->property-value-tx-m
|
||||
"Given a new block and its properties, creates a map of properties which have values of property value tx.
|
||||
This map is used for both creating the new property values and then adding them to a block"
|
||||
[new-block properties]
|
||||
(db-property-build/build-property-values-tx-m
|
||||
new-block
|
||||
(->> properties
|
||||
(keep (fn [[k v]]
|
||||
(when-let [built-in-type (get-in db-property/built-in-properties [k :schema :type])]
|
||||
(if (and (db-property-type/value-ref-property-types built-in-type)
|
||||
;; closed values are referenced by their :db/ident so no need to create values
|
||||
(not (get-in db-property/built-in-properties [k :closed-values])))
|
||||
(let [property-map {:db/ident k
|
||||
:logseq.property/type built-in-type}]
|
||||
[property-map v])
|
||||
(when-let [built-in-type' (get (:build/properties-ref-types new-block) built-in-type)]
|
||||
(let [property-map {:db/ident k
|
||||
:logseq.property/type built-in-type'}]
|
||||
[property-map v])))))))
|
||||
:pure? true))
|
||||
|
||||
(defn build-properties
|
||||
"Given a properties map in the format of db-property/built-in-properties, builds their properties tx"
|
||||
[built-in-properties]
|
||||
(mapcat
|
||||
(fn [[db-ident {:keys [attribute schema title closed-values properties]}]]
|
||||
(let [db-ident (or attribute db-ident)
|
||||
schema' (schema->qualified-property-keyword schema)
|
||||
[property & others] (if closed-values
|
||||
(db-property-build/build-closed-values
|
||||
db-ident
|
||||
title
|
||||
{:db/ident db-ident :schema schema' :closed-values closed-values}
|
||||
{})
|
||||
[(sqlite-util/build-new-property
|
||||
db-ident
|
||||
schema'
|
||||
{:title title})])
|
||||
pvalue-tx-m (->property-value-tx-m
|
||||
(merge property
|
||||
;; This config is for :logseq.property/default-value and may need to
|
||||
;; move to built-in-properties
|
||||
{:build/properties-ref-types {:entity :number}})
|
||||
(->> properties
|
||||
;; No need to create property value if it's an internal ident
|
||||
(remove (fn [[_k v]]
|
||||
(and (keyword? v) (db-malli-schema/internal-ident? v))))
|
||||
(into {})))
|
||||
;; _ (when (seq pvalue-tx-m) (prn :pvalue-tx-m db-ident pvalue-tx-m))
|
||||
|
||||
;; The order of tx matters. property and others must come first as
|
||||
;; they may contain idents and uuids that are referenced by properties
|
||||
tx
|
||||
(cond-> [property]
|
||||
(seq others)
|
||||
(into others)
|
||||
(seq pvalue-tx-m)
|
||||
(into (mapcat #(if (set? %) % [%]) (vals pvalue-tx-m)))
|
||||
(seq properties)
|
||||
(conj
|
||||
(merge {:block/uuid (:block/uuid property)}
|
||||
properties
|
||||
(db-property-build/build-properties-with-ref-values pvalue-tx-m))))]
|
||||
tx))
|
||||
built-in-properties))
|
||||
|
||||
(defn- build-bootstrap-property
|
||||
[db-ident]
|
||||
(sqlite-util/build-new-property
|
||||
db-ident
|
||||
(schema->qualified-property-keyword (get-in db-property/built-in-properties [db-ident :schema]))
|
||||
{:title (get-in db-property/built-in-properties [db-ident :title])}))
|
||||
|
||||
(defn- build-initial-properties
|
||||
"Builds initial properties and their closed values and marks them
|
||||
as built-in?. Returns their tx data as well as data needed for subsequent build steps"
|
||||
[]
|
||||
;; bootstrap-idents must either be necessary to define a property or be used on every property
|
||||
(let [bootstrap-idents #{:logseq.property/type :logseq.property/hide? :logseq.property/built-in?
|
||||
;; Required to define :properties on a property
|
||||
:logseq.property/created-from-property}
|
||||
bootstrap-properties (map build-bootstrap-property bootstrap-idents)
|
||||
;; First tx bootstrap properties so they can take affect. Then tx the bootstrap properties on themselves
|
||||
bootstrap-properties-tx (into (mapv #(apply dissoc % bootstrap-idents) bootstrap-properties)
|
||||
(mapv #(select-keys % (into [:block/uuid] bootstrap-idents))
|
||||
bootstrap-properties))
|
||||
properties-tx (build-properties (apply dissoc db-property/built-in-properties bootstrap-idents))
|
||||
mark-block-as-built-in' (fn [b] (mark-block-as-built-in {:block/uuid (:block/uuid b)}))
|
||||
;; Tx order matters
|
||||
tx (concat bootstrap-properties-tx
|
||||
properties-tx
|
||||
;; Adding built-ins must come after its properties are defined
|
||||
(map mark-block-as-built-in' bootstrap-properties)
|
||||
(map mark-block-as-built-in' properties-tx))]
|
||||
(doseq [m tx]
|
||||
(when-let [block-uuid (and (:db/ident m) (:block/uuid m))]
|
||||
(assert (string/starts-with? (str block-uuid) "00000002") m)))
|
||||
|
||||
{:tx tx
|
||||
:properties (filter entity-util/property? properties-tx)}))
|
||||
|
||||
(def built-in-pages-names
|
||||
#{common-config/library-page-name
|
||||
common-config/quick-add-page-name
|
||||
"Contents"})
|
||||
|
||||
(defn- validate-tx-for-duplicate-idents [tx]
|
||||
(when-let [conflicting-idents
|
||||
(->> (keep :db/ident tx)
|
||||
frequencies
|
||||
(keep (fn [[k v]] (when (> v 1) k)))
|
||||
seq)]
|
||||
(throw (ex-info (str "The following :db/idents are not unique and clobbered each other: "
|
||||
(vec conflicting-idents))
|
||||
{:idents conflicting-idents}))))
|
||||
|
||||
(defn build-initial-classes*
|
||||
[built-in-classes db-ident->properties]
|
||||
(map
|
||||
(fn [[db-ident {:keys [properties schema title]}]]
|
||||
(let [title' (or title (name db-ident))]
|
||||
(mark-block-as-built-in
|
||||
(sqlite-util/build-new-class
|
||||
(let [class-properties (mapv
|
||||
(fn [db-ident]
|
||||
(let [property (get db-ident->properties db-ident)]
|
||||
(assert property (str "Built-in property " db-ident " is not defined yet"))
|
||||
db-ident))
|
||||
(:properties schema))]
|
||||
(cond->
|
||||
{:block/title title'
|
||||
:block/name (common-util/page-name-sanity-lc title')
|
||||
:db/ident db-ident
|
||||
:block/uuid (common-uuid/gen-uuid :db-ident-block-uuid db-ident)}
|
||||
(seq class-properties)
|
||||
(assoc :logseq.property.class/properties class-properties)
|
||||
(seq properties)
|
||||
(merge properties)))))))
|
||||
built-in-classes))
|
||||
|
||||
(defn- build-initial-classes
|
||||
[db-ident->properties]
|
||||
(build-initial-classes* db-class/built-in-classes db-ident->properties))
|
||||
|
||||
(defn build-initial-views
|
||||
"Builds initial blocks used for storing views"
|
||||
[]
|
||||
(let [page-id (common-uuid/gen-uuid :builtin-block-uuid common-config/views-page-name)]
|
||||
[(sqlite-util/block-with-timestamps
|
||||
{:block/uuid page-id
|
||||
:block/name common-config/views-page-name
|
||||
:block/title common-config/views-page-name
|
||||
:block/tags [:logseq.class/Page]
|
||||
:logseq.property/hide? true
|
||||
:logseq.property/built-in? true})]))
|
||||
|
||||
(defn- build-recycle-page
|
||||
[]
|
||||
[(sqlite-util/block-with-timestamps
|
||||
{:block/uuid (common-uuid/gen-uuid :builtin-block-uuid "Recycle")
|
||||
:block/name (common-util/page-name-sanity-lc "Recycle")
|
||||
:block/title "Recycle"
|
||||
:block/tags [:logseq.class/Page]
|
||||
:logseq.property/hide? true
|
||||
:logseq.property/built-in? true})])
|
||||
|
||||
(defn- build-favorites-page
|
||||
[]
|
||||
[(sqlite-util/block-with-timestamps
|
||||
{:block/uuid (common-uuid/gen-uuid :builtin-block-uuid common-config/favorites-page-name)
|
||||
:block/name common-config/favorites-page-name
|
||||
:block/title common-config/favorites-page-name
|
||||
:block/tags [:logseq.class/Page]
|
||||
:logseq.property/hide? true
|
||||
:logseq.property/built-in? true})])
|
||||
|
||||
(defn- build-initial-files [config-content]
|
||||
[{:block/uuid (common-uuid/gen-uuid :builtin-block-uuid "logseq/config.edn")
|
||||
:file/path (str "logseq/" "config.edn")
|
||||
:file/content config-content
|
||||
:file/created-at (js/Date.)
|
||||
:file/last-modified-at (js/Date.)}
|
||||
{:block/uuid (common-uuid/gen-uuid :builtin-block-uuid "logseq/custom.css")
|
||||
:file/path (str "logseq/" "custom.css")
|
||||
:file/content ""
|
||||
:file/created-at (js/Date.)
|
||||
:file/last-modified-at (js/Date.)}
|
||||
{:block/uuid (common-uuid/gen-uuid :builtin-block-uuid "logseq/custom.js")
|
||||
:file/path (str "logseq/" "custom.js")
|
||||
:file/content ""
|
||||
:file/created-at (js/Date.)
|
||||
:file/last-modified-at (js/Date.)}
|
||||
{:block/uuid (common-uuid/gen-uuid :builtin-block-uuid "logseq/publish.css")
|
||||
:file/path (str "logseq/" "publish.css")
|
||||
:file/content ""
|
||||
:file/created-at (js/Date.)
|
||||
:file/last-modified-at (js/Date.)}
|
||||
{:block/uuid (common-uuid/gen-uuid :builtin-block-uuid "logseq/publish.js")
|
||||
:file/path (str "logseq/" "publish.js")
|
||||
:file/content ""
|
||||
:file/created-at (js/Date.)
|
||||
:file/last-modified-at (js/Date.)}])
|
||||
|
||||
(defn build-db-initial-data
|
||||
"Builds tx of initial data for a new graph including key values, initial files,
|
||||
built-in properties and built-in classes"
|
||||
[config-content & {:keys [import-type graph-git-sha creating-remote-graph?]}]
|
||||
(assert (string? config-content))
|
||||
(let [initial-data (cond->
|
||||
[(sqlite-util/kv :logseq.kv/db-type "db")
|
||||
(sqlite-util/kv :logseq.kv/schema-version db-schema/version)
|
||||
(sqlite-util/kv :logseq.kv/graph-initial-schema-version db-schema/version)
|
||||
(sqlite-util/kv :logseq.kv/graph-created-at (common-util/time-ms))
|
||||
;; Empty property value used by db.type/ref properties
|
||||
{:db/ident :logseq.property/empty-placeholder
|
||||
:block/uuid (common-uuid/gen-uuid :builtin-block-uuid :logseq.property/empty-placeholder)}]
|
||||
import-type
|
||||
(into (sqlite-util/import-tx import-type))
|
||||
graph-git-sha
|
||||
(conj (sqlite-util/kv :logseq.kv/graph-git-sha graph-git-sha))
|
||||
creating-remote-graph?
|
||||
(conj (sqlite-util/kv :logseq.kv/graph-remote? creating-remote-graph?))
|
||||
true
|
||||
(conj (sqlite-util/kv :logseq.kv/local-graph-uuid
|
||||
(uuid (str "00000000" (subs (str (common-uuid/gen-uuid)) 8))))))
|
||||
initial-files (build-initial-files config-content)
|
||||
{properties-tx :tx :keys [properties]} (build-initial-properties)
|
||||
db-ident->properties (zipmap (map :db/ident properties) properties)
|
||||
default-classes (build-initial-classes db-ident->properties)
|
||||
default-pages (->> (map sqlite-util/build-new-page built-in-pages-names)
|
||||
(map mark-block-as-built-in))
|
||||
hidden-pages (concat (build-initial-views) (build-favorites-page) (build-recycle-page))
|
||||
;; These classes bootstrap our tags and properties as they depend on each other e.g.
|
||||
;; Root <-> Tag, classes-tx depends on logseq.property.class/extends, properties-tx depends on Property
|
||||
bootstrap-class? (fn [c] (contains? #{:logseq.class/Root :logseq.class/Property :logseq.class/Tag
|
||||
:logseq.class/Page :logseq.class/Template} (:db/ident c)))
|
||||
bootstrap-classes (filter bootstrap-class? default-classes)
|
||||
bootstrap-class-ids (map #(select-keys % [:db/ident :block/uuid]) bootstrap-classes)
|
||||
classes-tx (concat (map #(dissoc % :db/ident) bootstrap-classes)
|
||||
(remove bootstrap-class? default-classes))
|
||||
;; Order of tx is critical. bootstrap-class-ids bootstraps properties-tx and classes-tx
|
||||
;; bootstrap-class-ids coming first is useful as Root, Tag and Property have stable :db/id's of 1, 2 and 3
|
||||
tx (vec (concat bootstrap-class-ids
|
||||
initial-data properties-tx classes-tx
|
||||
initial-files default-pages hidden-pages))]
|
||||
(validate-tx-for-duplicate-idents tx)
|
||||
tx))
|
||||
37
deps/db/src/logseq/db/sqlite/debug.cljs
vendored
Normal file
37
deps/db/src/logseq/db/sqlite/debug.cljs
vendored
Normal file
@@ -0,0 +1,37 @@
|
||||
(ns logseq.db.sqlite.debug
|
||||
"SQLite debug fns"
|
||||
(:require [cljs-bean.core :as bean]
|
||||
[clojure.set]
|
||||
[logseq.db.sqlite.util :as sqlite-util]))
|
||||
|
||||
(defn find-missing-addresses
|
||||
"WASM version to find missing addresses from the kvs table"
|
||||
[^Object db]
|
||||
(let [schema (some->> (.exec db #js {:sql "select content from kvs where addr = 0"
|
||||
:rowMode "array"})
|
||||
bean/->clj
|
||||
ffirst
|
||||
sqlite-util/read-transit-str)
|
||||
result (->> (.exec db #js {:sql "select addr, addresses from kvs"
|
||||
:rowMode "array"})
|
||||
bean/->clj
|
||||
(keep (fn [[addr addresses]]
|
||||
[addr (bean/->clj (js/JSON.parse addresses))])))
|
||||
used-addresses (set (concat (mapcat second result)
|
||||
[0 1 (:eavt schema) (:avet schema) (:aevt schema)]))]
|
||||
(clojure.set/difference used-addresses (set (map first result)))))
|
||||
|
||||
(defn find-missing-addresses-node-version
|
||||
"Node version to find missing addresses from the kvs table"
|
||||
[^Object db]
|
||||
(let [schema (let [stmt (.prepare db "select content from kvs where addr = ?")
|
||||
content (.-content (.get stmt 0))]
|
||||
(sqlite-util/read-transit-str content))
|
||||
stmt (.prepare db "select addr, addresses from kvs")
|
||||
result (->> (.all ^Object stmt)
|
||||
bean/->clj
|
||||
(keep (fn [{:keys [addr addresses]}]
|
||||
[addr (bean/->clj (js/JSON.parse addresses))])))
|
||||
used-addresses (set (concat (mapcat second result)
|
||||
[0 1 (:eavt schema) (:avet schema) (:aevt schema)]))]
|
||||
(clojure.set/difference used-addresses (set (map first result)))))
|
||||
1455
deps/db/src/logseq/db/sqlite/export.cljs
vendored
Normal file
1455
deps/db/src/logseq/db/sqlite/export.cljs
vendored
Normal file
File diff suppressed because it is too large
Load Diff
120
deps/db/src/logseq/db/sqlite/gc.cljs
vendored
Normal file
120
deps/db/src/logseq/db/sqlite/gc.cljs
vendored
Normal file
@@ -0,0 +1,120 @@
|
||||
(ns logseq.db.sqlite.gc
|
||||
"GC unused addresses from `kvs` table"
|
||||
(:require [cljs-bean.core :as bean]
|
||||
[clojure.set :as set]
|
||||
[logseq.db.sqlite.util :as sqlite-util]))
|
||||
|
||||
(defn- walk-addresses
|
||||
"Given a map of parent address to children addresses and a root address,
|
||||
returns a set of all used addresses including the root and its descendants."
|
||||
[root addr->children]
|
||||
(println :debug :walk-addresses :root root)
|
||||
(time
|
||||
(letfn [(collect-addresses [addr]
|
||||
(let [children (addr->children addr)]
|
||||
(into #{addr} (mapcat collect-addresses children))))]
|
||||
(collect-addresses root))))
|
||||
|
||||
(defonce get-non-refed-addrs-sql
|
||||
"WITH all_referenced AS (
|
||||
SELECT CAST(value AS INTEGER) AS addr
|
||||
FROM kvs, json_each(kvs.addresses)
|
||||
)
|
||||
SELECT kvs.addr
|
||||
FROM kvs
|
||||
WHERE kvs.addr NOT IN (SELECT addr FROM all_referenced)")
|
||||
|
||||
(defn- get-unused-addresses
|
||||
[db]
|
||||
(let [schema (some->> (.exec db #js {:sql "select content from kvs where addr = 0"
|
||||
:rowMode "array"})
|
||||
bean/->clj
|
||||
ffirst
|
||||
sqlite-util/read-transit-str)
|
||||
;; 0: Datascript sets 0 as the address to store the db's meta, including addresses for :eavt, :avet, and aevt index.
|
||||
;; 1: Datascript sets 1 for tail, to improve the performance
|
||||
internal-addrs (set [0 1 (:eavt schema) (:avet schema) (:aevt schema)])
|
||||
non-refed-addrs (->> (.exec db #js {:sql get-non-refed-addrs-sql
|
||||
:rowMode "array"})
|
||||
(map first)
|
||||
set)]
|
||||
(set/difference non-refed-addrs internal-addrs)))
|
||||
|
||||
(defn gc-kvs-table!
|
||||
"WASM version to GC kvs table to remove unused addresses"
|
||||
[^Object db {:keys [full-gc?] :as opts}]
|
||||
(when db
|
||||
(let [unused-addresses (get-unused-addresses db)]
|
||||
(if (seq unused-addresses)
|
||||
(do
|
||||
(println :debug :db-gc :unused-addresses unused-addresses)
|
||||
(.transaction db (fn [tx]
|
||||
(doseq [addr unused-addresses]
|
||||
(.exec tx #js {:sql "Delete from kvs where addr = ?"
|
||||
:bind #js [addr]}))))
|
||||
(when full-gc?
|
||||
(gc-kvs-table! db opts)))
|
||||
(println :debug :db-gc "There's no garbage data that's need to be collected.")))))
|
||||
|
||||
(defn- get-unused-addresses-node-version
|
||||
[^object db]
|
||||
(let [schema (let [^object stmt (.prepare db "select content from kvs where addr = ?")
|
||||
content (.-content (.get ^object stmt 0))]
|
||||
(sqlite-util/read-transit-str content))
|
||||
internal-addrs (set [0 1 (:eavt schema) (:avet schema) (:aevt schema)])
|
||||
non-refed-addrs (let [^object stmt (.prepare db get-non-refed-addrs-sql)]
|
||||
(->> (.all stmt)
|
||||
bean/->clj
|
||||
(map :addr)
|
||||
(set)))]
|
||||
(set/difference non-refed-addrs internal-addrs)))
|
||||
|
||||
(defn- get-unused-addresses-node-walk-version
|
||||
[^object db]
|
||||
(let [schema (let [^object stmt (.prepare db "select content from kvs where addr = ?")
|
||||
content (.-content (.get ^object stmt 0))]
|
||||
(sqlite-util/read-transit-str content))
|
||||
set-addresses #{(:eavt schema) (:avet schema) (:aevt schema)}
|
||||
internal-addresses (conj set-addresses 0 1)
|
||||
parent->children (let [^object stmt (.prepare db "select addr, addresses from kvs")]
|
||||
(->> (.all stmt)
|
||||
bean/->clj
|
||||
(map (fn [{:keys [addr addresses]}]
|
||||
[addr (bean/->clj (js/JSON.parse addresses))]))
|
||||
(into {})))
|
||||
used-addresses (->> (mapcat (fn [set-root-addr]
|
||||
(walk-addresses set-root-addr parent->children)) set-addresses)
|
||||
set
|
||||
(set/union internal-addresses))]
|
||||
(set/difference (set (keys parent->children)) used-addresses)))
|
||||
|
||||
(defn gc-kvs-table-node-version!
|
||||
"Node version to GC kvs table to remove unused addresses
|
||||
`walk?` - `true`: walk all used addresses, `false`: gc recursively"
|
||||
[^Object db walk?]
|
||||
(let [unused-addresses (if walk?
|
||||
(get-unused-addresses-node-walk-version db)
|
||||
(get-unused-addresses-node-version db))
|
||||
addrs-count (let [^object stmt (.prepare db "select count(*) as c from kvs")]
|
||||
(.-c (.get stmt)))]
|
||||
(println :debug "addrs total count: " addrs-count)
|
||||
(if (seq unused-addresses)
|
||||
(do
|
||||
(println :debug :db-gc :unused-addresses-count (count unused-addresses))
|
||||
(let [^object stmt (.prepare db "Delete from kvs where addr = ?")
|
||||
delete (.transaction
|
||||
db
|
||||
(fn [addrs]
|
||||
(doseq [addr addrs]
|
||||
(.run ^object stmt addr))))]
|
||||
(delete (bean/->js unused-addresses))
|
||||
(when-not walk?
|
||||
(gc-kvs-table-node-version! db false))))
|
||||
(println :debug :db-gc "There's no garbage data that's need to be collected."))))
|
||||
|
||||
(defn ensure-no-garbage
|
||||
[^Object db]
|
||||
(let [unused-addresses (get-unused-addresses-node-version db)]
|
||||
;; (println :debug :db-gc :unused-addresses-count (count unused-addresses))
|
||||
;; (println :debug :unused-addresses unused-addresses)
|
||||
(empty? unused-addresses)))
|
||||
134
deps/db/src/logseq/db/sqlite/util.cljs
vendored
Normal file
134
deps/db/src/logseq/db/sqlite/util.cljs
vendored
Normal file
@@ -0,0 +1,134 @@
|
||||
(ns logseq.db.sqlite.util
|
||||
"Utils fns for backend sqlite db"
|
||||
(:require [cljs-bean.transit]
|
||||
[clojure.string :as string]
|
||||
[cognitect.transit :as transit]
|
||||
[datascript.core]
|
||||
[datascript.impl.entity :as de]
|
||||
[datascript.transit :as dt]
|
||||
[logseq.common.config :as common-config]
|
||||
[logseq.common.util :as common-util]
|
||||
[logseq.common.uuid :as common-uuid]
|
||||
[logseq.db.common.order :as db-order]
|
||||
[logseq.db.frontend.property :as db-property]
|
||||
[logseq.db.frontend.property.type :as db-property-type]))
|
||||
|
||||
(defonce db-version-prefix common-config/db-version-prefix)
|
||||
|
||||
(def ^:private write-handlers (cljs-bean.transit/writer-handlers))
|
||||
(def ^:private read-handlers {})
|
||||
(def write-transit-str
|
||||
(let [write-handlers* (->> (assoc dt/write-handlers
|
||||
de/Entity (transit/write-handler (constantly "datascript/Entity")
|
||||
(fn [^de/entity entity]
|
||||
(assert (some? (:db/id entity)))
|
||||
(assoc (.-kv entity)
|
||||
:db/id (:db/id entity))))
|
||||
ExceptionInfo (transit/write-handler (constantly "error")
|
||||
(fn [e]
|
||||
{:message (ex-message e)
|
||||
:data (ex-data e)}))
|
||||
js/Error (transit/write-handler (constantly "js/Error")
|
||||
(fn [e] {:message (ex-message e)})))
|
||||
(merge write-handlers))
|
||||
writer (transit/writer :json {:handlers write-handlers*})]
|
||||
(fn write-transit-str* [o]
|
||||
(try (transit/write writer o)
|
||||
(catch :default e
|
||||
(prn ::write-transit-str (type o) o)
|
||||
(js/console.trace)
|
||||
(throw e))))))
|
||||
|
||||
(def read-transit-str
|
||||
(let [read-handlers* (->> (assoc dt/read-handlers
|
||||
"datascript/Entity" identity
|
||||
"error" (fn [m] (ex-info (:message m) (:data m)))
|
||||
"js/Error" (fn [m] (js/Error. (:message m))))
|
||||
(merge read-handlers))
|
||||
reader (transit/reader :json {:handlers read-handlers*})]
|
||||
(fn read-transit-str* [s]
|
||||
(transit/read reader s))))
|
||||
|
||||
(defn db-based-graph?
|
||||
[graph-name]
|
||||
(when graph-name
|
||||
(string/starts-with? graph-name db-version-prefix)))
|
||||
|
||||
(def block-with-timestamps common-util/block-with-timestamps)
|
||||
|
||||
(defn build-new-property
|
||||
"Build a standard new property so that it is is consistent across contexts. Takes
|
||||
an optional map with following keys:
|
||||
* :title - Case sensitive property name. Defaults to deriving this from db-ident
|
||||
* :block-uuid - :block/uuid for property"
|
||||
([db-ident prop-schema] (build-new-property db-ident prop-schema {}))
|
||||
([db-ident prop-schema {:keys [title block-uuid ref-type? properties]}]
|
||||
(let [db-ident' (if (qualified-keyword? db-ident)
|
||||
db-ident
|
||||
(db-property/create-user-property-ident-from-name (name db-ident)))
|
||||
prop-name (or title (name db-ident'))
|
||||
prop-type (get prop-schema :logseq.property/type :default)]
|
||||
(merge
|
||||
(dissoc prop-schema :db/cardinality)
|
||||
(block-with-timestamps
|
||||
(cond->
|
||||
{:db/ident db-ident'
|
||||
:block/tags #{:logseq.class/Property}
|
||||
:logseq.property/type prop-type
|
||||
:block/name (common-util/page-name-sanity-lc (name prop-name))
|
||||
:block/uuid (or block-uuid (common-uuid/gen-uuid :db-ident-block-uuid db-ident'))
|
||||
:block/title (name prop-name)
|
||||
:db/index true
|
||||
:db/cardinality (if (#{:many :db.cardinality/many} (:db/cardinality prop-schema))
|
||||
:db.cardinality/many
|
||||
:db.cardinality/one)
|
||||
:block/order (db-order/gen-key)}
|
||||
(or ref-type? (contains? db-property-type/all-ref-property-types prop-type))
|
||||
(assoc :db/valueType :db.type/ref)
|
||||
(seq properties)
|
||||
(merge properties)))))))
|
||||
|
||||
(defn build-new-class
|
||||
"Build a standard new class so that it is consistent across contexts"
|
||||
[block]
|
||||
{:pre [(qualified-keyword? (:db/ident block))]}
|
||||
(block-with-timestamps
|
||||
(cond-> (merge block
|
||||
{:block/tags (set (conj (:block/tags block) :logseq.class/Tag))})
|
||||
(and (not= (:db/ident block) :logseq.class/Root)
|
||||
(nil? (:logseq.property.class/extends block)))
|
||||
(assoc :logseq.property.class/extends :logseq.class/Root))))
|
||||
|
||||
(defn build-new-page
|
||||
"Builds a basic page to be transacted. A minimal version of gp-block/page-name->map"
|
||||
[title]
|
||||
(block-with-timestamps
|
||||
(cond->
|
||||
{:block/name (common-util/page-name-sanity-lc title)
|
||||
:block/title title
|
||||
:block/uuid (common-uuid/gen-uuid :builtin-block-uuid title)
|
||||
:block/tags #{:logseq.class/Page}}
|
||||
(contains? #{common-config/quick-add-page-name} title)
|
||||
(assoc :logseq.property/hide? true))))
|
||||
|
||||
(defn kv
|
||||
"Creates a key-value pair tx with the key and value respectively stored under
|
||||
:db/ident and :kv/value. The key must be under the namespace :logseq.kv"
|
||||
[k value]
|
||||
{:pre [(= "logseq.kv" (namespace k))]}
|
||||
{:db/ident k
|
||||
:kv/value value})
|
||||
|
||||
(defn import-tx
|
||||
"Creates tx for an import given an import-type"
|
||||
[import-type]
|
||||
(concat [(kv :logseq.kv/import-type import-type)
|
||||
;; Timestamp is useful as this can occur much later than :logseq.kv/graph-created-at
|
||||
(kv :logseq.kv/imported-at (common-util/time-ms))]
|
||||
(mapv
|
||||
(fn [db-ident] [:db/retractEntity db-ident])
|
||||
[:logseq.kv/graph-uuid ;rtc related
|
||||
:logseq.kv/graph-local-tx ;rtc related
|
||||
:logseq.kv/remote-schema-version ;rtc related
|
||||
:logseq.kv/graph-rtc-e2ee? ;rtc related
|
||||
])))
|
||||
105
deps/db/src/logseq/db/test/helper.cljs
vendored
Normal file
105
deps/db/src/logseq/db/test/helper.cljs
vendored
Normal file
@@ -0,0 +1,105 @@
|
||||
(ns ^:node-only logseq.db.test.helper
|
||||
"Main ns for providing test fns for DB graphs"
|
||||
(:require [datascript.core :as d]
|
||||
[datascript.impl.entity :as de]
|
||||
[logseq.db.frontend.property :as db-property]
|
||||
[logseq.db.sqlite.build :as sqlite-build]
|
||||
[logseq.db.sqlite.export :as sqlite-export]))
|
||||
|
||||
(defn find-block-by-content
|
||||
"Find first block by exact block string or by fuzzier regex"
|
||||
[db content]
|
||||
(if (instance? js/RegExp content)
|
||||
(->> content
|
||||
(d/q '[:find [?b ...]
|
||||
:in $ ?pattern
|
||||
:where
|
||||
[?b :block/title ?content]
|
||||
[?b :block/page]
|
||||
[(re-find ?pattern ?content)]]
|
||||
db)
|
||||
first
|
||||
(d/entity db))
|
||||
(->> content
|
||||
(d/q '[:find [?b ...]
|
||||
:in $ ?content
|
||||
:where
|
||||
[?b :block/title ?content]
|
||||
[?b :block/page]]
|
||||
db)
|
||||
first
|
||||
(d/entity db))))
|
||||
|
||||
(defn find-page-by-title
|
||||
"Find first page by its title"
|
||||
[db title]
|
||||
(->> title
|
||||
(d/q '[:find [?b ...]
|
||||
:in $ ?title
|
||||
:where [?b :block/title ?title]]
|
||||
db)
|
||||
first
|
||||
(d/entity db)))
|
||||
|
||||
(defn find-journal-by-journal-day
|
||||
[db journal-day]
|
||||
(->> journal-day
|
||||
(d/q
|
||||
'[:find [?page ...]
|
||||
:in $ ?journal-day
|
||||
:where
|
||||
[?page :block/journal-day ?journal-day]]
|
||||
db)
|
||||
first
|
||||
(d/entity db)))
|
||||
|
||||
(defn readable-properties
|
||||
"Returns an entity's properties and tags in readable form for assertions.
|
||||
tags are included here since they behave like properties on an ent"
|
||||
[ent]
|
||||
(->> (db-property/properties ent)
|
||||
(mapv (fn [[k v]]
|
||||
[k
|
||||
(cond
|
||||
(#{:block/tags :logseq.property.class/extends} k)
|
||||
(mapv :db/ident v)
|
||||
(and (set? v) (every? de/entity? v))
|
||||
(set (map db-property/property-value-content v))
|
||||
(de/entity? v)
|
||||
(or (:db/ident v) (db-property/property-value-content v))
|
||||
:else
|
||||
v)]))
|
||||
(into {})))
|
||||
|
||||
(def create-conn sqlite-export/create-conn)
|
||||
|
||||
(defn create-conn-with-blocks
|
||||
"Create a conn with create-conn and then create blocks using sqlite-build"
|
||||
[opts]
|
||||
(let [conn (create-conn)
|
||||
_ (sqlite-build/create-blocks conn opts)]
|
||||
conn))
|
||||
|
||||
(defn create-conn-with-import-map
|
||||
"Create a conn with create-conn and then create/upsert entities using sqlite-export/build-import.
|
||||
Unlike create-conn-with-blocks, this fn can upsert existing entities"
|
||||
[export-map]
|
||||
(let [conn (create-conn)
|
||||
{:keys [init-tx block-props-tx misc-tx]}
|
||||
(sqlite-export/build-import (dissoc export-map ::sqlite-export/graph-files) @conn {})
|
||||
_ (d/transact! conn (concat init-tx block-props-tx misc-tx))
|
||||
;; Handle graph-files separately b/c build-import can't upsert them
|
||||
_ (when (::sqlite-export/graph-files export-map)
|
||||
(d/transact! conn (::sqlite-export/graph-files export-map)))]
|
||||
conn))
|
||||
|
||||
(defmacro silence-stderr
|
||||
"Silence stderr as successful tests should print long stderr messages"
|
||||
[& body]
|
||||
`(let [orig-write# (.-write js/process.stderr)]
|
||||
(set! (.-write js/process.stderr)
|
||||
(fn [& _] true))
|
||||
(try
|
||||
~@body
|
||||
(finally
|
||||
(set! (.-write js/process.stderr) orig-write#)))))
|
||||
312
deps/db/test/logseq/db/common/delete_blocks_test.cljs
vendored
Normal file
312
deps/db/test/logseq/db/common/delete_blocks_test.cljs
vendored
Normal file
@@ -0,0 +1,312 @@
|
||||
(ns logseq.db.common.delete-blocks-test
|
||||
(:require [cljs.test :refer [deftest is testing]]
|
||||
[datascript.core :as d]
|
||||
[logseq.common.util :as common-util]
|
||||
[logseq.db :as ldb]
|
||||
[logseq.db.common.delete-blocks :as delete-blocks]
|
||||
[logseq.db.test.helper :as db-test]))
|
||||
|
||||
(deftest delete-blocks-removes-reactions
|
||||
(testing "reactions targeting deleted blocks are retracted"
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:pages-and-blocks
|
||||
[{:page {:block/title "Page"}
|
||||
:blocks [{:block/title "Block"}]}]})
|
||||
block (db-test/find-block-by-content @conn "Block")
|
||||
now (common-util/time-ms)
|
||||
reaction {:block/uuid (random-uuid)
|
||||
:block/created-at now
|
||||
:block/updated-at now
|
||||
:logseq.property.reaction/emoji-id "+1"
|
||||
:logseq.property.reaction/target (:db/id block)}
|
||||
_ (d/transact! conn [reaction])
|
||||
reaction-entity (first (:logseq.property.reaction/_target (d/entity @conn (:db/id block))))
|
||||
retracts [[:db/retractEntity (:db/id block)]]
|
||||
extra (delete-blocks/update-refs-history @conn retracts {})]
|
||||
(d/transact! conn (concat retracts extra))
|
||||
(is (nil? (d/entity @conn (:db/id reaction-entity)))))))
|
||||
|
||||
(deftest delete-blocks-expands-property-value-children
|
||||
(testing "delete-blocks retracts generated property value children"
|
||||
(let [property-value-uuid (random-uuid)
|
||||
conn (db-test/create-conn-with-blocks
|
||||
{:properties {:user.property/cli-http-prop {:logseq.property/type :default}}
|
||||
:pages-and-blocks
|
||||
[{:page {:block/title "Page"}
|
||||
:blocks [{:block/title "Parent"
|
||||
:build/properties
|
||||
{:user.property/cli-http-prop
|
||||
{:build/property-value :block
|
||||
:block/title "Property value"
|
||||
:block/uuid property-value-uuid
|
||||
:build/keep-uuid? true}}}]}]})
|
||||
parent (db-test/find-block-by-content @conn "Parent")
|
||||
property-value (d/entity @conn [:block/uuid property-value-uuid])
|
||||
txs [[:db/retractEntity (:db/id parent)]]
|
||||
expanded (delete-blocks/expand-delete-blocks-tx @conn txs {:outliner-op :delete-blocks})]
|
||||
(is (= (:db/id parent) (:db/id (:block/parent property-value))))
|
||||
(is (some #(= [:db/retractEntity (:db/id property-value)] %) expanded)))))
|
||||
|
||||
(deftest delete-blocks-removes-history-with-ref-value
|
||||
(testing "property history entries referencing a deleted block are retracted"
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:pages-and-blocks
|
||||
[{:page {:block/title "Page"}
|
||||
:blocks [{:block/title "Target block"}
|
||||
{:block/title "Choice value"}]}]})
|
||||
target-block (db-test/find-block-by-content @conn "Target block")
|
||||
choice-value-block (db-test/find-block-by-content @conn "Choice value")
|
||||
history-uuid (random-uuid)
|
||||
now (common-util/time-ms)
|
||||
_ (d/transact! conn [{:block/uuid history-uuid
|
||||
:block/created-at now
|
||||
:block/updated-at now
|
||||
:logseq.property.history/block (:db/id target-block)
|
||||
:logseq.property.history/property (:db/id (d/entity @conn :logseq.property/status))
|
||||
:logseq.property.history/ref-value (:db/id choice-value-block)}])
|
||||
history-entity (d/entity @conn [:block/uuid history-uuid])
|
||||
retracts [[:db/retractEntity (:db/id choice-value-block)]]
|
||||
extra (delete-blocks/update-refs-history @conn retracts {})]
|
||||
(d/transact! conn (concat retracts extra))
|
||||
(is (nil? (d/entity @conn (:db/id history-entity)))))))
|
||||
|
||||
(deftest property-history-block-updates-are-kept
|
||||
(testing "history block updates do not delete the history entity"
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:pages-and-blocks
|
||||
[{:page {:block/title "Page"}
|
||||
:blocks [{:block/title "Target block"}
|
||||
{:block/title "Choice value"}]}]})
|
||||
target-block (db-test/find-block-by-content @conn "Target block")
|
||||
choice-value-block (db-test/find-block-by-content @conn "Choice value")
|
||||
history-uuid (random-uuid)
|
||||
now (common-util/time-ms)
|
||||
_ (d/transact! conn [{:block/uuid history-uuid
|
||||
:block/title "History entry"
|
||||
:block/page (:db/id (:block/page target-block))
|
||||
:block/parent (:db/id (:block/page target-block))
|
||||
:block/order "a0"
|
||||
:block/created-at now
|
||||
:block/updated-at now
|
||||
:logseq.property.history/block (:db/id target-block)
|
||||
:logseq.property.history/property (:db/id (d/entity @conn :logseq.property/status))
|
||||
:logseq.property.history/ref-value (:db/id choice-value-block)}])
|
||||
history-entity (d/entity @conn [:block/uuid history-uuid])
|
||||
txs [[:db/retract (:db/id history-entity) :block/title "History entry"]
|
||||
[:db/add (:db/id history-entity) :block/title "Updated history entry"]
|
||||
[:db/retract (:db/id history-entity) :block/parent (:db/id (:block/page history-entity))]
|
||||
[:db/add (:db/id history-entity) :block/parent (:db/id target-block)]
|
||||
[:db/retract (:db/id history-entity) :block/order "a0"]
|
||||
[:db/add (:db/id history-entity) :block/order "b0"]]
|
||||
extra (delete-blocks/update-refs-history @conn txs {})]
|
||||
(d/transact! conn (concat txs extra))
|
||||
(let [updated-history (d/entity @conn (:db/id history-entity))]
|
||||
(is (= {:block/title "Updated history entry"
|
||||
:block/parent (:db/id target-block)
|
||||
:block/order "b0"}
|
||||
{:block/title (:block/title updated-history)
|
||||
:block/parent (:db/id (:block/parent updated-history))
|
||||
:block/order (:block/order updated-history)}))))))
|
||||
|
||||
(deftest remote-delete-blocks-removes-history-when-owner-ref-retracted
|
||||
(testing "remote delete-blocks datoms fully retract property history blocks"
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:pages-and-blocks
|
||||
[{:page {:block/title "Page"}
|
||||
:blocks [{:block/title "Target block"}
|
||||
{:block/title "Choice value"}]}]})
|
||||
target-block (db-test/find-block-by-content @conn "Target block")
|
||||
choice-value-block (db-test/find-block-by-content @conn "Choice value")
|
||||
history-uuid (random-uuid)
|
||||
now (common-util/time-ms)
|
||||
_ (d/transact! conn [{:block/uuid history-uuid
|
||||
:block/created-at now
|
||||
:block/updated-at now
|
||||
:logseq.property.history/block (:db/id target-block)
|
||||
:logseq.property.history/property (:db/id (d/entity @conn :logseq.property/status))
|
||||
:logseq.property.history/ref-value (:db/id choice-value-block)}])
|
||||
history-entity (d/entity @conn [:block/uuid history-uuid])
|
||||
txs [[:db/retract (:db/id history-entity) :logseq.property.history/block (:db/id target-block)]]
|
||||
extra (delete-blocks/update-refs-history @conn txs {:outliner-op :delete-blocks})]
|
||||
(d/transact! conn (concat txs extra))
|
||||
(is (nil? (d/entity @conn (:db/id history-entity)))))))
|
||||
|
||||
(deftest delete-blocks-removes-new-history-for-deleted-block
|
||||
(testing "history blocks created in the same tx as their deleted owner are retracted"
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:pages-and-blocks
|
||||
[{:page {:block/title "Page"}
|
||||
:blocks [{:block/title "Target block"}
|
||||
{:block/title "Choice value"}]}]})
|
||||
target-block (db-test/find-block-by-content @conn "Target block")
|
||||
choice-value-block (db-test/find-block-by-content @conn "Choice value")
|
||||
history-uuid (random-uuid)
|
||||
now (common-util/time-ms)
|
||||
txs [{:block/uuid history-uuid
|
||||
:block/created-at now
|
||||
:block/updated-at now
|
||||
:logseq.property.history/block (:db/id target-block)
|
||||
:logseq.property.history/property (:db/id (d/entity @conn :logseq.property/status))
|
||||
:logseq.property.history/ref-value (:db/id choice-value-block)}
|
||||
[:db/retractEntity (:db/id target-block)]]
|
||||
extra (delete-blocks/update-refs-history @conn txs {:outliner-op :delete-blocks})]
|
||||
(d/transact! conn (concat txs extra))
|
||||
(is (nil? (d/entity @conn [:block/uuid history-uuid]))))))
|
||||
|
||||
(deftest remote-delete-blocks-removes-new-normalized-history-when-owner-ref-retracted
|
||||
(testing "remote normalized delete-blocks datoms fully retract new property history blocks"
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:pages-and-blocks
|
||||
[{:page {:block/title "Page"}
|
||||
:blocks [{:block/title "Target block"}
|
||||
{:block/title "Choice value"}]}]})
|
||||
target-block (db-test/find-block-by-content @conn "Target block")
|
||||
choice-value-block (db-test/find-block-by-content @conn "Choice value")
|
||||
history-uuid (random-uuid)
|
||||
history-eid 1000000
|
||||
now (common-util/time-ms)
|
||||
txs [[:db/add history-eid :block/uuid history-uuid]
|
||||
[:db/add history-eid :block/created-at now]
|
||||
[:db/add history-eid :block/updated-at now]
|
||||
[:db/add history-eid :logseq.property.history/block (:db/id target-block)]
|
||||
[:db/add history-eid :logseq.property.history/property (:db/id (d/entity @conn :logseq.property/status))]
|
||||
[:db/add history-eid :logseq.property.history/ref-value (:db/id choice-value-block)]
|
||||
[:db/retract history-eid :logseq.property.history/block (:db/id target-block)]]
|
||||
extra (delete-blocks/update-refs-history @conn txs {:outliner-op :delete-blocks})]
|
||||
(d/transact! conn (concat txs extra))
|
||||
(is (nil? (d/entity @conn [:block/uuid history-uuid]))))))
|
||||
|
||||
(deftest delete-page-removes-history-with-ref-value
|
||||
(testing "deleting a page retracts property history entries that referenced that page"
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:pages-and-blocks
|
||||
[{:page {:block/title "Journal"}
|
||||
:blocks [{:block/title "b1"}]}
|
||||
{:page {:block/title "Page to delete"}}]})
|
||||
block (db-test/find-block-by-content @conn "b1")
|
||||
page (db-test/find-page-by-title @conn "Page to delete")
|
||||
history-uuid (random-uuid)
|
||||
now (common-util/time-ms)
|
||||
_ (d/transact! conn [{:block/uuid history-uuid
|
||||
:block/created-at now
|
||||
:block/updated-at now
|
||||
:logseq.property.history/block (:db/id block)
|
||||
:logseq.property.history/property (:db/id (d/entity @conn :logseq.property/status))
|
||||
:logseq.property.history/ref-value (:db/id page)}])
|
||||
history-entity (d/entity @conn [:block/uuid history-uuid])]
|
||||
(ldb/transact! conn [[:db/retractEntity (:db/id page)]])
|
||||
(is (nil? (d/entity @conn (:db/id history-entity)))))))
|
||||
|
||||
(deftest delete-property-removes-history-for-property
|
||||
(testing "property history entries for a deleted property are retracted"
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:pages-and-blocks
|
||||
[{:page {:block/title "Page"}
|
||||
:blocks [{:block/title "Target block"}]}]
|
||||
:properties {:obsolete {:logseq.property/type :string}}})
|
||||
block (db-test/find-block-by-content @conn "Target block")
|
||||
property (d/entity @conn :user.property/obsolete)
|
||||
history-uuid (random-uuid)
|
||||
now (common-util/time-ms)
|
||||
_ (d/transact! conn [{:block/uuid history-uuid
|
||||
:block/created-at now
|
||||
:block/updated-at now
|
||||
:logseq.property.history/block (:db/id block)
|
||||
:logseq.property.history/property (:db/id property)
|
||||
:logseq.property.history/scalar-value "old"}])
|
||||
history-entity (d/entity @conn [:block/uuid history-uuid])
|
||||
retracts [[:db/retractEntity (:db/id property)]]
|
||||
extra (delete-blocks/update-refs-history @conn retracts {})]
|
||||
(d/transact! conn (concat retracts extra))
|
||||
(is (nil? (d/entity @conn (:db/id history-entity)))))))
|
||||
|
||||
(deftest delete-blocks-removes-history-for-corresponding-views
|
||||
(testing "property history entries attached to a deleted block's view are retracted"
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:pages-and-blocks
|
||||
[{:page {:block/title "Page"}
|
||||
:blocks [{:block/title "Target block"}]}]})
|
||||
target-block (db-test/find-block-by-content @conn "Target block")
|
||||
view-uuid (random-uuid)
|
||||
target-history-uuid (random-uuid)
|
||||
view-history-uuid (random-uuid)
|
||||
now (common-util/time-ms)
|
||||
_ (d/transact! conn [{:block/uuid view-uuid
|
||||
:block/title "Target view"
|
||||
:block/created-at now
|
||||
:block/updated-at now
|
||||
:logseq.property/view-for (:db/id target-block)
|
||||
:logseq.property.view/type :logseq.property.view/type.table
|
||||
:logseq.property.view/feature-type :linked-references}
|
||||
{:block/uuid target-history-uuid
|
||||
:block/created-at now
|
||||
:block/updated-at now
|
||||
:logseq.property.history/block (:db/id target-block)
|
||||
:logseq.property.history/property (:db/id (d/entity @conn :logseq.property/status))
|
||||
:logseq.property.history/scalar-value "Todo"}
|
||||
{:block/uuid view-history-uuid
|
||||
:block/created-at now
|
||||
:block/updated-at now
|
||||
:logseq.property.history/block [:block/uuid view-uuid]
|
||||
:logseq.property.history/property (:db/id (d/entity @conn :logseq.property/status))
|
||||
:logseq.property.history/scalar-value "List"}])]
|
||||
(ldb/transact! conn [[:db/retractEntity (:db/id target-block)]])
|
||||
(is (nil? (d/entity @conn [:block/uuid view-uuid])))
|
||||
(is (nil? (d/entity @conn [:block/uuid target-history-uuid])))
|
||||
(is (nil? (d/entity @conn [:block/uuid view-history-uuid]))))))
|
||||
|
||||
(deftest delete-blocks-does-not-rewrite-title-for-deleted-view
|
||||
(testing "reference title rewrite should skip generated views deleted by the same cleanup"
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:pages-and-blocks
|
||||
[{:page {:block/title "Page"}
|
||||
:blocks [{:block/title "Asset target"}]}]})
|
||||
target-block (db-test/find-block-by-content @conn "Asset target")
|
||||
page (:block/page target-block)
|
||||
view-uuid (random-uuid)
|
||||
now (common-util/time-ms)
|
||||
_ (d/transact! conn [{:block/uuid view-uuid
|
||||
:block/title "Unlinked references"
|
||||
:block/raw-title "Unlinked references"
|
||||
:block/created-at now
|
||||
:block/updated-at now
|
||||
:block/page (:db/id page)
|
||||
:block/parent (:db/id page)
|
||||
:block/order "cD66"
|
||||
:block/refs (:db/id target-block)
|
||||
:logseq.property/view-for (:db/id target-block)
|
||||
:logseq.property.view/type :logseq.property.view/type.list
|
||||
:logseq.property.view/feature-type :unlinked-references}])
|
||||
view (d/entity @conn [:block/uuid view-uuid])
|
||||
retracts [[:db/retractEntity (:db/id target-block)]]
|
||||
extra (delete-blocks/update-refs-history @conn retracts {})]
|
||||
(is (some #(= [:db/retractEntity (:db/id view)] %) extra))
|
||||
(is (not-any? #(= [:db/add (:db/id view) :block/title "Unlinked references"] %) extra)))))
|
||||
|
||||
(deftest delete-blocks-does-not-rewrite-title-for-deleted-history
|
||||
(testing "reference title rewrite should skip history entities deleted by the same cleanup"
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:pages-and-blocks
|
||||
[{:page {:block/title "Page"}
|
||||
:blocks [{:block/title "Target block"}]}]})
|
||||
target-block (db-test/find-block-by-content @conn "Target block")
|
||||
page (:block/page target-block)
|
||||
history-uuid (random-uuid)
|
||||
now (common-util/time-ms)
|
||||
_ (d/transact! conn [{:block/uuid history-uuid
|
||||
:block/title "History entry"
|
||||
:block/raw-title "History entry"
|
||||
:block/created-at now
|
||||
:block/updated-at now
|
||||
:block/page (:db/id page)
|
||||
:block/parent (:db/id page)
|
||||
:block/order "a0"
|
||||
:block/refs (:db/id target-block)
|
||||
:logseq.property.history/block (:db/id target-block)
|
||||
:logseq.property.history/property (:db/id (d/entity @conn :logseq.property/status))
|
||||
:logseq.property.history/scalar-value "Todo"}])
|
||||
history (d/entity @conn [:block/uuid history-uuid])
|
||||
retracts [[:db/retractEntity (:db/id target-block)]]
|
||||
extra (delete-blocks/update-refs-history @conn retracts {})]
|
||||
(is (some #(= [:db/retractEntity (:db/id history)] %) extra))
|
||||
(is (not-any? #(= [:db/add (:db/id history) :block/title "History entry"] %) extra)))))
|
||||
49
deps/db/test/logseq/db/common/initial_data_refs_test.cljs
vendored
Normal file
49
deps/db/test/logseq/db/common/initial_data_refs_test.cljs
vendored
Normal file
@@ -0,0 +1,49 @@
|
||||
(ns logseq.db.common.initial-data-refs-test
|
||||
(:require [cljs.test :refer [deftest is]]
|
||||
[datascript.core :as d]
|
||||
[logseq.db.common.initial-data :as common-initial-data]
|
||||
[logseq.db.test.helper :as db-test]))
|
||||
|
||||
(deftest get-block-refs-count-matches-get-block-refs-for-class-page-test
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:classes {:Topic {:block/title "Topic"}}
|
||||
:pages-and-blocks
|
||||
[{:page {:block/title "Ref A"}}
|
||||
{:page {:block/title "Ref B"}}]})
|
||||
topic-id (:db/id (d/entity @conn :user.class/Topic))
|
||||
ref-a-id (d/q '[:find ?e .
|
||||
:in $ ?title
|
||||
:where [?e :block/title ?title]]
|
||||
@conn
|
||||
"Ref A")
|
||||
ref-b-id (d/q '[:find ?e .
|
||||
:in $ ?title
|
||||
:where [?e :block/title ?title]]
|
||||
@conn
|
||||
"Ref B")
|
||||
_ (d/transact! conn [[:db/add ref-a-id :block/refs topic-id]
|
||||
[:db/add ref-b-id :block/refs topic-id]
|
||||
[:db/add ref-b-id :logseq.property/hide? true]])
|
||||
refs (common-initial-data/get-block-refs @conn topic-id)
|
||||
count* (common-initial-data/get-block-refs-count @conn topic-id)]
|
||||
(is (= (count refs) count*))
|
||||
(is (= 1 count*))))
|
||||
|
||||
(deftest get-block-refs-count-page-without-db-ident-test
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:pages-and-blocks
|
||||
[{:page {:block/title "Foo"}}
|
||||
{:page {:block/title "Bar"}}]})
|
||||
foo-id (d/q '[:find ?e .
|
||||
:in $ ?title
|
||||
:where [?e :block/title ?title]]
|
||||
@conn
|
||||
"Foo")
|
||||
bar-id (d/q '[:find ?e .
|
||||
:in $ ?title
|
||||
:where [?e :block/title ?title]]
|
||||
@conn
|
||||
"Bar")
|
||||
_ (d/transact! conn [[:db/add bar-id :block/refs foo-id]])
|
||||
count* (common-initial-data/get-block-refs-count @conn foo-id)]
|
||||
(is (= 1 count*))))
|
||||
99
deps/db/test/logseq/db/common/initial_data_test.cljs
vendored
Normal file
99
deps/db/test/logseq/db/common/initial_data_test.cljs
vendored
Normal file
@@ -0,0 +1,99 @@
|
||||
(ns logseq.db.common.initial-data-test
|
||||
"This ns is the only one to test against file based datascript connections.
|
||||
These are useful integration tests"
|
||||
(:require ["fs" :as fs]
|
||||
["path" :as node-path]
|
||||
[cljs.test :refer [deftest async use-fixtures is testing]]
|
||||
[clojure.string :as string]
|
||||
[datascript.core :as d]
|
||||
[logseq.db.common.initial-data :as common-initial-data]
|
||||
[logseq.db.frontend.property :as db-property]
|
||||
[logseq.db.common.sqlite-cli :as sqlite-cli]
|
||||
[logseq.db.sqlite.build :as sqlite-build]
|
||||
[logseq.db.sqlite.create-graph :as sqlite-create-graph]
|
||||
[logseq.db.test.helper :as db-test]))
|
||||
|
||||
(use-fixtures
|
||||
:each
|
||||
;; Cleaning tmp/ before leaves last tmp/ after a test run for dev and debugging
|
||||
{:before
|
||||
#(async done
|
||||
(if (fs/existsSync "tmp")
|
||||
(fs/rm "tmp" #js {:recursive true} (fn [err]
|
||||
(when err (js/console.log err))
|
||||
(done)))
|
||||
(done)))})
|
||||
|
||||
(defn- create-graph-dir
|
||||
[dir db-name]
|
||||
(fs/mkdirSync (node-path/join dir db-name) #js {:recursive true}))
|
||||
|
||||
(deftest get-initial-data
|
||||
(testing "Fetches a defined block"
|
||||
(create-graph-dir "tmp/graphs" "test-db")
|
||||
|
||||
(let [conn* (sqlite-cli/open-db! "tmp/graphs" "test-db")
|
||||
_ (d/transact! conn* (sqlite-create-graph/build-db-initial-data "{}"))
|
||||
blocks [{:file/path "logseq/config.edn"
|
||||
:file/content "{:foo :bar}"}]
|
||||
_ (d/transact! conn* blocks)
|
||||
;; Simulate getting data from sqlite and restoring it for frontend
|
||||
{:keys [schema initial-data]} (common-initial-data/get-initial-data @conn*)
|
||||
conn (d/conn-from-datoms initial-data schema)]
|
||||
(is (= blocks
|
||||
(->> @conn
|
||||
(d/q '[:find (pull ?b [:file/path :file/content])
|
||||
:where [?b :file/content] [?b :file/path "logseq/config.edn"]])
|
||||
(map first)))
|
||||
"Correct file with content is found"))))
|
||||
|
||||
(deftest get-initial-data-includes-property-description-datoms
|
||||
(create-graph-dir "tmp/graphs" "test-db")
|
||||
(let [conn* (sqlite-cli/open-db! "tmp/graphs" "test-db")
|
||||
_ (d/transact! conn* (sqlite-create-graph/build-db-initial-data "{}"))
|
||||
{:keys [schema initial-data]} (common-initial-data/get-initial-data @conn*)
|
||||
conn (d/conn-from-datoms initial-data schema)
|
||||
description (-> (d/entity @conn :logseq.property/deadline)
|
||||
:logseq.property/description)]
|
||||
(is (string/includes? (str (db-property/property-value-content description))
|
||||
"finish something"))))
|
||||
|
||||
(deftest restore-initial-data
|
||||
(create-graph-dir "tmp/graphs" "test-db")
|
||||
(let [conn* (sqlite-cli/open-db! "tmp/graphs" "test-db")
|
||||
_ (d/transact! conn* (sqlite-create-graph/build-db-initial-data "{}"))
|
||||
{:keys [init-tx]}
|
||||
(sqlite-build/build-blocks-tx
|
||||
{:pages-and-blocks
|
||||
[{:page {:block/title "page1"}
|
||||
:blocks [{:block/title "b1"}]}]})
|
||||
_ (d/transact! conn* init-tx)
|
||||
;; Simulate getting data from sqlite and restoring it for frontend
|
||||
{:keys [schema initial-data]} (common-initial-data/get-initial-data @conn*)
|
||||
conn (d/conn-from-datoms initial-data schema)]
|
||||
(is (some? (db-test/find-page-by-title @conn "page1"))
|
||||
"Restores recently updated page")))
|
||||
|
||||
(deftest get-block-and-children-has-children-flag
|
||||
(testing "Top-level block with children has :block.temp/has-children? true"
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
[{:page {:block/title "test-page"}
|
||||
:blocks [{:block/title "parent"
|
||||
:build/children
|
||||
[{:block/title "child1"}
|
||||
{:block/title "child2"}]}]}])
|
||||
parent-block (db-test/find-block-by-content @conn "parent")
|
||||
result (common-initial-data/get-block-and-children
|
||||
@conn (:block/uuid parent-block) {:children? false})]
|
||||
(is (true? (:block.temp/has-children? (:block result)))
|
||||
"Top-level block with children should have :block.temp/has-children? true")))
|
||||
|
||||
(testing "Top-level block without children has :block.temp/has-children? false"
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
[{:page {:block/title "test-page2"}
|
||||
:blocks [{:block/title "leaf-block"}]}])
|
||||
leaf-block (db-test/find-block-by-content @conn "leaf-block")
|
||||
result (common-initial-data/get-block-and-children
|
||||
@conn (:block/uuid leaf-block) {:children? false})]
|
||||
(is (false? (:block.temp/has-children? (:block result)))
|
||||
"Top-level block without children should have :block.temp/has-children? false"))))
|
||||
303
deps/db/test/logseq/db/common/view_test.cljs
vendored
Normal file
303
deps/db/test/logseq/db/common/view_test.cljs
vendored
Normal file
@@ -0,0 +1,303 @@
|
||||
(ns logseq.db.common.view-test
|
||||
(:require [cljs.test :refer [deftest is]]
|
||||
[datascript.core :as d]
|
||||
[logseq.db.common.view :as db-view]
|
||||
[logseq.db.test.helper :as db-test]))
|
||||
|
||||
(defn- create-view-id
|
||||
[conn feature-type & {:keys [view-for-id]}]
|
||||
(let [tx (d/transact! conn [(cond-> {:db/id -100
|
||||
:block/title "Test view"
|
||||
:block/uuid (random-uuid)
|
||||
:logseq.property.view/feature-type feature-type
|
||||
:logseq.property.view/type :logseq.property.view/type.table}
|
||||
view-for-id
|
||||
(assoc :logseq.property/view-for view-for-id))])]
|
||||
(get-in tx [:tempids -100])))
|
||||
|
||||
(deftest get-view-data-journals-returns-ordered-compact-index-test
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:pages-and-blocks
|
||||
(mapv (fn [journal-day]
|
||||
{:page {:build/journal journal-day}
|
||||
:blocks [{:block/title (str "Block " journal-day)}]})
|
||||
[20260716 20260715 20260714 20260713 20260712])})
|
||||
result (db-view/get-view-data @conn nil {:journals? true})
|
||||
index (:data result)]
|
||||
(is (= 5 (:count result)))
|
||||
(is (= [20260716 20260715 20260714 20260713 20260712]
|
||||
(mapv :block/journal-day index)))
|
||||
(is (every? map? index))
|
||||
(when (every? map? index)
|
||||
(is (every? #(= #{:db/id :block/journal-day} (set (keys %))) index)
|
||||
"The journal index should include only the identity and placeholder title data."))
|
||||
(is (not (contains? result :selection-block-ids))
|
||||
"Blocks are loaded only for visible journals.")))
|
||||
|
||||
(deftest get-view-data-all-pages-sorts-and-filters-hidden-test
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:pages-and-blocks
|
||||
[{:page {:block/title "Alpha" :block/updated-at 10}}
|
||||
{:page {:block/title "Beta" :block/updated-at 20}}
|
||||
{:page {:block/title "Hidden" :block/updated-at 30 :logseq.property/hide? true}}
|
||||
{:page {:block/title "Deleted" :block/updated-at 40 :logseq.property/deleted-at 1}}]})
|
||||
view-id (create-view-id conn :all-pages)
|
||||
result (db-view/get-view-data @conn view-id {:view-feature-type :all-pages
|
||||
:sorting [{:id :block/updated-at :asc? false}]})
|
||||
ids (:data result)
|
||||
titles (map (fn [id] (:block/title (d/entity @conn id))) ids)]
|
||||
(is (= 2 (:count result)))
|
||||
(is (= ["Beta" "Alpha"] titles))))
|
||||
|
||||
(deftest get-view-data-all-pages-title-sort-test
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:pages-and-blocks
|
||||
[{:page {:block/title "gamma" :block/updated-at 1}}
|
||||
{:page {:block/title "alpha" :block/updated-at 2}}
|
||||
{:page {:block/title "beta" :block/updated-at 3}}]})
|
||||
view-id (create-view-id conn :all-pages)
|
||||
result (db-view/get-view-data @conn view-id {:view-feature-type :all-pages
|
||||
:sorting [{:id :block/title :asc? true}]})
|
||||
ids (:data result)
|
||||
titles (map (fn [id] (:block/title (d/entity @conn id))) ids)]
|
||||
(is (= ["alpha" "beta" "gamma"] titles))))
|
||||
|
||||
(deftest get-view-data-class-objects-sort-keeps-rows-with-missing-sort-value-test
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:classes {:Topic {:block/title "Topic"}}
|
||||
:pages-and-blocks
|
||||
[{:page {:block/title "With timestamp"
|
||||
:block/updated-at 20
|
||||
:build/tags [:Topic]}}
|
||||
{:page {:block/title "Without timestamp"
|
||||
:block/updated-at 10
|
||||
:build/tags [:Topic]}}]})
|
||||
class-id (:db/id (d/entity @conn :user.class/Topic))
|
||||
without-ts-id (d/q '[:find ?e .
|
||||
:in $ ?title
|
||||
:where [?e :block/title ?title]]
|
||||
@conn
|
||||
"Without timestamp")
|
||||
without-ts-value (:block/updated-at (d/entity @conn without-ts-id))
|
||||
_ (d/transact! conn [[:db/retract without-ts-id :block/updated-at without-ts-value]])
|
||||
view-id (create-view-id conn :class-objects :view-for-id class-id)
|
||||
result (db-view/get-view-data @conn view-id {:view-feature-type :class-objects
|
||||
:view-for-id class-id
|
||||
:sorting [{:id :block/updated-at :asc? false}]})
|
||||
titles (map (fn [id] (:block/title (d/entity @conn id))) (:data result))]
|
||||
(is (= 2 (:count result)))
|
||||
(is (= #{"With timestamp" "Without timestamp"} (set titles)))))
|
||||
|
||||
(deftest get-view-data-class-objects-simple-is-filter-test
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:classes {:Topic {:block/title "Topic"}}
|
||||
:pages-and-blocks
|
||||
[{:page {:block/title "A" :build/tags [:Topic]}}
|
||||
{:page {:block/title "B" :build/tags [:Topic]}}
|
||||
{:page {:block/title "C" :build/tags [:Topic]}}]})
|
||||
class-id (:db/id (d/entity @conn :user.class/Topic))
|
||||
view-id (create-view-id conn :class-objects :view-for-id class-id)
|
||||
result (db-view/get-view-data @conn view-id {:view-feature-type :class-objects
|
||||
:view-for-id class-id
|
||||
:filters {:or? false
|
||||
:filters [[:block/title :is #{"B"}]]}})
|
||||
titles (map (fn [id] (:block/title (d/entity @conn id))) (:data result))]
|
||||
(is (= 1 (:count result)))
|
||||
(is (= ["B"] titles))))
|
||||
|
||||
(deftest get-view-data-class-objects-groups-by-title-test
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:classes {:Topic {:block/title "Topic"}}
|
||||
:pages-and-blocks
|
||||
[{:page {:block/title "A" :build/tags [:Topic]}}
|
||||
{:page {:block/title "B" :build/tags [:Topic]}}]})
|
||||
class-id (:db/id (d/entity @conn :user.class/Topic))
|
||||
view-id (create-view-id conn :class-objects :view-for-id class-id)
|
||||
_ (d/transact! conn [[:db/add view-id :logseq.property.view/group-by-property :block/title]])
|
||||
result (db-view/get-view-data @conn view-id {:view-feature-type :class-objects
|
||||
:view-for-id class-id})
|
||||
group-titles (map first (:data result))]
|
||||
(is (= ["A" "B"] group-titles))))
|
||||
|
||||
(deftest get-view-data-class-objects-groups-by-many-values-test
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:classes {:Topic {:block/title "Topic"}
|
||||
:SciFi {:block/title "Sci-Fi"}
|
||||
:Drama {:block/title "Drama"}}
|
||||
:pages-and-blocks
|
||||
[{:page {:block/title "Movie A" :build/tags [:Topic :SciFi :Drama]}}
|
||||
{:page {:block/title "Movie B" :build/tags [:Topic :SciFi]}}]})
|
||||
class-id (:db/id (d/entity @conn :user.class/Topic))
|
||||
view-id (create-view-id conn :class-objects :view-for-id class-id)
|
||||
_ (d/transact! conn [[:db/add view-id :logseq.property.view/group-by-property :block/tags]])
|
||||
result (db-view/get-view-data @conn view-id {:view-feature-type :class-objects
|
||||
:view-for-id class-id})
|
||||
group->titles (into {}
|
||||
(map (fn [[group rows]]
|
||||
[(:block/title group)
|
||||
(set (map (fn [id] (:block/title (d/entity @conn id))) rows))]))
|
||||
(:data result))]
|
||||
(is (= #{"Movie A" "Movie B"} (get group->titles "Sci-Fi")))
|
||||
(is (= #{"Movie A"} (get group->titles "Drama")))))
|
||||
|
||||
(deftest get-view-data-list-view-keeps-one-row-shape-for-pages-and-blocks-test
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:classes {:Topic {:block/title "Topic"}}
|
||||
:pages-and-blocks
|
||||
[{:page {:block/title "Tagged page"
|
||||
:build/tags [:Topic]}}
|
||||
{:page {:block/title "Block page"}
|
||||
:blocks [{:block/title "Tagged block"
|
||||
:build/tags [:Topic]}]}]})
|
||||
class-id (:db/id (d/entity @conn :user.class/Topic))
|
||||
view-id (create-view-id conn :class-objects :view-for-id class-id)
|
||||
_ (d/transact! conn [[:db/add view-id
|
||||
:logseq.property.view/group-by-property
|
||||
:block/page]
|
||||
[:db/add view-id
|
||||
:logseq.property.view/type
|
||||
:logseq.property.view/type.list]])
|
||||
result (db-view/get-view-data @conn view-id
|
||||
{:view-feature-type :class-objects
|
||||
:view-for-id class-id})]
|
||||
(is (= 2 (:count result)))
|
||||
(is (every? (fn [[_group partitions]]
|
||||
(every? (fn [[breadcrumb-uuid rows]]
|
||||
(and (uuid? breadcrumb-uuid)
|
||||
(sequential? rows)
|
||||
(every? map? rows)))
|
||||
partitions))
|
||||
(:data result))
|
||||
"A list view must not mix flat row IDs with nested partitions.")))
|
||||
|
||||
(deftest get-view-data-linked-references-page-view-does-not-crash-on-missing-db-ident-test
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:pages-and-blocks
|
||||
[{:page {:block/title "Foo"}}
|
||||
{:page {:block/title "Bar"}}]})
|
||||
foo-id (d/q '[:find ?e .
|
||||
:in $ ?title
|
||||
:where [?e :block/title ?title]]
|
||||
@conn
|
||||
"Foo")
|
||||
bar-id (d/q '[:find ?e .
|
||||
:in $ ?title
|
||||
:where [?e :block/title ?title]]
|
||||
@conn
|
||||
"Bar")
|
||||
_ (d/transact! conn [[:db/add bar-id :block/refs foo-id]])
|
||||
view-id (create-view-id conn :linked-references :view-for-id foo-id)
|
||||
result (db-view/get-view-data @conn view-id {:view-feature-type :linked-references
|
||||
:view-for-id foo-id})]
|
||||
(is (number? (:count result)))
|
||||
(is (contains? (set (:data result)) bar-id))))
|
||||
|
||||
(deftest get-view-data-groups-page-level-linked-references-under-the-referring-page-test
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:pages-and-blocks
|
||||
[{:page {:block/title "Target"}}
|
||||
{:page {:block/title "Referring page"}}]})
|
||||
target-id (d/q '[:find ?e .
|
||||
:in $ ?title
|
||||
:where [?e :block/title ?title]]
|
||||
@conn
|
||||
"Target")
|
||||
referring-page-id (d/q '[:find ?e .
|
||||
:in $ ?title
|
||||
:where [?e :block/title ?title]]
|
||||
@conn
|
||||
"Referring page")
|
||||
target (d/entity @conn target-id)
|
||||
referring-page (d/entity @conn referring-page-id)
|
||||
view-id (create-view-id conn :linked-references
|
||||
:view-for-id (:db/id target))
|
||||
_ (d/transact! conn
|
||||
[[:db/add (:db/id referring-page)
|
||||
:block/refs
|
||||
(:db/id target)]
|
||||
[:db/add view-id
|
||||
:logseq.property.view/type
|
||||
:logseq.property.view/type.list]
|
||||
[:db/add view-id
|
||||
:logseq.property.view/group-by-property
|
||||
:block/page]])
|
||||
result (db-view/get-view-data
|
||||
@conn view-id
|
||||
{:view-feature-type :linked-references
|
||||
:view-for-id (:db/id target)})
|
||||
[[group partitions]] (:data result)]
|
||||
(is (= (:block/uuid referring-page) (:block/uuid group)))
|
||||
(is (= [[(:block/uuid referring-page)
|
||||
[{:db/id (:db/id referring-page)
|
||||
:block/parent nil}]]]
|
||||
(mapv (fn [[breadcrumb rows]]
|
||||
[breadcrumb (vec rows)])
|
||||
partitions)))))
|
||||
|
||||
(deftest get-view-data-class-objects-ref-filter-fast-path-test
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:classes {:Topic {:block/title "Topic"}}
|
||||
:pages-and-blocks
|
||||
[{:page {:block/title "Page A"}
|
||||
:blocks [{:block/title "Obj A"
|
||||
:build/tags [:Topic]}]}
|
||||
{:page {:block/title "Page B"}
|
||||
:blocks [{:block/title "Obj B"
|
||||
:build/tags [:Topic]}]}]})
|
||||
obj-a-id (d/q '[:find ?e .
|
||||
:in $ ?title
|
||||
:where [?e :block/title ?title]]
|
||||
@conn
|
||||
"Obj A")
|
||||
obj-b-id (d/q '[:find ?e .
|
||||
:in $ ?title
|
||||
:where [?e :block/title ?title]]
|
||||
@conn
|
||||
"Obj B")
|
||||
page-a-uuid (:block/uuid (d/entity @conn (d/q '[:find ?e .
|
||||
:in $ ?title
|
||||
:where [?e :block/title ?title]]
|
||||
@conn
|
||||
"Page A")))
|
||||
class-id (:db/id (d/entity @conn :user.class/Topic))
|
||||
view-id (create-view-id conn :class-objects :view-for-id class-id)
|
||||
is-result (db-view/get-view-data @conn view-id {:view-feature-type :class-objects
|
||||
:view-for-id class-id
|
||||
:filters {:or? false
|
||||
:filters [[:block/page :is #{page-a-uuid}]]}})
|
||||
is-titles (map (fn [id] (:block/title (d/entity @conn id))) (:data is-result))
|
||||
is-not-result (db-view/get-view-data @conn view-id {:view-feature-type :class-objects
|
||||
:view-for-id class-id
|
||||
:filters {:or? false
|
||||
:filters [[:block/page :is-not #{page-a-uuid}]]}})
|
||||
is-not-titles (set (map (fn [id] (:block/title (d/entity @conn id))) (:data is-not-result)))]
|
||||
(is (= #{"Obj A"} (set is-titles)))
|
||||
(is (= #{"Obj B"} is-not-titles))
|
||||
(is (= #{obj-a-id} (set (:data is-result))))
|
||||
(is (= #{obj-b-id} (set (:data is-not-result))))))
|
||||
|
||||
(deftest get-view-data-class-objects-groups-by-number-property-sorts-numerically-test
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:classes {:Topic {:block/title "Topic"}}
|
||||
:properties {:user.property/score {:logseq.property/type :number}}
|
||||
:pages-and-blocks
|
||||
[{:page {:block/title "A" :build/tags [:Topic]
|
||||
:build/properties {:user.property/score 2}}}
|
||||
{:page {:block/title "B" :build/tags [:Topic]
|
||||
:build/properties {:user.property/score 10}}}
|
||||
{:page {:block/title "C" :build/tags [:Topic]
|
||||
:build/properties {:user.property/score 1}}}]})
|
||||
class-id (:db/id (d/entity @conn :user.class/Topic))
|
||||
view-id (create-view-id conn :class-objects :view-for-id class-id)
|
||||
_ (d/transact! conn [[:db/add view-id :logseq.property.view/group-by-property :user.property/score]])
|
||||
asc-groups (map first (:data (db-view/get-view-data @conn view-id
|
||||
{:view-feature-type :class-objects
|
||||
:view-for-id class-id})))
|
||||
_ (d/transact! conn [[:db/add view-id :logseq.property.view/sort-groups-desc? true]])
|
||||
desc-groups (map first (:data (db-view/get-view-data @conn view-id
|
||||
{:view-feature-type :class-objects
|
||||
:view-for-id class-id})))]
|
||||
;; Number groups must sort numerically (1 2 10), not lexicographically (1 10 2)
|
||||
(is (= [1 2 10] asc-groups))
|
||||
;; "Sort groups order" (desc?) must reverse the numeric order
|
||||
(is (= [10 2 1] desc-groups))))
|
||||
59
deps/db/test/logseq/db/frontend/class_test.cljs
vendored
Normal file
59
deps/db/test/logseq/db/frontend/class_test.cljs
vendored
Normal file
@@ -0,0 +1,59 @@
|
||||
(ns logseq.db.frontend.class-test
|
||||
(:require [cljs.test :refer [deftest is testing]]
|
||||
[datascript.core :as d]
|
||||
[logseq.db.frontend.class :as db-class]
|
||||
[logseq.db.test.helper :as db-test]))
|
||||
|
||||
(defn- parent-class-id [db]
|
||||
(:db/id (d/entity db :user.class/Parent)))
|
||||
|
||||
(deftest get-class-objects-dedupes-inherited-tags-test
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:classes {:Parent {:block/title "Parent"}
|
||||
:Child {:block/title "Child"
|
||||
:build/class-extends [:Parent]}}
|
||||
:pages-and-blocks [{:page {:block/title "Object1"
|
||||
:build/tags [:Parent :Child]}}]})
|
||||
objects (db-class/get-class-objects @conn (parent-class-id @conn))
|
||||
ids (map :db/id objects)]
|
||||
(testing "an object tagged by both parent and child class is returned once"
|
||||
(is (= 1 (count ids)))
|
||||
(is (= 1 (count (distinct ids)))))))
|
||||
|
||||
(deftest get-class-objects-filters-hidden-objects-test
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:classes {:Parent {:block/title "Parent"}
|
||||
:Child {:block/title "Child"
|
||||
:build/class-extends [:Parent]}}
|
||||
:pages-and-blocks [{:page {:block/title "Visible"
|
||||
:build/tags [:Child]}}
|
||||
{:page {:block/title "Deleted"
|
||||
:build/tags [:Child]
|
||||
:logseq.property/deleted-at 1}}
|
||||
{:page {:block/title "Hidden"
|
||||
:build/tags [:Child]
|
||||
:logseq.property/hide? true}}]})
|
||||
titles (->> (db-class/get-class-objects @conn (parent-class-id @conn))
|
||||
(map :block/title)
|
||||
set)]
|
||||
(is (= #{"Visible"} titles))))
|
||||
|
||||
(deftest get-class-objects-includes-hide-by-default-properties-test
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:properties {:keywords {:logseq.property/type :default
|
||||
:logseq.property/hide? true}
|
||||
:author {:logseq.property/type :default}
|
||||
:deleted-prop {:logseq.property/type :default}}})
|
||||
deleted (d/entity @conn :user.property/deleted-prop)
|
||||
_ (d/transact! conn [[:db/add (:db/id deleted) :logseq.property/deleted-at 1]])
|
||||
property-class-id (:db/id (d/entity @conn :logseq.class/Property))
|
||||
titles (->> (db-class/get-class-objects @conn property-class-id)
|
||||
(map :block/title)
|
||||
set)]
|
||||
(testing "hide-by-default user properties still appear in the Property table"
|
||||
(is (contains? titles "keywords"))
|
||||
(is (contains? titles "author")))
|
||||
(testing "deleted properties stay out of the Property table"
|
||||
(is (not (contains? titles "deleted-prop"))))
|
||||
(testing "private built-in properties stay out of the Property table"
|
||||
(is (not (contains? titles "Property type"))))))
|
||||
104
deps/db/test/logseq/db/frontend/content_test.cljs
vendored
Normal file
104
deps/db/test/logseq/db/frontend/content_test.cljs
vendored
Normal file
@@ -0,0 +1,104 @@
|
||||
(ns logseq.db.frontend.content-test
|
||||
(:require [cljs.test :refer [deftest is testing]]
|
||||
[clojure.string :as string]
|
||||
[logseq.db.frontend.content :as db-content]
|
||||
[logseq.common.util.page-ref :as page-ref]))
|
||||
|
||||
(deftest replace-tags-with-page-refs
|
||||
(testing "tags with overlapping names get replaced correctly"
|
||||
(is (= "string [[foo]] string2 [[foo-bar]]"
|
||||
(db-content/replace-tags-with-id-refs
|
||||
"string #foo string2 #foo-bar"
|
||||
[{:block/title "foo" :block/uuid "foo"}
|
||||
{:block/title "foo-bar" :block/uuid "foo-bar"}])))))
|
||||
|
||||
(deftest title-ref->id-ref
|
||||
(let [page-uuid (random-uuid)]
|
||||
(is (= (str "some page ref " (page-ref/->page-ref page-uuid))
|
||||
(db-content/title-ref->id-ref "some page ref [[page1]]" [{:block/title "page1" :block/uuid page-uuid}]))
|
||||
"Replaces page ref name with uuid"))
|
||||
|
||||
(testing "uses original page name when journal title is normalized"
|
||||
(let [journal-uuid #uuid "00000001-2026-0615-0000-000000000000"]
|
||||
(is (= (str "some page ref " (page-ref/->page-ref journal-uuid))
|
||||
(db-content/title-ref->id-ref
|
||||
"some page ref [[2026-06-15]]"
|
||||
[{:block/title "Jun 15th, 2026"
|
||||
:block.temp/original-page-name "2026-06-15"
|
||||
:block/uuid journal-uuid}])))))
|
||||
|
||||
(testing "ignores refs without uuid"
|
||||
(is (= "some page ref [[2026-06-15]]"
|
||||
(db-content/title-ref->id-ref
|
||||
"some page ref [[2026-06-15]]"
|
||||
[{:block/title "2026-06-15"}]))))
|
||||
|
||||
(testing "does replace tag with :replace-tag? true"
|
||||
(is (= "some #[[5c6cd067-c602-4955-96b8-74b62e08113c]] tag"
|
||||
(db-content/title-ref->id-ref "some #test tag"
|
||||
[{:block/title "test" :block/uuid #uuid "5c6cd067-c602-4955-96b8-74b62e08113c"}]
|
||||
{:replace-tag? true})))
|
||||
(is (= "some #[[5c6cd067-c602-4955-96b8-74b62e08113c]] tag"
|
||||
(db-content/title-ref->id-ref "some #[[another test]] tag"
|
||||
[{:block/title "another test" :block/uuid #uuid "5c6cd067-c602-4955-96b8-74b62e08113c"}]
|
||||
{:replace-tag? true}))))
|
||||
|
||||
(testing "does not replace tag with :replace-tag? false"
|
||||
(is (= "some #test tag"
|
||||
(db-content/title-ref->id-ref "some #test tag"
|
||||
[{:block/title "test" :block/uuid (random-uuid)}]
|
||||
{:replace-tag? false})))
|
||||
(is (= "some #[[another test]] tag"
|
||||
(db-content/title-ref->id-ref "some #[[another test]] tag"
|
||||
[{:block/title "another test" :block/uuid (random-uuid)}]
|
||||
{:replace-tag? false})))))
|
||||
|
||||
(deftest recur-replace-uuid-in-block-title-test
|
||||
(let [uuid-a #uuid "11111111-1111-1111-1111-111111111111"
|
||||
uuid-b #uuid "22222222-2222-2222-2222-222222222222"
|
||||
uuid-c #uuid "33333333-3333-3333-3333-333333333333"
|
||||
id-ref-a (page-ref/->page-ref uuid-a)
|
||||
id-ref-b (page-ref/->page-ref uuid-b)
|
||||
id-ref-c (page-ref/->page-ref uuid-c)]
|
||||
(testing "replaces direct id-ref title"
|
||||
(is (= "[[Direct Page]]"
|
||||
(db-content/recur-replace-uuid-in-block-title
|
||||
{:block/title id-ref-a
|
||||
:block/refs [{:block/uuid uuid-a
|
||||
:block/title "Direct Page"}]}))))
|
||||
|
||||
(testing "replaces recursively through nested refs"
|
||||
(is (= "[[[[Leaf Page]]]]"
|
||||
(db-content/recur-replace-uuid-in-block-title
|
||||
{:block/title id-ref-a
|
||||
:block/refs [{:block/uuid uuid-a
|
||||
:block/title id-ref-b
|
||||
:block/refs [{:block/uuid uuid-b
|
||||
:block/title "Leaf Page"}]}]}))))
|
||||
|
||||
(testing "keeps hashtag style for simple page title"
|
||||
(is (= "#simple-tag"
|
||||
(db-content/recur-replace-uuid-in-block-title
|
||||
{:block/title (str "#" id-ref-a)
|
||||
:block/refs [{:block/uuid uuid-a
|
||||
:block/title "simple-tag"}]}))))
|
||||
|
||||
(testing "keeps page-ref style hashtag when title contains spaces"
|
||||
(is (= "#[[tag with space]]"
|
||||
(db-content/recur-replace-uuid-in-block-title
|
||||
{:block/title (str "#" id-ref-a)
|
||||
:block/refs [{:block/uuid uuid-a
|
||||
:block/title "tag with space"}]}))))
|
||||
|
||||
(testing "stops recursion when max-depth reached"
|
||||
(let [result (db-content/recur-replace-uuid-in-block-title
|
||||
{:block/title id-ref-a
|
||||
:block/refs [{:block/uuid uuid-a
|
||||
:block/title id-ref-b
|
||||
:block/refs [{:block/uuid uuid-b
|
||||
:block/title id-ref-c
|
||||
:block/refs [{:block/uuid uuid-c
|
||||
:block/title "Too Deep"}]}]}]}
|
||||
1)]
|
||||
(is (re-find db-content/id-ref-pattern result))
|
||||
(is (not (string/includes? result "Too Deep")))))))
|
||||
44
deps/db/test/logseq/db/frontend/db_ident_test.cljc
vendored
Normal file
44
deps/db/test/logseq/db/frontend/db_ident_test.cljc
vendored
Normal file
@@ -0,0 +1,44 @@
|
||||
(ns logseq.db.frontend.db-ident-test
|
||||
(:require [cljs.test :refer [deftest is testing]]
|
||||
[clojure.edn :as edn]
|
||||
[clojure.string :as string]
|
||||
[logseq.db.frontend.db-ident :as db-ident]))
|
||||
|
||||
(defn- valid-edn-keyword?
|
||||
"Like common-util/valid-edn-keyword? but with kw as argument"
|
||||
[kw]
|
||||
(try
|
||||
(boolean (edn/read-string (str "{" kw " nil}")))
|
||||
(catch :default _
|
||||
false)))
|
||||
|
||||
;; These tests are copied to and should be kept in sync with logseq.db.misc-test
|
||||
;; Tests edge cases as described in https://clojure.org/reference/reader#_symbols
|
||||
(deftest create-db-ident-from-name
|
||||
(testing "Symbols begin with a non-numeric character and can contain alphanumeric characters and *, +, !, -, _, ', ?, <, > and ="
|
||||
(is (valid-edn-keyword? (db-ident/create-db-ident-from-name "user.property" "f@!{h[#"))
|
||||
"Kw created from name containing invalid special characters is valid edn")
|
||||
(is (valid-edn-keyword? (db-ident/create-db-ident-from-name "user.property" "foo*+!_'?<>=-"))
|
||||
"Kw created from name with all valid special characters is valid edn")
|
||||
(is (string/includes? (name (db-ident/create-db-ident-from-name "user.property" "foo*+!_'?<>=-"))
|
||||
"*+!_'?<>=-")
|
||||
"Kw created from name with all valid special characters contains all special characters")
|
||||
(is (valid-edn-keyword? (db-ident/create-db-ident-from-name "user.property" "2ndCity"))
|
||||
"Kw created from name starting with number is valid edn")
|
||||
(is (string/starts-with? (name (db-ident/create-db-ident-from-name "user.property" "2ndCity"))
|
||||
"NUM-2nd")
|
||||
"Kw created from name starting with number does not start with that number"))
|
||||
|
||||
(testing "All other special characters"
|
||||
(is (valid-edn-keyword? (db-ident/create-db-ident-from-name "user.property" ":name"))
|
||||
"Kw created from name with ':' is valid edn")
|
||||
(is (not (string/includes? (name (db-ident/create-db-ident-from-name "user.property" ":name")) ":"))
|
||||
"Kw created from name with ':' doesn't contain ':'")
|
||||
(is (valid-edn-keyword? (db-ident/create-db-ident-from-name "user.property" "foo/bar"))
|
||||
"Kw created from name with '/' is valid edn")
|
||||
(is (not (string/includes? (name (db-ident/create-db-ident-from-name "user.property" "foo/bar")) "/"))
|
||||
"Kw created from name with '/' doesn't contain '/'")
|
||||
(is (valid-edn-keyword? (db-ident/create-db-ident-from-name "user.property" "foo.bar"))
|
||||
"Kw created from name with '.' is valid edn")
|
||||
(is (not (string/includes? (name (db-ident/create-db-ident-from-name "user.property" "foo.bar")) "."))
|
||||
"Kw created from name with '.' doesn't contain '.'")))
|
||||
207
deps/db/test/logseq/db/frontend/inputs_test.cljs
vendored
Normal file
207
deps/db/test/logseq/db/frontend/inputs_test.cljs
vendored
Normal file
@@ -0,0 +1,207 @@
|
||||
(ns logseq.db.frontend.inputs-test
|
||||
(:require [cljs-time.core :as t]
|
||||
[cljs.test :refer [deftest is]]
|
||||
[datascript.core :as d]
|
||||
[logseq.common.util.date-time :as date-time-util]
|
||||
[logseq.db.frontend.inputs :as db-inputs]
|
||||
[logseq.db.frontend.rules :as rules]
|
||||
[logseq.db.frontend.schema :as db-schema]
|
||||
[logseq.db.sqlite.build :as sqlite-build]
|
||||
[logseq.db.test.helper :as db-test]))
|
||||
|
||||
(defn- custom-query [db {:keys [inputs query input-options]}]
|
||||
(let [q-args (cond-> (mapv #(db-inputs/resolve-input db % input-options) inputs)
|
||||
(contains? (set query) '%)
|
||||
(conj (rules/extract-rules rules/db-query-dsl-rules [:between])))]
|
||||
(->> (apply d/q query db q-args)
|
||||
(map first))))
|
||||
|
||||
(deftest resolve-input-for-page-and-block-inputs
|
||||
(let [conn (d/create-conn db-schema/schema)
|
||||
_ (d/transact! conn [{:db/ident :logseq.class/Page}])
|
||||
_ (sqlite-build/create-blocks
|
||||
conn
|
||||
[{:page {:block/title "page1"}
|
||||
:blocks [{:block/title "parent"
|
||||
:build/children
|
||||
[{:block/title "child 1"}
|
||||
{:block/title "child 2"}]}]}])]
|
||||
(is (= ["child 2" "child 1" "parent"]
|
||||
(map :block/title
|
||||
(custom-query @conn
|
||||
{:inputs [:current-page]
|
||||
:query '[:find (pull ?b [*])
|
||||
:in $ ?current-page
|
||||
:where [?b :block/page ?bp]
|
||||
[?bp :block/name ?current-page]]
|
||||
:input-options {:current-page-fn (constantly "page1")}})))
|
||||
":current-page input resolves to current page name")
|
||||
|
||||
(is (= []
|
||||
(map :block/title
|
||||
(custom-query @conn
|
||||
{:inputs [:current-page]
|
||||
:query '[:find (pull ?b [*])
|
||||
:in $ ?current-page
|
||||
:where [?b :block/page ?bp]
|
||||
[?bp :block/name ?current-page]]})))
|
||||
":current-page input doesn't resolve when :current-page-fn not provided")
|
||||
|
||||
(is (= ["child 1" "child 2"]
|
||||
(let [block-uuid (:block/uuid (db-test/find-block-by-content @conn "parent"))]
|
||||
(map :block/title
|
||||
(custom-query @conn
|
||||
{:inputs [:current-block]
|
||||
:query '[:find (pull ?b [*])
|
||||
:in $ ?current-block
|
||||
:where [?b :block/parent ?current-block]]
|
||||
:input-options {:current-block-uuid block-uuid}}))))
|
||||
":current-block input resolves to current block's :db/id")
|
||||
|
||||
(is (thrown-with-msg?
|
||||
js/Error
|
||||
#"Nothing found for entity"
|
||||
(custom-query @conn
|
||||
{:inputs [:current-block]
|
||||
:query '[:find (pull ?b [*])
|
||||
:in $ ?current-block
|
||||
:where [?b :block/parent ?current-block]]
|
||||
:input-options {:current-block-uuid nil}}))
|
||||
":current-block input doesn't resolve and fails when :current-block-uuid is not provided")
|
||||
|
||||
(is (thrown-with-msg?
|
||||
js/Error
|
||||
#"Nothing found for entity"
|
||||
(custom-query @conn
|
||||
{:inputs [:current-block]
|
||||
:query '[:find (pull ?b [*])
|
||||
:in $ ?current-block
|
||||
:where [?b :block/parent ?current-block]]
|
||||
:input-options {:current-block-uuid :magic}}))
|
||||
":current-block input doesn't resolve and fails when :current-block-uuid is invalid")
|
||||
|
||||
(is (= ["parent"]
|
||||
(let [block-uuid (:block/uuid (db-test/find-block-by-content @conn "child 1"))]
|
||||
(map :block/title
|
||||
(custom-query @conn
|
||||
{:inputs [:parent-block]
|
||||
:query '[:find (pull ?parent-block [*])
|
||||
:in $ ?parent-block
|
||||
:where [?parent-block :block/parent]]
|
||||
:input-options {:current-block-uuid block-uuid}}))))
|
||||
":parent-block input resolves to parent of current blocks's :db/id")))
|
||||
|
||||
(deftest resolve-input-for-journal-date-inputs
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
[{:page {:build/journal 20230101}
|
||||
:blocks [{:block/title "b1"}]}
|
||||
{:page {:build/journal 20230107}
|
||||
:blocks [{:block/title "b2"}]}])]
|
||||
(is (= ["b2"]
|
||||
(with-redefs [t/today (constantly (t/date-time 2023 1 7))]
|
||||
(map :block/title
|
||||
(custom-query @conn
|
||||
{:inputs [:3d-before :today]
|
||||
:query '[:find (pull ?b [*])
|
||||
:in $ ?start ?end %
|
||||
:where (between ?b ?start ?end)]}))))
|
||||
":Xd-before and :today resolve to correct journal range")
|
||||
|
||||
(is (= ["b1"]
|
||||
(with-redefs [t/today (constantly (t/date-time 2022 12 31))]
|
||||
(map :block/title
|
||||
(custom-query @conn
|
||||
{:inputs [:tomorrow :4d-after]
|
||||
:query '[:find (pull ?b [*])
|
||||
:in $ ?start ?end %
|
||||
:where (between ?b ?start ?end)]}))))
|
||||
":tomorrow and :Xd-after resolve to correct journal range")))
|
||||
|
||||
(defn- blocks-on-journal-page-from-block-with-content [db page-input block-content current-page-date]
|
||||
(map :block/title
|
||||
(custom-query db
|
||||
{:inputs [page-input]
|
||||
:query '[:find (pull ?b [*])
|
||||
:in $ ?page
|
||||
:where [?b :block/page ?e]
|
||||
[?e :block/name ?page]]
|
||||
:input-options
|
||||
{:current-block-uuid (:block/uuid (db-test/find-block-by-content db block-content))
|
||||
:current-page-fn (constantly
|
||||
(date-time-util/int->journal-title (date-time-util/date->int current-page-date)
|
||||
"MMM do, yyyy"))}})))
|
||||
|
||||
(deftest resolve-input-for-query-page
|
||||
(let [current-date (t/date-time 2023 1 1)
|
||||
conn (db-test/create-conn-with-blocks
|
||||
[{:page {:build/journal 20221231} :blocks [{:block/title "-1d"}]}
|
||||
{:page {:build/journal 20230101} :blocks [{:block/title "now"}]}
|
||||
{:page {:build/journal 20230102} :blocks [{:block/title "+1d"}]}])
|
||||
db @conn]
|
||||
(is (= ["now"] (blocks-on-journal-page-from-block-with-content db :current-page "now" current-date))
|
||||
":current-page resolves to the stateful page when called from a block on the stateful page")
|
||||
|
||||
(is (= ["now"] (blocks-on-journal-page-from-block-with-content db :query-page "now" current-date))
|
||||
":query-page resolves to the stateful page when called from a block on the stateful page")
|
||||
|
||||
(is (= ["now"] (blocks-on-journal-page-from-block-with-content db :current-page "+1d" current-date))
|
||||
":current-page resolves to the stateful page when called from a block on another page")
|
||||
|
||||
(is (= ["+1d"] (blocks-on-journal-page-from-block-with-content db :query-page "+1d" current-date))
|
||||
":query-page resolves to the parent page when called from another page")))
|
||||
|
||||
(defn- blocks-journaled-between-inputs [db a b]
|
||||
;; reverse is for sort order and may be brittle
|
||||
(reverse
|
||||
(map :block/title
|
||||
(custom-query db
|
||||
{:inputs [a b]
|
||||
:query '[:find (pull ?b [*])
|
||||
:in $ ?start ?end %
|
||||
:where (between ?b ?start ?end)]}))))
|
||||
|
||||
(deftest resolve-input-for-relative-date-queries
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
[{:page {:build/journal 20220101} :blocks [{:block/title "-1y"}]}
|
||||
{:page {:build/journal 20221201} :blocks [{:block/title "-1m"}]}
|
||||
{:page {:build/journal 20221225} :blocks [{:block/title "-1w"}]}
|
||||
{:page {:build/journal 20221231} :blocks [{:block/title "-1d"}]}
|
||||
{:page {:build/journal 20230101} :blocks [{:block/title "now"}]}
|
||||
{:page {:build/journal 20230102} :blocks [{:block/title "+1d"}]}
|
||||
{:page {:build/journal 20230108} :blocks [{:block/title "+1w"}]}
|
||||
{:page {:build/journal 20230201} :blocks [{:block/title "+1m"}]}
|
||||
{:page {:build/journal 20240101} :blocks [{:block/title "+1y"}]}])
|
||||
db @conn]
|
||||
(with-redefs [t/today (constantly (t/date-time 2023 1 1))]
|
||||
(is (= ["now" "-1d" "-1w" "-1m" "-1y"] (blocks-journaled-between-inputs db :-365d :today))
|
||||
":-365d and today resolve to correct journal range")
|
||||
|
||||
(is (= ["now" "-1d" "-1w" "-1m" "-1y"] (blocks-journaled-between-inputs db :-1y :today))
|
||||
":-1y and today resolve to correct journal range")
|
||||
|
||||
(is (= ["now" "-1d" "-1w" "-1m"] (blocks-journaled-between-inputs db :-1m :today))
|
||||
":-1m and today resolve to correct journal range")
|
||||
|
||||
(is (= ["now" "-1d" "-1w"] (blocks-journaled-between-inputs db :-1w :today))
|
||||
":-1w and today resolve to correct journal range")
|
||||
|
||||
(is (= ["now" "-1d"] (blocks-journaled-between-inputs db :-1d :today))
|
||||
":-1d and today resolve to correct journal range")
|
||||
|
||||
(is (= ["+1y" "+1m" "+1w" "+1d" "now"] (blocks-journaled-between-inputs db :today :+365d))
|
||||
":+365d and today resolve to correct journal range")
|
||||
|
||||
(is (= ["+1y" "+1m" "+1w" "+1d" "now"] (blocks-journaled-between-inputs db :today :+1y))
|
||||
":+1y and today resolve to correct journal range")
|
||||
|
||||
(is (= ["+1m" "+1w" "+1d" "now"] (blocks-journaled-between-inputs db :today :+1m))
|
||||
":+1m and today resolve to correct journal range")
|
||||
|
||||
(is (= ["+1w" "+1d" "now"] (blocks-journaled-between-inputs db :today :+1w))
|
||||
":+1w and today resolve to correct journal range")
|
||||
|
||||
(is (= ["+1d" "now"] (blocks-journaled-between-inputs db :today :+1d))
|
||||
":+1d and today resolve to correct journal range")
|
||||
|
||||
(is (= ["+1d" "now"] (blocks-journaled-between-inputs db :today :today/+1d))
|
||||
":today/+1d and today resolve to correct journal range"))))
|
||||
42
deps/db/test/logseq/db/frontend/property/type_test.cljs
vendored
Normal file
42
deps/db/test/logseq/db/frontend/property/type_test.cljs
vendored
Normal file
@@ -0,0 +1,42 @@
|
||||
(ns logseq.db.frontend.property.type-test
|
||||
(:require [cljs.test :refer [deftest is testing]]
|
||||
[logseq.db.frontend.property.type :as db-property-type]
|
||||
[logseq.db.test.helper :as db-test]))
|
||||
|
||||
(deftest asset-property-type-registered
|
||||
(testing ":asset is exposed to users"
|
||||
(is (some #{:asset} db-property-type/user-built-in-property-types)))
|
||||
|
||||
(testing ":asset is treated as a ref type"
|
||||
(is (contains? db-property-type/user-ref-property-types :asset))
|
||||
(is (contains? db-property-type/all-ref-property-types :asset)))
|
||||
|
||||
(testing ":asset validator requires a datascript db"
|
||||
(is (contains? db-property-type/property-types-with-db :asset)))
|
||||
|
||||
(testing ":asset can be configured as multiple values"
|
||||
(is (contains? db-property-type/cardinality-property-types :asset))
|
||||
(is (not (contains? db-property-type/closed-value-property-types :asset))
|
||||
":asset should not support closed values")))
|
||||
|
||||
(deftest asset-entity-validator
|
||||
(testing "asset-entity? accepts blocks tagged :logseq.class/Asset and rejects others"
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:pages-and-blocks
|
||||
[{:page {:block/title "Page"}
|
||||
:blocks [{:block/title "plain-block"}
|
||||
{:block/title "my-asset"
|
||||
:build/tags [:logseq.class/Asset]}]}]})
|
||||
db @conn
|
||||
asset-block (db-test/find-block-by-content db "my-asset")
|
||||
plain-block (db-test/find-block-by-content db "plain-block")
|
||||
;; asset-entity? is private; reach through the registered schema
|
||||
asset-fn (last (:asset db-property-type/built-in-validation-schemas))]
|
||||
(is (some? asset-block))
|
||||
(is (some? plain-block))
|
||||
(testing "asset block validates"
|
||||
(is (true? (boolean (asset-fn db (:db/id asset-block))))))
|
||||
(testing "non-asset block does not validate"
|
||||
(is (not (asset-fn db (:db/id plain-block)))))
|
||||
(testing "nonexistent id does not validate"
|
||||
(is (not (asset-fn db 9999999)))))))
|
||||
89
deps/db/test/logseq/db/frontend/property_test.cljs
vendored
Normal file
89
deps/db/test/logseq/db/frontend/property_test.cljs
vendored
Normal file
@@ -0,0 +1,89 @@
|
||||
(ns logseq.db.frontend.property-test
|
||||
(:require [cljs.test :refer [deftest is testing]]
|
||||
[logseq.db.frontend.property :as db-property]))
|
||||
|
||||
(deftest sort-properties
|
||||
(let [p1 {:db/id 1, :block/order "a", :block/uuid "uuid-a"}
|
||||
p2 {:db/id 2, :block/order "b", :block/uuid "uuid-b"}
|
||||
p3 {:db/id 3, :block/order nil, :block/uuid "uuid-d"}
|
||||
p4 {:db/id 4, :block/order "b", :block/uuid "uuid-c"}
|
||||
p5 {:db/id 5, :block/order nil, :block/uuid "uuid-e"}]
|
||||
(is (= [p1 p2 p4 p3 p5]
|
||||
(db-property/sort-properties [p3 p1 p5 p2 p4])))))
|
||||
|
||||
(deftest normalize-block-order-tx-data-test
|
||||
(testing "Generate appropriate :block/order values for sorted-blocks with :block/order value = nil or duplicated"
|
||||
(let [p1 {:db/id 1, :block/order "a0"}
|
||||
p2 {:db/id 2, :block/order "bbb"}
|
||||
p3 {:db/id 3, :block/order "bbb"}
|
||||
p4 {:db/id 4, :block/order nil}
|
||||
p5 {:db/id 5, :block/order nil}
|
||||
sorted-entities [p1 p2 p3 p4 p5]
|
||||
tx-data (db-property/normalize-sorted-entities-block-order sorted-entities)
|
||||
;; apply tx-data to entities
|
||||
tx-map (into {} (map (juxt :db/id identity) tx-data))
|
||||
updated-entities (map (fn [ent]
|
||||
(if-let [tx (get tx-map (:db/id ent))]
|
||||
(merge ent tx)
|
||||
ent))
|
||||
sorted-entities)
|
||||
;; sort again and test
|
||||
final-sorted (db-property/sort-properties updated-entities)]
|
||||
(is (= 5 (count final-sorted)))
|
||||
;; Check that all orders are now strings
|
||||
(is (every? string? (map :block/order final-sorted)))
|
||||
;; Check that all orders are unique
|
||||
(is (= 5 (count (set (map :block/order final-sorted)))))
|
||||
;; Check that the final list is sorted correctly by the new orders
|
||||
(is (= final-sorted (sort-by :block/order final-sorted)))))
|
||||
|
||||
(testing "No changes needed for already valid orders"
|
||||
(let [p1 {:db/id 1, :block/order "b00"}
|
||||
p2 {:db/id 2, :block/order "b01"}
|
||||
sorted-entities [p1 p2]
|
||||
tx-data (db-property/normalize-sorted-entities-block-order sorted-entities)]
|
||||
(is (empty? tx-data)))))
|
||||
|
||||
(deftest reaction-built-in-properties
|
||||
(let [props db-property/built-in-properties]
|
||||
(testing "entries exist"
|
||||
(is (contains? props :logseq.property.reaction/emoji-id))
|
||||
(is (contains? props :logseq.property.reaction/target)))
|
||||
|
||||
(testing "schema types"
|
||||
(is (= :string (get-in props [:logseq.property.reaction/emoji-id :schema :type])))
|
||||
(is (= :node (get-in props [:logseq.property.reaction/target :schema :type]))))
|
||||
|
||||
(testing "internal visibility"
|
||||
(is (= false (get-in props [:logseq.property.reaction/emoji-id :schema :public?])))
|
||||
(is (= false (get-in props [:logseq.property.reaction/target :schema :public?])))
|
||||
(is (= true (get-in props [:logseq.property.reaction/emoji-id :schema :hide?])))
|
||||
(is (= true (get-in props [:logseq.property.reaction/target :schema :hide?]))))
|
||||
|
||||
(testing "logseq property namespace"
|
||||
(is (db-property/logseq-property? :logseq.property.reaction/emoji-id))
|
||||
(is (db-property/logseq-property? :logseq.property.reaction/target)))))
|
||||
|
||||
(deftest comments-built-in-properties
|
||||
(let [props db-property/built-in-properties
|
||||
property :logseq.property.comments/blocks]
|
||||
(is (contains? props property))
|
||||
(is (= "Commented blocks" (get-in props [property :title])))
|
||||
(is (= :node (get-in props [property :schema :type])))
|
||||
(is (= :many (get-in props [property :schema :cardinality])))
|
||||
(is (= false (get-in props [property :schema :public?])))
|
||||
(is (= true (get-in props [property :schema :hide?])))
|
||||
(is (db-property/logseq-property? property))))
|
||||
|
||||
(deftest assignee-built-in-property
|
||||
(let [property (get db-property/built-in-properties :logseq.property/assignee)]
|
||||
(testing "schema"
|
||||
(is (= "Assignee" (:title property)))
|
||||
(is (= :node (get-in property [:schema :type])))
|
||||
(is (= :many (get-in property [:schema :cardinality])))
|
||||
(is (= true (get-in property [:schema :public?]))))
|
||||
|
||||
(testing "queryable built-in logseq property"
|
||||
(is (= true (:queryable? property)))
|
||||
(is (contains? db-property/public-built-in-properties :logseq.property/assignee))
|
||||
(is (db-property/logseq-property? :logseq.property/assignee)))))
|
||||
21
deps/db/test/logseq/db/frontend/reaction_test.cljs
vendored
Normal file
21
deps/db/test/logseq/db/frontend/reaction_test.cljs
vendored
Normal file
@@ -0,0 +1,21 @@
|
||||
(ns logseq.db.frontend.reaction-test
|
||||
(:require [cljs.test :refer [deftest is testing]]
|
||||
[datascript.core :as d]
|
||||
[logseq.common.util :as common-util]
|
||||
[logseq.db.frontend.validate :as db-validate]
|
||||
[logseq.db.test.helper :as db-test]))
|
||||
|
||||
(deftest reaction-entity-valid
|
||||
(testing "reaction entity passes db validation"
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:pages-and-blocks
|
||||
[{:page {:block/title "Page"}
|
||||
:blocks [{:block/title "Block"}]}]})
|
||||
block (db-test/find-block-by-content @conn "Block")
|
||||
now (common-util/time-ms)
|
||||
reaction {:block/uuid (random-uuid)
|
||||
:block/created-at now
|
||||
:logseq.property.reaction/emoji-id "+1"
|
||||
:logseq.property.reaction/target (:db/id block)}]
|
||||
(d/transact! conn [reaction])
|
||||
(is (empty? (:errors (db-validate/validate-local-db! @conn)))))))
|
||||
266
deps/db/test/logseq/db/frontend/rules_test.cljs
vendored
Normal file
266
deps/db/test/logseq/db/frontend/rules_test.cljs
vendored
Normal file
@@ -0,0 +1,266 @@
|
||||
(ns logseq.db.frontend.rules-test
|
||||
(:require [cljs.test :refer [deftest is testing are]]
|
||||
[datascript.core :as d]
|
||||
[logseq.db.frontend.rules :as rules]
|
||||
[logseq.db.test.helper :as db-test]))
|
||||
|
||||
(defn q-with-rules [query db]
|
||||
;; query assumes no :in given
|
||||
(d/q (into query [:in '$ '%])
|
||||
db
|
||||
(rules/extract-rules rules/db-query-dsl-rules)))
|
||||
|
||||
(deftest get-full-deps
|
||||
(let [property-value-deps #{:ref->val :class-extends :object-has-class-property :property-missing-value :ref-property-value :ref-property-value-with-default}
|
||||
property-deps (conj property-value-deps :ref-property-with-default)
|
||||
task-deps (conj property-deps :task)
|
||||
priority-deps (conj property-deps :priority)
|
||||
task-priority-deps (into priority-deps task-deps)]
|
||||
(are [x y] (= y (#'rules/get-full-deps x rules/rules-dependencies))
|
||||
[:ref-property-value-with-default] property-value-deps
|
||||
[:ref-property-with-default] property-deps
|
||||
[:task] task-deps
|
||||
[:priority] priority-deps
|
||||
[:task :priority] task-priority-deps)))
|
||||
|
||||
(deftest has-property-rule
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:properties {:foo {:logseq.property/type :default}
|
||||
:foo2 {:logseq.property/type :default}}
|
||||
:pages-and-blocks
|
||||
[{:page {:block/title "Page1"
|
||||
:build/properties {:foo "bar"}}}]})]
|
||||
|
||||
(is (= ["Page1"]
|
||||
(->> (q-with-rules '[:find (pull ?b [:block/title]) :where (has-property ?b :user.property/foo)]
|
||||
@conn)
|
||||
(map (comp :block/title first))))
|
||||
"has-property returns result when block has property")
|
||||
(is (= []
|
||||
(->> (q-with-rules '[:find (pull ?b [:block/title]) :where (has-property ?b :user.property/foo2)]
|
||||
@conn)
|
||||
(map (comp :block/title first))))
|
||||
"has-property returns no result when block doesn't have property")
|
||||
(is (= [:block/tags :user.property/foo]
|
||||
(q-with-rules '[:find [?p ...]
|
||||
:where (has-property ?b ?p) [?b :block/title "Page1"]]
|
||||
@conn))
|
||||
"has-property can bind to property arg")))
|
||||
|
||||
(deftest ref-property-rule
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:properties {:foo {:logseq.property/type :default}
|
||||
:foo2 {:logseq.property/type :default}
|
||||
:number-many {:logseq.property/type :number
|
||||
:db/cardinality :many}
|
||||
:page-many {:logseq.property/type :node
|
||||
:db/cardinality :many}}
|
||||
:pages-and-blocks
|
||||
[{:page {:block/title "Page1"
|
||||
:build/properties {:foo "bar" :number-many #{5 10} :page-many #{[:build/page {:block/title "Page A"}]}}}}
|
||||
{:page {:block/title "Page A"
|
||||
:build/properties {:foo "bar A"}}}]})]
|
||||
(testing "cardinality :one property"
|
||||
(is (= ["Page1"]
|
||||
(->> (q-with-rules '[:find (pull ?b [:block/title]) :where (ref-property ?b :user.property/foo "bar")]
|
||||
@conn)
|
||||
(map (comp :block/title first))))
|
||||
"property returns result when page has property")
|
||||
(is (= []
|
||||
(->> (q-with-rules '[:find (pull ?b [:block/title]) :where (ref-property ?b :user.property/foo "baz")]
|
||||
@conn)
|
||||
(map (comp :block/title first))))
|
||||
"property returns no result when page doesn't have property value")
|
||||
(is (= #{:user.property/foo}
|
||||
(->> (q-with-rules '[:find [?p ...]
|
||||
:where (ref-property ?b ?p "bar") [?b :block/title "Page1"]]
|
||||
@conn)
|
||||
set))
|
||||
"property can bind to property arg with bound property value"))
|
||||
|
||||
(testing "cardinality :many property"
|
||||
(is (= ["Page1"]
|
||||
(->> (q-with-rules '[:find (pull ?b [:block/title]) :where (ref-property ?b :user.property/number-many 5)]
|
||||
@conn)
|
||||
(map (comp :block/title first))))
|
||||
"property returns result when page has property")
|
||||
(is (= []
|
||||
(->> (q-with-rules '[:find (pull ?b [:block/title]) :where (ref-property ?b :user.property/number-many 20)]
|
||||
@conn)
|
||||
(map (comp :block/title first))))
|
||||
"property returns no result when page doesn't have property value")
|
||||
(is (= #{:user.property/number-many}
|
||||
(->> (q-with-rules '[:find [?p ...]
|
||||
:where (ref-property ?b ?p 5) [?b :block/title "Page1"]]
|
||||
@conn)
|
||||
set))
|
||||
"property can bind to property arg with bound property value"))
|
||||
|
||||
;; NOTE: Querying a ref's name is different than before and requires more than just the rule
|
||||
(testing ":ref property"
|
||||
(is (= ["Page1"]
|
||||
(->> (q-with-rules '[:find (pull ?b [:block/title])
|
||||
:where (ref-property ?b :user.property/page-many "Page A")]
|
||||
@conn)
|
||||
(map (comp :block/title first))))
|
||||
"property returns result when page has property")
|
||||
(is (= []
|
||||
(->> (q-with-rules '[:find (pull ?b [:block/title])
|
||||
:where [?b :user.property/page-many ?pv] [?pv :block/title "Page B"]]
|
||||
@conn)
|
||||
(map (comp :block/title first))))
|
||||
"property returns no result when page doesn't have property value"))
|
||||
|
||||
(testing "bindings with property value"
|
||||
(is (= #{:user.property/foo :user.property/number-many :user.property/page-many :block/tags}
|
||||
(->> (q-with-rules '[:find [?p ...]
|
||||
:where (ref-property ?b ?p _) [?b :block/title "Page1"]]
|
||||
@conn)
|
||||
set))
|
||||
"property can bind to property arg with unbound property value")
|
||||
(is (= #{[:user.property/number-many 10]
|
||||
[:user.property/number-many 5]
|
||||
[:user.property/foo "bar"]
|
||||
[:user.property/page-many "Page A"]
|
||||
[:block/tags "Page"]}
|
||||
(->> (q-with-rules '[:find ?p ?val
|
||||
:where (ref-property ?b ?p ?val) [?b :block/title "Page1"]]
|
||||
@conn)
|
||||
set))
|
||||
"property can bind to property and property value args")
|
||||
(is (= #{"Page1"}
|
||||
(->> (q-with-rules '[:find (pull ?b [:block/title])
|
||||
:where
|
||||
[?b :user.property/page-many ?pv]
|
||||
(ref-property ?pv :user.property/foo "bar A")]
|
||||
@conn)
|
||||
(map (comp :block/title first))
|
||||
set))
|
||||
"property can be used multiple times to query a property value's property"))))
|
||||
|
||||
(deftest property-rule
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:properties {:foo {:logseq.property/type :default}
|
||||
:foo2 {:logseq.property/type :default}
|
||||
:number-many {:logseq.property/type :number
|
||||
:db/cardinality :many}
|
||||
:page-many {:logseq.property/type :node
|
||||
:db/cardinality :many}}
|
||||
:pages-and-blocks
|
||||
[{:page {:block/title "Page1"
|
||||
:build/properties {:foo "bar" :number-many #{5 10} :page-many #{[:build/page {:block/title "Page A"}]}}}}
|
||||
{:page {:block/title "Page A"
|
||||
:build/properties {:foo "bar A"}}}]})]
|
||||
(testing "cardinality :one property"
|
||||
(is (= ["Page1"]
|
||||
(->> (q-with-rules '[:find (pull ?b [:block/title]) :where (property ?b :user.property/foo "bar")]
|
||||
@conn)
|
||||
(map (comp :block/title first))))
|
||||
"property returns result when page has property")
|
||||
(is (= []
|
||||
(->> (q-with-rules '[:find (pull ?b [:block/title]) :where (property ?b :user.property/foo "baz")]
|
||||
@conn)
|
||||
(map (comp :block/title first))))
|
||||
"property returns no result when page doesn't have property value")
|
||||
(is (= #{:user.property/foo}
|
||||
(->> (q-with-rules '[:find [?p ...]
|
||||
:where (property ?b ?p "bar") [?b :block/title "Page1"]]
|
||||
@conn)
|
||||
set))
|
||||
"property can bind to property arg with bound property value"))
|
||||
|
||||
(testing "cardinality :many property"
|
||||
(is (= ["Page1"]
|
||||
(->> (q-with-rules '[:find (pull ?b [:block/title]) :where (property ?b :user.property/number-many 5)]
|
||||
@conn)
|
||||
(map (comp :block/title first))))
|
||||
"property returns result when page has property")
|
||||
(is (= []
|
||||
(->> (q-with-rules '[:find (pull ?b [:block/title]) :where (property ?b :user.property/number-many 20)]
|
||||
@conn)
|
||||
(map (comp :block/title first))))
|
||||
"property returns no result when page doesn't have property value")
|
||||
(is (= #{:user.property/number-many}
|
||||
(->> (q-with-rules '[:find [?p ...]
|
||||
:where (property ?b ?p 5) [?b :block/title "Page1"]]
|
||||
@conn)
|
||||
set))
|
||||
"property can bind to property arg with bound property value"))
|
||||
|
||||
;; NOTE: Querying a ref's name is different than before and requires more than just the rule
|
||||
(testing ":ref property"
|
||||
(is (= ["Page1"]
|
||||
(->> (q-with-rules '[:find (pull ?b [:block/title])
|
||||
:where (property ?b :user.property/page-many "Page A")]
|
||||
@conn)
|
||||
(map (comp :block/title first))))
|
||||
"property returns result when page has property")
|
||||
(is (= []
|
||||
(->> (q-with-rules '[:find (pull ?b [:block/title])
|
||||
:where [?b :user.property/page-many ?pv] [?pv :block/title "Page B"]]
|
||||
@conn)
|
||||
(map (comp :block/title first))))
|
||||
"property returns no result when page doesn't have property value"))
|
||||
|
||||
(testing "bindings with property value"
|
||||
(is (= #{:user.property/foo :user.property/number-many :user.property/page-many :block/tags}
|
||||
(->> (q-with-rules '[:find [?p ...]
|
||||
:where (property ?b ?p _) [?b :block/title "Page1"]]
|
||||
@conn)
|
||||
set))
|
||||
"property can bind to property arg with unbound property value")
|
||||
(is (= #{[:user.property/number-many 10]
|
||||
[:user.property/number-many 5]
|
||||
[:user.property/foo "bar"]
|
||||
[:user.property/page-many "Page A"]
|
||||
[:block/tags "Page"]}
|
||||
(->> (q-with-rules '[:find ?p ?val
|
||||
:where (property ?b ?p ?val) [?b :block/title "Page1"]]
|
||||
@conn)
|
||||
set))
|
||||
"property can bind to property and property value args")
|
||||
(is (= #{"Page1"}
|
||||
(->> (q-with-rules '[:find (pull ?b [:block/title])
|
||||
:where
|
||||
[?b :user.property/page-many ?pv]
|
||||
(property ?pv :user.property/foo "bar A")]
|
||||
@conn)
|
||||
(map (comp :block/title first))
|
||||
set))
|
||||
"property can be used multiple times to query a property value's property"))))
|
||||
|
||||
(deftest tags-test
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:pages-and-blocks
|
||||
[{:page {:block/title "Page1"
|
||||
:build/tags [:Person]}}
|
||||
{:page {:block/title "Page2"
|
||||
:build/tags [:Person]}}
|
||||
{:page {:block/title "Page3"
|
||||
:build/tags [:Employee]}}]})
|
||||
person-eid (:db/id (d/entity @conn :user.class/Person))]
|
||||
(d/transact! conn [{:db/ident :user.class/Employee
|
||||
:logseq.property.class/extends :user.class/Person}])
|
||||
(testing "tags query with eid"
|
||||
(is (= #{"Page1" "Page2" "Page3"}
|
||||
(->> (d/q
|
||||
'[:find (pull ?b [:block/title])
|
||||
:in $ % ?tag-ids
|
||||
:where (tags ?b ?tag-ids)]
|
||||
@conn
|
||||
(rules/extract-rules rules/db-query-dsl-rules)
|
||||
#{person-eid})
|
||||
(map (comp :block/title first))
|
||||
set))))
|
||||
(testing "tags query with db/ident"
|
||||
(is (= #{"Page1" "Page2" "Page3"}
|
||||
(->> (q-with-rules '[:find (pull ?b [:block/title]) :where (tags ?b #{:user.class/Person})]
|
||||
@conn)
|
||||
(map (comp :block/title first))
|
||||
set))))
|
||||
(testing "tags query with block/title"
|
||||
(is (= #{"Page1" "Page2" "Page3"}
|
||||
(->> (q-with-rules '[:find (pull ?b [:block/title]) :where (tags ?b #{"Person"})]
|
||||
@conn)
|
||||
(map (comp :block/title first))
|
||||
set))))))
|
||||
342
deps/db/test/logseq/db/sqlite/build_test.cljs
vendored
Normal file
342
deps/db/test/logseq/db/sqlite/build_test.cljs
vendored
Normal file
@@ -0,0 +1,342 @@
|
||||
(ns logseq.db.sqlite.build-test
|
||||
(:require [cljs.test :refer [deftest is testing]]
|
||||
[datascript.core :as d]
|
||||
[logseq.common.util.page-ref :as page-ref]
|
||||
[logseq.db :as ldb]
|
||||
[logseq.db.frontend.entity-util :as entity-util]
|
||||
[logseq.db.frontend.property :as db-property]
|
||||
[logseq.db.sqlite.build :as sqlite-build]
|
||||
[logseq.db.sqlite.export :as sqlite-export]
|
||||
[logseq.db.test.helper :as db-test]))
|
||||
|
||||
(deftest build-tags
|
||||
(let [conn (db-test/create-conn)
|
||||
_ (sqlite-build/create-blocks
|
||||
conn
|
||||
[{:page {:block/title "page1"}
|
||||
:blocks [{:block/title "Jrue Holiday" :build/tags [:Person]}
|
||||
{:block/title "some task" :build/tags [:logseq.class/Task]}]}
|
||||
{:page {:block/title "Jayson Tatum" :build/tags [:Person]}}])]
|
||||
(is (= [:user.class/Person]
|
||||
(mapv :db/ident (:block/tags (db-test/find-block-by-content @conn "Jrue Holiday"))))
|
||||
"Person class is created and correctly associated to a block")
|
||||
|
||||
(is (contains?
|
||||
(set (map :db/ident (:block/tags (db-test/find-page-by-title @conn "Jayson Tatum"))))
|
||||
:user.class/Person)
|
||||
"Person class is created and correctly associated to a page")
|
||||
|
||||
(is (= [:logseq.class/Task]
|
||||
(mapv :db/ident (:block/tags (db-test/find-block-by-content @conn "some task"))))
|
||||
"Built-in class is associatedly correctly")))
|
||||
|
||||
(deftest build-properties-user
|
||||
(let [conn (db-test/create-conn)
|
||||
_ (sqlite-build/create-blocks
|
||||
conn
|
||||
[{:page {:block/title "page1"}
|
||||
:blocks [{:block/title "Jrue Holiday" :build/properties {:description "Clutch defense"}}]}
|
||||
{:page {:block/title "Jayson Tatum" :build/properties {:description "Awesome selfless basketball"}}}])]
|
||||
(is (= "Clutch defense"
|
||||
(->> (db-test/find-block-by-content @conn "Jrue Holiday")
|
||||
:user.property/description
|
||||
db-property/property-value-content))
|
||||
"description property is created and correctly associated to a block")
|
||||
|
||||
(is (= "Awesome selfless basketball"
|
||||
(->> (db-test/find-page-by-title @conn "Jayson Tatum")
|
||||
:user.property/description
|
||||
db-property/property-value-content))
|
||||
"description property is created and correctly associated to a page")))
|
||||
|
||||
(deftest build-properties-built-in
|
||||
(let [conn (db-test/create-conn)
|
||||
_ (sqlite-build/create-blocks
|
||||
conn
|
||||
[{:page {:block/title "page1"}
|
||||
:blocks [{:block/title "some todo"
|
||||
:build/properties {:logseq.property/status :logseq.property/status.doing}}
|
||||
{:block/title "rojo"
|
||||
:build/properties {:logseq.property/background-color "red"}}]}])]
|
||||
(is (= :logseq.property/status.doing
|
||||
(->> (db-test/find-block-by-content @conn "some todo")
|
||||
:logseq.property/status
|
||||
:db/ident))
|
||||
"built-in property with closed value is created and correctly associated to a block")
|
||||
|
||||
(is (= "red"
|
||||
(->> (db-test/find-block-by-content @conn "rojo")
|
||||
:logseq.property/background-color
|
||||
db-property/property-value-content))
|
||||
"built-in :default property is created and correctly associated to a block")))
|
||||
|
||||
(deftest build-properties-with-build-page
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
[{:page {:block/title "page1"
|
||||
:build/properties
|
||||
{:date [:build/page {:build/journal 20250223}]
|
||||
:page #{[:build/page {:block/title "page object"
|
||||
:build/properties {:p1 "foo"
|
||||
:date [:build/page {:build/journal 20250224}]}}]}}}}])]
|
||||
(is (= "foo"
|
||||
(->> (db-test/find-page-by-title @conn "page object")
|
||||
:user.property/p1
|
||||
db-property/property-value-content))
|
||||
":build/page page can have a :default property")
|
||||
(is (= 20250224
|
||||
(->> (db-test/find-page-by-title @conn "page object")
|
||||
:user.property/date
|
||||
:block/journal-day))
|
||||
":build/page page can have a :date property defined by another :build/page")))
|
||||
|
||||
(deftest build-for-existing-blocks
|
||||
(let [conn (db-test/create-conn)
|
||||
_ (sqlite-build/create-blocks
|
||||
conn
|
||||
{:properties {:p1 {}}
|
||||
:classes {:MyClass {}}
|
||||
:pages-and-blocks
|
||||
[{:page {:block/title "page1"}
|
||||
:blocks [{:block/title "block 1"}
|
||||
{:block/title "block 2"}]}]})
|
||||
block (db-test/find-block-by-content @conn "block 1")
|
||||
block2 (db-test/find-block-by-content @conn "block 2")
|
||||
page1 (db-test/find-page-by-title @conn "page1")
|
||||
built-in-page (db-test/find-page-by-title @conn "Quick add")
|
||||
{:keys [init-tx block-props-tx]}
|
||||
(sqlite-build/build-blocks-tx
|
||||
{:pages-and-blocks [{:page
|
||||
{:block/uuid (:block/uuid built-in-page)
|
||||
:build/keep-uuid? true
|
||||
:build/properties {:logseq.property/description "foo"}
|
||||
:block/title "Quick add"}
|
||||
:blocks []}
|
||||
{:page (select-keys (:block/page block) [:block/uuid])
|
||||
:blocks [(merge {:block/title "imported task" :block/uuid (:block/uuid block)}
|
||||
{:build/properties {:logseq.property/status :logseq.property/status.todo}
|
||||
:build/tags [:logseq.class/Task]})]}]
|
||||
:build-existing-tx? true})
|
||||
_ (d/transact! conn init-tx)
|
||||
_ (d/transact! conn block-props-tx)
|
||||
updated-block (d/entity @conn [:block/uuid (:block/uuid block)])
|
||||
{init-tx2 :init-tx block-props-tx2 :block-props-tx :as _tx}
|
||||
(sqlite-build/build-blocks-tx
|
||||
{:pages-and-blocks [{:page (select-keys (:block/page block2) [:block/uuid])
|
||||
:blocks [(merge {:block/title "imported block" :block/uuid (:block/uuid block2)}
|
||||
{:build/properties {:user.property/p1 "foo"}
|
||||
:build/tags [:user.class/MyClass]})]}]
|
||||
:properties {:user.property/p1 (select-keys (d/entity @conn :user.property/p1)
|
||||
[:logseq.property/type :block/uuid])}
|
||||
:build-existing-tx? true})
|
||||
_ (d/transact! conn init-tx2)
|
||||
_ (d/transact! conn block-props-tx2)
|
||||
updated-block2 (d/entity @conn [:block/uuid (:block/uuid block2)])]
|
||||
;; (cljs.pprint/pprint _tx)
|
||||
(testing "existing page cases"
|
||||
(is (= (:block/updated-at page1)
|
||||
(:block/updated-at (db-test/find-page-by-title @conn "page1")))
|
||||
"Existing page with no property changes didn't get updated")
|
||||
(is (not= (:block/updated-at built-in-page)
|
||||
(:block/updated-at (db-test/find-page-by-title @conn "Quick add")))
|
||||
"Existing page with property changes does get updated"))
|
||||
|
||||
(testing "block with built-in properties and tags"
|
||||
(is (= []
|
||||
(filter #(and (not= (:block/uuid %) (:block/uuid built-in-page))
|
||||
(or (:db/id %) (:db/ident %)))
|
||||
(concat init-tx block-props-tx)))
|
||||
"Tx doesn't try to create new blocks or modify existing idents")
|
||||
(is (= "imported task" (:block/title updated-block)))
|
||||
(is (= {:block/tags [:logseq.class/Task]
|
||||
:logseq.property/status :logseq.property/status.todo}
|
||||
(db-test/readable-properties updated-block))
|
||||
"Block's properties and tags are updated"))
|
||||
|
||||
(testing "block with existing user properties and tags"
|
||||
(is (= "imported block" (:block/title updated-block2)))
|
||||
(is (= {:block/tags [:user.class/MyClass]
|
||||
:user.property/p1 "foo"}
|
||||
(db-test/readable-properties updated-block2))
|
||||
"Block's properties and tags are updated"))))
|
||||
|
||||
(deftest build-blocks-with-refs
|
||||
(let [block-uuid (random-uuid)
|
||||
class-uuid (random-uuid)
|
||||
page-uuid (random-uuid)
|
||||
property-uuid (random-uuid)
|
||||
conn (db-test/create-conn-with-blocks
|
||||
{:classes {:C1 {:block/uuid class-uuid :build/keep-uuid? true}}
|
||||
:properties {:p1 {:block/uuid property-uuid :build/keep-uuid? true}}
|
||||
:build-existing-tx? true
|
||||
:pages-and-blocks
|
||||
[{:page {:block/title "page 1"}
|
||||
:blocks [{:block/title "named page ref to [[named page]]"}
|
||||
{:block/title (str "page ref to " (page-ref/->page-ref page-uuid))}
|
||||
{:block/title (str "block ref to " (page-ref/->page-ref block-uuid))}
|
||||
{:block/title (str "class ref to " (page-ref/->page-ref class-uuid))}
|
||||
{:block/title (str "inline class ref to #" (page-ref/->page-ref class-uuid))}
|
||||
{:block/title (str "property ref to " (page-ref/->page-ref property-uuid))}
|
||||
{:block/title "hi" :block/uuid block-uuid :build/keep-uuid? true}]}
|
||||
{:page {:block/title "another page" :block/uuid page-uuid :build/keep-uuid? true}}]})
|
||||
block-with-named-page-ref (db-test/find-block-by-content @conn #"^named page ref")
|
||||
block-with-page-ref (db-test/find-block-by-content @conn #"^page ref")
|
||||
block-with-class-ref (db-test/find-block-by-content @conn #"^class ref")
|
||||
block-with-inline-class-ref (db-test/find-block-by-content @conn #"^inline class ref")
|
||||
block-with-property-ref (db-test/find-block-by-content @conn #"^property ref")
|
||||
block-with-block-ref (db-test/find-block-by-content @conn #"^block ref")]
|
||||
(is (ldb/internal-page? (db-test/find-page-by-title @conn "named page")))
|
||||
(is (contains? (:block/refs block-with-named-page-ref) (db-test/find-page-by-title @conn "named page")))
|
||||
|
||||
(is (= page-uuid (:block/uuid (db-test/find-page-by-title @conn "another page"))))
|
||||
(is (contains? (:block/refs block-with-page-ref) (db-test/find-page-by-title @conn "another page")))
|
||||
|
||||
(is (= class-uuid (:block/uuid (d/entity @conn :user.class/C1))))
|
||||
(is (contains? (:block/refs block-with-class-ref) (d/entity @conn :user.class/C1)))
|
||||
(is (contains? (:block/refs block-with-inline-class-ref) (d/entity @conn :user.class/C1)))
|
||||
|
||||
(is (= property-uuid (:block/uuid (d/entity @conn :user.property/p1))))
|
||||
(is (contains? (:block/refs block-with-property-ref) (d/entity @conn :user.property/p1)))
|
||||
|
||||
(is (= block-uuid (:block/uuid (db-test/find-block-by-content @conn "hi"))))
|
||||
(is (contains? (:block/refs block-with-block-ref) (db-test/find-block-by-content @conn "hi")))))
|
||||
|
||||
(deftest build-class-and-property-pages
|
||||
(let [class-uuid (random-uuid)
|
||||
property-uuid (random-uuid)
|
||||
conn (db-test/create-conn-with-blocks
|
||||
{:classes {:C1 {:block/uuid class-uuid :build/keep-uuid? true}}
|
||||
:properties {:p1 {:block/uuid property-uuid :build/keep-uuid? true}}
|
||||
:pages-and-blocks
|
||||
[{:page {:block/uuid class-uuid}
|
||||
:blocks [{:block/title "b1"
|
||||
:build/children [{:block/title "b2"}]}]}
|
||||
{:page {:block/uuid property-uuid}
|
||||
:blocks [{:block/title "b3"
|
||||
:build/children [{:block/title "b4"}]}]}]
|
||||
:build-existing-tx? true})]
|
||||
(is (= ["b1" "b2"]
|
||||
(->> (d/q '[:find [?b ...] :in $ ?page-id :where [?b :block/page ?page-id]]
|
||||
@conn [:block/uuid class-uuid])
|
||||
(map #(:block/title (d/entity @conn %)))))
|
||||
"Class page has correct blocks")
|
||||
|
||||
(is (= ["b3" "b4"]
|
||||
(->> (d/q '[:find [?b ...] :in $ ?page-id :where [?b :block/page ?page-id]]
|
||||
@conn [:block/uuid property-uuid])
|
||||
(map #(:block/title (d/entity @conn %)))))
|
||||
"Property page has correct blocks")))
|
||||
|
||||
(deftest build-class-extends-rejects-cycles
|
||||
(testing "self cycle"
|
||||
(is (thrown-with-msg?
|
||||
js/Error
|
||||
#"Cycle detected in :build/class-extends"
|
||||
(sqlite-build/build-blocks-tx
|
||||
{:classes {:user.class/A {:build/class-extends [:user.class/A]}}}))))
|
||||
(testing "deprecated class parent self cycle"
|
||||
(is (thrown-with-msg?
|
||||
js/Error
|
||||
#"Cycle detected in :build/class-extends"
|
||||
(sqlite-build/build-blocks-tx
|
||||
{:classes {:user.class/A {:build/class-parent :user.class/A}}}))))
|
||||
(testing "deprecated class parent takes precedence"
|
||||
(let [txs (sqlite-build/build-blocks-tx
|
||||
{:classes {:user.class/A {:build/class-parent :user.class/B
|
||||
:build/class-extends [:user.class/A]}
|
||||
:user.class/B {}}})
|
||||
class-a (some #(when (= :user.class/A (:db/ident %)) %) (:init-tx txs))
|
||||
class-b (some #(when (= :user.class/B (:db/ident %)) %) (:init-tx txs))]
|
||||
(is (= [(:db/id class-b)]
|
||||
(:logseq.property.class/extends class-a)))))
|
||||
(testing "multi-class cycle"
|
||||
(is (thrown-with-msg?
|
||||
js/Error
|
||||
#"Cycle detected in :build/class-extends"
|
||||
(sqlite-build/build-blocks-tx
|
||||
{:classes {:user.class/A {:build/class-extends [:user.class/B]}
|
||||
:user.class/B {:build/class-extends [:user.class/C]}
|
||||
:user.class/C {:build/class-extends [:user.class/A]}}}))))
|
||||
(testing "diamond inheritance without a cycle"
|
||||
(is (map?
|
||||
(sqlite-build/build-blocks-tx
|
||||
{:classes {:user.class/A {}
|
||||
:user.class/B {:build/class-extends [:user.class/A]}
|
||||
:user.class/C {:build/class-extends [:user.class/A]}
|
||||
:user.class/D {:build/class-extends [:user.class/B :user.class/C]}}})))))
|
||||
|
||||
(deftest property-value-with-properties-and-tags
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:properties {:p1 {:logseq.property/type :default}}
|
||||
:classes {:C1 {}}
|
||||
:pages-and-blocks
|
||||
[{:page {:block/title "page1"}
|
||||
:blocks [{:block/title "block has pvalue with built-in tag"
|
||||
:build/properties
|
||||
{:p1 {:build/property-value :block
|
||||
:block/title "t1"
|
||||
:build/tags [:logseq.class/Task]}}}
|
||||
{:block/title "block has pvalue with user tag"
|
||||
:build/properties
|
||||
{:p1 {:build/property-value :block
|
||||
:block/title "u1"
|
||||
:build/tags [:C1]}}}
|
||||
{:block/title "Todo query",
|
||||
:build/tags [:logseq.class/Query],
|
||||
:build/properties
|
||||
{:logseq.property/query
|
||||
{:build/property-value :block
|
||||
:block/title "{:query (task Todo)}"
|
||||
:build/properties
|
||||
{:logseq.property.code/lang "clojure"
|
||||
:logseq.property.node/display-type :code}}}}]}]})]
|
||||
(is (= {:logseq.property.node/display-type :code
|
||||
:logseq.property.code/lang "clojure"}
|
||||
(-> (db-test/find-block-by-content @conn "{:query (task Todo)}")
|
||||
db-test/readable-properties
|
||||
(dissoc :logseq.property/created-from-property))))
|
||||
(is (= {:block/tags [:logseq.class/Task]}
|
||||
(-> (db-test/find-block-by-content @conn "t1")
|
||||
db-test/readable-properties
|
||||
(dissoc :logseq.property/created-from-property))))
|
||||
(is (= {:block/tags [:user.class/C1]}
|
||||
(-> (db-test/find-block-by-content @conn "u1")
|
||||
db-test/readable-properties
|
||||
(dissoc :logseq.property/created-from-property))))))
|
||||
|
||||
(deftest build-ontology-with-multiple-namespaces
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:properties {:user.property/p1 {:logseq.property/type :default}
|
||||
:other.property/p1 {:logseq.property/type :default}}
|
||||
:classes {:user.class/C1 {}
|
||||
:other.class/C1 {}}})]
|
||||
(is (entity-util/property? (d/entity @conn :user.property/p1)))
|
||||
(is (entity-util/property? (d/entity @conn :other.property/p1)))
|
||||
(is (entity-util/class? (d/entity @conn :user.class/C1)))
|
||||
(is (entity-util/class? (d/entity @conn :other.class/C1)))))
|
||||
|
||||
(deftest build-preserves-class-property-ordering-for-export
|
||||
(let [class-properties-c1 [:user.property/p2 :user.property/p1 :user.property/p3]
|
||||
class-properties-c2 [:user.property/p4 :user.property/p2 :user.property/p3]
|
||||
another-class-properties-c1 [:user.property/p5]
|
||||
another-class-properties-c2 [:user.property/p6]
|
||||
another-class-properties-c3 [:user.property/p6 :user.property/p5]
|
||||
conn (db-test/create-conn-with-blocks
|
||||
{:properties {:user.property/p1 {:logseq.property/type :default}
|
||||
:user.property/p2 {:logseq.property/type :default}
|
||||
:user.property/p3 {:logseq.property/type :default}
|
||||
:user.property/p4 {:logseq.property/type :default}
|
||||
:user.property/p5 {:logseq.property/type :default}
|
||||
:user.property/p6 {:logseq.property/type :default}}
|
||||
:classes {:user.class/C1 {:build/class-properties class-properties-c1}
|
||||
:user.class/C2 {:build/class-properties class-properties-c2}
|
||||
:user.class/AnotherC1 {:build/class-properties another-class-properties-c1}
|
||||
:user.class/AnotherC2 {:build/class-properties another-class-properties-c2}
|
||||
:user.class/AnotherC3 {:build/class-properties another-class-properties-c3}}})
|
||||
export-map (sqlite-export/build-export @conn {:export-type :graph-ontology})]
|
||||
(is (= class-properties-c1
|
||||
(get-in export-map [:classes :user.class/C1 :build/class-properties])))
|
||||
(is (= class-properties-c2
|
||||
(get-in export-map [:classes :user.class/C2 :build/class-properties])))
|
||||
(is (= another-class-properties-c3
|
||||
(get-in export-map [:classes :user.class/AnotherC3 :build/class-properties]))
|
||||
"Later class-level ordering constraint :p6 before :p5 is preserved")))
|
||||
173
deps/db/test/logseq/db/sqlite/create_graph_test.cljs
vendored
Normal file
173
deps/db/test/logseq/db/sqlite/create_graph_test.cljs
vendored
Normal file
@@ -0,0 +1,173 @@
|
||||
(ns logseq.db.sqlite.create-graph-test
|
||||
(:require [cljs.test :refer [deftest is testing]]
|
||||
[clojure.data :as data]
|
||||
[clojure.set :as set]
|
||||
[clojure.string :as string]
|
||||
[datascript.core :as d]
|
||||
[logseq.db :as ldb]
|
||||
[logseq.db.frontend.class :as db-class]
|
||||
[logseq.db.frontend.property :as db-property]
|
||||
[logseq.db.frontend.schema :as db-schema]
|
||||
[logseq.db.frontend.validate :as db-validate]
|
||||
[logseq.db.sqlite.build :as sqlite-build]
|
||||
[logseq.db.sqlite.create-graph :as sqlite-create-graph]
|
||||
[logseq.db.test.helper :as db-test]))
|
||||
|
||||
(deftest new-graph-db-idents
|
||||
(testing "a new graph follows :db/ident conventions for"
|
||||
(let [conn (db-test/create-conn)
|
||||
ident-ents (->> (d/q '[:find [?b ...]
|
||||
:where [?b :db/ident]]
|
||||
@conn)
|
||||
(map (fn [id] (d/entity @conn id))))
|
||||
default-idents (map :db/ident ident-ents)]
|
||||
(is (> (count default-idents) 45)
|
||||
"Approximate number of default idents is correct")
|
||||
|
||||
(testing "namespaces"
|
||||
(is (= '() (remove namespace default-idents))
|
||||
"All default :db/ident's have namespaces")
|
||||
(is (= []
|
||||
(->> (remove db-property/db-attribute-properties default-idents)
|
||||
(keep namespace)
|
||||
(remove #(string/starts-with? % "logseq"))))
|
||||
"All default :db/ident namespaces start with logseq."))
|
||||
|
||||
(testing "closed values"
|
||||
(let [closed-value-ents (filter #(string/includes? (name (:db/ident %)) ".") ident-ents)
|
||||
closed-value-properties (->> closed-value-ents
|
||||
(map :db/ident)
|
||||
(map #(keyword (namespace %) (string/replace (name %) #".[^.]+$" "")))
|
||||
set)]
|
||||
(is (= []
|
||||
(remove ldb/closed-value? closed-value-ents))
|
||||
"All property names that contain a '.' are closed values")
|
||||
(is (= #{}
|
||||
(set/difference
|
||||
(set (remove #{:logseq.property/color} closed-value-properties))
|
||||
(set default-idents)))
|
||||
"All closed values start with a prefix that is a property name"))))))
|
||||
|
||||
(deftest new-graph-marks-built-ins
|
||||
(let [conn (db-test/create-conn)
|
||||
idents (->> (d/q '[:find [(pull ?b [:db/ident :logseq.property/built-in?]) ...]
|
||||
:where [?b :db/ident]]
|
||||
@conn)
|
||||
;; only kv's and empty property value aren't marked because
|
||||
;; they aren't user facing
|
||||
(remove #(or (= "logseq.kv" (namespace (:db/ident %)))
|
||||
(= :logseq.property/empty-placeholder (:db/ident %)))))
|
||||
pages (d/q '[:find [(pull ?b [:logseq.property/built-in? :block/title]) ...]
|
||||
:where [?b :block/tags :logseq.class/Page]]
|
||||
@conn)]
|
||||
(is (= [] (remove :logseq.property/built-in? idents))
|
||||
"All entities with :db/ident have built-in property (except for kv idents)")
|
||||
(is (= [] (remove :logseq.property/built-in? pages))
|
||||
"All default internal pages should have built-in property")))
|
||||
|
||||
(deftest new-graph-creates-class
|
||||
(let [conn (db-test/create-conn)
|
||||
task (d/entity @conn :logseq.class/Task)]
|
||||
(is (ldb/class? task)
|
||||
"Task class has correct type")
|
||||
(is (= 4 (count (:logseq.property.class/properties task)))
|
||||
"Has correct number of task properties")
|
||||
(is (every? ldb/property? (:logseq.property.class/properties task))
|
||||
"Each task property has correct type")))
|
||||
|
||||
(deftest new-graph-initializes-default-classes-correctly
|
||||
(let [conn (db-test/create-conn)]
|
||||
(is (= (set (keys db-class/built-in-classes))
|
||||
(set (map #(:db/ident (d/entity @conn (:e %))) (d/datoms @conn :avet :block/tags :logseq.class/Tag))))
|
||||
"All built-in classes are indexed by :block/tags with :logseq.class/Tag")
|
||||
|
||||
(is (= (count (dissoc db-class/built-in-classes :logseq.class/Root))
|
||||
(count (->> (d/datoms @conn :avet :block/tags :logseq.class/Tag)
|
||||
(map #(d/entity @conn (:e %)))
|
||||
(mapcat :logseq.property.class/_extends)
|
||||
set)))
|
||||
"Reverse lookup of :logseq.property.class/extends correctly fetches number of child classes")))
|
||||
|
||||
(deftest new-graph-initializes-default-properties-correctly
|
||||
(let [conn (db-test/create-conn)]
|
||||
(is (= (set (keys db-property/built-in-properties))
|
||||
(set (map #(:db/ident (d/entity @conn (:e %))) (d/datoms @conn :avet :block/tags :logseq.class/Property))))
|
||||
"All built-in properties are indexed by :block/tags with :logseq.class/Property")
|
||||
(is (= (set (keys db-property/built-in-properties))
|
||||
(set (map #(:db/ident (d/entity @conn (:e %))) (d/datoms @conn :avet :logseq.property/type))))
|
||||
"All built-in properties have and are indexed by :logseq.property/type")
|
||||
(is (= #{}
|
||||
(set/difference
|
||||
(set (keys db-property/built-in-properties))
|
||||
(set (map #(:db/ident (d/entity @conn (:e %))) (d/datoms @conn :avet :logseq.property/built-in?)))))
|
||||
"All built-in properties have and are indexed by :logseq.property/built-in?")
|
||||
|
||||
;; testing :properties config
|
||||
(testing "A built-in property that has"
|
||||
(is (= :logseq.property/status.todo
|
||||
(-> (d/entity @conn :logseq.property/status)
|
||||
:logseq.property/default-value
|
||||
:db/ident))
|
||||
"A property with a :db/ident property value is created correctly")
|
||||
(is (-> (d/entity @conn :logseq.property/deadline)
|
||||
:logseq.property/description
|
||||
db-property/property-value-content
|
||||
str
|
||||
(string/includes? "finish something"))
|
||||
"A :default property is created correctly")
|
||||
(is (= true
|
||||
(-> (d/entity @conn :logseq.property/status)
|
||||
:logseq.property/enable-history?))
|
||||
"A :checkbox property is created correctly")
|
||||
(is (= 1
|
||||
(-> (d/entity @conn :logseq.property.repeat/recur-frequency)
|
||||
:logseq.property/default-value
|
||||
db-property/property-value-content))
|
||||
"A numeric property is created correctly"))))
|
||||
|
||||
(deftest new-graph-is-valid
|
||||
(let [conn (db-test/create-conn)
|
||||
validation (db-validate/validate-local-db! @conn)]
|
||||
;; For debugging
|
||||
;; (println (count (:errors validation)) "errors of" (count (:entities validation)))
|
||||
;; (cljs.pprint/pprint (:errors validation))
|
||||
(is (empty? (map :entity (:errors validation)))
|
||||
"New graph has no validation errors")))
|
||||
|
||||
(deftest property-types
|
||||
(let [conn (d/create-conn db-schema/schema)
|
||||
_ (d/transact! conn (sqlite-create-graph/build-db-initial-data
|
||||
(pr-str {:macros {"docs-base-url" "https://docs.logseq.com/#/page/$1"}})))]
|
||||
|
||||
(testing ":url property"
|
||||
(sqlite-build/create-blocks
|
||||
conn
|
||||
{:properties {:url {:logseq.property/type :url}}
|
||||
:pages-and-blocks
|
||||
[{:page {:block/title "page1"}
|
||||
:blocks [{:block/title "b1" :build/properties {:url "https://logseq.com"}}
|
||||
;; :url macros are used for consistently building urls with the same hostname e.g. docs graph
|
||||
{:block/title "b2" :build/properties {:url "{{docs-base-url test}}"}}]}]})
|
||||
|
||||
(is (empty? (map :entity (:errors (db-validate/validate-local-db! @conn))))
|
||||
"Graph with different :url blocks has no validation errors"))))
|
||||
|
||||
(deftest build-db-initial-data-test
|
||||
(testing "idempotent initial-data"
|
||||
(letfn [(remove-ignored-attrs&entities [init-data]
|
||||
(let [ignored-idents #{:logseq.kv/graph-created-at :logseq.kv/graph-uuid :logseq.kv/local-graph-uuid}
|
||||
init-data* (remove (fn [ent]
|
||||
(contains? ignored-idents (:db/ident ent)))
|
||||
init-data)]
|
||||
(map (fn [ent] (dissoc ent
|
||||
:block/created-at :block/updated-at
|
||||
:file/last-modified-at :file/created-at
|
||||
:block/order ;; TODO: block/order should be same as well
|
||||
))
|
||||
init-data*)))]
|
||||
(let [[first-only second-only _common]
|
||||
(data/diff (remove-ignored-attrs&entities (sqlite-create-graph/build-db-initial-data ""))
|
||||
(remove-ignored-attrs&entities (sqlite-create-graph/build-db-initial-data "")))]
|
||||
;; (pr-str [first-only second-only _common])
|
||||
(is (and (every? nil? first-only)
|
||||
(every? nil? second-only)))))))
|
||||
1469
deps/db/test/logseq/db/sqlite/export_test.cljs
vendored
Normal file
1469
deps/db/test/logseq/db/sqlite/export_test.cljs
vendored
Normal file
File diff suppressed because it is too large
Load Diff
49
deps/db/test/logseq/db/sqlite/gc_test.cljs
vendored
Normal file
49
deps/db/test/logseq/db/sqlite/gc_test.cljs
vendored
Normal file
@@ -0,0 +1,49 @@
|
||||
(ns logseq.db.sqlite.gc-test
|
||||
(:require ["fs" :as fs]
|
||||
["path" :as node-path]
|
||||
[cljs.test :refer [deftest async use-fixtures is testing]]
|
||||
[datascript.core :as d]
|
||||
[logseq.db.common.sqlite-cli :as sqlite-cli]
|
||||
[logseq.db.sqlite.debug :as sqlite-debug]
|
||||
[logseq.db.sqlite.gc :as sqlite-gc]))
|
||||
|
||||
(use-fixtures
|
||||
:each
|
||||
;; Cleaning tmp/ before leaves last tmp/ after a test run for dev and debugging
|
||||
{:before
|
||||
#(async done
|
||||
(if (fs/existsSync "tmp")
|
||||
(fs/rm "tmp" #js {:recursive true} (fn [err]
|
||||
(when err (js/console.log err))
|
||||
(done)))
|
||||
(done)))})
|
||||
|
||||
(defn- create-graph-dir
|
||||
[dir db-name]
|
||||
(fs/mkdirSync (node-path/join dir db-name) #js {:recursive true}))
|
||||
|
||||
(deftest ^:long gc-kvs-table-test
|
||||
(testing "Create a datascript db, gc it and ensure there's no missing addrs and garbage addrs"
|
||||
(create-graph-dir "tmp/graphs" "test-db")
|
||||
|
||||
(let [{:keys [conn sqlite]} (sqlite-cli/open-sqlite-datascript! "tmp/graphs" "test-db")
|
||||
tx-data (map (fn [i] {:block/uuid (random-uuid)
|
||||
:block/title (str "title " i)})
|
||||
(range 0 500000))]
|
||||
(println "DB start transacting")
|
||||
(d/transact! conn tx-data)
|
||||
(println "DB transacted")
|
||||
(let [non-ordered-tx (->> (shuffle tx-data)
|
||||
(take 100000)
|
||||
(map (fn [block] [:db/retractEntity [:block/uuid (:block/uuid block)]])))]
|
||||
(d/transact! conn non-ordered-tx))
|
||||
(println "gc time")
|
||||
;; `true` to walk addresses and `false` to recursively run gc
|
||||
(time (sqlite-gc/gc-kvs-table-node-version! sqlite false))
|
||||
|
||||
;; ensure there's no missing address (broken db)
|
||||
(is (empty? (sqlite-debug/find-missing-addresses-node-version sqlite))
|
||||
"Found missing addresses!")
|
||||
|
||||
(is (true? (sqlite-gc/ensure-no-garbage sqlite))
|
||||
"Found garbage addresses!"))))
|
||||
6
deps/db/test/logseq/db/test_runner.cljs
vendored
Normal file
6
deps/db/test/logseq/db/test_runner.cljs
vendored
Normal file
@@ -0,0 +1,6 @@
|
||||
(ns logseq.db.test-runner
|
||||
"Test runner which enables https://github.com/PEZ/baldr by default"
|
||||
(:require [nextjournal.test-runner :as next-runner]
|
||||
[pez.baldr]))
|
||||
|
||||
(def -main next-runner/-main)
|
||||
376
deps/db/test/logseq/db_test.cljs
vendored
Normal file
376
deps/db/test/logseq/db_test.cljs
vendored
Normal file
@@ -0,0 +1,376 @@
|
||||
(ns logseq.db-test
|
||||
(:require [cljs.test :refer [deftest is testing]]
|
||||
[datascript.core :as d]
|
||||
[datascript.storage :as ds-storage :refer [IStorage]]
|
||||
[logseq.db :as ldb]
|
||||
[logseq.db.frontend.validate :as db-validate]
|
||||
[logseq.db.test.helper :as db-test]))
|
||||
|
||||
(defrecord InMemoryStorage [*disk]
|
||||
IStorage
|
||||
(-store [_ addr+data-seq _delete-addrs]
|
||||
(doseq [[addr data] addr+data-seq]
|
||||
(vswap! *disk assoc addr data)))
|
||||
(-restore [_ addr]
|
||||
(get @*disk addr)))
|
||||
|
||||
(defn- make-storage []
|
||||
(->InMemoryStorage (volatile! {})))
|
||||
|
||||
;;; datoms
|
||||
;;; - 1 <----+
|
||||
;;; - 2 |
|
||||
;;; - 3 -+
|
||||
(comment
|
||||
;; TODO: throw error or fix broken data when cycle detected
|
||||
(def broken-outliner-data-with-cycle
|
||||
[{:db/id 1
|
||||
:block/uuid #uuid"e538d319-48d4-4a6d-ae70-c03bb55b6fe4"
|
||||
:block/parent 3}
|
||||
{:db/id 2
|
||||
:block/uuid #uuid"c46664c0-ea45-4998-adf0-4c36486bb2e5"
|
||||
:block/parent 1}
|
||||
{:db/id 3
|
||||
:block/uuid #uuid"2b736ac4-fd49-4e04-b00f-48997d2c61a2"
|
||||
:block/parent 2}]))
|
||||
|
||||
(def class-parents-data
|
||||
[{:block/tags :logseq.class/Tag
|
||||
:block/title "x"
|
||||
:block/name "x"
|
||||
:block/uuid #uuid "6c353967-f79b-4785-b804-a39b81d72461"}
|
||||
{:block/tags :logseq.class/Tag
|
||||
:block/title "y"
|
||||
:block/name "y"
|
||||
:block/uuid #uuid "7008db08-ba0c-4aa9-afc6-7e4783e40a99"
|
||||
:logseq.property.class/extends [:block/uuid #uuid "6c353967-f79b-4785-b804-a39b81d72461"]}
|
||||
{:block/tags :logseq.class/Tag
|
||||
:block/title "z"
|
||||
:block/name "z"
|
||||
:block/uuid #uuid "d95f2912-a7af-41b9-8ed5-28861f7fc0be"
|
||||
:logseq.property.class/extends [:block/uuid #uuid "7008db08-ba0c-4aa9-afc6-7e4783e40a99"]}])
|
||||
|
||||
(deftest get-class-extends
|
||||
(let [conn (db-test/create-conn)]
|
||||
(d/transact! conn class-parents-data)
|
||||
(is (= #{"x" "y"}
|
||||
(->> (ldb/get-class-extends (ldb/get-page @conn "z"))
|
||||
(map :block/title)
|
||||
set)))))
|
||||
|
||||
(deftest get-case-page
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:properties
|
||||
{:foo {:logseq.property/type :default}
|
||||
:Foo {:logseq.property/type :default}}
|
||||
:classes {:movie {} :Movie {}}})]
|
||||
;; Case sensitive properties
|
||||
(is (= "foo" (:block/title (ldb/get-case-page @conn "foo"))))
|
||||
(is (= "Foo" (:block/title (ldb/get-case-page @conn "Foo"))))
|
||||
;; Case sensitive classes
|
||||
(is (= "movie" (:block/title (ldb/get-case-page @conn "movie"))))
|
||||
(is (= "Movie" (:block/title (ldb/get-case-page @conn "Movie"))))))
|
||||
|
||||
(deftest get-journal-page-by-day
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:pages-and-blocks
|
||||
[{:page {:build/journal 20260410}}
|
||||
{:page {:build/journal 20260411}}]})]
|
||||
(is (= "Apr 10th, 2026"
|
||||
(:block/title (ldb/get-journal-page-by-day @conn 20260410))))
|
||||
(is (= "Apr 11th, 2026"
|
||||
(:block/title (ldb/get-journal-page-by-day @conn 20260411))))))
|
||||
|
||||
(deftest page-exists
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:properties
|
||||
{:foo {:logseq.property/type :default}
|
||||
:Foo {:logseq.property/type :default}}
|
||||
:classes {:movie {} :Movie {}}})]
|
||||
(is (= ["foo"]
|
||||
(map #(:block/title (d/entity @conn %)) (ldb/page-exists? @conn "foo" #{:logseq.class/Property})))
|
||||
"Property pages correctly found for given class")
|
||||
(is (= nil
|
||||
(ldb/page-exists? @conn "foo" #{:logseq.class/Tag}))
|
||||
"Property pages correctly not found for given class")
|
||||
(is (= ["movie"]
|
||||
(map #(:block/title (d/entity @conn %)) (ldb/page-exists? @conn "movie" #{:logseq.class/Tag})))
|
||||
"Class pages correctly found for given class")
|
||||
(is (= nil
|
||||
(ldb/page-exists? @conn "movie" #{:logseq.class/Property}))
|
||||
"Class pages correctly not found for given class")))
|
||||
|
||||
(deftest test-transact-with-multiple-tx-datoms
|
||||
(testing "last write wins with same tx"
|
||||
(let [conn (d/create-conn)]
|
||||
(d/transact! conn [[:db/add -1 :property :v1]])
|
||||
(let [tx (:max-tx @conn)]
|
||||
(ldb/transact! conn
|
||||
[(d/datom 1 :property :v1 (inc tx) false)
|
||||
(d/datom 1 :property :v1 (inc tx) true)]))
|
||||
(is (= :v1 (:property (d/entity @conn 1))))))
|
||||
(testing "last write wins with different tx"
|
||||
(let [conn (d/create-conn)]
|
||||
(d/transact! conn [[:db/add -1 :property :v1]])
|
||||
(let [tx (:max-tx @conn)]
|
||||
(ldb/transact! conn
|
||||
[(d/datom 1 :property :v1 (inc tx) false)
|
||||
(d/datom 1 :property :v1 (+ tx 2) true)]))
|
||||
(is (= :v1 (:property (d/entity @conn 1)))))))
|
||||
|
||||
(deftest test-batch-transact!
|
||||
(testing "DB validation should be running after the whole transaction"
|
||||
(let [conn (db-test/create-conn)]
|
||||
(testing "#Task shouldn't be converted to property"
|
||||
(is (thrown? js/Error
|
||||
(with-out-str
|
||||
(db-test/silence-stderr
|
||||
(ldb/transact! conn [{:db/ident :logseq.class/Task
|
||||
:block/tags :logseq.class/Property}]))))))
|
||||
(ldb/batch-transact-with-temp-conn!
|
||||
conn
|
||||
{}
|
||||
(fn [temp-conn]
|
||||
(ldb/transact! temp-conn [{:db/ident :logseq.class/Task
|
||||
:block/tags :logseq.class/Property}])
|
||||
(ldb/transact! temp-conn [[:db/retract :logseq.class/Task :block/tags :logseq.class/Property]]))))))
|
||||
|
||||
(deftest batch-transact-with-temp-conn-preserves-retracts-test
|
||||
(testing "temp conn batch replays retractions instead of treating every datom as an add"
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:pages-and-blocks [{:page {:block/title "page 1"}
|
||||
:blocks [{:block/title "old"}]}]})
|
||||
block (db-test/find-block-by-content @conn "old")
|
||||
block-id (:db/id block)]
|
||||
(ldb/batch-transact-with-temp-conn!
|
||||
conn
|
||||
{}
|
||||
(fn [temp-conn]
|
||||
(ldb/transact! temp-conn [[:db/retract [:block/uuid (:block/uuid block)] :block/title "old"]])
|
||||
(ldb/transact! temp-conn [[:db/add [:block/uuid (:block/uuid block)] :block/title "new"]])))
|
||||
(is (= "new" (:block/title (d/entity @conn block-id)))))))
|
||||
|
||||
(deftest batch-transact-with-temp-conn-preserves-cardinality-one-schema-test
|
||||
(testing "temp conn emits replacement retractions for cardinality-one attrs"
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:pages-and-blocks [{:page {:block/title "page 1"}
|
||||
:blocks [{:block/title "old"}]}]})
|
||||
block (db-test/find-block-by-content @conn "old")
|
||||
block-id (:db/id block)]
|
||||
(ldb/transact! conn [[:db/add block-id :block/order "a0"]])
|
||||
(ldb/batch-transact-with-temp-conn!
|
||||
conn
|
||||
{}
|
||||
(fn [temp-conn]
|
||||
(ldb/transact! temp-conn [[:db/add [:block/uuid (:block/uuid block)] :block/order "a1"]])))
|
||||
(is (= "a1" (:block/order (d/entity @conn block-id)))))))
|
||||
|
||||
(deftest batch-transact-with-temp-conn-before-commit-can-abort-live-commit-test
|
||||
(testing "before-commit runs after temp work and before the live conn is modified"
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:pages-and-blocks [{:page {:block/title "page 1"}
|
||||
:blocks [{:block/title "old"}]}]})
|
||||
block (db-test/find-block-by-content @conn "old")
|
||||
block-id (:db/id block)]
|
||||
(is (thrown? js/Error
|
||||
(ldb/batch-transact-with-temp-conn!
|
||||
conn
|
||||
{}
|
||||
(fn [temp-conn]
|
||||
(ldb/transact! temp-conn [[:db/add [:block/uuid (:block/uuid block)] :block/title "new"]]))
|
||||
{:before-commit #(throw (js/Error. "abort before commit"))})))
|
||||
(is (= "old" (:block/title (d/entity @conn block-id)))))))
|
||||
|
||||
(deftest validated-transact-retries-when-live-conn-changes-before-commit-test
|
||||
(testing "validated transact does not overwrite a concurrent live commit"
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:pages-and-blocks [{:page {:block/title "page 1"}
|
||||
:blocks [{:block/title "first"}
|
||||
{:block/title "second"}]}]})
|
||||
first-block (db-test/find-block-by-content @conn "first")
|
||||
second-block (db-test/find-block-by-content @conn "second")
|
||||
injected? (atom false)
|
||||
original-pipeline @ldb/*transact-pipeline-fn]
|
||||
(try
|
||||
(ldb/register-transact-pipeline-fn!
|
||||
(fn [tx-report]
|
||||
(when (and (not @injected?)
|
||||
(some (fn [datom]
|
||||
(and (= :block/title (:a datom))
|
||||
(= "first updated" (:v datom))))
|
||||
(:tx-data tx-report)))
|
||||
(reset! injected? true)
|
||||
(ldb/transact! conn [[:db/add (:db/id second-block) :block/title "second updated"]]))
|
||||
tx-report))
|
||||
(ldb/transact! conn [[:db/add (:db/id first-block) :block/title "first updated"]])
|
||||
(is (= "first updated" (:block/title (d/entity @conn (:db/id first-block)))))
|
||||
(is (= "second updated" (:block/title (d/entity @conn (:db/id second-block)))))
|
||||
(finally
|
||||
(reset! ldb/*transact-pipeline-fn original-pipeline))))))
|
||||
|
||||
(deftest fix-db-transact-runs-pipeline-without-recursive-validation-test
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:pages-and-blocks [{:page {:block/title "page 1"}
|
||||
:blocks [{:block/title "before"}]}]})
|
||||
block (db-test/find-block-by-content @conn "before")
|
||||
block-id (:db/id block)
|
||||
original-pipeline @ldb/*transact-pipeline-fn]
|
||||
(try
|
||||
(ldb/register-transact-pipeline-fn!
|
||||
(fn [{:keys [db-before db-after tx-data tx-meta]}]
|
||||
(let [stamp-report (d/with db-after
|
||||
[{:db/id block-id
|
||||
:block/tx-id 42}])]
|
||||
(assoc stamp-report
|
||||
:db-before db-before
|
||||
:tx-data (concat tx-data (:tx-data stamp-report))
|
||||
:tx-meta tx-meta))))
|
||||
(with-redefs [db-validate/validate-tx-report
|
||||
(fn [& _]
|
||||
(throw (js/Error. "Repair transactions must not recursively validate")))]
|
||||
(ldb/transact! conn
|
||||
[{:db/id block-id
|
||||
:block/title "after"}]
|
||||
{:fix-db? true}))
|
||||
(is (= "after" (:block/title (d/entity @conn block-id))))
|
||||
(is (= 42 (:block/tx-id (d/entity @conn block-id)))
|
||||
"Repair transactions commit canonical data produced by the pipeline.")
|
||||
(finally
|
||||
(reset! ldb/*transact-pipeline-fn original-pipeline)))))
|
||||
|
||||
(deftest test-batch-transact-clears-stale-tx-tail-before-next-store-tail
|
||||
(let [block-uuid #uuid "00000001-2026-0421-0000-000000000000"
|
||||
schema {:block/uuid {:db/unique :db.unique/identity}}
|
||||
storage (make-storage)
|
||||
conn (d/create-conn schema {:storage storage})]
|
||||
;; Valid history: unique value moves to another entity.
|
||||
(d/transact! conn [[:db/add 28446 :block/uuid block-uuid]])
|
||||
(ldb/batch-transact!
|
||||
conn
|
||||
{}
|
||||
(fn [batch-conn]
|
||||
(d/transact! batch-conn [[:db/retract 28446 :block/uuid block-uuid]])
|
||||
(d/transact! batch-conn [[:db/add 28447 :block/uuid block-uuid]])))
|
||||
;; Next tx uses tail-only persistence.
|
||||
(d/transact! conn [[:db/add 1 :filler "x"]])
|
||||
(let [tail (get @(:*disk storage) @#'ds-storage/tail-addr)
|
||||
stale? (some (fn [[e a v _tx]]
|
||||
(and (= 28446 e)
|
||||
(= :block/uuid a)
|
||||
(= block-uuid v)))
|
||||
(apply concat tail))
|
||||
restored (d/restore-conn storage)]
|
||||
(is (nil? stale?)
|
||||
"stale pre-batch datoms should not leak into tail after batch-transact!")
|
||||
(is (= [28447]
|
||||
(mapv :e (d/datoms @restored :avet :block/uuid block-uuid)))))))
|
||||
|
||||
(deftest get-bidirectional-properties
|
||||
(testing "disabled by default"
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:properties {:friend {:logseq.property/type :node
|
||||
:build/property-classes [:Person]}}
|
||||
:classes {:Person {}
|
||||
:Project {}}
|
||||
:pages-and-blocks
|
||||
[{:page {:block/title "Alice"
|
||||
:build/tags [:Person]
|
||||
:build/properties {:friend [:build/page {:block/title "Bob"}]}}}
|
||||
{:page {:block/title "Bob"}}
|
||||
{:page {:block/title "Charlie"
|
||||
:build/tags [:Project]
|
||||
:build/properties {:friend [:build/page {:block/title "Bob"}]}}}]})
|
||||
target (db-test/find-page-by-title @conn "Bob")]
|
||||
(is (empty? (ldb/get-bidirectional-properties @conn (:db/id target))))))
|
||||
|
||||
(testing "enabled per class"
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:properties {:friend {:logseq.property/type :node
|
||||
:build/property-classes [:Person]}}
|
||||
:classes {:Person {:build/properties {:logseq.property.class/enable-bidirectional? true}}
|
||||
:Project {}}
|
||||
:pages-and-blocks
|
||||
[{:page {:block/title "Alice"
|
||||
:build/tags [:Person]
|
||||
:build/properties {:friend [:build/page {:block/title "Bob"}]}}}
|
||||
{:page {:block/title "Bob"}}
|
||||
{:page {:block/title "Charlie"
|
||||
:build/tags [:Project]
|
||||
:build/properties {:friend [:build/page {:block/title "Bob"}]}}}]})
|
||||
target (db-test/find-page-by-title @conn "Bob")
|
||||
results (ldb/get-bidirectional-properties @conn (:db/id target))]
|
||||
(is (= 1 (count results)))
|
||||
(is (= "People" (:title (first results))))
|
||||
(is (= ["Alice"]
|
||||
(map :block/title (:entities (first results))))))))
|
||||
|
||||
(deftest get-bidirectional-properties-ignores-recycled-entities
|
||||
(let [conn (db-test/create-conn-with-blocks
|
||||
{:properties {:friend {:logseq.property/type :node
|
||||
:build/property-classes [:Person]}}
|
||||
:classes {:Person {:build/properties {:logseq.property.class/enable-bidirectional? true}}}
|
||||
:pages-and-blocks
|
||||
[{:page {:block/title "Alice"
|
||||
:build/tags [:Person]
|
||||
:build/properties {:friend [:build/page {:block/title "Bob"}]}}}
|
||||
{:page {:block/title "Bob"}}]})
|
||||
alice (db-test/find-page-by-title @conn "Alice")
|
||||
target (db-test/find-page-by-title @conn "Bob")]
|
||||
(d/transact! conn [{:db/id (:db/id alice)
|
||||
:logseq.property/deleted-at 1}])
|
||||
(is (empty? (ldb/get-bidirectional-properties @conn (:db/id target))))))
|
||||
|
||||
(defn- bidirectional-perf-conn
|
||||
[n property-titles]
|
||||
(let [target-page {:page {:block/title "Target"}}
|
||||
properties (into {}
|
||||
(map (fn [property-title]
|
||||
[property-title {:logseq.property/type :node
|
||||
:build/property-classes [:Person]}]))
|
||||
property-titles)
|
||||
person-properties (into {}
|
||||
(map (fn [property-title]
|
||||
[property-title [:build/page {:block/title "Target"}]]))
|
||||
property-titles)
|
||||
pages (vec (concat [target-page]
|
||||
(map (fn [i]
|
||||
{:page {:block/title (str "Person " i)
|
||||
:build/tags [:Person]
|
||||
:build/properties person-properties}})
|
||||
(range n))))]
|
||||
(db-test/create-conn-with-blocks
|
||||
{:properties properties
|
||||
:classes {:Person {:build/properties {:logseq.property.class/enable-bidirectional? true}}}
|
||||
:pages-and-blocks pages})))
|
||||
|
||||
(deftest ^:long get-bidirectional-properties-performance-single-property
|
||||
(testing "attribute lookups scale with unique properties, not entities"
|
||||
(let [conn (bidirectional-perf-conn 400 [:friend])
|
||||
target-id (:db/id (db-test/find-page-by-title @conn "Target"))
|
||||
original-entity d/entity
|
||||
attr-lookups (atom 0)
|
||||
results (with-redefs [d/entity (fn [db eid]
|
||||
(when (keyword? eid)
|
||||
(swap! attr-lookups inc))
|
||||
(original-entity db eid))]
|
||||
(ldb/get-bidirectional-properties @conn target-id))]
|
||||
(is (= 1 (count results)))
|
||||
(is (= 400 (count (:entities (first results)))))
|
||||
(is (<= @attr-lookups 8)
|
||||
(str "expected bounded attr lookups, got " @attr-lookups)))))
|
||||
|
||||
(deftest ^:long get-bidirectional-properties-performance-multi-property
|
||||
(testing "attribute lookups stay bounded with multiple matching properties"
|
||||
(let [conn (bidirectional-perf-conn 300 [:friend :colleague])
|
||||
target-id (:db/id (db-test/find-page-by-title @conn "Target"))
|
||||
original-entity d/entity
|
||||
attr-lookups (atom 0)
|
||||
results (with-redefs [d/entity (fn [db eid]
|
||||
(when (keyword? eid)
|
||||
(swap! attr-lookups inc))
|
||||
(original-entity db eid))]
|
||||
(ldb/get-bidirectional-properties @conn target-id))]
|
||||
(is (= 1 (count results)))
|
||||
(is (= 300 (count (:entities (first results)))))
|
||||
(is (<= @attr-lookups 12)
|
||||
(str "expected bounded attr lookups, got " @attr-lookups)))))
|
||||
Reference in New Issue
Block a user