Initial import of LogDarioSeq baseline
Establishes version control for the personal Logseq DB fork before starting the server-client architecture migration and domain features described in docs/agent-guide and the approved implementation plan. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
242
scripts/src/logseq/libs/sdk_generator.clj
Normal file
242
scripts/src/logseq/libs/sdk_generator.clj
Normal file
@@ -0,0 +1,242 @@
|
||||
(ns logseq.libs.sdk-generator
|
||||
(:refer-clojure :exclude [run!])
|
||||
(:require [babashka.fs :as fs]
|
||||
[cheshire.core :as json]
|
||||
[clojure.string :as string]))
|
||||
|
||||
(def default-schema "libs/dist/logseq-sdk-schema.json")
|
||||
(def default-output-dir "libs/cljs-sdk/src")
|
||||
(def default-ns-prefix "com.logseq")
|
||||
(def core-namespace "core")
|
||||
(def clojure-core-conflict-fns #{"get"})
|
||||
|
||||
(defn parse-args
|
||||
[args]
|
||||
(loop [opts {}
|
||||
tokens args]
|
||||
(if (empty? tokens)
|
||||
opts
|
||||
(let [[flag value & more] tokens]
|
||||
(case flag
|
||||
"--schema" (recur (assoc opts :schema value) more)
|
||||
"--out-dir" (recur (assoc opts :out-dir value) more)
|
||||
"--out" (recur (assoc opts :out-dir value) more)
|
||||
"--ns-prefix" (recur (assoc opts :ns-prefix value) more)
|
||||
(throw (ex-info (str "Unknown flag: " flag) {:flag flag})))))))
|
||||
|
||||
(defn camel->kebab [s]
|
||||
(-> s
|
||||
(string/replace #"([a-z0-9])([A-Z])" "$1-$2")
|
||||
(string/replace #"([A-Z]+)([A-Z][a-z])" "$1-$2")
|
||||
(string/lower-case)
|
||||
(string/replace #"[^a-z0-9]+" "-")
|
||||
(string/replace #"(^-|-$)" "")))
|
||||
|
||||
(defn getter->namespace [ns-prefix getter-name]
|
||||
(str ns-prefix "." (camel->kebab getter-name)))
|
||||
|
||||
(defn getter->interface-name [return-type]
|
||||
(some->> (re-find #"\.(I[A-Za-z0-9]+)" return-type)
|
||||
second))
|
||||
|
||||
(defn getter->class-name [return-type]
|
||||
(some->> (re-find #"\.(LSPlugin[A-Za-z0-9]+)" return-type)
|
||||
second))
|
||||
|
||||
(defn format-docstring [doc]
|
||||
(when (and doc (not (string/blank? doc)))
|
||||
(str " " (pr-str doc) "\n")))
|
||||
|
||||
(defn param->info
|
||||
[{:keys [name optional rest?] rest-param :rest}]
|
||||
(let [sym (camel->kebab name)]
|
||||
{:name name
|
||||
:sym sym
|
||||
:optional (boolean optional)
|
||||
:rest (boolean (or rest-param rest?))}))
|
||||
|
||||
(defn emit-rest-binding [{:keys [sym]}]
|
||||
(let [rest-var (str "rest-" sym)
|
||||
line (str " " rest-var " (vec " sym ")\n")]
|
||||
{:binding line
|
||||
:var rest-var}))
|
||||
|
||||
(defn format-param-vector [params]
|
||||
(str "[" (string/join " " params) "]"))
|
||||
|
||||
(defn emit-method-body
|
||||
[method-name params {:keys [call]}]
|
||||
(let [rest-param (some #(when (:rest %) %) params)
|
||||
fixed-params (->> (if rest-param (vec (remove :rest params)) params)
|
||||
(map :sym))
|
||||
{:keys [binding var]} (when rest-param (emit-rest-binding rest-param))
|
||||
rest-lines (if binding [binding] [])
|
||||
args-expr (if rest-param
|
||||
(str "(into [" (string/join " " fixed-params) "] " var ")")
|
||||
(str "[" (string/join " " fixed-params) "]"))]
|
||||
(str (format " (let [method (aget api-proxy \"%s\")\n" method-name)
|
||||
(apply str rest-lines)
|
||||
" args " args-expr "]\n"
|
||||
" (" call " api-proxy method args)))\n")))
|
||||
|
||||
(defn emit-optional-def
|
||||
[fn-name doc-str params impl-name helpers method-name]
|
||||
(let [required (take-while (complement :optional) params)
|
||||
total (count params)
|
||||
param-syms (map :sym params)
|
||||
arities (range (count required) (inc total))
|
||||
header (str "\n(defn " fn-name "\n"
|
||||
(or doc-str ""))]
|
||||
(str "\n(defn- " impl-name "\n"
|
||||
" " (format-param-vector param-syms) "\n"
|
||||
(emit-method-body method-name params helpers)
|
||||
header
|
||||
(apply str
|
||||
(map-indexed
|
||||
(fn [idx arity]
|
||||
(let [provided (take arity param-syms)
|
||||
missing (- total arity)
|
||||
call-args (concat provided (repeat missing "nil"))
|
||||
param-vector (format-param-vector provided)
|
||||
call-arg-str (string/join " " call-args)
|
||||
call-arg-str (if (string/blank? call-arg-str) "" (str " " call-arg-str))]
|
||||
(str " (" param-vector "\n"
|
||||
" (" impl-name call-arg-str "))"
|
||||
(when (not= (inc idx) (count arities))
|
||||
"\n"))))
|
||||
arities))
|
||||
")\n")))
|
||||
|
||||
(defn emit-method
|
||||
[{:keys [name documentation signatures]}
|
||||
helpers]
|
||||
(let [{:keys [parameters]} (apply max-key #(count (:parameters %)) signatures)
|
||||
params (map param->info parameters)
|
||||
fn-name (camel->kebab name)
|
||||
doc-str (format-docstring documentation)
|
||||
rest-param (some #(when (:rest %) %) params)
|
||||
optional-params (filter :optional params)
|
||||
impl-name (str fn-name "-impl")
|
||||
method-body (emit-method-body name params helpers)]
|
||||
(when-not (string/starts-with? name "_") ; system methods
|
||||
(cond
|
||||
rest-param
|
||||
(let [fixed-syms (map :sym (vec (remove :rest params)))
|
||||
param-vector (format-param-vector (concat fixed-syms ["&" (:sym rest-param)]))]
|
||||
(str "\n(defn " fn-name "\n"
|
||||
(or doc-str "")
|
||||
" " param-vector "\n"
|
||||
method-body))
|
||||
|
||||
(seq optional-params)
|
||||
(emit-optional-def fn-name doc-str params impl-name helpers name)
|
||||
|
||||
:else
|
||||
(let [param-vector (format-param-vector (map :sym params))]
|
||||
(str "\n(defn " fn-name "\n"
|
||||
(or doc-str "")
|
||||
" " param-vector "\n"
|
||||
method-body))))))
|
||||
|
||||
(defn emit-core-namespace
|
||||
[ns-prefix {api-methods :methods}]
|
||||
(let [ns (str ns-prefix "." core-namespace)
|
||||
header (str ";; Auto-generated via `bb libs:generate-cljs-sdk`\n"
|
||||
"(ns " ns "\n"
|
||||
" (:require [\"@logseq/libs\"]
|
||||
[cljs-bean.core :as bean]
|
||||
[com.logseq.util :as util]))\n\n"
|
||||
"(defn- normalize-result [result]\n"
|
||||
" (if (instance? js/Promise result)\n"
|
||||
" (.then result (fn [value] (normalize-result value)))\n"
|
||||
" (util/->clj-tagged result)))\n\n"
|
||||
"(defn call-method [owner method args]
|
||||
(when-not method
|
||||
(throw (js/Error. \"Missing method on logseq namespace\")))
|
||||
(normalize-result (.apply method owner (bean/->js args))))\n")
|
||||
helpers {:call "call-method"}
|
||||
owner "\n(def api-proxy js/logseq)\n"
|
||||
methods-str (->> api-methods
|
||||
(map #(emit-method % helpers))
|
||||
(apply str))]
|
||||
[ns (str header owner methods-str)]))
|
||||
|
||||
(defn emit-getter-namespace
|
||||
[ns-prefix getter-name api]
|
||||
(let [ns (getter->namespace ns-prefix getter-name)
|
||||
owner-expr (format "(aget js/logseq \"%s\")" getter-name)
|
||||
excluded-core-fns (->> (:methods api)
|
||||
(map (comp camel->kebab :name))
|
||||
(filter clojure-core-conflict-fns)
|
||||
distinct)
|
||||
header (str ";; Auto-generated via `bb libs:generate-cljs-sdk`\n"
|
||||
"(ns " ns "\n"
|
||||
(when (seq excluded-core-fns)
|
||||
(str " (:refer-clojure :exclude [" (string/join " " excluded-core-fns) "])\n"))
|
||||
" (:require [com.logseq.core :as core]))\n")
|
||||
helpers {:call "core/call-method"}
|
||||
owner (format "\n(def api-proxy %s)\n" owner-expr)
|
||||
methods-str (->> (:methods api)
|
||||
(keep #(emit-method % helpers))
|
||||
(apply str))]
|
||||
(when-not (string/blank? methods-str)
|
||||
[ns (str header owner methods-str)])))
|
||||
|
||||
(defn namespace->file
|
||||
[out-dir ns]
|
||||
(let [parts (string/split ns #"\.")
|
||||
dir-parts (butlast parts)
|
||||
file-name (str (last parts) ".cljs")]
|
||||
(apply fs/path out-dir (concat dir-parts [file-name]))))
|
||||
|
||||
(defn ensure-schema! [schema-path]
|
||||
(when-not (fs/exists? schema-path)
|
||||
(throw (ex-info (str "Schema not found, run `pnpm --dir libs generate:schema` first: " schema-path)
|
||||
{:schema schema-path}))))
|
||||
|
||||
(defn write-namespaces!
|
||||
[out-dir namespaces]
|
||||
(doseq [[ns content] namespaces]
|
||||
(when ns
|
||||
(let [file (namespace->file out-dir ns)]
|
||||
(fs/create-dirs (fs/parent file))
|
||||
(spit (str file) content)
|
||||
(println "Generated" (str file))))))
|
||||
|
||||
(defn run!
|
||||
([] (run! {}))
|
||||
([opts]
|
||||
(let [schema-path (fs/absolutize (or (:schema opts) default-schema))
|
||||
out-dir (fs/absolutize (or (:out-dir opts) default-output-dir))
|
||||
ns-prefix (or (:ns-prefix opts) default-ns-prefix)]
|
||||
(ensure-schema! schema-path)
|
||||
(let [schema (json/parse-string (slurp (str schema-path)) true)
|
||||
interfaces (:interfaces schema)
|
||||
classes (:classes schema)
|
||||
ls-user (get-in schema [:classes :LSPluginUser])
|
||||
_ (when-not ls-user
|
||||
(throw (ex-info "Missing LSPluginUser metadata in schema" {:schema schema-path})))
|
||||
getters (:getters ls-user)
|
||||
modules (->> getters
|
||||
(remove #(= "Utils" (:name %)))
|
||||
(keep (fn [{:keys [name returnType]}]
|
||||
(let [iface-key (some-> (getter->interface-name returnType) keyword)
|
||||
class-key (some-> (getter->class-name returnType) keyword)
|
||||
api (or (get interfaces iface-key)
|
||||
(get classes class-key))]
|
||||
(when api
|
||||
{:getter-name name
|
||||
:api api}))))
|
||||
distinct)
|
||||
proxies (keep (fn [{:keys [getter-name api]}]
|
||||
(emit-getter-namespace ns-prefix getter-name api))
|
||||
modules)
|
||||
core (emit-core-namespace ns-prefix ls-user)
|
||||
namespaces (cons core proxies)]
|
||||
(fs/create-dirs out-dir)
|
||||
(write-namespaces! out-dir namespaces)
|
||||
out-dir))))
|
||||
|
||||
(defn -main [& args]
|
||||
(let [opts (parse-args args)]
|
||||
(run! opts)))
|
||||
15
scripts/src/logseq/tasks/common_errors.clj
Normal file
15
scripts/src/logseq/tasks/common_errors.clj
Normal file
@@ -0,0 +1,15 @@
|
||||
(ns logseq.tasks.common-errors
|
||||
"Task to use AI to detect common errors"
|
||||
(:require [babashka.fs :as fs]
|
||||
[babashka.process :refer [shell]]
|
||||
[clojure.string :as string]))
|
||||
|
||||
(defn check-common-errors
|
||||
[]
|
||||
(let [prompt (String. (fs/read-all-bytes "prompts/review.md"))
|
||||
diff (:out (shell {:out :string :shutdown nil} "git diff --no-prefix -U100 -- '*.cljs'"))]
|
||||
(when-not (string/blank? diff)
|
||||
(let [command (format "gh models run openai/gpt-5 \"%s\""
|
||||
(str prompt
|
||||
(format "\n\n <diff>%s</diff>" diff)))]
|
||||
(shell {:shutdown nil} command)))))
|
||||
@@ -0,0 +1,287 @@
|
||||
(ns logseq.tasks.db-graph.create-graph-with-clojure-irc-history
|
||||
"Import local #clojure IRC log HTML pages into a Logseq DB graph."
|
||||
(:require ["fs" :as fs]
|
||||
["path" :as node-path]
|
||||
[babashka.cli :as cli]
|
||||
[clojure.string :as string]
|
||||
[datascript.core :as d]
|
||||
[logseq.db.common.sqlite-cli :as sqlite-cli]
|
||||
[logseq.outliner.cli :as outliner-cli]
|
||||
[nbb.classpath :as cp]
|
||||
[nbb.core :as nbb]))
|
||||
|
||||
(def ^:private default-base-url "https://chouser.us/clojure-log/")
|
||||
(def ^:private default-source-dir
|
||||
(or (.. js/process -env -CLOJURE_LOG_SOURCE_DIR)
|
||||
"/Users/tiensonqin/Codes/projects/clojure-irc-history"))
|
||||
|
||||
(def spec
|
||||
"Options spec"
|
||||
{:help {:alias :h
|
||||
:desc "Print help"}
|
||||
:base-url {:alias :u
|
||||
:default default-base-url
|
||||
:desc "Base URL for log pages"}
|
||||
:source-dir {:alias :m
|
||||
:default default-source-dir
|
||||
:desc "Local mirror root or clojure-log directory containing index.html and year folders"}
|
||||
:limit-days {:alias :l
|
||||
:desc "Maximum number of day pages to import"}
|
||||
:batch-days {:alias :b
|
||||
:default 200
|
||||
:desc "Number of day pages transacted per batch"}
|
||||
:verbose {:alias :v
|
||||
:desc "Verbose mode"}})
|
||||
|
||||
(def ^:private html-entity-map
|
||||
{""" "\""
|
||||
"&" "&"
|
||||
"<" "<"
|
||||
">" ">"
|
||||
"'" "'"
|
||||
"'" "'"
|
||||
" " " "})
|
||||
|
||||
(defn- parse-long-option
|
||||
[value]
|
||||
(if (string? value)
|
||||
(js/parseInt value 10)
|
||||
value))
|
||||
|
||||
(defn- normalize-base-url
|
||||
[base-url]
|
||||
(if (string/ends-with? base-url "/")
|
||||
base-url
|
||||
(str base-url "/")))
|
||||
|
||||
(defn- normalize-options
|
||||
[options]
|
||||
(-> options
|
||||
(update :limit-days parse-long-option)
|
||||
(update :batch-days parse-long-option)
|
||||
(update :base-url normalize-base-url)))
|
||||
|
||||
(defn parse-args
|
||||
[args]
|
||||
{:graph-dir (first args)
|
||||
:options (normalize-options (cli/parse-opts (rest args) {:spec spec}))})
|
||||
|
||||
(defn- decode-html-entity
|
||||
[entity]
|
||||
(or (html-entity-map entity)
|
||||
(when-let [[_ code] (re-matches #"&#(\d+);" entity)]
|
||||
(js/String.fromCodePoint (js/parseInt code 10)))
|
||||
(when-let [[_ code] (re-matches #"&#x([0-9a-fA-F]+);" entity)]
|
||||
(js/String.fromCodePoint (js/parseInt code 16)))
|
||||
entity))
|
||||
|
||||
(defn- decode-html-entities
|
||||
[s]
|
||||
(string/replace s #"&(?:#\d+|#x[0-9a-fA-F]+|[a-zA-Z]+);" decode-html-entity))
|
||||
|
||||
(defn- strip-html
|
||||
[s]
|
||||
(-> s
|
||||
(string/replace #"(?is)<a[^>]*>(.*?)</a>" "$1")
|
||||
(string/replace #"(?is)<[^>]+>" "")
|
||||
decode-html-entities
|
||||
string/trim))
|
||||
|
||||
(defn extract-day-paths
|
||||
[index-html]
|
||||
(->> (re-seq #"href=\"((\d{4})/\2-\d{2}-\d{2}\.html)\"" index-html)
|
||||
(map second)
|
||||
distinct
|
||||
sort
|
||||
vec))
|
||||
|
||||
(defn- day-path->date
|
||||
[day-path]
|
||||
(some-> (re-find #"\d{4}-\d{2}-\d{2}" day-path)
|
||||
str))
|
||||
|
||||
(defn select-day-paths
|
||||
[day-paths {:keys [limit-days]}]
|
||||
(->> day-paths
|
||||
(take (or limit-days js/Number.MAX_SAFE_INTEGER))
|
||||
vec))
|
||||
|
||||
(defn- date->journal-day
|
||||
[date]
|
||||
(some-> date
|
||||
(string/replace "-" "")
|
||||
(js/parseInt 10)))
|
||||
|
||||
(defn extract-chat-rows
|
||||
[day-html]
|
||||
(->> (re-seq #"(?s)<tr[^>]*>\s*<td><a[^>]*>([^<]+)</a></td><td[^>]*>(.*?)</td><td>(.*?)</td></tr>" day-html)
|
||||
(map (fn [[_ time nick-html message-html]]
|
||||
{:time (string/trim time)
|
||||
:nick (strip-html nick-html)
|
||||
:message (strip-html message-html)}))
|
||||
vec))
|
||||
|
||||
(defn date-time->timestamp-ms
|
||||
[date time]
|
||||
(let [[_ year month day] (re-matches #"(\d{4})-(\d{2})-(\d{2})" date)
|
||||
[_ hour minute] (re-matches #"(\d{1,2}):(\d{2})" time)]
|
||||
(when (and year month day hour minute)
|
||||
(js/Date.UTC (js/parseInt year 10)
|
||||
(dec (js/parseInt month 10))
|
||||
(js/parseInt day 10)
|
||||
(js/parseInt hour 10)
|
||||
(js/parseInt minute 10)
|
||||
0
|
||||
0))))
|
||||
|
||||
(defn build-user-pages
|
||||
[user->uuid]
|
||||
(->> user->uuid
|
||||
(sort-by key)
|
||||
(mapv (fn [[user-name user-uuid]]
|
||||
{:page {:block/uuid user-uuid
|
||||
:build/keep-uuid? true
|
||||
:block/title user-name
|
||||
:build/properties {:logseq.property.user/name user-name}}
|
||||
:blocks []}))))
|
||||
|
||||
(defn build-page-and-blocks
|
||||
[{:keys [date day-path base-url user->uuid chat-rows]}]
|
||||
{:page {:build/journal (date->journal-day date)}
|
||||
:blocks
|
||||
(mapv (fn [{:keys [time nick message]}]
|
||||
(let [timestamp (date-time->timestamp-ms date time)
|
||||
user-uuid (get user->uuid nick)]
|
||||
{:block/title (str time " " nick ": " message)
|
||||
:block/created-at timestamp
|
||||
:block/updated-at timestamp
|
||||
:build/properties {:logseq.property/created-by-ref [:block/uuid user-uuid]}}))
|
||||
chat-rows)})
|
||||
|
||||
(defn- extract-day-date
|
||||
[day-html day-path]
|
||||
(or (some-> (re-find #"<p class=\"dh\">([^<]+)</p>" day-html)
|
||||
second)
|
||||
(day-path->date day-path)))
|
||||
|
||||
(defn- read-text-file
|
||||
[path]
|
||||
(str (fs/readFileSync path "utf8")))
|
||||
|
||||
(defn- error-message
|
||||
[error]
|
||||
(or (ex-message error)
|
||||
(some-> error .-message)
|
||||
(str error)))
|
||||
|
||||
(defn resolve-source-dir
|
||||
[source-dir]
|
||||
(let [candidates [source-dir
|
||||
(node-path/join source-dir "clojure-log")
|
||||
(node-path/join source-dir "chouser.github.io-master" "clojure-log")]
|
||||
found (some (fn [candidate]
|
||||
(when (fs/existsSync (node-path/join candidate "index.html"))
|
||||
candidate))
|
||||
candidates)]
|
||||
(or found
|
||||
(throw (ex-info (str "Could not find clojure-log source directory from: " source-dir)
|
||||
{:source-dir source-dir
|
||||
:candidates candidates})))))
|
||||
|
||||
(defn- fetch-day-page-data
|
||||
[base-url day-path day-html]
|
||||
(let [day-url (str base-url day-path)
|
||||
date (extract-day-date day-html day-path)
|
||||
chat-rows (extract-chat-rows day-html)]
|
||||
{:date date
|
||||
:day-path day-path
|
||||
:base-url base-url
|
||||
:chat-rows chat-rows}))
|
||||
|
||||
(defn- load-source-day-page-data
|
||||
[base-url source-dir day-path]
|
||||
(fetch-day-page-data base-url
|
||||
day-path
|
||||
(read-text-file (node-path/join source-dir day-path))))
|
||||
|
||||
(defn- ensure-user!
|
||||
[user->uuid* new-users* user-name]
|
||||
(if-let [user-uuid (get @user->uuid* user-name)]
|
||||
user-uuid
|
||||
(let [user-uuid (random-uuid)]
|
||||
(swap! user->uuid* assoc user-name user-uuid)
|
||||
(swap! new-users* conj user-name)
|
||||
user-uuid)))
|
||||
|
||||
(defn- transact-batch!
|
||||
[conn pages-and-blocks]
|
||||
(let [{:keys [init-tx block-props-tx]}
|
||||
(outliner-cli/build-blocks-tx {:pages-and-blocks pages-and-blocks})]
|
||||
(d/transact! conn init-tx)
|
||||
(when (seq block-props-tx)
|
||||
(d/transact! conn block-props-tx))))
|
||||
|
||||
(defn -main [args]
|
||||
(let [{:keys [graph-dir options]} (parse-args args)
|
||||
_ (when (or (nil? graph-dir) (:help options))
|
||||
(println (str "Usage: $0 GRAPH-NAME [OPTIONS]\nOptions:\n"
|
||||
(cli/format-opts {:spec spec})))
|
||||
(js/process.exit 1))
|
||||
{:keys [base-url source-dir batch-days verbose]} options
|
||||
resolved-source-dir (resolve-source-dir source-dir)
|
||||
index-html (read-text-file (node-path/join resolved-source-dir "index.html"))
|
||||
all-day-paths (extract-day-paths index-html)
|
||||
selected-day-paths (select-day-paths all-day-paths options)
|
||||
init-conn-args (sqlite-cli/->open-db-args graph-dir)
|
||||
db-name (if (= 1 (count init-conn-args))
|
||||
(first init-conn-args)
|
||||
(second init-conn-args))
|
||||
conn (apply outliner-cli/init-conn
|
||||
(conj init-conn-args {:classpath (cp/get-classpath)}))
|
||||
total-day-count (count selected-day-paths)
|
||||
total-batches (js/Math.ceil (/ total-day-count batch-days))
|
||||
user->uuid* (atom {})
|
||||
imported-stats (atom {:days 0 :messages 0})]
|
||||
(println "Found" (count all-day-paths) "day pages in index")
|
||||
(println "Using source directory:" resolved-source-dir)
|
||||
(println "Importing" total-day-count "day pages to graph" (str "'" db-name "'")
|
||||
"in" total-batches "batch(es)")
|
||||
(doseq [[batch-index day-path-batch] (map-indexed vector (partition-all batch-days selected-day-paths))]
|
||||
(println "Processing batch" (inc batch-index) "of" total-batches
|
||||
"with" (count day-path-batch) "day pages")
|
||||
(let [new-users* (atom #{})
|
||||
day-pages-and-blocks
|
||||
(->> day-path-batch
|
||||
(map (fn [day-path]
|
||||
(try
|
||||
(when verbose
|
||||
(println "Loading source log" day-path "..."))
|
||||
(let [day-page-data (load-source-day-page-data base-url resolved-source-dir day-path)
|
||||
_ (doseq [{:keys [nick]} (:chat-rows day-page-data)]
|
||||
(ensure-user! user->uuid* new-users* nick))]
|
||||
(when (seq (:chat-rows day-page-data))
|
||||
(build-page-and-blocks (assoc day-page-data
|
||||
:user->uuid @user->uuid*))))
|
||||
(catch :default e
|
||||
(println "Skipping" day-path ":" (error-message e))
|
||||
nil))))
|
||||
(remove nil?)
|
||||
vec)
|
||||
new-user-pages-and-blocks (build-user-pages (select-keys @user->uuid* @new-users*))
|
||||
pages-and-blocks (vec (concat new-user-pages-and-blocks day-pages-and-blocks))]
|
||||
(swap! imported-stats
|
||||
(fn [{:keys [days messages]}]
|
||||
{:days (+ days (count day-pages-and-blocks))
|
||||
:messages (+ messages
|
||||
(reduce
|
||||
(fn [acc {:keys [blocks]}]
|
||||
(+ acc (count blocks)))
|
||||
0
|
||||
day-pages-and-blocks))}))
|
||||
(when (seq pages-and-blocks)
|
||||
(transact-batch! conn pages-and-blocks))))
|
||||
(let [{:keys [days messages]} @imported-stats]
|
||||
(println "Imported" days "pages and" messages "chat messages into graph" (str "'" db-name "'") "."))))
|
||||
|
||||
(when (= nbb/*file* (nbb/invoked-file))
|
||||
(-main *command-line-args*))
|
||||
@@ -0,0 +1,200 @@
|
||||
(ns logseq.tasks.db-graph.create-graph-with-clojuredocs
|
||||
"Script that convert clojuredocs-export.json into logseq graph data."
|
||||
(:require ["fs" :as fs]
|
||||
[babashka.cli :as cli]
|
||||
[cljs.pprint :as pp]
|
||||
[clojure.edn :as edn]
|
||||
[clojure.string :as string]
|
||||
[datascript.core :as d]
|
||||
[logseq.db.common.sqlite-cli :as sqlite-cli]
|
||||
[logseq.outliner.cli :as outliner-cli]
|
||||
[nbb.classpath :as cp]
|
||||
[nbb.core :as nbb]))
|
||||
|
||||
(def spec
|
||||
"Options spec"
|
||||
{:help {:alias :h
|
||||
:desc "Print help"}
|
||||
:config {:alias :c
|
||||
:coerce edn/read-string
|
||||
:desc "EDN map to add to config.edn"}
|
||||
:export {:alias :e
|
||||
:desc "Exports graph to clojuredocs.edn"}
|
||||
:verbose {:alias :v
|
||||
:desc "Verbose mode"}})
|
||||
|
||||
(defn example=>block
|
||||
[author->block-uuid example]
|
||||
(when-let [body (:body example)]
|
||||
(let [author-block-uuid (author->block-uuid (:login (:author example)))]
|
||||
{:block/title body
|
||||
:build/tags [:logseq.class/Code-block]
|
||||
:build/properties (cond-> {:logseq.property.node/display-type :code
|
||||
:logseq.property.code/lang "Clojure"}
|
||||
author-block-uuid
|
||||
(assoc :user.property/author-X_lTJqwD [:block/uuid author-block-uuid]))})))
|
||||
|
||||
(defn convert-var-to-page
|
||||
[type->block-uuid author->block-uuid
|
||||
{:keys [ns name type see-alsos examples notes arglists doc library-url] :as _clj-var}]
|
||||
{:page
|
||||
{:build/properties
|
||||
{:user.property/library-url-ip8W5T7M library-url
|
||||
:user.property/type-Un-Aypix
|
||||
[:block/uuid (type->block-uuid type)]},
|
||||
:block/title (str "`" name "` (" ns ")")}
|
||||
:blocks
|
||||
[{:block/title "Doc",
|
||||
:build/properties {:logseq.property/heading 3},
|
||||
:build/children
|
||||
(mapv
|
||||
(fn [line]
|
||||
{:block/title line
|
||||
:build/properties {}})
|
||||
(string/split-lines doc))}
|
||||
{:block/title "Examples",
|
||||
:build/properties {:logseq.property/heading 3},
|
||||
:build/children
|
||||
(vec (keep (partial example=>block author->block-uuid) examples))}
|
||||
{:block/title "Notes",
|
||||
:build/properties {:logseq.property/heading 3}
|
||||
:build/children
|
||||
(mapv
|
||||
(fn [note]
|
||||
(let [body (or (:body note) "")
|
||||
author-block-uuid (author->block-uuid (:login (:author note)))]
|
||||
{:block/title body
|
||||
:build/tags [:logseq.class/Quote-block]
|
||||
:build/properties (cond-> {:logseq.property.node/display-type :quote}
|
||||
author-block-uuid
|
||||
(assoc :user.property/author-X_lTJqwD [:block/uuid author-block-uuid]))}))
|
||||
notes)}
|
||||
{:block/title "Arglists",
|
||||
:build/properties {:logseq.property/heading 3}
|
||||
:build/children
|
||||
(vec
|
||||
(when-let [arglists-content (with-out-str (pp/pprint arglists))]
|
||||
[{:block/title arglists-content
|
||||
:build/tags [:logseq.class/Code-block]
|
||||
:build/properties {:logseq.property.node/display-type :code
|
||||
:logseq.property.code/lang "Clojure"}}]))}
|
||||
;; {:block/title "See also",
|
||||
;; :build/properties {:logseq.property/heading 3}
|
||||
;; :build/children
|
||||
;; (vec
|
||||
;; (when-let [arglists-content (with-out-str (pp/pprint see-alsos))]
|
||||
;; [{:block/title arglists-content
|
||||
;; :build/tags [:logseq.class/Code-block]
|
||||
;; :build/properties {:logseq.property.node/display-type :code
|
||||
;; :logseq.property.code/lang "Clojure"}}]))}
|
||||
]})
|
||||
|
||||
(defn convert-type-pages
|
||||
"return {:pages ..., :type->block-uuid ...}"
|
||||
[clj-vars]
|
||||
(let [all-types (set (map :type clj-vars))
|
||||
type->block-uuid
|
||||
(into {} (map (fn [tp] [tp (random-uuid)])) all-types)
|
||||
pages
|
||||
(map
|
||||
(fn [[tp block-uuid]]
|
||||
{:page
|
||||
{:block/uuid block-uuid
|
||||
:build/keep-uuid? true
|
||||
:block/title tp},
|
||||
:blocks
|
||||
[]})
|
||||
type->block-uuid)]
|
||||
{:pages pages :type->block-uuid type->block-uuid}))
|
||||
|
||||
(defn convert-author-pages
|
||||
"return {:pages ..., :author->block-uuid ...}"
|
||||
[clj-vars]
|
||||
(let [author-name (comp :login :author)
|
||||
author-names
|
||||
(set
|
||||
(mapcat
|
||||
(fn [clj-var]
|
||||
(set
|
||||
(map
|
||||
author-name
|
||||
(concat (:see-alsos clj-var)
|
||||
(:examples clj-var)
|
||||
(:notes clj-var)))))
|
||||
clj-vars))
|
||||
author->block-uuid
|
||||
(into {} (map (fn [author] [author (random-uuid)])) author-names)
|
||||
pages
|
||||
(map
|
||||
(fn [[author block-uuid]]
|
||||
{:page
|
||||
{:block/uuid block-uuid
|
||||
:build/keep-uuid? true
|
||||
:block/title author}
|
||||
:blocks
|
||||
[]})
|
||||
author->block-uuid)]
|
||||
{:pages pages :author->block-uuid author->block-uuid}))
|
||||
|
||||
(def properties
|
||||
{:user.property/type-Un-Aypix
|
||||
{:logseq.property/type :node,
|
||||
:block/title "type",
|
||||
:db/cardinality :db.cardinality/one},
|
||||
:user.property/library-url-ip8W5T7M
|
||||
{:db/cardinality :db.cardinality/one,
|
||||
:logseq.property/type :url,
|
||||
:block/title "library-url"}
|
||||
:user.property/author-X_lTJqwD
|
||||
{:logseq.property/type :node
|
||||
:block/title "author"
|
||||
:db/cardinality :db.cardinality/one}})
|
||||
|
||||
(defn convert
|
||||
[clj-vars]
|
||||
(let [{type-pages :pages type->block-uuid :type->block-uuid} (convert-type-pages clj-vars)
|
||||
{author-pages :pages author->block-uuid :author->block-uuid} (convert-author-pages clj-vars)]
|
||||
{:properties properties
|
||||
:classes {}
|
||||
:pages-and-blocks
|
||||
(vec
|
||||
(concat
|
||||
type-pages
|
||||
author-pages
|
||||
(map (partial convert-var-to-page type->block-uuid author->block-uuid) clj-vars)))}))
|
||||
|
||||
(comment
|
||||
(def clojuredocs-json (js->clj (js/JSON.parse (fs/readFileSync "resources/clojuredocs-export.json"))
|
||||
:keywordize-keys true))
|
||||
(def result (convert (filter #(= "clojure.core" (:ns %)) (:vars clojuredocs-json))))
|
||||
(fs/writeFileSync "cljdocs.edn" (with-out-str (pp/pprint result))))
|
||||
|
||||
(defn -main [args]
|
||||
(let [[graph-dir] args
|
||||
options (cli/parse-opts args {:spec spec})
|
||||
_ (when (or (nil? graph-dir) (:help options))
|
||||
(println (str "Usage: $0 GRAPH-NAME [OPTIONS]\nOptions:\n"
|
||||
(cli/format-opts {:spec spec})))
|
||||
(js/process.exit 1))
|
||||
init-conn-args (sqlite-cli/->open-db-args graph-dir)
|
||||
db-name (if (= 1 (count init-conn-args)) (first init-conn-args) (second init-conn-args))
|
||||
conn (apply outliner-cli/init-conn
|
||||
(conj init-conn-args {:additional-config (:config options)
|
||||
:classpath (cp/get-classpath)}))
|
||||
clojuredocs-json (js->clj (js/JSON.parse (fs/readFileSync "resources/clojuredocs-export.json"))
|
||||
:keywordize-keys true)
|
||||
clj-vars (:vars clojuredocs-json)
|
||||
result-edn (convert clj-vars)]
|
||||
(println "Generating" (count (:pages-and-blocks result-edn)) "pages")
|
||||
(if (:export options)
|
||||
(do (println "Generating clojuredocs.edn ...")
|
||||
(fs/writeFileSync "clojuredocs.edn" (with-out-str (pp/pprint result-edn))))
|
||||
(do (println "Transacting to graph ...")
|
||||
(let [{:keys [init-tx block-props-tx]} (outliner-cli/build-blocks-tx result-edn)]
|
||||
(d/transact! conn init-tx)
|
||||
(d/transact! conn block-props-tx)
|
||||
(println "Transacted" (count (d/datoms @conn :eavt)) "datoms")
|
||||
(println "Created graph " (str "'" db-name "'") "!"))))))
|
||||
|
||||
(when (= nbb/*file* (nbb/invoked-file))
|
||||
(-main *command-line-args*))
|
||||
@@ -0,0 +1,142 @@
|
||||
(ns logseq.tasks.db-graph.create-graph-with-large-sizes
|
||||
"Script that generates graphs at large sizes"
|
||||
(:require ["os" :as os]
|
||||
["path" :as node-path]
|
||||
[babashka.cli :as cli]
|
||||
[clojure.string :as string]
|
||||
[datascript.core :as d]
|
||||
[logseq.outliner.cli :as outliner-cli]
|
||||
[nbb.classpath :as cp]
|
||||
[nbb.core :as nbb]))
|
||||
|
||||
(def ^:private target-entities-per-batch 25000)
|
||||
(def ^:private max-pages-per-batch 1000)
|
||||
(def ^:private min-block-title-length 24)
|
||||
(def ^:private max-block-title-length 256)
|
||||
(def ^:private block-title-fragment
|
||||
"captures project updates, implementation details, follow-up notes, and review context ")
|
||||
|
||||
(defn- parse-long-option
|
||||
[value]
|
||||
(if (string? value)
|
||||
(js/parseInt value 10)
|
||||
value))
|
||||
|
||||
(defn- normalize-options
|
||||
[options]
|
||||
(update-vals options parse-long-option))
|
||||
|
||||
(defn default-batch-pages
|
||||
[blocks-per-page]
|
||||
(-> (quot target-entities-per-batch (max 1 (inc blocks-per-page)))
|
||||
(max 1)
|
||||
(min max-pages-per-batch)))
|
||||
|
||||
(defn- build-block-title
|
||||
[page-idx block-idx]
|
||||
(let [target-length (+ min-block-title-length
|
||||
(mod (+ (* page-idx 17) (* block-idx 31))
|
||||
(inc (- max-block-title-length min-block-title-length))))
|
||||
prefix (str "Page " page-idx " block " block-idx " summary ")
|
||||
raw-title (str prefix
|
||||
(apply str (repeat (inc (quot target-length (count block-title-fragment)))
|
||||
block-title-fragment)))]
|
||||
(subs raw-title 0 target-length)))
|
||||
|
||||
(defn- build-blocks
|
||||
[page-idx blocks-per-page next-id]
|
||||
(loop [block-idx 0
|
||||
blocks (transient [])]
|
||||
(if (= block-idx blocks-per-page)
|
||||
(persistent! blocks)
|
||||
(recur (inc block-idx)
|
||||
(conj! blocks
|
||||
{:block/uuid (next-id)
|
||||
:block/title (build-block-title page-idx block-idx)})))))
|
||||
|
||||
(defn build-page-and-blocks-batch
|
||||
([start-idx page-count blocks-per-page]
|
||||
(build-page-and-blocks-batch start-idx page-count blocks-per-page random-uuid))
|
||||
([start-idx page-count blocks-per-page next-id]
|
||||
(loop [page-idx 0
|
||||
pages-and-blocks (transient [])]
|
||||
(if (= page-idx page-count)
|
||||
(persistent! pages-and-blocks)
|
||||
(recur (inc page-idx)
|
||||
(conj! pages-and-blocks
|
||||
{:page {:block/uuid (next-id)
|
||||
:block/title (str "Page-" (+ start-idx page-idx))}
|
||||
:blocks (build-blocks (+ start-idx page-idx) blocks-per-page next-id)}))))))
|
||||
|
||||
(defn page-and-block-batches
|
||||
([{:keys [pages blocks batch-pages]}]
|
||||
(page-and-block-batches {:pages pages
|
||||
:blocks blocks
|
||||
:batch-pages batch-pages}
|
||||
random-uuid))
|
||||
([{:keys [pages blocks batch-pages]} next-id]
|
||||
(let [batch-pages' (or batch-pages (default-batch-pages blocks))]
|
||||
((fn step [start-idx]
|
||||
(lazy-seq
|
||||
(when (< start-idx pages)
|
||||
(cons (build-page-and-blocks-batch start-idx
|
||||
(min batch-pages' (- pages start-idx))
|
||||
blocks
|
||||
next-id)
|
||||
(step (+ start-idx batch-pages'))))))
|
||||
0))))
|
||||
|
||||
(defn- transact-batch!
|
||||
[conn pages-and-blocks]
|
||||
(let [{:keys [init-tx block-props-tx]} (outliner-cli/build-blocks-tx {:pages-and-blocks pages-and-blocks})]
|
||||
(d/transact! conn init-tx)
|
||||
(when (seq block-props-tx)
|
||||
(d/transact! conn block-props-tx))))
|
||||
|
||||
(defn- total-batches
|
||||
[{:keys [pages blocks batch-pages]}]
|
||||
(let [batch-pages' (or batch-pages (default-batch-pages blocks))]
|
||||
(js/Math.ceil (/ pages batch-pages'))))
|
||||
|
||||
(def spec
|
||||
"Options spec"
|
||||
{:help {:alias :h
|
||||
:desc "Print help"}
|
||||
:pages {:alias :p
|
||||
:default 1000
|
||||
:desc "Number of pages to create"}
|
||||
:blocks {:alias :b
|
||||
:default 20
|
||||
:desc "Number of blocks to create per page"}
|
||||
:batch-pages {:alias :t
|
||||
:desc "Number of pages to build and transact per batch"}})
|
||||
|
||||
(defn parse-args
|
||||
[args]
|
||||
{:graph-dir (first args)
|
||||
:options (normalize-options (cli/parse-opts (rest args) {:spec spec}))})
|
||||
|
||||
(defn -main [args]
|
||||
(let [{:keys [graph-dir options]} (parse-args args)
|
||||
_ (when (or (nil? graph-dir) (:help options))
|
||||
(println (str "Usage: $0 GRAPH-NAME [OPTIONS]\nOptions:\n"
|
||||
(cli/format-opts {:spec spec})))
|
||||
(js/process.exit 1))
|
||||
{:keys [pages blocks batch-pages]} options
|
||||
[dir db-name] (if (string/includes? graph-dir "/")
|
||||
((juxt node-path/dirname node-path/basename) graph-dir)
|
||||
[(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
|
||||
conn (outliner-cli/init-conn dir db-name {:classpath (cp/get-classpath)})
|
||||
total-batches' (total-batches options)
|
||||
pages-per-batch (or batch-pages (default-batch-pages blocks))
|
||||
total-blocks (* pages blocks)]
|
||||
(println "Creating graph with" pages "pages and" total-blocks "blocks"
|
||||
"using" total-batches' "batch(es) of up to" pages-per-batch "pages ...")
|
||||
(doseq [[batch-num pages-and-blocks] (map-indexed vector (page-and-block-batches options))]
|
||||
(println "Transacting batch" (inc batch-num) "of" total-batches'
|
||||
"with" (count pages-and-blocks) "pages")
|
||||
(transact-batch! conn pages-and-blocks))
|
||||
(println "Created graph" db-name "with" pages "pages and" total-blocks "blocks.")))
|
||||
|
||||
(when (= nbb/*file* (nbb/invoked-file))
|
||||
(-main *command-line-args*))
|
||||
@@ -0,0 +1,237 @@
|
||||
(ns logseq.tasks.db-graph.create-graph-with-properties
|
||||
"Script that generates all the permutations of property types and cardinality.
|
||||
Also creates a page of queries that exercises most properties
|
||||
NOTE: This script is also used in CI to confirm graph creation works"
|
||||
(:require ["fs" :as fs]
|
||||
["fs-extra$default" :as fse]
|
||||
["path" :as node-path]
|
||||
[babashka.cli :as cli]
|
||||
[cljs.pprint :as pprint]
|
||||
[clojure.edn :as edn]
|
||||
[clojure.set :as set]
|
||||
[clojure.string :as string]
|
||||
[datascript.core :as d]
|
||||
[logseq.common.util :as common-util]
|
||||
[logseq.common.util.date-time :as date-time-util]
|
||||
[logseq.common.util.page-ref :as page-ref]
|
||||
[logseq.db.common.sqlite-cli :as sqlite-cli]
|
||||
[logseq.db.frontend.property.type :as db-property-type]
|
||||
[logseq.outliner.cli :as outliner-cli]
|
||||
[nbb.classpath :as cp]
|
||||
[nbb.core :as nbb]))
|
||||
|
||||
(defn- date-journal-title [date]
|
||||
(date-time-util/int->journal-title (date-time-util/date->int date) "MMM do, yyyy"))
|
||||
|
||||
(defn- subtract-days
|
||||
[date days]
|
||||
(new js/Date (- (.getTime date) (* days 24 60 60 1000))))
|
||||
|
||||
(defn- build-closed-values-config
|
||||
[]
|
||||
{:default-closed
|
||||
(mapv #(hash-map :value %
|
||||
:uuid (random-uuid)
|
||||
:icon {:id % :type :emoji})
|
||||
["joy" "sob" "upside_down_face"])
|
||||
:url-closed
|
||||
(mapv #(hash-map :value %
|
||||
:uuid (random-uuid))
|
||||
["https://logseq.com" "https://docs.logseq.com" "https://github.com/logseq/logseq"])
|
||||
:number-closed
|
||||
(mapv #(hash-map :value %
|
||||
:uuid (random-uuid))
|
||||
[10 42 (rand 100)])})
|
||||
|
||||
(defn- query [query-string]
|
||||
{:block/title query-string
|
||||
:build/properties {:logseq.property/query query-string}
|
||||
:block/tags [{:db/ident :logseq.class/Query}]})
|
||||
|
||||
(defn- create-init-data
|
||||
[]
|
||||
(let [today (new js/Date)
|
||||
yesterday (subtract-days today 1)
|
||||
[today-int yesterday-int] (map date-time-util/date->int [today yesterday])
|
||||
two-days-ago (subtract-days today 2)
|
||||
closed-values-config (build-closed-values-config)
|
||||
;; Stores random closed values for use with queries
|
||||
closed-values (atom {})
|
||||
random-closed-value #(let [val (-> closed-values-config % rand-nth)]
|
||||
(swap! closed-values assoc % val)
|
||||
[:block/uuid (:uuid val)])
|
||||
object-uuid (random-uuid)
|
||||
get-closed-value #(:value (get @closed-values %))
|
||||
get-closed-value-ref #(vector :block/uuid (:uuid (get @closed-values %)))
|
||||
timestamp (common-util/time-ms)]
|
||||
{:pages-and-blocks
|
||||
(vec
|
||||
(concat
|
||||
;; Page property values needs to be before b/c they are referenced by everything else
|
||||
[{:page {:block/title "Page 1"}}]
|
||||
|
||||
;; Objects
|
||||
[{:page {:block/title "Page object"
|
||||
:build/tags [:TestClass]}}
|
||||
{:page {:block/title "Blocks"}
|
||||
:blocks
|
||||
[{:block/title "block object"
|
||||
:block/uuid object-uuid
|
||||
:build/tags [:TestClass]}]}]
|
||||
|
||||
;; Journals
|
||||
[{:page
|
||||
{:build/journal today-int}
|
||||
:blocks
|
||||
[{:block/title "[[Block Properties]]"}
|
||||
{:block/title "[[Property Queries]]"}
|
||||
{:block/title "[[Has Property Queries]]"}]}
|
||||
{:page {:build/journal yesterday-int}}
|
||||
{:page {:build/journal (date-time-util/date->int two-days-ago)}}
|
||||
|
||||
;; Block property blocks and queries
|
||||
{:page {:block/title "Block Properties"
|
||||
:build/properties
|
||||
{:logseq.property/description "This page demonstrates all the combinations of property types and single/multiple values that are possible."}}
|
||||
:blocks
|
||||
[{:block/title "default property block" :build/properties {:default "haha"}}
|
||||
{:block/title "default property block" :build/properties {:default-many #{"yee" "haw" "sir"}}}
|
||||
{:block/title "default-closed property block" :build/properties {:default-closed (random-closed-value :default-closed)}}
|
||||
{:block/title "url property block" :build/properties {:url "https://logseq.com"}}
|
||||
{:block/title "url-many property block" :build/properties {:url-many #{"https://logseq.com" "https://docs.logseq.com"}}}
|
||||
{:block/title "url-closed property block" :build/properties {:url-closed (random-closed-value :url-closed)}}
|
||||
{:block/title "checkbox property block" :build/properties {:checkbox true}}
|
||||
{:block/title "number property block" :build/properties {:number 5}}
|
||||
{:block/title "number-many property block" :build/properties {:number-many #{5 10}}}
|
||||
{:block/title "number-closed property block" :build/properties {:number-closed (random-closed-value :number-closed)}}
|
||||
{:block/title "node property block" :build/properties {:node [:block/uuid object-uuid]}}
|
||||
{:block/title "node without classes property block" :build/properties {:node-without-classes [:build/page {:block/title "Page 1"}]}}
|
||||
{:block/title "node-many property block" :build/properties {:node-many #{[:block/uuid object-uuid] [:build/page {:block/title "Page object"}]}}}
|
||||
{:block/title "date property block" :build/properties {:date [:build/page {:build/journal today-int}]}}
|
||||
{:block/title "date-many property block" :build/properties {:date-many #{[:build/page {:build/journal today-int}]
|
||||
[:build/page {:build/journal yesterday-int}]}}}
|
||||
{:block/title "datetime property block" :build/properties {:datetime timestamp}}]}
|
||||
{:page {:block/title "Property Queries"
|
||||
:build/properties
|
||||
{:logseq.property/description "This page demonstrates all property type combinations being queried for a specific value. There should be 2 results for each query, one block and one page."}}
|
||||
:blocks
|
||||
[(query "(property default \"haha\")")
|
||||
(query "(property default-many \"haw\")")
|
||||
(query (str "(property default-closed " (pr-str (get-closed-value :default-closed)) ")"))
|
||||
(query "(property url \"https://logseq.com\")")
|
||||
(query "(property url-many \"https://logseq.com\")")
|
||||
(query (str "(property url-closed " (pr-str (get-closed-value :url-closed)) ")"))
|
||||
(query "(property checkbox true)")
|
||||
(query "(property number 5)")
|
||||
(query "(property number-many 10)")
|
||||
(query (str "(property number-closed " (pr-str (get-closed-value :number-closed)) ")"))
|
||||
(query "(property node \"block object\")")
|
||||
(query "(property node-without-classes [[Page 1]])")
|
||||
(query "(property node-many [[Page object]])")
|
||||
(query (str "(property date " (page-ref/->page-ref (string/capitalize (date-journal-title today))) ")"))
|
||||
(query (str "(property date-many " (page-ref/->page-ref (string/capitalize (date-journal-title yesterday))) ")"))
|
||||
(query (str "(property datetime " timestamp ")"))]}
|
||||
|
||||
;; Page property pages and queries
|
||||
{:page {:block/title "default page" :build/properties {:default "haha"}}}
|
||||
{:page {:block/title "default-many page" :build/properties {:default-many #{"yee" "haw" "sir"}}}}
|
||||
{:page {:block/title "default-closed page" :build/properties {:default-closed (get-closed-value-ref :default-closed)}}}
|
||||
{:page {:block/title "url page" :build/properties {:url "https://logseq.com"}}}
|
||||
{:page {:block/title "url-many page" :build/properties {:url-many #{"https://logseq.com" "https://docs.logseq.com"}}}}
|
||||
{:page {:block/title "url-closed page" :build/properties {:url-closed (get-closed-value-ref :url-closed)}}}
|
||||
{:page {:block/title "checkbox page" :build/properties {:checkbox true}}}
|
||||
{:page {:block/title "number page" :build/properties {:number 5}}}
|
||||
{:page {:block/title "number-many page" :build/properties {:number-many #{5 10}}}}
|
||||
{:page {:block/title "number-closed page" :build/properties {:number-closed (get-closed-value-ref :number-closed)}}}
|
||||
{:page {:block/title "node page" :build/properties {:node [:block/uuid object-uuid]}}}
|
||||
{:page {:block/title "node without classes page" :build/properties {:node-without-classes [:build/page {:block/title "Page 1"}]}}}
|
||||
{:page {:block/title "node-many page" :build/properties {:node-many #{[:block/uuid object-uuid] [:build/page {:block/title "Page object"}]}}}}
|
||||
{:page {:block/title "date page" :build/properties {:date [:build/page {:build/journal today-int}]}}}
|
||||
{:page {:block/title "date-many page" :build/properties {:date-many #{[:build/page {:build/journal today-int}]
|
||||
[:build/page {:build/journal yesterday-int}]}}}}
|
||||
{:page {:block/title "datetime page" :build/properties {:datetime timestamp}}}
|
||||
|
||||
{:page {:block/title "Has Property Queries"
|
||||
:build/properties
|
||||
{:logseq.property/description "This page demonstrates all property type combinations being queried for having a specific property. There should be 2 results for each query, one block and one page."}}
|
||||
:blocks
|
||||
[(query "(property default)")
|
||||
(query "(property default-many)")
|
||||
(query "(property default-closed)")
|
||||
(query "(property url)")
|
||||
(query "(property url-many)")
|
||||
(query "(property url-closed)")
|
||||
(query "(property checkbox)")
|
||||
(query "(property number)")
|
||||
(query "(property number-many)")
|
||||
(query "(property number-closed)")
|
||||
(query "(property node)")
|
||||
(query "(property node-without-classes)")
|
||||
(query "(property node-many)")
|
||||
(query "(property date)")
|
||||
(query "(property date-many)")
|
||||
(query "(property datetime)")]}]))
|
||||
|
||||
:classes {:TestClass {}}
|
||||
|
||||
;; Properties
|
||||
:properties
|
||||
(->> db-property-type/user-built-in-property-types
|
||||
(mapcat #(cond-> (if (= :node %)
|
||||
[[% {:logseq.property/type % :build/property-classes [:TestClass]}]
|
||||
[:node-without-classes {:logseq.property/type %}]]
|
||||
[[% {:logseq.property/type %}]])
|
||||
(contains? db-property-type/cardinality-property-types %)
|
||||
(conj [(keyword (str (name %) "-many"))
|
||||
(cond-> {:logseq.property/type %
|
||||
:db/cardinality :many}
|
||||
(= :node %)
|
||||
(assoc :build/property-classes [:TestClass]))])))
|
||||
(into (mapv #(vector (keyword (str (name %) "-closed"))
|
||||
{:logseq.property/type %
|
||||
:build/closed-values (closed-values-config (keyword (str (name %) "-closed")))})
|
||||
[:default :url :number]))
|
||||
(into {}))}))
|
||||
|
||||
(def spec
|
||||
"Options spec"
|
||||
{:help {:alias :h
|
||||
:desc "Print help"}
|
||||
:file {:alias :f
|
||||
:desc "File to save generated sqlite.build EDN"}
|
||||
:config {:alias :c
|
||||
:coerce edn/read-string
|
||||
:desc "EDN map to add to config.edn"}})
|
||||
|
||||
(defn -main [args]
|
||||
(let [graph-dir (first args)
|
||||
options (cli/parse-opts args {:spec spec})
|
||||
_ (when (or (nil? graph-dir) (:help options))
|
||||
(println (str "Usage: $0 GRAPH-NAME [OPTIONS]\nOptions:\n"
|
||||
(cli/format-opts {:spec spec})))
|
||||
(js/process.exit 1))
|
||||
init-conn-args (sqlite-cli/->open-db-args graph-dir)
|
||||
db-name (if (= 1 (count init-conn-args)) (first init-conn-args) (second init-conn-args))
|
||||
db-path (apply node-path/join init-conn-args)
|
||||
;; Only remove the directory if the directory is being overwritten
|
||||
_ (when (and (= 2 (count init-conn-args)) (fs/existsSync db-path))
|
||||
(fse/removeSync db-path))
|
||||
conn (apply outliner-cli/init-conn
|
||||
(conj init-conn-args {:additional-config (:config options)
|
||||
:classpath (cp/get-classpath)}))
|
||||
init-data (create-init-data)
|
||||
_ (when (:file options) (fs/writeFileSync (:file options) (with-out-str (pprint/pprint init-data))))
|
||||
{:keys [init-tx block-props-tx]} (outliner-cli/build-blocks-tx init-data)
|
||||
existing-names (set (map :v (d/datoms @conn :avet :block/title)))
|
||||
conflicting-names (set/intersection existing-names (set (keep :block/title init-tx)))]
|
||||
(when (seq conflicting-names)
|
||||
(println "Error: Following names conflict -" (string/join "," conflicting-names))
|
||||
(js/process.exit 1))
|
||||
(println "Generating" (count (filter :block/name init-tx)) "pages and"
|
||||
(count (filter :block/title init-tx)) "blocks ...")
|
||||
(d/transact! conn init-tx)
|
||||
(d/transact! conn block-props-tx)
|
||||
(println "Created graph" (str db-name " with " (count (d/datoms @conn :eavt)) " datoms!"))))
|
||||
|
||||
(when (= nbb/*file* (nbb/invoked-file))
|
||||
(-main *command-line-args*))
|
||||
@@ -0,0 +1,366 @@
|
||||
(ns logseq.tasks.db-graph.create-graph-with-schema-org
|
||||
"Script that converts the jsonld version of schema.org into Logseq classes and
|
||||
properties. Initially works with 900 classes and 1368 properties! The script
|
||||
currently provides the following in a Logseq graph:
|
||||
* All schema.org classes with their name, url, parent class (namespace) and properties
|
||||
* Some classes are renamed due to naming conflicts
|
||||
* All properties with their property type, url, description
|
||||
* Property type is determined by looking for the first range value that is
|
||||
a subclass of https://schema.org/DataType and then falling back to :node.
|
||||
* Some properties are skipped because they are superseded/deprecated or because they have a property
|
||||
type logseq doesnt' support yet
|
||||
* schema.org assumes no cardinality. For now, only :node properties are given a :cardinality :many"
|
||||
(:require ["fs" :as fs]
|
||||
[babashka.cli :as cli]
|
||||
[clojure.edn :as edn]
|
||||
[clojure.pprint :as pprint]
|
||||
[clojure.set :as set]
|
||||
[clojure.string :as string]
|
||||
[clojure.walk :as w]
|
||||
[datascript.core :as d]
|
||||
[logseq.db.common.sqlite-cli :as sqlite-cli]
|
||||
[logseq.db.sqlite.export :as sqlite-export]
|
||||
[logseq.outliner.cli :as outliner-cli]
|
||||
[nbb.classpath :as cp]
|
||||
[nbb.core :as nbb]))
|
||||
|
||||
(defn- get-comment-string
|
||||
[rdfs-comment renamed-pages]
|
||||
(let [desc* (if (map? rdfs-comment)
|
||||
(get rdfs-comment "@value")
|
||||
rdfs-comment)
|
||||
;; Update refs to renamed classes
|
||||
regex (re-pattern (str "\\[\\[(" (string/join "|" (keys renamed-pages)) ")\\]\\]"))
|
||||
desc (string/replace desc* regex #(str "[[" (get renamed-pages (second %)) "]]"))]
|
||||
;; Fix markdown and html links to schema website docs
|
||||
(string/replace desc #"(\(|\")/docs" "$1https://schema.org/docs")))
|
||||
|
||||
(defn- strip-schema-prefix [s]
|
||||
(string/replace-first s "schema:" ""))
|
||||
|
||||
(defn- ->class-page [class-m class-properties {:keys [renamed-classes renamed-pages]}]
|
||||
(let [parent-classes* (class-m "rdfs:subClassOf")
|
||||
parent-classes (cond
|
||||
(map? parent-classes*)
|
||||
[(parent-classes* "@id")]
|
||||
(vector? parent-classes*)
|
||||
(mapv #(get % "@id") parent-classes*)
|
||||
;; DataTypes are weird in that they are subclassed from
|
||||
;; rdfs:class but that info is omitted. It seems schema
|
||||
;; does this on purpose to display it as a separate tree but
|
||||
;; we want all classes under one tree
|
||||
(contains? (set (as-> (class-m "@type") type'
|
||||
(if (string? type') [type'] type')))
|
||||
"schema:DataType")
|
||||
["schema:DataType"])
|
||||
;; Map of owl:equivalentClass exceptions
|
||||
parent-classes' (mapv #(get {"rdfs:Class" "Class"} % %) parent-classes)
|
||||
properties (class-properties (class-m "@id"))
|
||||
inverted-renamed-classes (set/map-invert renamed-classes)
|
||||
class-name (strip-schema-prefix (class-m "@id"))
|
||||
url (str "https://schema.org/" (get inverted-renamed-classes class-name class-name))]
|
||||
(cond-> {:block/title class-name
|
||||
:build/properties (cond-> {:url url}
|
||||
(class-m "rdfs:comment")
|
||||
(assoc :logseq.property/description (get-comment-string (class-m "rdfs:comment") renamed-pages)))}
|
||||
(seq parent-classes')
|
||||
(assoc :build/class-extends (mapv #(keyword (strip-schema-prefix %)) parent-classes'))
|
||||
(seq properties)
|
||||
(assoc :build/class-properties (mapv (comp keyword strip-schema-prefix) properties)))))
|
||||
|
||||
(def schema->logseq-data-types
|
||||
"Schema datatypes, https://schema.org/DataType, mapped to their Logseq equivalents"
|
||||
{"schema:Integer" :number
|
||||
"schema:Float" :number
|
||||
"schema:Number" :number
|
||||
"schema:Text" :default
|
||||
"schema:URL" :url
|
||||
"schema:Boolean" :checkbox
|
||||
"schema:Date" :date
|
||||
"schema:DateTime" :datetime})
|
||||
|
||||
(def unsupported-data-types
|
||||
"Schema datatypes, https://schema.org/DataType, that don't have Logseq equivalents"
|
||||
#{"schema:Time"})
|
||||
|
||||
(defn- get-range-includes [property-m]
|
||||
(let [range-includes (as-> (property-m "schema:rangeIncludes") range-includes*
|
||||
(map (fn [m] (m "@id"))
|
||||
(if (map? range-includes*) [range-includes*] range-includes*)))]
|
||||
;; Prioritize (sort first) DataType subclasses because they are easier to enter in the app
|
||||
(sort-by #(if (schema->logseq-data-types %) -1 1) range-includes)))
|
||||
|
||||
(defn- get-schema-type [range-includes class-map]
|
||||
(some #(or (schema->logseq-data-types %)
|
||||
(when (class-map %) :node))
|
||||
range-includes))
|
||||
|
||||
(defn- ->property-page [property-m class-map {:keys [verbose renamed-pages]}]
|
||||
(let [range-includes (get-range-includes property-m)
|
||||
schema-type (get-schema-type range-includes class-map)
|
||||
;; Notify when a property is only using one of the available property types (rangeIncludes)
|
||||
;; Doesn't apply to :node since they do include all rangeIncludes
|
||||
_ (when (and verbose (> (count range-includes) 1) (not= schema-type :node))
|
||||
(println "Picked property type:"
|
||||
{:property (property-m "@id") :type schema-type :range-includes (vec range-includes)}))
|
||||
_ (assert schema-type (str "No schema found for property " (property-m "@id")))
|
||||
_ (when (= schema-type :node)
|
||||
(when-let [datatype-classes (not-empty (set/intersection (set range-includes)
|
||||
(set (keys schema->logseq-data-types))))]
|
||||
(throw (ex-info (str "property " (pr-str (property-m "@id"))
|
||||
" with type :node has DataType class values which aren't supported: " datatype-classes) {}))))
|
||||
|
||||
url (str "https://schema.org/" (strip-schema-prefix (property-m "@id")))]
|
||||
{(keyword (strip-schema-prefix (property-m "@id")))
|
||||
(cond-> {:logseq.property/type schema-type
|
||||
:build/properties (cond-> {:url url}
|
||||
(property-m "rdfs:comment")
|
||||
(assoc :logseq.property/description (get-comment-string (property-m "rdfs:comment") renamed-pages)))}
|
||||
;; This cardinality rule should be adjusted as we use schema.org more
|
||||
(= schema-type :node)
|
||||
(assoc :db/cardinality :many)
|
||||
(= schema-type :node)
|
||||
(assoc :build/property-classes (mapv (comp keyword strip-schema-prefix) range-includes)))}))
|
||||
|
||||
(defn- get-class-to-properties
|
||||
"Given a vec of class ids and a vec of properties map to process, return a map of
|
||||
class ids to their property ids"
|
||||
[all-classes all-properties]
|
||||
(let [;; build map of class id to all their properties
|
||||
all-class-properties (reduce
|
||||
(fn [acc prop]
|
||||
(let [domains* (prop "schema:domainIncludes")
|
||||
domains (if (map? domains*) [domains*] domains*)]
|
||||
(merge-with into
|
||||
acc
|
||||
(->> domains
|
||||
(map (fn [class] (vector (class "@id") [(prop "@id")])))
|
||||
(into {})))))
|
||||
{}
|
||||
all-properties)]
|
||||
(->> all-classes
|
||||
(map #(vector % (vec (all-class-properties %))))
|
||||
(into {}))))
|
||||
|
||||
(defn property-with-unsupported-type? [prop]
|
||||
(let [range-includes
|
||||
(as-> (prop "schema:rangeIncludes") range-includes*
|
||||
(set (map (fn [m] (m "@id")) (if (map? range-includes*) [range-includes*] range-includes*))))
|
||||
unsupported-data-types'
|
||||
(set/intersection range-includes unsupported-data-types)]
|
||||
(and (seq range-includes)
|
||||
(every? (fn [x] (contains? unsupported-data-types' x)) range-includes))))
|
||||
|
||||
(defn- get-vector-conflicts
|
||||
"Given a seq of tuples returns a seq of tuples that conflict i.e. their first element
|
||||
has a case sensitive conflict/duplicate with another. An example conflict:
|
||||
[[\"schema:status\" :property] [\"schema:status\" :node]]"
|
||||
[tuples-seq]
|
||||
(->> tuples-seq
|
||||
(group-by first)
|
||||
(filter #(> (count (val %)) 1))
|
||||
vals))
|
||||
|
||||
(defn- detect-final-conflicts
|
||||
"Does one final detection for conflicts after everything has been renamed"
|
||||
[all-classes page-tuples]
|
||||
(let [class-ids (map #(vector (% "@id") :class) all-classes)
|
||||
existing-conflicts (get-vector-conflicts (concat class-ids page-tuples))]
|
||||
(when (seq existing-conflicts) (prn :CONFLICTS existing-conflicts))
|
||||
(assert (empty? existing-conflicts)
|
||||
"There are no conflicts between existing pages, schema classes and properties")))
|
||||
|
||||
;; NOTE: There are currently no class conflicts but if some come up this fn can be updated to exclude built-in properties
|
||||
(defn- detect-id-conflicts-and-get-renamed-classes
|
||||
"Detects conflicts between classes AND properties and existing
|
||||
pages. Renames any detected conflicts. Properties and class names conflict in
|
||||
Logseq because schema.org names are case sensitive whereas Logseq's
|
||||
:block/name is case insensitive. This is dealt with by appending a '_Class'
|
||||
suffix to conflicting classes. If this strategy changes, be sure to update
|
||||
schema->logseq-data-types"
|
||||
[property-ids class-ids existing-pages {:keys [verbose]}]
|
||||
(let [conflicts (get-vector-conflicts (concat property-ids class-ids))
|
||||
;; If this assertion fails then renamed-classes approach to resolving
|
||||
;; conflicts may need to be revisited
|
||||
_ (assert (every? #(= 2 (count %)) conflicts) "All conflicts must only be between two elements")
|
||||
existing-conflicts (get-vector-conflicts (concat class-ids existing-pages))
|
||||
_ (when (seq existing-conflicts) (prn :EXISTING-CLASS-CONFLICTS existing-conflicts))
|
||||
;; Add existing-conflicts to conflicts if this ever fails
|
||||
_ (assert (empty? existing-conflicts)
|
||||
"There are no conflicts between existing pages and schema classes and properties")
|
||||
renamed-classes (->> conflicts
|
||||
(map #(-> % second first))
|
||||
;; Renaming classes with '_Class' suffix guarantees uniqueness
|
||||
;; b/c schema.org doesn't use '_' in their names
|
||||
(map #(vector % (str % "_Class")))
|
||||
(into {}))]
|
||||
(when (seq renamed-classes)
|
||||
(if verbose
|
||||
(println "Renaming the following classes because they have names that conflict with Logseq's case insensitive :block/name:"
|
||||
(keys renamed-classes) "\n")
|
||||
(when (pos? (count renamed-classes))
|
||||
(println "Renaming" (count renamed-classes) "classes due to page name conflicts"))))
|
||||
renamed-classes))
|
||||
|
||||
(defn- get-all-properties [schema-data {:keys [verbose]}]
|
||||
(let [all-properties** (filter #(= "rdf:Property" (% "@type")) schema-data)
|
||||
[superseded-properties all-properties*] ((juxt filter remove) #(% "schema:supersededBy") all-properties**)
|
||||
_ (if verbose
|
||||
(println "Skipping the following superseded properties:" (mapv #(% "@id") superseded-properties) "\n")
|
||||
(println "Skipping" (count superseded-properties) "superseded properties"))
|
||||
[unsupported-properties all-properties] ((juxt filter remove) property-with-unsupported-type? all-properties*)
|
||||
_ (if verbose
|
||||
(println "Skipping the following unsupported properties:" (mapv #(% "@id") unsupported-properties) "\n")
|
||||
(println "Skipping" (count unsupported-properties) "properties with unsupported data types"))]
|
||||
all-properties))
|
||||
|
||||
(defn- generate-classes
|
||||
[select-classes class-to-properties options]
|
||||
(let [classes (->> select-classes
|
||||
(map #(vector (keyword (strip-schema-prefix (get % "@id")))
|
||||
(->class-page % class-to-properties options)))
|
||||
(into {}))]
|
||||
(assert (= ["Thing"] (keep #(when-not (:build/class-extends %)
|
||||
(:block/title %))
|
||||
(vals classes)))
|
||||
"Thing is the only class that doesn't have a schema.org parent class")
|
||||
classes))
|
||||
|
||||
(defn- generate-properties
|
||||
[select-properties class-map options]
|
||||
(when (:verbose options)
|
||||
(println "Properties by type:"
|
||||
(->> select-properties
|
||||
(mapv (fn [property-m]
|
||||
(get-schema-type (get-range-includes property-m) class-map)))
|
||||
frequencies)
|
||||
"\n"))
|
||||
(assoc
|
||||
(apply merge
|
||||
(mapv #(->property-page % class-map options) select-properties))
|
||||
;; Have to update schema for now as validation doesn't take into account existing properties
|
||||
:logseq.property/description {:logseq.property/public? true
|
||||
:logseq.property/type :default
|
||||
:build/properties {:url "https://schema.org/description"
|
||||
:logseq.property/description "A description of the item."}}))
|
||||
|
||||
(defn- get-all-classes-and-properties
|
||||
"Get all classes and properties from raw json file"
|
||||
[schema-data existing-pages options]
|
||||
(let [;; TODO: See if it's worth pulling in non-types like schema:MusicReleaseFormatType
|
||||
all-classes* (filter #(contains? (set (as-> (% "@type") type'
|
||||
(if (string? type') [type'] type')))
|
||||
"rdfs:Class")
|
||||
schema-data)
|
||||
;; Use built-in description
|
||||
all-properties (remove #(= "schema:description" (% "@id")) (get-all-properties schema-data options))
|
||||
property-tuples (map #(vector (% "@id") :property) all-properties)
|
||||
class-tuples (map #(vector (% "@id") :class) all-classes*)
|
||||
page-tuples (map #(vector (str "schema:" %) :node) existing-pages)
|
||||
renamed-classes (detect-id-conflicts-and-get-renamed-classes
|
||||
property-tuples class-tuples page-tuples options)
|
||||
;; Note: schema:description refs don't get renamed but they aren't used
|
||||
;; Updates keys like @id, @subClassOf
|
||||
rename-class-ids (fn [m]
|
||||
(w/postwalk (fn [x]
|
||||
(if-let [new-page (and (map? x) (renamed-classes (x "@id")))]
|
||||
(merge x {"@id" new-page})
|
||||
x)) m))
|
||||
;; Updates keys like @id, @rangeIncludes, @domainIncludes
|
||||
all-classes (map rename-class-ids all-classes*)]
|
||||
(detect-final-conflicts all-classes page-tuples)
|
||||
{:all-classes all-classes
|
||||
:all-properties all-properties
|
||||
:renamed-classes (->> renamed-classes
|
||||
(map (fn [[k v]] [(strip-schema-prefix k) (strip-schema-prefix v)]))
|
||||
(into {}))}))
|
||||
|
||||
(defn- create-init-data [existing-pages options]
|
||||
(let [schema-data (-> (str (fs/readFileSync "resources/schemaorg-current-https.json"))
|
||||
js/JSON.parse
|
||||
(js->clj)
|
||||
(get "@graph"))
|
||||
{:keys [all-classes all-properties renamed-classes]}
|
||||
(get-all-classes-and-properties schema-data existing-pages options)
|
||||
;; Generate data shared across pages and properties
|
||||
class-map (->> all-classes
|
||||
(map #(vector (% "@id") %))
|
||||
(into {}))
|
||||
select-class-ids
|
||||
(if (:subset options)
|
||||
["schema:Person" "schema:CreativeWorkSeries" "schema:Organization"
|
||||
"schema:Intangible" "schema:Series" "schema:TVSeries" ; Test class multiple inheritance
|
||||
"schema:Movie" "schema:CreativeWork" "schema:Thing" "schema:Comment"]
|
||||
(keys class-map))
|
||||
class-to-properties (get-class-to-properties select-class-ids all-properties)
|
||||
select-properties (set (mapcat val class-to-properties))
|
||||
options' (assoc options
|
||||
:renamed-classes renamed-classes
|
||||
:renamed-pages renamed-classes)
|
||||
;; Generate pages and properties
|
||||
properties (generate-properties
|
||||
(filter #(contains? select-properties (% "@id")) all-properties)
|
||||
class-map options')
|
||||
properties'
|
||||
(if (:subset options)
|
||||
;; only keep classes that are in subset to keep graph valid
|
||||
(let [select-class-ids' (->> select-class-ids (map (comp keyword strip-schema-prefix)) set)]
|
||||
(-> properties
|
||||
(update-vals (fn [m]
|
||||
(if (:build/property-classes m)
|
||||
(update m :build/property-classes
|
||||
(fn [cs] (vec (set (filter #(contains? select-class-ids' %) cs)))))
|
||||
m)))))
|
||||
properties)
|
||||
classes (generate-classes
|
||||
(map #(class-map %) select-class-ids)
|
||||
class-to-properties options')]
|
||||
{:graph-namespace :schema
|
||||
:classes classes
|
||||
:properties properties'}))
|
||||
|
||||
(def spec
|
||||
"Options spec"
|
||||
{:help {:alias :h
|
||||
:desc "Print help"}
|
||||
:config {:alias :c
|
||||
:coerce edn/read-string
|
||||
:desc "EDN map to add to config.edn"}
|
||||
:export {:alias :e
|
||||
:desc "Exports graph to schema.edn"}
|
||||
:subset {:alias :s
|
||||
:desc "Only generate a subset of data for testing purposes"}
|
||||
:verbose {:alias :v
|
||||
:desc "Verbose mode"}})
|
||||
|
||||
(defn- write-export-file [db]
|
||||
(let [export-map (sqlite-export/build-export db {:export-type :graph-ontology})]
|
||||
(fs/writeFileSync "schema.edn"
|
||||
(with-out-str (pprint/pprint export-map)))))
|
||||
|
||||
(defn -main [args]
|
||||
(let [[graph-dir] args
|
||||
options (cli/parse-opts args {:spec spec})
|
||||
_ (when (or (nil? graph-dir) (:help options))
|
||||
(println (str "Usage: $0 GRAPH-NAME [OPTIONS]\nOptions:\n"
|
||||
(cli/format-opts {:spec spec})))
|
||||
(js/process.exit 1))
|
||||
init-conn-args (sqlite-cli/->open-db-args graph-dir)
|
||||
db-name (if (= 1 (count init-conn-args)) (first init-conn-args) (second init-conn-args))
|
||||
conn (apply outliner-cli/init-conn
|
||||
(conj init-conn-args {:additional-config (:config options)
|
||||
:classpath (cp/get-classpath)}))
|
||||
init-data (create-init-data (d/q '[:find [?name ...] :where [?b :block/name ?name]] @conn)
|
||||
options)
|
||||
{:keys [init-tx block-props-tx]} (outliner-cli/build-blocks-tx init-data)]
|
||||
(println "Generating" (str (count (filter :block/name init-tx)) " pages with "
|
||||
(count (:classes init-data)) " tags and "
|
||||
(count (:properties init-data)) " properties ..."))
|
||||
(d/transact! conn init-tx)
|
||||
(d/transact! conn block-props-tx)
|
||||
(when (:verbose options) (println "Transacted" (count (d/datoms @conn :eavt)) "datoms"))
|
||||
(when (:export options) (write-export-file @conn))
|
||||
(println "Created graph" (str db-name "!"))))
|
||||
|
||||
(when (= nbb/*file* (nbb/invoked-file))
|
||||
(-main *command-line-args*))
|
||||
251
scripts/src/logseq/tasks/dev.clj
Normal file
251
scripts/src/logseq/tasks/dev.clj
Normal file
@@ -0,0 +1,251 @@
|
||||
(ns logseq.tasks.dev
|
||||
"Tasks for general development. For desktop or mobile development see their
|
||||
namespaces"
|
||||
(:refer-clojure :exclude [test])
|
||||
(:require [babashka.cli :as cli]
|
||||
[babashka.fs :as fs]
|
||||
[babashka.process :refer [shell]]
|
||||
[babashka.tasks :refer [clojure]]
|
||||
[clojure.data :as data]
|
||||
[clojure.edn :as edn]
|
||||
[clojure.java.io :as io]
|
||||
[clojure.pprint :as pp]
|
||||
[clojure.string :as string]
|
||||
[logseq.tasks.dev.lint :as dev-lint]
|
||||
[logseq.tasks.util :as task-util]))
|
||||
|
||||
(defn run-shell
|
||||
[& args]
|
||||
(apply shell args))
|
||||
|
||||
(defn test
|
||||
"Run tests. Pass args through to cmd 'pnpm cljs:run-test'"
|
||||
[& args]
|
||||
(shell {:shutdown nil} "pnpm cljs:test")
|
||||
(apply shell {:shutdown nil} "pnpm cljs:run-test" args))
|
||||
|
||||
(defn test-no-worker
|
||||
"Run tests without compiling worker namespaces. Pass args through to cmd 'pnpm cljs:run-test-no-worker'"
|
||||
[& args]
|
||||
(shell "pnpm cljs:test-no-worker")
|
||||
(apply shell "pnpm cljs:run-test-no-worker" args))
|
||||
|
||||
(def test-jobs (min 5 (.availableProcessors (Runtime/getRuntime))))
|
||||
(def test-batches-per-job 6)
|
||||
|
||||
(def default-test-args ["-r" "^(?!logseq.db-sync.).*"])
|
||||
|
||||
(def isolated-test-namespaces
|
||||
#{"frontend.components.block.drop-boundary-test"
|
||||
"frontend.handler.db-based.page-test"
|
||||
"frontend.handler.editor-async-test"
|
||||
"frontend.handler.editor-lifecycle-test"
|
||||
"frontend.handler.editor-test"
|
||||
"frontend.handler.route-test"
|
||||
"frontend.rfx-test"
|
||||
"frontend.worker.db-core-test"})
|
||||
|
||||
(def serial-test-namespaces
|
||||
#{"frontend.db.query-dsl-test"
|
||||
"frontend.worker.search-test"})
|
||||
|
||||
(defn isolated-test-namespace?
|
||||
[test-ns]
|
||||
(contains? isolated-test-namespaces test-ns))
|
||||
|
||||
(defn- run-parallel!
|
||||
([tasks]
|
||||
(run-parallel! (count tasks) tasks))
|
||||
([jobs tasks]
|
||||
(let [tasks (vec tasks)]
|
||||
(when (seq tasks)
|
||||
(let [task-queue (atom (seq tasks))
|
||||
next-task! (fn []
|
||||
(loop []
|
||||
(let [tasks @task-queue]
|
||||
(when (seq tasks)
|
||||
(if (compare-and-set! task-queue tasks (rest tasks))
|
||||
(first tasks)
|
||||
(recur))))))
|
||||
worker (fn []
|
||||
(loop [error nil]
|
||||
(if-let [task (next-task!)]
|
||||
(recur (try
|
||||
(task)
|
||||
error
|
||||
(catch Throwable e
|
||||
(or error e))))
|
||||
error)))
|
||||
results (->> #(future (worker))
|
||||
(repeatedly (min jobs (count tasks)))
|
||||
doall)]
|
||||
(when-let [error (some deref results)]
|
||||
(throw error)))))))
|
||||
|
||||
(defn- selected-test-namespaces
|
||||
[args]
|
||||
(->> (:out (apply run-shell {:out :string
|
||||
:shutdown nil
|
||||
:extra-env {"LOGSEQ_STABLE_IDENTS" "1"}}
|
||||
"node" "static/tests.js" "--list-namespaces" args))
|
||||
string/split-lines
|
||||
(remove string/blank?)
|
||||
vec))
|
||||
|
||||
(defn- round-robin-batches
|
||||
[items batch-count]
|
||||
(when (pos? batch-count)
|
||||
(->> items
|
||||
(map-indexed (fn [idx item]
|
||||
[(mod idx batch-count) item]))
|
||||
(group-by first)
|
||||
(sort-by first)
|
||||
(mapv (fn [[_ indexed-items]]
|
||||
(mapv second indexed-items))))))
|
||||
|
||||
(defn- namespace-batches
|
||||
[test-namespaces]
|
||||
(let [priority-namespaces ["frontend.handler.editor-test"
|
||||
"frontend.worker.db-core-test"
|
||||
"frontend.handler.editor-async-test"]
|
||||
priority-set (set priority-namespaces)
|
||||
priority (->> priority-namespaces
|
||||
(filter (set test-namespaces))
|
||||
(mapv vector))
|
||||
isolated (->> test-namespaces
|
||||
(filter isolated-test-namespace?)
|
||||
(remove priority-set)
|
||||
(mapv vector))
|
||||
shared (remove isolated-test-namespace? test-namespaces)
|
||||
batch-count (min (* test-jobs test-batches-per-job) (count shared))
|
||||
batches (round-robin-batches shared batch-count)]
|
||||
(into priority
|
||||
(loop [batches (seq batches)
|
||||
isolated (seq isolated)
|
||||
result []]
|
||||
(if (or batches isolated)
|
||||
(recur (next batches)
|
||||
(next isolated)
|
||||
(cond-> result
|
||||
batches (conj (first batches))
|
||||
isolated (conj (first isolated))))
|
||||
result)))))
|
||||
|
||||
(defn- namespace-args
|
||||
[test-namespaces]
|
||||
(mapcat (fn [test-ns] ["-n" test-ns]) test-namespaces))
|
||||
|
||||
(defn run-test-namespaces
|
||||
"Run compiled tests in isolated Node processes, in parallel by namespace."
|
||||
[& args]
|
||||
(let [test-namespaces (selected-test-namespaces args)
|
||||
parallel-namespaces (remove serial-test-namespaces test-namespaces)
|
||||
serial-namespaces (filter serial-test-namespaces test-namespaces)
|
||||
run-namespaces! (fn [namespaces]
|
||||
(apply run-shell {:shutdown nil
|
||||
:extra-env {"LOGSEQ_STABLE_IDENTS" "1"}}
|
||||
"node" "static/tests.js"
|
||||
(concat (namespace-args namespaces) args)))]
|
||||
(run-parallel! test-jobs
|
||||
(mapv (fn [test-namespaces]
|
||||
#(run-namespaces! test-namespaces))
|
||||
(namespace-batches parallel-namespaces)))
|
||||
(doseq [test-ns serial-namespaces]
|
||||
(run-namespaces! [test-ns]))))
|
||||
|
||||
(defn parallel-test
|
||||
"Compile tests, then run them in parallel by namespace."
|
||||
[& args]
|
||||
(run-shell {:shutdown nil} "pnpm cljs:test")
|
||||
(apply run-test-namespaces (concat default-test-args args)))
|
||||
|
||||
(defn lint-and-test
|
||||
"Run all lint tasks and tests excluding testcases tagged by :long and :fix-me."
|
||||
[]
|
||||
(run-parallel! [dev-lint/dev
|
||||
#(parallel-test "-e" "long" "-e" "fix-me")]))
|
||||
|
||||
(defn e2e-basic-test
|
||||
"Run e2e basic tests. HTTP server should be available at localhost:3001"
|
||||
[& _]
|
||||
(clojure {:dir "clj-e2e"} "-X:dev-run-all-basic-test"))
|
||||
|
||||
(defn e2e-rtc-extra-test
|
||||
"Run e2e rtc extra tests. HTTP server should be available at localhost:3001"
|
||||
[& _]
|
||||
(clojure {:dir "clj-e2e"} "-X:dev-run-rtc-extra-test"))
|
||||
|
||||
(defn gen-malli-kondo-config
|
||||
"Generate clj-kondo type-mismatch config from malli schema
|
||||
.clj-kondo/metosin/malli-types/config.edn"
|
||||
[]
|
||||
(let [config-edn ".clj-kondo/metosin/malli-types/config.edn"
|
||||
compile-cmd "clojure -M:cljs compile gen-malli-kondo-config"]
|
||||
(println compile-cmd)
|
||||
(shell {:shutdown nil} compile-cmd)
|
||||
(println "generate kondo config: " config-edn)
|
||||
(io/make-parents config-edn)
|
||||
(let [config (with-out-str
|
||||
(pp/pprint (edn/read-string (:out (shell {:out :string :shutdown nil}
|
||||
"node ./static/gen-malli-kondo-config.js")))))]
|
||||
(spit config-edn config))))
|
||||
|
||||
(defn diff-datoms
|
||||
"Runs data/diff on two edn files written by dev:datoms"
|
||||
[file1 file2 & args]
|
||||
(let [spec {:ignored-attributes
|
||||
;; Ignores some attributes by default that are expected to change often
|
||||
{:alias :i :coerce #{:keyword} :default #{:block/tx-id :block/order :block/updated-at}}}
|
||||
{{:keys [ignored-attributes]} :opts} (cli/parse-args args {:spec spec})
|
||||
datom-filter (fn [[_e a _ _ _]] (contains? ignored-attributes a))
|
||||
data-diff* (apply data/diff (map (fn [x] (->> x slurp edn/read-string (remove datom-filter))) [file1 file2]))
|
||||
data-diff (->> data-diff*
|
||||
;; Drop common as we're only interested in differences
|
||||
drop-last
|
||||
;; Remove nils as we're only interested in diffs
|
||||
(mapv #(vec (remove nil? %))))]
|
||||
(pp/pprint data-diff)))
|
||||
|
||||
(defn build-publishing-frontend
|
||||
"Builds frontend release publishing asset when files have changed"
|
||||
[& _args]
|
||||
(if-let [_files (and (not (System/getenv "SKIP_ASSET"))
|
||||
(seq (set (fs/modified-since (fs/file "static/js/publishing/main.js")
|
||||
(fs/glob "." "{src/main,deps/graph-parser/src}/**")))))]
|
||||
(do
|
||||
(println "Building publishing js asset...")
|
||||
(shell {:shutdown nil} "clojure -M:cljs release publishing db-worker"))
|
||||
(println "Publishing js asset is up to date")))
|
||||
|
||||
(defn publishing-backend
|
||||
"Builds publishing backend and copies over supporting frontend assets"
|
||||
[& args]
|
||||
(apply shell {:dir "deps/publishing" :extra-env {"ORIGINAL_PWD" (fs/cwd)} :shutdown nil}
|
||||
"pnpm exec nbb-logseq -cp src:../graph-parser/src script/publishing.cljs"
|
||||
(into ["static"] args)))
|
||||
|
||||
(defn watch-publishing-frontend
|
||||
[& _args]
|
||||
(shell {:shutdown nil} "pnpm exec shadow-cljs watch publishing"))
|
||||
|
||||
(defn watch-publishing-backend
|
||||
"Builds publishing backend once watch-publishing-frontend has built initial frontend"
|
||||
[& args]
|
||||
(let [start-time (java.time.Instant/now)]
|
||||
(Thread/sleep 3000)
|
||||
(loop [n 1000]
|
||||
(if (and (fs/exists? "static/js/publishing/main.js")
|
||||
(task-util/file-modified-later-than? "static/js/publishing/main.js" start-time))
|
||||
(apply publishing-backend args)
|
||||
(do (println "Waiting for publishing frontend to build...")
|
||||
(Thread/sleep 1000)
|
||||
(recur (inc n)))))))
|
||||
|
||||
(defn db-import-many
|
||||
[& args]
|
||||
(let [parent-graph-dir "./out"
|
||||
[file-graphs import-options] (split-with #(not (string/starts-with? % "-")) args)]
|
||||
(doseq [file-graph file-graphs]
|
||||
(let [db-graph (fs/path parent-graph-dir (fs/file-name file-graph))]
|
||||
(println "Importing" (str db-graph) "...")
|
||||
(apply shell {:shutdown nil} "bb" "dev:db-import" file-graph db-graph (concat import-options ["--validate"]))))))
|
||||
25
scripts/src/logseq/tasks/dev/db_sync.clj
Normal file
25
scripts/src/logseq/tasks/dev/db_sync.clj
Normal file
@@ -0,0 +1,25 @@
|
||||
(ns logseq.tasks.dev.db-sync
|
||||
"Tasks for db-sync dev processes"
|
||||
(:require [babashka.process :refer [process]]))
|
||||
|
||||
(def processes
|
||||
[{:name "db-sync-watch"
|
||||
:dir "deps/db-sync"
|
||||
:cmd "clojure -M:cljs watch db-sync"}
|
||||
{:name "wrangler-dev"
|
||||
:dir "deps/db-sync/worker"
|
||||
:cmd "wrangler dev"}
|
||||
{:name "pnpm-watch"
|
||||
:dir "."
|
||||
:cmd "ENABLE_DB_SYNC_LOCAL=true pnpm watch"}])
|
||||
|
||||
(defn start
|
||||
[]
|
||||
(println "Starting db-sync processes in foreground.")
|
||||
(println "Use Ctrl-C to stop.")
|
||||
(let [procs (mapv (fn [{:keys [name dir cmd]}]
|
||||
(println "Running:" name "-" cmd)
|
||||
(process ["bash" "-lc" cmd] {:dir dir :inherit true}))
|
||||
processes)]
|
||||
(doseq [proc procs]
|
||||
@proc)))
|
||||
21
scripts/src/logseq/tasks/dev/desktop.clj
Normal file
21
scripts/src/logseq/tasks/dev/desktop.clj
Normal file
@@ -0,0 +1,21 @@
|
||||
(ns logseq.tasks.dev.desktop
|
||||
"Tasks for desktop (electron) development"
|
||||
(:require [babashka.tasks :refer [shell]]
|
||||
[babashka.fs :as fs]
|
||||
[logseq.tasks.util :as task-util]))
|
||||
|
||||
(defn watch
|
||||
"Watches environment to reload cljs, css and other assets"
|
||||
[]
|
||||
(shell {:shutdown nil} "pnpm electron-watch"))
|
||||
|
||||
(defn open-dev-electron-app
|
||||
"Opens dev-electron-app when watch process has built main.js"
|
||||
[]
|
||||
(let [start-time (java.time.Instant/now)]
|
||||
(dotimes [_n 1000]
|
||||
(if (and (fs/exists? "static/js/main.js")
|
||||
(task-util/file-modified-later-than? "static/js/main.js" start-time))
|
||||
(shell {:shutdown nil} "pnpm dev-electron-app")
|
||||
(println "Waiting for app to build..."))
|
||||
(Thread/sleep 1000))))
|
||||
93
scripts/src/logseq/tasks/dev/lint.clj
Normal file
93
scripts/src/logseq/tasks/dev/lint.clj
Normal file
@@ -0,0 +1,93 @@
|
||||
(ns logseq.tasks.dev.lint
|
||||
(:require [babashka.process :refer [shell]]
|
||||
[clojure.string :as string]))
|
||||
|
||||
(defn dev
|
||||
"Run all lint tasks
|
||||
- clj-kondo lint
|
||||
- carve lint for unused vars
|
||||
- lint for vars that are too large
|
||||
- lint invalid translation entries
|
||||
- lint to ensure file and db graph remain separate
|
||||
- lint to ensure worker and frontend are separate"
|
||||
[]
|
||||
(doseq [cmd ["clojure -M:clj-kondo --parallel --lint src --cache false"
|
||||
"bb lint:carve"
|
||||
"bb lint:large-vars"
|
||||
"bb lint:worker-and-frontend-separate"
|
||||
"bb lang:validate-translations"
|
||||
"bb lint:ns-docstrings"]]
|
||||
(println cmd)
|
||||
(shell {:shutdown nil} cmd)))
|
||||
|
||||
(defn kondo-git-changes
|
||||
"Run clj-kondo across dirs and only for files that git diff detects as unstaged changes"
|
||||
[]
|
||||
(let [kondo-dirs ["src" "deps/common" "deps/db" "deps/graph-parser" "deps/outliner" "deps/publishing" "deps/publish"]
|
||||
dir-regex (re-pattern (str "^(" (string/join "|" kondo-dirs) ")"))
|
||||
dir-to-files (->> (shell {:out :string :shutdown nil} "git diff --name-only")
|
||||
:out
|
||||
string/split-lines
|
||||
(filter #(re-find #"\.(cljs|clj|cljc)$" %))
|
||||
(group-by #(first (re-find dir-regex %)))
|
||||
;; remove files that aren't in a kondo dir
|
||||
((fn [x] (dissoc x nil))))]
|
||||
(if (seq dir-to-files)
|
||||
(doseq [[dir* files*] dir-to-files]
|
||||
(let [dir (if (= dir* "src") "." dir*)
|
||||
files (mapv #(string/replace-first % (str dir "/") "") files*)
|
||||
cmd (str "cd " dir " && clj-kondo --lint " (string/join " " files))
|
||||
_ (println cmd)
|
||||
res (apply shell {:dir dir :continue :true :shutdown nil} "clj-kondo --lint" files)]
|
||||
(when (pos? (:exit res)) (System/exit (:exit res)))))
|
||||
(println "No clj* files have changed to lint."))))
|
||||
|
||||
(defn- validate-frontend-not-in-workers
|
||||
[]
|
||||
(let [res (shell {:out :string :shutdown nil}
|
||||
"git grep -h" "\\[frontend.*:as"
|
||||
"src/main/frontend/worker" "src/main/frontend/worker_common")
|
||||
allowed-export-requires #{" [frontend.handler.export.common-impl :as common-impl]"
|
||||
" [frontend.handler.export.html :as export-html]"
|
||||
" [frontend.handler.export.opml :as export-opml]"
|
||||
" [frontend.handler.export.text-impl :as export-text]"}
|
||||
req-lines (->> (:out res)
|
||||
string/split-lines
|
||||
(remove #(re-find #"frontend\.worker|frontend\.common" %))
|
||||
(remove allowed-export-requires))]
|
||||
|
||||
(if (seq req-lines)
|
||||
(do
|
||||
(println "The following frontend requires should not be in worker namespaces:")
|
||||
(println (string/join "\n" req-lines))
|
||||
(System/exit 1))
|
||||
(println "Valid worker namespaces!"))))
|
||||
|
||||
(defn- validate-workers-not-in-frontend
|
||||
[]
|
||||
(let [res (shell {:out :string :continue true :shutdown nil}
|
||||
"git grep --untracked --exclude-standard"
|
||||
"\\[frontend.worker.*:" "--" "src/main/frontend")
|
||||
;; allow reset-file b/c it's only affects tests
|
||||
allowed-exceptions #{"src/main/frontend/handler/file_based/file.cljs: [frontend.worker.file.reset :as file-reset]"}
|
||||
excluded-path-prefixes ["src/main/frontend/worker/"]
|
||||
invalid-lines (when (= 0 (:exit res))
|
||||
(->> (:out res)
|
||||
string/split-lines
|
||||
(remove (fn [line]
|
||||
(let [path (first (string/split line #":" 2))]
|
||||
(or (contains? allowed-exceptions line)
|
||||
(some #(string/starts-with? path %)
|
||||
excluded-path-prefixes)))))))
|
||||
_ (when (> (:exit res) 1) (System/exit 1))]
|
||||
(if (and (= 0 (:exit res)) (seq invalid-lines))
|
||||
(do (println "The following worker requires should not be in frontend namespaces:")
|
||||
(println (string/join "\n" invalid-lines))
|
||||
(System/exit 1))
|
||||
(println "Valid frontend namespaces!"))))
|
||||
|
||||
(defn worker-and-frontend-separate
|
||||
"Ensures workers are independent of frontend"
|
||||
[]
|
||||
(validate-frontend-not-in-workers)
|
||||
(validate-workers-not-in-frontend))
|
||||
111
scripts/src/logseq/tasks/dev/lint_test_deps.clj
Normal file
111
scripts/src/logseq/tasks/dev/lint_test_deps.clj
Normal file
@@ -0,0 +1,111 @@
|
||||
(ns logseq.tasks.dev.lint-test-deps
|
||||
"Runs lint/test tasks for selected deps with readable progress output."
|
||||
(:require [babashka.process :refer [shell]]))
|
||||
|
||||
(def ^:private kondo-src-test-step
|
||||
{:runner :cmd :name "clj-kondo (src test)" :cmd "clojure -M:clj-kondo --lint src test --cache false"})
|
||||
|
||||
(def ^:private dep-plan
|
||||
[{:dep "deps/common"
|
||||
:steps [kondo-src-test-step
|
||||
{:runner :bb :name "lint:large-vars" :cmd "lint:large-vars"}
|
||||
{:runner :bb :name "lint:carve" :cmd "lint:carve"}
|
||||
{:runner :bb :name "lint:ns-docstrings" :cmd "lint:ns-docstrings"}
|
||||
{:runner :cmd :name "pnpm exec nbb-logseq (-e long)" :cmd "pnpm exec nbb-logseq -cp test -m nextjournal.test-runner -e long"}]}
|
||||
{:dep "deps/db"
|
||||
:steps [kondo-src-test-step
|
||||
{:runner :bb :name "lint:large-vars" :cmd "lint:large-vars"}
|
||||
{:runner :bb :name "lint:carve" :cmd "lint:carve"}
|
||||
{:runner :bb :name "lint:ns-docstrings" :cmd "lint:ns-docstrings"}
|
||||
{:runner :bb :name "lint:rules" :cmd "lint:rules"}
|
||||
{:runner :cmd :name "pnpm exec nbb-logseq (-e long)" :cmd "pnpm exec nbb-logseq -cp test -m nextjournal.test-runner -e long"}]}
|
||||
{:dep "deps/db-sync"
|
||||
:steps [kondo-src-test-step
|
||||
{:runner :bb :name "lint:large-vars" :cmd "lint:large-vars"}
|
||||
{:runner :bb :name "lint:carve" :cmd "lint:carve"}
|
||||
;; {:runner :bb :name "lint:ns-docstrings" :cmd "lint:ns-docstrings"}
|
||||
;; {:runner :bb :name "lint:minimize-public-vars" :cmd "lint:minimize-public-vars"}
|
||||
]}
|
||||
{:dep "."
|
||||
:steps [{:runner :bb :name "dev:db-sync-test" :cmd "dev:db-sync-test"}]}
|
||||
{:dep "deps/outliner"
|
||||
:steps [kondo-src-test-step
|
||||
{:runner :bb :name "lint:large-vars" :cmd "lint:large-vars"}
|
||||
{:runner :bb :name "lint:carve" :cmd "lint:carve"}
|
||||
{:runner :bb :name "lint:ns-docstrings" :cmd "lint:ns-docstrings"}
|
||||
{:runner :bb :name "lint:minimize-public-vars" :cmd "lint:minimize-public-vars"}
|
||||
{:runner :cmd :name "pnpm exec nbb-logseq (-e long)" :cmd "pnpm exec nbb-logseq -cp test -m nextjournal.test-runner -e long"}]}
|
||||
{:dep "deps/graph-parser"
|
||||
:steps [kondo-src-test-step
|
||||
{:runner :bb :name "lint:large-vars" :cmd "lint:large-vars"}
|
||||
{:runner :bb :name "lint:carve" :cmd "lint:carve"}
|
||||
{:runner :bb :name "lint:ns-docstrings" :cmd "lint:ns-docstrings"}
|
||||
;; {:runner :bb :name "lint:minimize-public-vars" :cmd "lint:minimize-public-vars"}
|
||||
{:runner :cmd :name "pnpm exec nbb-logseq (-e long)" :cmd "pnpm exec nbb-logseq -cp test -m nextjournal.test-runner -e long"}]}
|
||||
{:dep "deps/publish"
|
||||
:steps [kondo-src-test-step
|
||||
{:runner :bb :name "lint:large-vars" :cmd "lint:large-vars"}
|
||||
{:runner :bb :name "lint:carve" :cmd "lint:carve"}
|
||||
{:runner :bb :name "lint:ns-docstrings" :cmd "lint:ns-docstrings"}
|
||||
{:runner :cmd :name "pnpm test" :cmd "pnpm test"}]}
|
||||
{:dep "deps/publishing"
|
||||
:steps [kondo-src-test-step
|
||||
{:runner :bb :name "lint:large-vars" :cmd "lint:large-vars"}
|
||||
{:runner :bb :name "lint:carve" :cmd "lint:carve"}
|
||||
{:runner :bb :name "lint:ns-docstrings" :cmd "lint:ns-docstrings"}
|
||||
{:runner :bb :name "lint:minimize-public-vars" :cmd "lint:minimize-public-vars"}]}])
|
||||
|
||||
(defn- run-step!
|
||||
[dep {:keys [runner name cmd reason]}]
|
||||
(case runner
|
||||
:skip
|
||||
(do
|
||||
(println (str "\n==> [" dep "] " name " (skip)"))
|
||||
(println (str "SKIP[" dep "] " name " - " reason))
|
||||
{:dep dep :task name :exit 0 :skipped true})
|
||||
|
||||
:bb
|
||||
(do
|
||||
(println (str "\n==> [" dep "] bb " cmd))
|
||||
(let [result (shell {:dir dep :continue true} (str "bb " cmd))
|
||||
success? (zero? (:exit result))]
|
||||
(println (str (if success? "OK " "ERR ")
|
||||
"[" dep "] "
|
||||
name
|
||||
(when-not success?
|
||||
(str " (exit " (:exit result) ")"))))
|
||||
{:dep dep :task name :exit (:exit result)}))
|
||||
|
||||
:cmd
|
||||
(do
|
||||
(println (str "\n==> [" dep "] " cmd))
|
||||
(let [result (shell {:dir dep :continue true} cmd)
|
||||
success? (zero? (:exit result))]
|
||||
(println (str (if success? "OK " "ERR ")
|
||||
"[" dep "] "
|
||||
name
|
||||
(when-not success?
|
||||
(str " (exit " (:exit result) ")"))))
|
||||
{:dep dep :task name :exit (:exit result)}))))
|
||||
|
||||
(defn run
|
||||
"Lint and test selected deps projects with per-step progress and summary."
|
||||
[& _]
|
||||
(println "Starting deps lint/test suite...")
|
||||
(let [results (vec
|
||||
(mapcat (fn [{:keys [dep steps]}]
|
||||
(map #(run-step! dep %) steps))
|
||||
dep-plan))
|
||||
skipped-count (count (filter :skipped results))
|
||||
failures (filter #(pos? (:exit %)) results)]
|
||||
(println "\nSummary:")
|
||||
(println (str " Total tasks: " (count results)))
|
||||
(println (str " Passed: " (- (count results) skipped-count (count failures))))
|
||||
(println (str " Skipped: " skipped-count))
|
||||
(println (str " Failed: " (count failures)))
|
||||
(when (seq failures)
|
||||
(println "\nFailed tasks:")
|
||||
(doseq [{:keys [dep task exit]} failures]
|
||||
(println (str " - [" dep "] " task " (exit " exit ")")))
|
||||
(System/exit 1))
|
||||
(println "\nAll deps lint/test tasks passed.")))
|
||||
223
scripts/src/logseq/tasks/dev/mobile.clj
Normal file
223
scripts/src/logseq/tasks/dev/mobile.clj
Normal file
@@ -0,0 +1,223 @@
|
||||
(ns logseq.tasks.dev.mobile
|
||||
"Tasks for mobile development"
|
||||
(:require [babashka.fs :as fs]
|
||||
[babashka.tasks :refer [shell]]
|
||||
[clojure.string :as string]
|
||||
[logseq.tasks.util :as task-util])
|
||||
(:import [java.nio.file FileAlreadyExistsException]))
|
||||
|
||||
(def ^:private dev-server-port "3002")
|
||||
(def ^:private ssl-dir "ssl/mobile-dev")
|
||||
(def ^:private ssl-lock-file "ssl/mobile-dev/.cert.lock")
|
||||
(def ^:private ssl-ca-keystore "ssl/mobile-dev/ca-keystore.jks")
|
||||
(def ^:private ssl-ca-cert "ssl/mobile-dev/logseq-dev-ca.cer")
|
||||
(def ^:private ssl-keystore "ssl/mobile-dev/keystore.jks")
|
||||
(def ^:private ssl-server-cert "ssl/mobile-dev/logseq-dev-server.cer")
|
||||
(def ^:private ssl-server-csr "ssl/mobile-dev/logseq-dev-server.csr")
|
||||
(def ^:private ssl-password "shadow-cljs")
|
||||
|
||||
(defn- local-ip
|
||||
[]
|
||||
(letfn [(interface-ip [interface]
|
||||
(let [{:keys [exit out]} (shell {:out :string
|
||||
:err :string
|
||||
:continue true
|
||||
:shutdown nil}
|
||||
(str "ipconfig getifaddr " interface))
|
||||
ip (string/trim (or out ""))]
|
||||
(when (and (zero? exit) (not (string/blank? ip)))
|
||||
ip)))]
|
||||
(or (interface-ip "en0")
|
||||
(interface-ip "en1")
|
||||
(throw (ex-info "Failed to detect local network IP" {})))))
|
||||
|
||||
(defn dev-server-url
|
||||
[ip]
|
||||
(format "https://%s:%s/mobile/" ip dev-server-port))
|
||||
|
||||
(defn- with-ssl-cert-lock!
|
||||
[f]
|
||||
(fs/create-dirs ssl-dir)
|
||||
(loop [attempt 0]
|
||||
(let [locked? (try
|
||||
(fs/create-file ssl-lock-file)
|
||||
true
|
||||
(catch FileAlreadyExistsException _
|
||||
false))]
|
||||
(when-not locked?
|
||||
(when (> attempt 600)
|
||||
(throw (ex-info "Timed out waiting for mobile HTTPS certificate lock"
|
||||
{:lock-file ssl-lock-file})))
|
||||
(Thread/sleep 100)
|
||||
(recur (inc attempt)))))
|
||||
(try
|
||||
(f)
|
||||
(finally
|
||||
(fs/delete-if-exists ssl-lock-file))))
|
||||
|
||||
(defn- usable-keystore?
|
||||
[]
|
||||
(and (fs/exists? ssl-keystore)
|
||||
(fs/exists? ssl-ca-keystore)
|
||||
(fs/exists? ssl-ca-cert)
|
||||
(zero? (:exit (shell {:continue true :out :string :err :string :shutdown nil}
|
||||
(str "keytool -list"
|
||||
" -keystore " ssl-keystore
|
||||
" -storepass " ssl-password
|
||||
" -alias logseq-dev"))))
|
||||
(zero? (:exit (shell {:continue true :out :string :err :string :shutdown nil}
|
||||
(str "keytool -list"
|
||||
" -keystore " ssl-ca-keystore
|
||||
" -storepass " ssl-password
|
||||
" -alias logseq-dev-ca"))))))
|
||||
|
||||
(defn- backup-existing-ssl-files!
|
||||
[]
|
||||
(when (and (fs/exists? ssl-keystore)
|
||||
(not (usable-keystore?)))
|
||||
(let [backup (str ssl-keystore ".bak." (System/currentTimeMillis))]
|
||||
(println "Backing up incompatible HTTPS keystore to" backup)
|
||||
(fs/move ssl-keystore backup {:replace-existing true})))
|
||||
(when-not (usable-keystore?)
|
||||
(doseq [path [ssl-ca-keystore ssl-ca-cert ssl-server-cert ssl-server-csr]]
|
||||
(fs/delete-if-exists path))))
|
||||
|
||||
(defn- generate-dev-ssl-cert!
|
||||
[ip]
|
||||
(let [san (str "SAN=ip:" ip ",dns:localhost,ip:127.0.0.1")]
|
||||
(shell {:shutdown nil}
|
||||
(str "keytool -genkeypair"
|
||||
" -alias logseq-dev-ca"
|
||||
" -keyalg RSA"
|
||||
" -keysize 2048"
|
||||
" -validity 3650"
|
||||
" -keystore " ssl-ca-keystore
|
||||
" -storepass " ssl-password
|
||||
" -keypass " ssl-password
|
||||
" -dname \"CN=Logseq Mobile Dev CA, OU=Logseq Development, O=Logseq, L=Local, ST=Local, C=US\""
|
||||
" -ext bc=ca:true"
|
||||
" -ext ku=keyCertSign,cRLSign"))
|
||||
(shell {:shutdown nil}
|
||||
(str "keytool -exportcert"
|
||||
" -rfc"
|
||||
" -alias logseq-dev-ca"
|
||||
" -keystore " ssl-ca-keystore
|
||||
" -storepass " ssl-password
|
||||
" -file " ssl-ca-cert))
|
||||
(shell {:shutdown nil}
|
||||
(str "keytool -genkeypair"
|
||||
" -alias logseq-dev"
|
||||
" -keyalg RSA"
|
||||
" -keysize 2048"
|
||||
" -validity 365"
|
||||
" -keystore " ssl-keystore
|
||||
" -storepass " ssl-password
|
||||
" -keypass " ssl-password
|
||||
" -dname \"CN=" ip ", OU=Logseq Development, O=Logseq, L=Local, ST=Local, C=US\""
|
||||
" -ext " san))
|
||||
(shell {:shutdown nil}
|
||||
(str "keytool -certreq"
|
||||
" -alias logseq-dev"
|
||||
" -keystore " ssl-keystore
|
||||
" -storepass " ssl-password
|
||||
" -file " ssl-server-csr))
|
||||
(shell {:shutdown nil}
|
||||
(str "keytool -gencert"
|
||||
" -alias logseq-dev-ca"
|
||||
" -keystore " ssl-ca-keystore
|
||||
" -storepass " ssl-password
|
||||
" -infile " ssl-server-csr
|
||||
" -outfile " ssl-server-cert
|
||||
" -rfc"
|
||||
" -validity 365"
|
||||
" -ext " san
|
||||
" -ext KU=digitalSignature,keyEncipherment"
|
||||
" -ext EKU=serverAuth"))
|
||||
(shell {:shutdown nil}
|
||||
(str "keytool -importcert"
|
||||
" -noprompt"
|
||||
" -trustcacerts"
|
||||
" -alias logseq-dev-ca"
|
||||
" -file " ssl-ca-cert
|
||||
" -keystore " ssl-keystore
|
||||
" -storepass " ssl-password))
|
||||
(shell {:shutdown nil}
|
||||
(str "keytool -importcert"
|
||||
" -noprompt"
|
||||
" -trustcacerts"
|
||||
" -alias logseq-dev"
|
||||
" -file " ssl-server-cert
|
||||
" -keystore " ssl-keystore
|
||||
" -storepass " ssl-password))))
|
||||
|
||||
(defn ensure-dev-ssl-cert!
|
||||
"Creates the local Shadow CLJS dev HTTPS keystore and exported iOS trust CA."
|
||||
[]
|
||||
(with-ssl-cert-lock!
|
||||
(fn []
|
||||
(let [ip (local-ip)]
|
||||
(backup-existing-ssl-files!)
|
||||
(when-not (usable-keystore?)
|
||||
(generate-dev-ssl-cert! ip))
|
||||
(println "Shadow CLJS HTTPS keystore:" ssl-keystore)
|
||||
(println "Install and fully trust this CA certificate on iOS:" ssl-ca-cert)
|
||||
(println "Mobile dev server URL:" (dev-server-url ip))))))
|
||||
|
||||
(defn ensure-dev-ssl-cert-task
|
||||
[]
|
||||
(ensure-dev-ssl-cert!))
|
||||
|
||||
(defn- open-dev-app
|
||||
"Opens mobile app when watch process has built main.js"
|
||||
[shell-opts cmd]
|
||||
(let [start-time (java.time.Instant/now)]
|
||||
(loop [n 1000]
|
||||
(if (and (fs/exists? "static/mobile/js/main.js")
|
||||
(task-util/file-modified-later-than? "static/mobile/js/main.js" start-time))
|
||||
(shell (merge {:shutdown nil} shell-opts) cmd)
|
||||
(println "Waiting for app to build..."))
|
||||
(Thread/sleep 1000)
|
||||
(when-not (or (and (fs/exists? "ios/App/App/public/js/main.js")
|
||||
(task-util/file-modified-later-than? "ios/App/App/public/js/main.js" start-time))
|
||||
(and (fs/exists? "android/App/src/main/assets/public/js/main.js")
|
||||
(task-util/file-modified-later-than? "android/App/src/main/assets/public/js/main.js" start-time)))
|
||||
(recur (dec n))))))
|
||||
|
||||
(defn app-watch
|
||||
"Watches environment to reload cljs, css and other assets for mobile"
|
||||
[]
|
||||
(ensure-dev-ssl-cert!)
|
||||
(doseq [cmd ["pnpm clean"
|
||||
"pnpm mobile-watch"]]
|
||||
(println cmd)
|
||||
(shell {:shutdown nil
|
||||
:extra-env {"LOGSEQ_SHADOW_HTTPS" "true"}}
|
||||
cmd)))
|
||||
|
||||
(defn cap-run-ios
|
||||
"Copy assets files to iOS build directory, and run app in Xcode"
|
||||
[]
|
||||
(ensure-dev-ssl-cert!)
|
||||
(open-dev-app {:extra-env {"LOGSEQ_APP_SERVER_URL" (dev-server-url (local-ip))}}
|
||||
"pnpm exec cap sync ios")
|
||||
(shell {:shutdown nil} "pnpm exec cap open ios"))
|
||||
|
||||
(defn cap-run-android
|
||||
"Copy assets files to Android build directory, and run app in Android Studio"
|
||||
[]
|
||||
(ensure-dev-ssl-cert!)
|
||||
(open-dev-app {:extra-env {"LOGSEQ_APP_SERVER_URL" (dev-server-url (local-ip))}}
|
||||
"pnpm exec cap sync android")
|
||||
(shell {:shutdown nil} "pnpm exec cap open android"))
|
||||
|
||||
(defn run-ios-release
|
||||
"Build iOS app release"
|
||||
[]
|
||||
(shell {:shutdown nil} "git checkout capacitor.config.ts")
|
||||
(shell {:shutdown nil} "pnpm run-ios-release"))
|
||||
|
||||
(defn run-android-release
|
||||
"Build Android app release"
|
||||
[]
|
||||
(shell {:shutdown nil} "git checkout capacitor.config.ts")
|
||||
(shell {:shutdown nil} "pnpm run-android-release"))
|
||||
334
scripts/src/logseq/tasks/lang.clj
Normal file
334
scripts/src/logseq/tasks/lang.clj
Normal file
@@ -0,0 +1,334 @@
|
||||
(ns logseq.tasks.lang
|
||||
"Tasks related to language translations"
|
||||
(:require [babashka.cli :as cli]
|
||||
[babashka.fs :as fs]
|
||||
[babashka.process :refer [shell]]
|
||||
[borkdude.rewrite-edn :as rewrite]
|
||||
[clojure.set :as set]
|
||||
[clojure.string :as string]
|
||||
[frontend.dicts :as dicts]
|
||||
[logseq.tasks.lang-lint :as lang-lint]
|
||||
[logseq.tasks.util :as task-util]
|
||||
[rewrite-clj.node :as node]))
|
||||
|
||||
(defn- get-dicts
|
||||
[]
|
||||
dicts/dicts)
|
||||
|
||||
(defn- get-languages
|
||||
[]
|
||||
(->> dicts/languages
|
||||
(map (juxt :value :label))
|
||||
(into {})))
|
||||
|
||||
(defn- shorten [s length]
|
||||
(if (< (count s) length)
|
||||
s
|
||||
(string/replace (str (subs s 0 length) "...")
|
||||
;; Keep shortened table rows single-line for multi-line translations.
|
||||
"\n" "\\n")))
|
||||
|
||||
(defn list-langs
|
||||
"List translated languages with their number of translations"
|
||||
[]
|
||||
(let [dicts (get-dicts)
|
||||
langs (get-languages)]
|
||||
(->> (lang-lint/translation-summary-stats dicts)
|
||||
(lang-lint/sort-translation-summary-stats)
|
||||
(map (fn [{:keys [lang translation-count untranslated-count same-as-en-count]}]
|
||||
{:locale lang
|
||||
:translation-count translation-count
|
||||
:untranslated-count (if (= lang :en) "-" untranslated-count)
|
||||
:same-as-en-count (if (= lang :en) "-" same-as-en-count)
|
||||
:language (langs lang)}))
|
||||
task-util/print-table)))
|
||||
|
||||
(defn list-pseudo
|
||||
"List translations for LOCALE whose localized value is identical to English."
|
||||
[& args]
|
||||
(let [lang (or (some-> (first args) keyword)
|
||||
(task-util/print-usage "LOCALE"))
|
||||
langs (get-languages)
|
||||
dicts (get-dicts)]
|
||||
(when-not (contains? langs lang)
|
||||
(println "Language" lang "does not have an entry in frontend.dicts/languages")
|
||||
(System/exit 1))
|
||||
(let [findings (->> (lang-lint/identical-translation-findings dicts lang)
|
||||
(map (fn [{:keys [translation-key default-value]}]
|
||||
{:translation-key translation-key
|
||||
:same-as-en-value default-value
|
||||
:file (str "dicts/" (-> lang name string/lower-case) ".edn")}))
|
||||
(sort-by (juxt :file :translation-key)))]
|
||||
(if (empty? findings)
|
||||
(println "Language" lang "does not contain translations identical to English!")
|
||||
(task-util/print-table
|
||||
(map #(update % :same-as-en-value shorten 50) findings))))))
|
||||
|
||||
(defn list-missing
|
||||
"List missing translations for a given language"
|
||||
[& args]
|
||||
(let [lang (or (keyword (first args))
|
||||
(task-util/print-usage "LOCALE [--copy]"))
|
||||
options (cli/parse-opts (rest args) {:coerce {:copy :boolean}})
|
||||
_ (when-not (contains? (get-languages) lang)
|
||||
(println "Language" lang "does not have an entry in frontend.dicts/languages")
|
||||
(System/exit 1))
|
||||
dicts (get-dicts)
|
||||
all-missing (select-keys (dicts :en)
|
||||
(set/difference (set (keys (dicts :en)))
|
||||
(set (keys (dicts lang)))))]
|
||||
(if (-> all-missing count zero?)
|
||||
(println "Language" lang "is fully translated!")
|
||||
(let [sorted-missing (->> all-missing
|
||||
(map (fn [[k v]]
|
||||
{:translation-key k
|
||||
:string-to-translate v
|
||||
:file (str "dicts/" (-> lang name string/lower-case) ".edn")}))
|
||||
(sort-by (juxt :file :translation-key)))]
|
||||
(if (:copy options)
|
||||
(doseq [[file missing-for-file] (group-by :file sorted-missing)]
|
||||
(println "\n;; For" file)
|
||||
(doseq [{:keys [translation-key string-to-translate]} missing-for-file]
|
||||
(println translation-key (pr-str string-to-translate))))
|
||||
(task-util/print-table
|
||||
;; Shorten values
|
||||
(map #(update % :string-to-translate shorten 50) sorted-missing)))))))
|
||||
|
||||
(defn- delete-invalid-non-default-languages
|
||||
[invalid-keys-by-lang]
|
||||
(doseq [[lang invalid-keys] invalid-keys-by-lang]
|
||||
(let [path (fs/path "src/resources/dicts" (str (name lang) ".edn"))
|
||||
result (rewrite/parse-string (String. (fs/read-all-bytes path)))
|
||||
new-content (str (reduce
|
||||
(fn [result k]
|
||||
(rewrite/dissoc result k))
|
||||
result invalid-keys))]
|
||||
(spit (fs/file path) new-content))))
|
||||
|
||||
(def ^:private dicts-dir
|
||||
(fs/path "src/resources/dicts"))
|
||||
|
||||
(def ^:private ignored-dict-node-tags
|
||||
#{:comment :newline :whitespace})
|
||||
|
||||
(defn- ignored-dict-node?
|
||||
[node]
|
||||
(contains? ignored-dict-node-tags (node/tag node)))
|
||||
|
||||
(defn- dict-map-node
|
||||
[root]
|
||||
(->> (:children root)
|
||||
(remove ignored-dict-node?)
|
||||
first))
|
||||
|
||||
(defn- parse-dict-entries
|
||||
[text]
|
||||
(let [root (rewrite/parse-string text)
|
||||
map-node (dict-map-node root)]
|
||||
(when-not (= :map (node/tag map-node))
|
||||
(println "Expected a top-level map in dictionary file.")
|
||||
(System/exit 1))
|
||||
(let [entry-nodes (->> (:children map-node)
|
||||
(remove ignored-dict-node?))]
|
||||
(when (odd? (count entry-nodes))
|
||||
(println "Encountered an uneven number of top-level dictionary nodes.")
|
||||
(System/exit 1))
|
||||
(mapv (fn [[key-node value-node]]
|
||||
{:key (rewrite/sexpr key-node)
|
||||
:value-node value-node})
|
||||
(partition 2 entry-nodes)))))
|
||||
|
||||
(defn- render-dict-entry
|
||||
[{:keys [key value-node]}]
|
||||
(str " " key " " value-node))
|
||||
|
||||
(defn- key-namespace-root
|
||||
[key]
|
||||
(some-> key namespace (string/split #"\.") first))
|
||||
|
||||
(defn- key-leaf
|
||||
[key]
|
||||
(name key))
|
||||
|
||||
(defn- compare-dict-keys
|
||||
[key-a key-b]
|
||||
(let [namespace-a (namespace key-a)
|
||||
namespace-b (namespace key-b)
|
||||
root-a (key-namespace-root key-a)
|
||||
root-b (key-namespace-root key-b)
|
||||
root-diff (compare root-a root-b)]
|
||||
(cond
|
||||
(not= 0 root-diff)
|
||||
root-diff
|
||||
|
||||
(not= namespace-a root-a)
|
||||
(if (= namespace-b root-b) 1
|
||||
(let [namespace-diff (compare namespace-a namespace-b)]
|
||||
(if (zero? namespace-diff)
|
||||
(compare (key-leaf key-a) (key-leaf key-b))
|
||||
namespace-diff)))
|
||||
|
||||
(not= namespace-b root-b)
|
||||
-1
|
||||
|
||||
:else
|
||||
(compare (key-leaf key-a) (key-leaf key-b)))))
|
||||
|
||||
(defn- render-dict
|
||||
[entries]
|
||||
(let [sorted-entries (sort #(neg? (compare-dict-keys (:key %1) (:key %2))) entries)
|
||||
lines (loop [remaining sorted-entries
|
||||
previous-namespace nil
|
||||
acc ["{"]]
|
||||
(if-let [{:keys [key] :as entry} (first remaining)]
|
||||
(let [current-namespace (namespace key)
|
||||
acc (cond-> acc
|
||||
(and previous-namespace
|
||||
(not= previous-namespace current-namespace))
|
||||
(conj "")
|
||||
true
|
||||
(conj (render-dict-entry entry)))]
|
||||
(recur (next remaining) current-namespace acc))
|
||||
(conj acc "}")))]
|
||||
(str (string/join "\n" lines) "\n")))
|
||||
|
||||
(defn- dict-file-paths
|
||||
[]
|
||||
(->> (fs/list-dir dicts-dir)
|
||||
(filter #(string/ends-with? (str %) ".edn"))
|
||||
(sort-by fs/file-name)))
|
||||
|
||||
(defn format-dicts
|
||||
"Formats dictionary files by full-key sort order and inserts a blank line
|
||||
between namespace groups. Use --check to fail when any file would change."
|
||||
[& args]
|
||||
(let [check? (contains? (set args) "--check")
|
||||
changed? (volatile! false)]
|
||||
(doseq [path (dict-file-paths)]
|
||||
(let [file-name (fs/file-name path)
|
||||
current-text (slurp (str path))
|
||||
output-text (-> current-text
|
||||
parse-dict-entries
|
||||
render-dict)]
|
||||
(if (= current-text output-text)
|
||||
(println file-name ": already formatted")
|
||||
(do
|
||||
(vreset! changed? true)
|
||||
(if check?
|
||||
(println file-name "would change")
|
||||
(do
|
||||
(spit (str path) output-text)
|
||||
(println file-name ": formatted")))))))
|
||||
(when (and check? @changed?)
|
||||
(System/exit 1))))
|
||||
|
||||
(defn- validate-non-default-languages
|
||||
"This validation finds any translation keys that don't exist in the default
|
||||
language English. Logseq needs to work out of the box with its default
|
||||
language. This catches mistakes where another language has accidentally typoed
|
||||
keys or added ones without updating :en"
|
||||
[fix?]
|
||||
(let [dicts (get-dicts)
|
||||
;; For now defined as :en but clj-kondo analysis could be more thorough
|
||||
valid-keys (set (keys (dicts :en)))
|
||||
invalid-dicts
|
||||
(->> (dissoc dicts :en)
|
||||
(mapcat (fn [[lang lang-dicts]]
|
||||
(map
|
||||
#(hash-map :language lang :invalid-key %)
|
||||
(set/difference (set (keys lang-dicts))
|
||||
valid-keys)))))]
|
||||
(if (empty? invalid-dicts)
|
||||
(println "All non-default translations have valid keys!")
|
||||
(do
|
||||
(println "\nThese translation keys are invalid because they don't exist in English:")
|
||||
(task-util/print-table invalid-dicts)
|
||||
(when fix?
|
||||
(delete-invalid-non-default-languages
|
||||
(update-vals (group-by :language invalid-dicts) #(map :invalid-key %)))
|
||||
(println "These invalid translation keys have been removed from non-default dictionaries."))
|
||||
(System/exit 1)))))
|
||||
|
||||
(def ^:private i18n-lint-launcher-path
|
||||
(fs/absolutize "bin/logseq-i18n-lint"))
|
||||
|
||||
(def ^:private i18n-lint-config-path
|
||||
(fs/absolutize ".i18n-lint.toml"))
|
||||
|
||||
(defn- ensure-i18n-lint-ready!
|
||||
[]
|
||||
(when-not (fs/exists? i18n-lint-launcher-path)
|
||||
(println "logseq-i18n-lint launcher not found at" (str i18n-lint-launcher-path))
|
||||
(System/exit 1))
|
||||
(when-not (fs/exists? i18n-lint-config-path)
|
||||
(println "i18n lint config not found at" (str i18n-lint-config-path))
|
||||
(System/exit 1)))
|
||||
|
||||
(defn- run-i18n-lint-command!
|
||||
[subcommand cli-args]
|
||||
(ensure-i18n-lint-ready!)
|
||||
(let [cmd (into ["bash"
|
||||
(str i18n-lint-launcher-path)
|
||||
"-c"
|
||||
(str i18n-lint-config-path)
|
||||
subcommand]
|
||||
cli-args)
|
||||
result (apply shell {:continue true
|
||||
:out :inherit
|
||||
:err :inherit}
|
||||
cmd)]
|
||||
(when (pos? (:exit result))
|
||||
(System/exit (:exit result)))))
|
||||
|
||||
(defn- check-missing-translations
|
||||
"Use logseq-i18n-lint to fail fast on missing translations before other checks."
|
||||
[]
|
||||
(run-i18n-lint-command! "check-missing" []))
|
||||
|
||||
(defn- check-translation-keys
|
||||
"Use logseq-i18n-lint to detect unused translation keys."
|
||||
[args]
|
||||
(run-i18n-lint-command! "check-keys" args))
|
||||
|
||||
(defn- validate-rich-translations
|
||||
"Checks that localized rich translations remain rich zero-arg functions.
|
||||
Missing translations are allowed, but once a locale defines a rich key it
|
||||
must preserve the same renderable contract as English."
|
||||
[]
|
||||
(let [invalid-dicts (lang-lint/rich-translation-mismatch-findings (get-dicts))]
|
||||
(if (empty? invalid-dicts)
|
||||
(println "All rich translations preserve English render contracts!")
|
||||
(do
|
||||
(println "These translation keys are invalid because they no longer preserve English rich render contracts:")
|
||||
(task-util/print-table invalid-dicts)
|
||||
(System/exit 1)))))
|
||||
|
||||
(defn- validate-translation-placeholders
|
||||
"Checks that every localized string uses the same placeholder set as English.
|
||||
Missing translations are allowed because Tongue falls back to :en, but once
|
||||
a locale defines a string it must preserve the placeholder contract."
|
||||
[]
|
||||
(let [invalid-dicts (lang-lint/placeholder-mismatch-findings (get-dicts))]
|
||||
(if (empty? invalid-dicts)
|
||||
(println "All translations preserve English placeholder contracts!")
|
||||
(do
|
||||
(println "These translation keys are invalid because their placeholders do not match English:")
|
||||
(task-util/print-table
|
||||
(map #(dissoc % :default-value :localized-value) invalid-dicts))
|
||||
(System/exit 1)))))
|
||||
|
||||
(defn validate-translations
|
||||
"Runs multiple translation validations that fail fast if one of them is invalid"
|
||||
[& args]
|
||||
(check-missing-translations)
|
||||
(validate-non-default-languages (contains? (set args) "--fix"))
|
||||
(check-translation-keys args)
|
||||
(validate-rich-translations)
|
||||
(validate-translation-placeholders))
|
||||
|
||||
(defn lint-hardcoded
|
||||
"Run logseq-i18n-lint to lint likely hardcoded user-facing strings in UI-oriented source files.
|
||||
Use -w or --warn-only to report findings without failing and -g or --git-changed to scan
|
||||
only files changed in git status."
|
||||
[& args]
|
||||
(run-i18n-lint-command! "lint" args))
|
||||
149
scripts/src/logseq/tasks/lang_lint.cljc
Normal file
149
scripts/src/logseq/tasks/lang_lint.cljc
Normal file
@@ -0,0 +1,149 @@
|
||||
(ns logseq.tasks.lang-lint)
|
||||
|
||||
;; Matches numbered placeholders like `{1}` in translation strings.
|
||||
(def ^:private translation-placeholder-pattern
|
||||
#"\{(\d+)\}")
|
||||
|
||||
(defn translation-placeholders
|
||||
"Return the placeholder indexes referenced by translation string `value`.
|
||||
|
||||
Non-string values return an empty set."
|
||||
[value]
|
||||
(if (string? value)
|
||||
(->> (re-seq translation-placeholder-pattern value)
|
||||
(map second)
|
||||
set)
|
||||
#{}))
|
||||
|
||||
(defn- placeholders-compatible?
|
||||
[default-value localized-value]
|
||||
(or (not (string? default-value))
|
||||
(not (string? localized-value))
|
||||
(= (translation-placeholders default-value)
|
||||
(translation-placeholders localized-value))))
|
||||
|
||||
(defn placeholder-mismatch-findings
|
||||
"Return localized string findings whose placeholder set diverges from
|
||||
English."
|
||||
[dicts]
|
||||
(let [en-dicts (:en dicts)]
|
||||
(->> (dissoc dicts :en)
|
||||
(mapcat
|
||||
(fn [[lang lang-dicts]]
|
||||
(keep (fn [[translation-key localized-value]]
|
||||
(let [default-value (get en-dicts translation-key)]
|
||||
(when (and (string? default-value)
|
||||
(string? localized-value)
|
||||
(not (placeholders-compatible? default-value localized-value)))
|
||||
{:lang lang
|
||||
:translation-key translation-key
|
||||
:expected-placeholders (sort (translation-placeholders default-value))
|
||||
:actual-placeholders (sort (translation-placeholders localized-value))
|
||||
:default-value default-value
|
||||
:localized-value localized-value})))
|
||||
lang-dicts)))
|
||||
(sort-by (juxt :lang :translation-key))
|
||||
vec)))
|
||||
|
||||
(defn- rich-translation-value?
|
||||
"Return true when `value` preserves the rich zero-arg translation contract.
|
||||
|
||||
Babashka reads dictionary `fn` forms as lists, while tests may pass actual
|
||||
function values, so both representations are treated as rich translations."
|
||||
[value]
|
||||
(or (fn? value)
|
||||
(and (seq? value)
|
||||
(= 'fn (first value))
|
||||
(= [] (second value)))))
|
||||
|
||||
(defn- value-kind
|
||||
[value]
|
||||
(cond
|
||||
(rich-translation-value? value) :fn
|
||||
(string? value) :string
|
||||
(nil? value) :nil
|
||||
:else :other))
|
||||
|
||||
(defn rich-translation-mismatch-findings
|
||||
"Return localized rich-translation findings whose value kind no longer
|
||||
matches the English zero-arg function contract."
|
||||
[dicts]
|
||||
(let [en-dicts (:en dicts)]
|
||||
(->> (dissoc dicts :en)
|
||||
(mapcat
|
||||
(fn [[lang lang-dicts]]
|
||||
(keep (fn [[translation-key default-value]]
|
||||
(let [localized-present? (contains? lang-dicts translation-key)
|
||||
localized-value (get lang-dicts translation-key)]
|
||||
(when (and localized-present?
|
||||
(rich-translation-value? default-value)
|
||||
(not (rich-translation-value? localized-value)))
|
||||
{:lang lang
|
||||
:translation-key translation-key
|
||||
:expected-value-kind :fn
|
||||
:actual-value-kind (value-kind localized-value)})))
|
||||
en-dicts)))
|
||||
(sort-by (juxt :lang :translation-key))
|
||||
vec)))
|
||||
|
||||
(defn identical-translation-findings
|
||||
"Return localized translation findings whose defined value is identical to
|
||||
English for the same key."
|
||||
[dicts lang]
|
||||
(let [en-dicts (:en dicts)
|
||||
lang-dicts (get dicts lang)]
|
||||
(->> lang-dicts
|
||||
(keep (fn [[translation-key localized-value]]
|
||||
(let [default-value (get en-dicts translation-key ::missing)]
|
||||
(when (= default-value localized-value)
|
||||
{:lang lang
|
||||
:translation-key translation-key
|
||||
:default-value default-value}))))
|
||||
(sort-by :translation-key)
|
||||
vec)))
|
||||
|
||||
(defn identical-translation-stats
|
||||
"Return per-locale identical-to-English counts for defined translations."
|
||||
[dicts]
|
||||
(let [en-dicts (:en dicts)]
|
||||
(->> dicts
|
||||
(map (fn [[lang lang-dicts]]
|
||||
{:lang lang
|
||||
:translation-count (count lang-dicts)
|
||||
:same-as-en-count
|
||||
(count
|
||||
(filter (fn [[translation-key localized-value]]
|
||||
(= (get en-dicts translation-key ::missing)
|
||||
localized-value))
|
||||
lang-dicts))}))
|
||||
(sort-by (juxt (comp - :same-as-en-count) (comp - :translation-count) :lang))
|
||||
vec)))
|
||||
|
||||
(defn translation-summary-stats
|
||||
"Return per-locale translation summary stats for overview tables."
|
||||
[dicts]
|
||||
(let [en-count (count (:en dicts))
|
||||
same-as-en-counts (->> (identical-translation-stats dicts)
|
||||
(map (juxt :lang :same-as-en-count))
|
||||
(into {}))]
|
||||
(->> dicts
|
||||
(map (fn [[lang lang-dicts]]
|
||||
{:lang lang
|
||||
:translation-count (count lang-dicts)
|
||||
:untranslated-count (when-not (= lang :en)
|
||||
(max 0 (- en-count (count lang-dicts))))
|
||||
:same-as-en-count (when-not (= lang :en)
|
||||
(get same-as-en-counts lang 0))}))
|
||||
vec)))
|
||||
|
||||
(defn sort-translation-summary-stats
|
||||
"Sort translation summary stats with English first, then by untranslated
|
||||
count descending, then by identical-to-English count descending."
|
||||
[stats]
|
||||
(let [en-stats (filter #(= :en (:lang %)) stats)
|
||||
other-stats (remove #(= :en (:lang %)) stats)]
|
||||
(into (vec en-stats)
|
||||
(sort-by (juxt (comp - :untranslated-count)
|
||||
(comp - :same-as-en-count)
|
||||
:lang)
|
||||
other-stats))))
|
||||
72
scripts/src/logseq/tasks/malli.clj
Normal file
72
scripts/src/logseq/tasks/malli.clj
Normal file
@@ -0,0 +1,72 @@
|
||||
(ns logseq.tasks.malli
|
||||
"Malli related tasks"
|
||||
(:require [malli.core :as m]
|
||||
[malli.error :as me]
|
||||
[frontend.schema.handler.plugin-config :as plugin-config-schema]
|
||||
[frontend.schema.handler.global-config :as global-config-schema]
|
||||
[frontend.schema.handler.repo-config :as repo-config-schema]
|
||||
[logseq.graph-parser.schema.mldoc :as mldoc-schema]
|
||||
[babashka.fs :as fs]
|
||||
[clojure.pprint :as pprint]
|
||||
[clojure.edn :as edn]))
|
||||
|
||||
(defn validate-plugins-edn
|
||||
"Validate a plugins.edn file"
|
||||
[file]
|
||||
(if-let [errors (->> file
|
||||
slurp
|
||||
edn/read-string
|
||||
(m/explain plugin-config-schema/Plugins-edn)
|
||||
me/humanize)]
|
||||
(do
|
||||
(println "Found errors:")
|
||||
(pprint/pprint errors))
|
||||
(println "Valid!")))
|
||||
|
||||
(defn- validate-file-with-schema
|
||||
"Validate a file given its schema"
|
||||
[file schema]
|
||||
(if-let [errors (->> file
|
||||
slurp
|
||||
edn/read-string
|
||||
(m/explain schema)
|
||||
me/humanize)]
|
||||
(do
|
||||
(println "Found errors:")
|
||||
(pprint/pprint errors))
|
||||
(println "Valid!")))
|
||||
|
||||
(defn validate-repo-config-edn
|
||||
"Validate a repo config.edn"
|
||||
[file]
|
||||
(validate-file-with-schema file global-config-schema/Config-edn))
|
||||
|
||||
(defn validate-global-config-edn
|
||||
"Validate a global config.edn"
|
||||
[file]
|
||||
(validate-file-with-schema file repo-config-schema/Config-edn))
|
||||
|
||||
(defn validate-ast
|
||||
"Validate mldoc ast(s) in a file or as an EDN arg"
|
||||
[file-or-edn]
|
||||
(let [edn (edn/read-string
|
||||
(if (fs/exists? file-or-edn) (slurp file-or-edn) file-or-edn))]
|
||||
(if (and (sequential? edn) (:ast (first edn)))
|
||||
;; Validate multiple asts in the format [{:file "" :ast []} ...]
|
||||
;; Produced by https://github.com/logseq/nbb-logseq/tree/main/examples/from-js#graph_astmjs
|
||||
(do
|
||||
(println "Validating" (count edn) "files...")
|
||||
(if-let [errors-by-file (seq (keep
|
||||
#(when-let [errors (m/explain mldoc-schema/block-ast-with-pos-coll-schema (:ast %))]
|
||||
{:file (:file %)
|
||||
:errors errors})
|
||||
edn))]
|
||||
(do
|
||||
(println "Found errors:")
|
||||
(pprint/pprint errors-by-file))
|
||||
(println "All files valid!")))
|
||||
(if-let [errors (m/explain mldoc-schema/block-ast-with-pos-coll-schema edn)]
|
||||
(do
|
||||
(println "Found errors:")
|
||||
(pprint/pprint errors))
|
||||
(println "Valid!")))))
|
||||
19
scripts/src/logseq/tasks/spec.clj
Normal file
19
scripts/src/logseq/tasks/spec.clj
Normal file
@@ -0,0 +1,19 @@
|
||||
(ns logseq.tasks.spec
|
||||
"Clojure spec related tasks"
|
||||
(:require [clojure.spec.alpha :as s]
|
||||
[cheshire.core :as json]
|
||||
[frontend.spec.storage :as storage-spec]
|
||||
[clojure.edn :as edn]))
|
||||
|
||||
;; To create file for validation, `JSON.stringify(localStorage)` in the js
|
||||
;; console and copy string to file
|
||||
(defn validate-local-storage
|
||||
"Validate a localStorage json file"
|
||||
[file]
|
||||
(let [local-storage
|
||||
(update-vals (json/parse-string (slurp file) keyword)
|
||||
;; Not all localStorage values are edn so gracefully return.
|
||||
;; For example, logseq-plugin-tabs stores data as json
|
||||
#(try (edn/read-string %) (catch Throwable _ %)))]
|
||||
(s/assert ::storage-spec/local-storage local-storage)
|
||||
(println "Success!")))
|
||||
123
scripts/src/logseq/tasks/util.cljc
Normal file
123
scripts/src/logseq/tasks/util.cljc
Normal file
@@ -0,0 +1,123 @@
|
||||
(ns logseq.tasks.util
|
||||
"Utils for tasks"
|
||||
(:require [clojure.string :as string]
|
||||
#?(:clj [babashka.fs :as fs])))
|
||||
|
||||
(defn- in-range?
|
||||
[code-point [start end]]
|
||||
(<= start code-point end))
|
||||
|
||||
(def ^:private zero-width-ranges
|
||||
[[0x0300 0x036F]
|
||||
[0x1AB0 0x1AFF]
|
||||
[0x1DC0 0x1DFF]
|
||||
[0x200C 0x200F]
|
||||
[0x202A 0x202E]
|
||||
[0x2060 0x206F]
|
||||
[0x20D0 0x20FF]
|
||||
[0xFE00 0xFE0F]
|
||||
[0xFE20 0xFE2F]])
|
||||
|
||||
(def ^:private wide-ranges
|
||||
[[0x1100 0x115F]
|
||||
[0x2329 0x232A]
|
||||
[0x2E80 0xA4CF]
|
||||
[0xAC00 0xD7A3]
|
||||
[0xF900 0xFAFF]
|
||||
[0xFE10 0xFE19]
|
||||
[0xFE30 0xFE6F]
|
||||
[0xFF00 0xFF60]
|
||||
[0xFFE0 0xFFE6]
|
||||
[0x1F300 0x1FAFF]
|
||||
[0x20000 0x3FFFD]])
|
||||
|
||||
(defn- code-point-width
|
||||
[code-point]
|
||||
(cond
|
||||
(or (<= 0x0000 code-point 0x001F)
|
||||
(<= 0x007F code-point 0x009F)
|
||||
(some #(in-range? code-point %) zero-width-ranges))
|
||||
0
|
||||
|
||||
(some #(in-range? code-point %) wide-ranges)
|
||||
2
|
||||
|
||||
:else
|
||||
1))
|
||||
|
||||
(defn display-width
|
||||
[value]
|
||||
(let [text (str value)]
|
||||
(loop [index 0
|
||||
width 0]
|
||||
(if (< index #?(:clj (.length text) :cljs (.-length text)))
|
||||
(let [code-point #?(:clj (.codePointAt text index)
|
||||
:cljs (.codePointAt text index))]
|
||||
(recur (+ index #?(:clj (Character/charCount code-point)
|
||||
:cljs (if (> code-point 0xFFFF) 2 1)))
|
||||
(+ width (code-point-width code-point))))
|
||||
width))))
|
||||
|
||||
(defn- pad-left
|
||||
[text width]
|
||||
(let [padding (- width (display-width text))]
|
||||
(str (apply str (repeat (max 0 padding) " ")) text)))
|
||||
|
||||
(defn- column-widths
|
||||
[columns rows]
|
||||
(reduce
|
||||
(fn [widths column]
|
||||
(assoc widths
|
||||
column
|
||||
(apply max
|
||||
(display-width (str column))
|
||||
(map #(display-width (str (get % column ""))) rows))))
|
||||
{}
|
||||
columns))
|
||||
|
||||
(defn- render-separator
|
||||
[columns widths]
|
||||
(str "|" (string/join "+" (map #(apply str (repeat (+ 2 (get widths %)) "-")) columns)) "|"))
|
||||
|
||||
(defn- render-row
|
||||
[columns widths row]
|
||||
(str "|"
|
||||
(string/join "|" (map #(str " "
|
||||
(pad-left (str (get row % "")) (get widths %))
|
||||
" ")
|
||||
columns))
|
||||
"|"))
|
||||
|
||||
(defn render-table
|
||||
[rows]
|
||||
(when-let [columns (seq (keys (first rows)))]
|
||||
(let [rows (vec rows)
|
||||
widths (column-widths columns rows)
|
||||
header-row (zipmap columns columns)]
|
||||
(str (string/join
|
||||
"\n"
|
||||
(concat [(render-row columns widths header-row)
|
||||
(render-separator columns widths)]
|
||||
(map #(render-row columns widths %) rows)))
|
||||
"\n"))))
|
||||
|
||||
#?(:clj
|
||||
(defn file-modified-later-than?
|
||||
[file comparison-instant]
|
||||
(pos? (.compareTo (fs/file-time->instant (fs/last-modified-time file))
|
||||
comparison-instant))))
|
||||
|
||||
(defn print-usage [arg-str]
|
||||
(println (str "Usage: bb "
|
||||
#?(:clj (System/getProperty "babashka.task")
|
||||
:cljs "task")
|
||||
" "
|
||||
arg-str))
|
||||
#?(:clj (System/exit 1)
|
||||
:cljs (js/process.exit 1)))
|
||||
|
||||
(defn print-table
|
||||
[rows]
|
||||
(when-some [rendered-table (render-table rows)]
|
||||
(print rendered-table))
|
||||
(println "Total:" (count rows)))
|
||||
Reference in New Issue
Block a user