diff --git a/src/main/frontend/fs.cljs b/src/main/frontend/fs.cljs
index 8e0c0b7..c22ee2d 100644
--- a/src/main/frontend/fs.cljs
+++ b/src/main/frontend/fs.cljs
@@ -8,6 +8,7 @@
[frontend.fs.memory-fs :as memory-fs]
[frontend.fs.node :as node]
[frontend.fs.protocol :as protocol]
+ [frontend.persist-db.remote :as remote]
[frontend.state :as state]
[frontend.util :as util]
[lambdaisland.glogi :as log]
@@ -115,15 +116,34 @@
:user-agent (when js/navigator js/navigator.userAgent)
:content-length (count content)}}])))))))
+(defn- remote-server-configured?
+ "True for a web/mobile client pointed at a self-hosted db-worker-node
+ server (Fase 3) -- it has no local filesystem to write assets to at all,
+ unlike Electron (writes straight to disk below) or a plain local browser
+ (falls through to memory-backend). Duplicated from the equivalent check in
+ frontend.handler.assets, which can't be required here without a cycle
+ (it already requires this ns)."
+ []
+ (and (not (util/electron?))
+ (some? (config/get-server-base-url))))
+
(defn write-asset-file!
[repo file-name data]
(let [repo-dir (config/get-repo-dir repo)]
- (if (util/electron?)
+ (cond
+ (util/electron?)
(let [assets-dir (path/path-join repo-dir common-config/local-assets-dir)
file-path (path/path-join assets-dir file-name)]
;; Use writeFileBytes directly instead of ipc/ipc (write-file!) because
;; binary data like ArrayBuffer can't be transit-serialized
(js/window.apis.writeFileBytes file-path data))
+
+ (remote-server-configured?)
+ (remote/upload-asset! {:base-url (config/get-server-base-url)
+ :auth-token (config/get-server-token)}
+ repo file-name data)
+
+ :else
(let [file-path (path/path-join common-config/local-assets-dir file-name)]
(write-plain-text-file! repo repo-dir file-path data {:skip-transact? true
:skip-compare? true})))))
diff --git a/src/main/frontend/handler/assets.cljs b/src/main/frontend/handler/assets.cljs
index a225701..0b588af 100644
--- a/src/main/frontend/handler/assets.cljs
+++ b/src/main/frontend/handler/assets.cljs
@@ -76,12 +76,52 @@
(string/replace-first "/logseq__colon/" ":/"))
url))
+(defn remote-server-configured?
+ "True for a web/mobile client pointed at a self-hosted db-worker-node
+ server (Fase 3) -- the counterpart to util/electron? for asset URL
+ resolution below, since such a client has no local filesystem to read
+ assets from at all (see docs/agent-guide Fase 5)."
+ []
+ (and (not (util/electron?))
+ (some? (config/get-server-base-url))))
+
+(defn- bare-asset-file-name
+ "Strips a leading assets/ (or ./assets/, ../assets/) prefix, if any --
+ the daemon's /v1/assets/:repo/:name route stores files flat per repo, so
+ it only wants the bare file name, not a path relative to the graph root."
+ [rpath]
+ (let [rpath (string/replace-first rpath #"^\.{1,2}/" "")
+ prefix (str common-config/local-assets-dir "/")]
+ (if (string/starts-with? rpath prefix)
+ (subs rpath (count prefix))
+ rpath)))
+
+(defn remote-asset-url
+ "Builds a directly-loadable URL (
, browser navigation, plain
+ fetch) for an asset served by a remote db-worker-node daemon. The token
+ travels as a query param -- the same interim mechanism SSE used before
+ the Fase 4 WebSocket migration -- since a browser navigating straight to
+ this URL (rather than fetching it from JS) can't set an Authorization
+ header. See asset-get-request? in db_worker_node.cljs."
+ [repo rpath]
+ (let [base-url (string/replace (config/get-server-base-url) #"/$" "")
+ file-name (bare-asset-file-name rpath)
+ token (config/get-server-token)]
+ (cond-> (str base-url "/v1/assets/" (js/encodeURIComponent repo) "/" (js/encodeURIComponent file-name))
+ (seq token) (str "?token=" (js/encodeURIComponent token)))))
+
(defn resolve-asset-real-path-url
[repo rpath]
(when-let [rpath (and (string? rpath)
(string/replace rpath #"^[.\/\\]+" ""))]
- (if config/publishing?
+ (cond
+ config/publishing?
(str "./" rpath)
+
+ (remote-server-configured?)
+ (remote-asset-url repo rpath)
+
+ :else
(let [ret (let [rpath (if-not (string/starts-with? rpath common-config/local-assets-dir)
(path/path-join common-config/local-assets-dir rpath)
rpath)
@@ -176,6 +216,9 @@
;; fullpath will be encoded
(path/prepend-protocol "assets:" (protect-windows-drive-in-assets-path full-path)))
+ (remote-server-configured?)
+ (remote-asset-url repo path)
+
:else
(p/let [binary (fs/read-file-raw repo-dir path {})
svg? (string/ends-with? path ".svg")
diff --git a/src/main/frontend/persist_db/remote.cljs b/src/main/frontend/persist_db/remote.cljs
index b893323..c6d0f47 100644
--- a/src/main/frontend/persist_db/remote.cljs
+++ b/src/main/frontend/persist_db/remote.cljs
@@ -23,6 +23,14 @@
"/v1/import-db-binary?repo="
(js/encodeURIComponent repo)))
+(defn- asset-url
+ [base-url repo file-name]
+ (str (normalize-base-url base-url)
+ "/v1/assets/"
+ (js/encodeURIComponent repo)
+ "/"
+ (js/encodeURIComponent file-name)))
+
(defn- events-url
"EventSource has no header API, so the bearer token travels as a query
param here instead of the `Authorization` header every other request
@@ -102,6 +110,26 @@
:code (normalize-code (:code error))}
error (assoc :error error))))))))
+(defn upload-asset!
+ "PUTs raw asset bytes to a db-worker-node daemon's /v1/assets/:repo/:name
+ (see Fase 5 in docs/agent-guide). Used by frontend.fs/write-asset-file! for
+ a web/mobile client, which has no local filesystem to write assets to --
+ Electron keeps writing straight to disk via its own IPC instead."
+ [{:keys [base-url auth-token fetch-fn]} repo file-name data]
+ (p/let [{:keys [status body]} ((or fetch-fn default-fetch-json!)
+ {:method "PUT"
+ :url (asset-url base-url repo file-name)
+ :headers (binary-headers auth-token)
+ :body data})]
+ (if (<= 200 status 299)
+ true
+ (let [parsed (parse-response-body body)
+ error (:error parsed)]
+ (throw (ex-info (or (:message error) "asset upload failed")
+ (cond-> {:status status
+ :code (normalize-code (:code error))}
+ error (assoc :error error))))))))
+
(defn- decode-event
[{:keys [type payload]}]
(let [decoded (when (some? payload)
diff --git a/src/main/frontend/worker/db_core.cljs b/src/main/frontend/worker/db_core.cljs
index e49222f..5876ec4 100644
--- a/src/main/frontend/worker/db_core.cljs
+++ b/src/main/frontend/worker/db_core.cljs
@@ -888,6 +888,25 @@
[repo config-file files opts]
( (platform/asset-read-bytes! (platform/current) repo file-name)
+ (p/catch (fn [_error] nil))))
+
+(def-thread-api :thread-api/asset-write-bytes
+ [repo file-name payload]
+ (platform/asset-write-bytes! (platform/current) repo file-name payload))
+
+(def-thread-api :thread-api/asset-stat
+ [repo file-name]
+ (platform/asset-stat (platform/current) repo file-name))
+
(comment
(def-thread-api :general/dangerousRemoveAllDbs
[]
diff --git a/src/main/frontend/worker/db_worker_node.cljs b/src/main/frontend/worker/db_worker_node.cljs
index 7b31b65..706413c 100644
--- a/src/main/frontend/worker/db_worker_node.cljs
+++ b/src/main/frontend/worker/db_worker_node.cljs
@@ -1,6 +1,7 @@
(ns frontend.worker.db-worker-node
"Node.js daemon entrypoint for db-worker."
- (:require ["http" :as http]
+ (:require ["crypto" :as crypto]
+ ["http" :as http]
["ws" :as ws]
[clojure.string :as string]
[frontend.worker.db-core :as db-core]
@@ -384,17 +385,68 @@
[^js req allowed-hosts]
(contains? allowed-hosts (some-> req .-headers .-host)))
+(defn- asset-get-request?
+ [request-path method]
+ (and (= method "GET") (string/starts-with? request-path "/v1/assets/")))
+
(defn- request-authorized?
- "/v1/events (SSE) can't carry an Authorization header -- EventSource has no
- header API -- so it accepts the bearer token as a query param instead.
- This is interim: the WebSocket migration (see docs/agent-guide) replaces
- it with Sec-WebSocket-Protocol, which doesn't leak into proxy/access logs
- the way a query string does."
- [^js req parsed-url request-path]
- (if (= request-path "/v1/events")
- (auth/authorized? (some->> (.get (.-searchParams parsed-url) "token")
- (str "Bearer ")))
- (auth/authorized? (.. req -headers -authorization))))
+ "The Authorization header works for every route, including these two --
+ a plain JS fetch() can set it fine. It's only a *browser navigating
+ directly to a URL* that can't: EventSource (/v1/events, SSE, now a
+ fallback since the Fase 4 WS migration) has no header API, and an
+
/direct navigation to an asset GET can't set custom headers
+ either. Both accept ?token= as a fallback in place of Authorization, never
+ as a replacement for it."
+ [^js req parsed-url request-path method]
+ (or (auth/authorized? (.. req -headers -authorization))
+ (and (or (= request-path "/v1/events")
+ (asset-get-request? request-path method))
+ (auth/authorized? (some->> (.get (.-searchParams parsed-url) "token")
+ (str "Bearer "))))))
+
+(def ^:private asset-content-types
+ {"png" "image/png"
+ "jpg" "image/jpeg"
+ "jpeg" "image/jpeg"
+ "gif" "image/gif"
+ "webp" "image/webp"
+ "svg" "image/svg+xml"
+ "avif" "image/avif"
+ "bmp" "image/bmp"
+ "ico" "image/x-icon"
+ "pdf" "application/pdf"
+ "mp3" "audio/mpeg"
+ "wav" "audio/wav"
+ "ogg" "audio/ogg"
+ "mp4" "video/mp4"
+ "webm" "video/webm"
+ "mov" "video/quicktime"
+ "json" "application/json"
+ "txt" "text/plain"})
+
+(defn- guess-asset-content-type
+ [file-name]
+ (let [ext (some-> (re-find #"\.([A-Za-z0-9]+)$" (or file-name ""))
+ second
+ string/lower-case)]
+ (get asset-content-types ext "application/octet-stream")))
+
+(defn- parse-asset-path
+ "/v1/assets/:repo/:name -- both segments are URI-encoded by the client
+ (frontend.persist-db.remote), since a repo or asset file name could
+ contain characters that aren't legal in a URL path segment as-is."
+ [request-path]
+ (let [rest (subs request-path (count "/v1/assets/"))
+ slash-idx (string/index-of rest "/")]
+ (when (and slash-idx (pos? slash-idx) (< (inc slash-idx) (count rest)))
+ (try
+ {:repo (js/decodeURIComponent (subs rest 0 slash-idx))
+ :file-name (js/decodeURIComponent (subs rest (inc slash-idx)))}
+ (catch :default _ nil)))))
+
+(defn- asset-etag
+ [^js buf]
+ (str "\"" (-> (crypto/createHash "sha1") (.update buf) (.digest "hex")) "\""))
(defn- ws-upgrade-handler
[^js wss bound-repo port extra-allowed-hosts]
@@ -470,7 +522,7 @@
(send-json! res 400 {:ok false :error {:code :invalid-request
:message "invalid login request body"}}))))
- (not (request-authorized? req parsed-url request-path))
+ (not (request-authorized? req parsed-url request-path method))
(send-json! res 401 {:ok false :error {:code :unauthorized
:message "missing or invalid credentials"}})
@@ -514,6 +566,51 @@
(send-json! res status payload))))))
(send-text! res 405 "method-not-allowed"))
+ (and (string/starts-with? request-path "/v1/assets/") (= method "GET"))
+ (if-let [{:keys [repo file-name]} (parse-asset-path request-path)]
+ (let [method-kw :thread-api/asset-read-bytes
+ method-str (normalize-method-str method-kw)
+ args [repo file-name]]
+ (if-let [{:keys [status error]} (repo-error method-kw args bound-repo)]
+ (send-json! res status {:ok false :error error})
+ (-> (p/let [result ( (p/let [binary (
diff --git a/src/test/frontend/worker/db_worker_node_test.cljs b/src/test/frontend/worker/db_worker_node_test.cljs
index 64b28db..e1c9d01 100644
--- a/src/test/frontend/worker/db_worker_node_test.cljs
+++ b/src/test/frontend/worker/db_worker_node_test.cljs
@@ -195,6 +195,51 @@
[^js client payload]
(.send client (js/JSON.stringify (clj->js payload))))
+(defn- http-request-binary
+ "Like http-request, but keeps the raw response bytes instead of decoding
+ them as utf8 -- utf8-decoding arbitrary binary content (asset bytes) would
+ be lossy and non-invertible."
+ [opts body]
+ (p/create
+ (fn [resolve reject]
+ (let [req (.request http (clj->js opts)
+ (fn [^js res]
+ (let [chunks (array)]
+ (.on res "data" (fn [chunk] (.push chunks chunk)))
+ (.on res "end" (fn []
+ (resolve {:status (.-statusCode res)
+ :headers (js->clj (.-headers res) :keywordize-keys true)
+ :body-buffer (js/Buffer.concat chunks)}))))))
+ finish! (fn []
+ (when body (.write req body))
+ (.end req))]
+ (.on req "error" reject)
+ (finish!)))))
+
+(defn- asset-path
+ [repo file-name]
+ (str "/v1/assets/" (js/encodeURIComponent repo) "/" (js/encodeURIComponent file-name)))
+
+(defn- asset-put!
+ [host port repo file-name binary]
+ (p/let [headers (auth-headers! host port)]
+ (http-request-binary {:hostname host
+ :port port
+ :path (asset-path repo file-name)
+ :method "PUT"
+ :headers headers}
+ binary)))
+
+(defn- asset-get
+ [host port repo file-name & [extra-headers]]
+ (p/let [headers (auth-headers! host port)]
+ (http-request-binary {:hostname host
+ :port port
+ :path (asset-path repo file-name)
+ :method "GET"
+ :headers (merge headers extra-headers)}
+ nil)))
+
(defn- lock-path
[root-dir repo]
(db-lock/lock-path root-dir repo))
@@ -1442,6 +1487,101 @@
:else
(done)))))))))
+(deftest db-worker-node-asset-put-get-round-trips-bytes-with-caching-headers
+ (async done
+ (let [daemon (atom nil)
+ data-dir (node-helper/create-tmp-dir "db-worker-asset-roundtrip")
+ repo (str "logseq_db_asset_roundtrip_" (subs (str (random-uuid)) 0 8))
+ binary (js/Buffer.from (js/Uint8Array. (into-array (range 256))))]
+ (-> (p/let [{:keys [host port stop!]} (start-daemon! {:root-dir data-dir :repo repo})
+ _ (reset! daemon {:stop! stop!})
+ _ (invoke host port "thread-api/create-or-open-db" [repo {}])
+ put-result (asset-put! host port repo "pic1.png" binary)
+ _ (is (= 200 (:status put-result)))
+ get-result (asset-get host port repo "pic1.png")
+ {:keys [status headers body-buffer]} get-result]
+ (is (= 200 status))
+ (is (= "image/png" (:content-type headers)))
+ (is (string? (:etag headers)))
+ (is (string/starts-with? (:cache-control headers) "public, max-age=31536000, immutable"))
+ (is (= (.toString binary "base64") (.toString body-buffer "base64")))
+ (p/let [conditional-result (asset-get host port repo "pic1.png"
+ {"If-None-Match" (:etag headers)})]
+ (is (= 304 (:status conditional-result)))))
+ (p/catch (fn [e]
+ (println "[db-worker-node-test] asset-roundtrip error:" e)
+ (is false (str e))))
+ (p/finally (fn []
+ (if-let [stop! (:stop! @daemon)]
+ (-> (stop!) (p/finally done))
+ (done))))))))
+
+(deftest db-worker-node-asset-get-missing-file-returns-404
+ (async done
+ (let [daemon (atom nil)
+ data-dir (node-helper/create-tmp-dir "db-worker-asset-missing")
+ repo (str "logseq_db_asset_missing_" (subs (str (random-uuid)) 0 8))]
+ (-> (p/let [{:keys [host port stop!]} (start-daemon! {:root-dir data-dir :repo repo})
+ _ (reset! daemon {:stop! stop!})
+ _ (invoke host port "thread-api/create-or-open-db" [repo {}])
+ {:keys [status]} (asset-get host port repo "does-not-exist.png")]
+ (is (= 404 status)))
+ (p/catch (fn [e]
+ (println "[db-worker-node-test] asset-missing error:" e)
+ (is false (str e))))
+ (p/finally (fn []
+ (if-let [stop! (:stop! @daemon)]
+ (-> (stop!) (p/finally done))
+ (done))))))))
+
+(deftest db-worker-node-asset-repo-mismatch-returns-409
+ (async done
+ (let [daemon (atom nil)
+ data-dir (node-helper/create-tmp-dir "db-worker-asset-repo-mismatch")
+ repo (str "logseq_db_asset_repo_mismatch_" (subs (str (random-uuid)) 0 8))
+ binary (js/Buffer.from #js [1 2 3])]
+ (-> (p/let [{:keys [host port stop!]} (start-daemon! {:root-dir data-dir :repo repo})
+ _ (reset! daemon {:stop! stop!})
+ _ (invoke host port "thread-api/create-or-open-db" [repo {}])
+ get-result (asset-get host port "some_other_repo" "pic1.png")
+ put-result (asset-put! host port "some_other_repo" "pic1.png" binary)]
+ (is (= 409 (:status get-result)))
+ (is (= 409 (:status put-result))))
+ (p/catch (fn [e]
+ (println "[db-worker-node-test] asset-repo-mismatch error:" e)
+ (is false (str e))))
+ (p/finally (fn []
+ (if-let [stop! (:stop! @daemon)]
+ (-> (stop!) (p/finally done))
+ (done))))))))
+
+(deftest db-worker-node-asset-requires-auth
+ (async done
+ (let [daemon (atom nil)
+ data-dir (node-helper/create-tmp-dir "db-worker-asset-auth")
+ repo (str "logseq_db_asset_auth_" (subs (str (random-uuid)) 0 8))
+ binary (js/Buffer.from #js [1 2 3])]
+ (-> (p/let [{:keys [host port stop!]} (start-daemon! {:root-dir data-dir :repo repo})
+ _ (reset! daemon {:stop! stop!})
+ _ (invoke host port "thread-api/create-or-open-db" [repo {}])
+ get-result (http-request-binary {:hostname host :port port
+ :path (asset-path repo "pic1.png")
+ :method "GET"}
+ nil)
+ put-result (http-request-binary {:hostname host :port port
+ :path (asset-path repo "pic1.png")
+ :method "PUT"}
+ binary)]
+ (is (= 401 (:status get-result)))
+ (is (= 401 (:status put-result))))
+ (p/catch (fn [e]
+ (println "[db-worker-node-test] asset-auth error:" e)
+ (is false (str e))))
+ (p/finally (fn []
+ (if-let [stop! (:stop! @daemon)]
+ (-> (stop!) (p/finally done))
+ (done))))))))
+
(deftest db-worker-node-export-client-ops-db-binary
(async done
(let [daemon (atom nil)