Fase 5: serve and accept assets over HTTP on the daemon

Closes the gap the plan flagged as blocking the sync deletion: a web/mobile
client had no way to read or write asset bytes at all (only db-sync's now
end-of-life handlers touched platform/asset-*). Adds GET/PUT
/v1/assets/:repo/:name on db_worker_node.cljs, backed by three new
thread-api ops (asset-read-bytes/write-bytes/stat) that just call the
existing Node platform.cljs asset-*-bytes! functions -- reusing the same
JSON+transit /v1/invoke path already proven for export-db-binary, so no new
binary RPC mechanism was needed.

GET responses carry a real Content-Type (guessed from extension), a sha1
ETag, and immutable Cache-Control -- assets are content-addressed by
filename, so this is safe to cache forever once fetched. Auth accepts
?token= as a fallback for a direct <img src>/browser navigation, same
rationale as the SSE interim token, but as a genuine fallback alongside the
Authorization header rather than replacing it (an early version of this
made query-token exclusive with header auth, which 401'd every JS-driven
fetch -- caught before committing, real end-to-end curl testing against a
live daemon also caught a 500-instead-of-404 on missing assets and a
never-firing 304 from a hyphenated JS property access that needed `aget`
instead of `.-`).

Client side: frontend.handler.assets/resolve-asset-real-path-url and
<make-asset-url now resolve to the daemon's asset URL (with the server
token) instead of an unreachable file:// path when a web/mobile client has
a server configured (frontend.config/get-server-base-url) -- covers viewing
existing assets. frontend.fs/write-asset-file! gets the matching upload
path via a new frontend.persist-db.remote/upload-asset!, so pasting an
asset from a web/mobile client no longer silently lands in
frontend.fs.memory-fs and vanishes on reload.

Server-side routes and round-trip verified end-to-end against a real
daemon (curl) and with new automated tests (byte equality, content-type,
etag/304, 404, repo-mismatch, auth-required). The client-side URL
resolution changes could not be visually verified -- no browser in this
environment -- only reviewed and compiled clean.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
2026-08-26 10:33:42 +02:00
parent cf61c46c35
commit 902bf7f568
6 changed files with 361 additions and 14 deletions

View File

@@ -8,6 +8,7 @@
[frontend.fs.memory-fs :as memory-fs]
[frontend.fs.node :as node]
[frontend.fs.protocol :as protocol]
[frontend.persist-db.remote :as remote]
[frontend.state :as state]
[frontend.util :as util]
[lambdaisland.glogi :as log]
@@ -115,15 +116,34 @@
:user-agent (when js/navigator js/navigator.userAgent)
:content-length (count content)}}])))))))
(defn- remote-server-configured?
"True for a web/mobile client pointed at a self-hosted db-worker-node
server (Fase 3) -- it has no local filesystem to write assets to at all,
unlike Electron (writes straight to disk below) or a plain local browser
(falls through to memory-backend). Duplicated from the equivalent check in
frontend.handler.assets, which can't be required here without a cycle
(it already requires this ns)."
[]
(and (not (util/electron?))
(some? (config/get-server-base-url))))
(defn write-asset-file!
[repo file-name data]
(let [repo-dir (config/get-repo-dir repo)]
(if (util/electron?)
(cond
(util/electron?)
(let [assets-dir (path/path-join repo-dir common-config/local-assets-dir)
file-path (path/path-join assets-dir file-name)]
;; Use writeFileBytes directly instead of ipc/ipc (write-file!) because
;; binary data like ArrayBuffer can't be transit-serialized
(js/window.apis.writeFileBytes file-path data))
(remote-server-configured?)
(remote/upload-asset! {:base-url (config/get-server-base-url)
:auth-token (config/get-server-token)}
repo file-name data)
:else
(let [file-path (path/path-join common-config/local-assets-dir file-name)]
(write-plain-text-file! repo repo-dir file-path data {:skip-transact? true
:skip-compare? true})))))

View File

@@ -76,12 +76,52 @@
(string/replace-first "/logseq__colon/" ":/"))
url))
(defn remote-server-configured?
"True for a web/mobile client pointed at a self-hosted db-worker-node
server (Fase 3) -- the counterpart to util/electron? for asset URL
resolution below, since such a client has no local filesystem to read
assets from at all (see docs/agent-guide Fase 5)."
[]
(and (not (util/electron?))
(some? (config/get-server-base-url))))
(defn- bare-asset-file-name
"Strips a leading assets/ (or ./assets/, ../assets/) prefix, if any --
the daemon's /v1/assets/:repo/:name route stores files flat per repo, so
it only wants the bare file name, not a path relative to the graph root."
[rpath]
(let [rpath (string/replace-first rpath #"^\.{1,2}/" "")
prefix (str common-config/local-assets-dir "/")]
(if (string/starts-with? rpath prefix)
(subs rpath (count prefix))
rpath)))
(defn remote-asset-url
"Builds a directly-loadable URL (<img src>, browser navigation, plain
fetch) for an asset served by a remote db-worker-node daemon. The token
travels as a query param -- the same interim mechanism SSE used before
the Fase 4 WebSocket migration -- since a browser navigating straight to
this URL (rather than fetching it from JS) can't set an Authorization
header. See asset-get-request? in db_worker_node.cljs."
[repo rpath]
(let [base-url (string/replace (config/get-server-base-url) #"/$" "")
file-name (bare-asset-file-name rpath)
token (config/get-server-token)]
(cond-> (str base-url "/v1/assets/" (js/encodeURIComponent repo) "/" (js/encodeURIComponent file-name))
(seq token) (str "?token=" (js/encodeURIComponent token)))))
(defn resolve-asset-real-path-url
[repo rpath]
(when-let [rpath (and (string? rpath)
(string/replace rpath #"^[.\/\\]+" ""))]
(if config/publishing?
(cond
config/publishing?
(str "./" rpath)
(remote-server-configured?)
(remote-asset-url repo rpath)
:else
(let [ret (let [rpath (if-not (string/starts-with? rpath common-config/local-assets-dir)
(path/path-join common-config/local-assets-dir rpath)
rpath)
@@ -176,6 +216,9 @@
;; fullpath will be encoded
(path/prepend-protocol "assets:" (protect-windows-drive-in-assets-path full-path)))
(remote-server-configured?)
(remote-asset-url repo path)
:else
(p/let [binary (fs/read-file-raw repo-dir path {})
svg? (string/ends-with? path ".svg")

View File

@@ -23,6 +23,14 @@
"/v1/import-db-binary?repo="
(js/encodeURIComponent repo)))
(defn- asset-url
[base-url repo file-name]
(str (normalize-base-url base-url)
"/v1/assets/"
(js/encodeURIComponent repo)
"/"
(js/encodeURIComponent file-name)))
(defn- events-url
"EventSource has no header API, so the bearer token travels as a query
param here instead of the `Authorization` header every other request
@@ -102,6 +110,26 @@
:code (normalize-code (:code error))}
error (assoc :error error))))))))
(defn upload-asset!
"PUTs raw asset bytes to a db-worker-node daemon's /v1/assets/:repo/:name
(see Fase 5 in docs/agent-guide). Used by frontend.fs/write-asset-file! for
a web/mobile client, which has no local filesystem to write assets to --
Electron keeps writing straight to disk via its own IPC instead."
[{:keys [base-url auth-token fetch-fn]} repo file-name data]
(p/let [{:keys [status body]} ((or fetch-fn default-fetch-json!)
{:method "PUT"
:url (asset-url base-url repo file-name)
:headers (binary-headers auth-token)
:body data})]
(if (<= 200 status 299)
true
(let [parsed (parse-response-body body)
error (:error parsed)]
(throw (ex-info (or (:message error) "asset upload failed")
(cond-> {:status status
:code (normalize-code (:code error))}
error (assoc :error error))))))))
(defn- decode-event
[{:keys [type payload]}]
(let [decoded (when (some? payload)

View File

@@ -888,6 +888,25 @@
[repo config-file files opts]
(<import-file-graph! repo config-file files opts))
;; Fase 5 (server-client migration): let a remote client read/write asset
;; bytes over HTTP instead of a local filesystem it doesn't have -- see
;; the /v1/assets/:repo/:name routes in db_worker_node.cljs, which call
;; these via the regular JSON+transit /v1/invoke path (same as
;; export-db-binary/import-db-binary; transit already round-trips
;; Uint8Array, so no separate binary RPC is needed here).
(def-thread-api :thread-api/asset-read-bytes
[repo file-name]
(-> (platform/asset-read-bytes! (platform/current) repo file-name)
(p/catch (fn [_error] nil))))
(def-thread-api :thread-api/asset-write-bytes
[repo file-name payload]
(platform/asset-write-bytes! (platform/current) repo file-name payload))
(def-thread-api :thread-api/asset-stat
[repo file-name]
(platform/asset-stat (platform/current) repo file-name))
(comment
(def-thread-api :general/dangerousRemoveAllDbs
[]

View File

@@ -1,6 +1,7 @@
(ns frontend.worker.db-worker-node
"Node.js daemon entrypoint for db-worker."
(:require ["http" :as http]
(:require ["crypto" :as crypto]
["http" :as http]
["ws" :as ws]
[clojure.string :as string]
[frontend.worker.db-core :as db-core]
@@ -384,17 +385,68 @@
[^js req allowed-hosts]
(contains? allowed-hosts (some-> req .-headers .-host)))
(defn- asset-get-request?
[request-path method]
(and (= method "GET") (string/starts-with? request-path "/v1/assets/")))
(defn- request-authorized?
"/v1/events (SSE) can't carry an Authorization header -- EventSource has no
header API -- so it accepts the bearer token as a query param instead.
This is interim: the WebSocket migration (see docs/agent-guide) replaces
it with Sec-WebSocket-Protocol, which doesn't leak into proxy/access logs
the way a query string does."
[^js req parsed-url request-path]
(if (= request-path "/v1/events")
(auth/authorized? (some->> (.get (.-searchParams parsed-url) "token")
(str "Bearer ")))
(auth/authorized? (.. req -headers -authorization))))
"The Authorization header works for every route, including these two --
a plain JS fetch() can set it fine. It's only a *browser navigating
directly to a URL* that can't: EventSource (/v1/events, SSE, now a
fallback since the Fase 4 WS migration) has no header API, and an
<img src>/direct navigation to an asset GET can't set custom headers
either. Both accept ?token= as a fallback in place of Authorization, never
as a replacement for it."
[^js req parsed-url request-path method]
(or (auth/authorized? (.. req -headers -authorization))
(and (or (= request-path "/v1/events")
(asset-get-request? request-path method))
(auth/authorized? (some->> (.get (.-searchParams parsed-url) "token")
(str "Bearer "))))))
(def ^:private asset-content-types
{"png" "image/png"
"jpg" "image/jpeg"
"jpeg" "image/jpeg"
"gif" "image/gif"
"webp" "image/webp"
"svg" "image/svg+xml"
"avif" "image/avif"
"bmp" "image/bmp"
"ico" "image/x-icon"
"pdf" "application/pdf"
"mp3" "audio/mpeg"
"wav" "audio/wav"
"ogg" "audio/ogg"
"mp4" "video/mp4"
"webm" "video/webm"
"mov" "video/quicktime"
"json" "application/json"
"txt" "text/plain"})
(defn- guess-asset-content-type
[file-name]
(let [ext (some-> (re-find #"\.([A-Za-z0-9]+)$" (or file-name ""))
second
string/lower-case)]
(get asset-content-types ext "application/octet-stream")))
(defn- parse-asset-path
"/v1/assets/:repo/:name -- both segments are URI-encoded by the client
(frontend.persist-db.remote), since a repo or asset file name could
contain characters that aren't legal in a URL path segment as-is."
[request-path]
(let [rest (subs request-path (count "/v1/assets/"))
slash-idx (string/index-of rest "/")]
(when (and slash-idx (pos? slash-idx) (< (inc slash-idx) (count rest)))
(try
{:repo (js/decodeURIComponent (subs rest 0 slash-idx))
:file-name (js/decodeURIComponent (subs rest (inc slash-idx)))}
(catch :default _ nil)))))
(defn- asset-etag
[^js buf]
(str "\"" (-> (crypto/createHash "sha1") (.update buf) (.digest "hex")) "\""))
(defn- ws-upgrade-handler
[^js wss bound-repo port extra-allowed-hosts]
@@ -470,7 +522,7 @@
(send-json! res 400 {:ok false :error {:code :invalid-request
:message "invalid login request body"}}))))
(not (request-authorized? req parsed-url request-path))
(not (request-authorized? req parsed-url request-path method))
(send-json! res 401 {:ok false :error {:code :unauthorized
:message "missing or invalid credentials"}})
@@ -514,6 +566,51 @@
(send-json! res status payload))))))
(send-text! res 405 "method-not-allowed"))
(and (string/starts-with? request-path "/v1/assets/") (= method "GET"))
(if-let [{:keys [repo file-name]} (parse-asset-path request-path)]
(let [method-kw :thread-api/asset-read-bytes
method-str (normalize-method-str method-kw)
args [repo file-name]]
(if-let [{:keys [status error]} (repo-error method-kw args bound-repo)]
(send-json! res status {:ok false :error error})
(-> (p/let [result (<invoke! proxy method-str method-kw args)
bytes (ldb/read-transit-str result)]
(if (some? bytes)
(let [buf (js/Buffer.from bytes)
etag (asset-etag buf)]
(if (= etag (aget (.-headers req) "if-none-match"))
(do
(.writeHead res 304 (response-headers res #js {"ETag" etag
"Cache-Control" "public, max-age=31536000, immutable"}))
(.end res))
(do
(.writeHead res 200 (response-headers
res #js {"Content-Type" (guess-asset-content-type file-name)
"Content-Length" (.-length buf)
"ETag" etag
"Cache-Control" "public, max-age=31536000, immutable"}))
(.end res buf))))
(send-json! res 404 {:ok false :error {:code :not-found :message "asset not found"}})))
(p/catch (fn [error]
(log-invoke-error! res error method-kw))))))
(send-json! res 400 {:ok false :error {:code :invalid-request :message "invalid asset path"}}))
(and (string/starts-with? request-path "/v1/assets/") (= method "PUT"))
(if-let [{:keys [repo file-name]} (parse-asset-path request-path)]
(-> (p/let [binary (<read-body-buffer req)
method-kw :thread-api/asset-write-bytes
method-str (normalize-method-str method-kw)
args [repo file-name binary]]
(if-let [{:keys [status error]} (repo-error method-kw args bound-repo)]
(send-json! res status {:ok false :error error})
(p/let [_ (let [{:keys [path lock]} @*lock-info]
(db-lock/assert-lock-owner! path lock))
_ (<invoke! proxy method-str method-kw args)]
(send-json! res 200 {:ok true}))))
(p/catch (fn [error]
(log-invoke-error! res error :thread-api/asset-write-bytes))))
(send-json! res 400 {:ok false :error {:code :invalid-request :message "invalid asset path"}}))
(= request-path "/v1/invoke")
(if (= method "POST")
(->

View File

@@ -195,6 +195,51 @@
[^js client payload]
(.send client (js/JSON.stringify (clj->js payload))))
(defn- http-request-binary
"Like http-request, but keeps the raw response bytes instead of decoding
them as utf8 -- utf8-decoding arbitrary binary content (asset bytes) would
be lossy and non-invertible."
[opts body]
(p/create
(fn [resolve reject]
(let [req (.request http (clj->js opts)
(fn [^js res]
(let [chunks (array)]
(.on res "data" (fn [chunk] (.push chunks chunk)))
(.on res "end" (fn []
(resolve {:status (.-statusCode res)
:headers (js->clj (.-headers res) :keywordize-keys true)
:body-buffer (js/Buffer.concat chunks)}))))))
finish! (fn []
(when body (.write req body))
(.end req))]
(.on req "error" reject)
(finish!)))))
(defn- asset-path
[repo file-name]
(str "/v1/assets/" (js/encodeURIComponent repo) "/" (js/encodeURIComponent file-name)))
(defn- asset-put!
[host port repo file-name binary]
(p/let [headers (auth-headers! host port)]
(http-request-binary {:hostname host
:port port
:path (asset-path repo file-name)
:method "PUT"
:headers headers}
binary)))
(defn- asset-get
[host port repo file-name & [extra-headers]]
(p/let [headers (auth-headers! host port)]
(http-request-binary {:hostname host
:port port
:path (asset-path repo file-name)
:method "GET"
:headers (merge headers extra-headers)}
nil)))
(defn- lock-path
[root-dir repo]
(db-lock/lock-path root-dir repo))
@@ -1442,6 +1487,101 @@
:else
(done)))))))))
(deftest db-worker-node-asset-put-get-round-trips-bytes-with-caching-headers
(async done
(let [daemon (atom nil)
data-dir (node-helper/create-tmp-dir "db-worker-asset-roundtrip")
repo (str "logseq_db_asset_roundtrip_" (subs (str (random-uuid)) 0 8))
binary (js/Buffer.from (js/Uint8Array. (into-array (range 256))))]
(-> (p/let [{:keys [host port stop!]} (start-daemon! {:root-dir data-dir :repo repo})
_ (reset! daemon {:stop! stop!})
_ (invoke host port "thread-api/create-or-open-db" [repo {}])
put-result (asset-put! host port repo "pic1.png" binary)
_ (is (= 200 (:status put-result)))
get-result (asset-get host port repo "pic1.png")
{:keys [status headers body-buffer]} get-result]
(is (= 200 status))
(is (= "image/png" (:content-type headers)))
(is (string? (:etag headers)))
(is (string/starts-with? (:cache-control headers) "public, max-age=31536000, immutable"))
(is (= (.toString binary "base64") (.toString body-buffer "base64")))
(p/let [conditional-result (asset-get host port repo "pic1.png"
{"If-None-Match" (:etag headers)})]
(is (= 304 (:status conditional-result)))))
(p/catch (fn [e]
(println "[db-worker-node-test] asset-roundtrip error:" e)
(is false (str e))))
(p/finally (fn []
(if-let [stop! (:stop! @daemon)]
(-> (stop!) (p/finally done))
(done))))))))
(deftest db-worker-node-asset-get-missing-file-returns-404
(async done
(let [daemon (atom nil)
data-dir (node-helper/create-tmp-dir "db-worker-asset-missing")
repo (str "logseq_db_asset_missing_" (subs (str (random-uuid)) 0 8))]
(-> (p/let [{:keys [host port stop!]} (start-daemon! {:root-dir data-dir :repo repo})
_ (reset! daemon {:stop! stop!})
_ (invoke host port "thread-api/create-or-open-db" [repo {}])
{:keys [status]} (asset-get host port repo "does-not-exist.png")]
(is (= 404 status)))
(p/catch (fn [e]
(println "[db-worker-node-test] asset-missing error:" e)
(is false (str e))))
(p/finally (fn []
(if-let [stop! (:stop! @daemon)]
(-> (stop!) (p/finally done))
(done))))))))
(deftest db-worker-node-asset-repo-mismatch-returns-409
(async done
(let [daemon (atom nil)
data-dir (node-helper/create-tmp-dir "db-worker-asset-repo-mismatch")
repo (str "logseq_db_asset_repo_mismatch_" (subs (str (random-uuid)) 0 8))
binary (js/Buffer.from #js [1 2 3])]
(-> (p/let [{:keys [host port stop!]} (start-daemon! {:root-dir data-dir :repo repo})
_ (reset! daemon {:stop! stop!})
_ (invoke host port "thread-api/create-or-open-db" [repo {}])
get-result (asset-get host port "some_other_repo" "pic1.png")
put-result (asset-put! host port "some_other_repo" "pic1.png" binary)]
(is (= 409 (:status get-result)))
(is (= 409 (:status put-result))))
(p/catch (fn [e]
(println "[db-worker-node-test] asset-repo-mismatch error:" e)
(is false (str e))))
(p/finally (fn []
(if-let [stop! (:stop! @daemon)]
(-> (stop!) (p/finally done))
(done))))))))
(deftest db-worker-node-asset-requires-auth
(async done
(let [daemon (atom nil)
data-dir (node-helper/create-tmp-dir "db-worker-asset-auth")
repo (str "logseq_db_asset_auth_" (subs (str (random-uuid)) 0 8))
binary (js/Buffer.from #js [1 2 3])]
(-> (p/let [{:keys [host port stop!]} (start-daemon! {:root-dir data-dir :repo repo})
_ (reset! daemon {:stop! stop!})
_ (invoke host port "thread-api/create-or-open-db" [repo {}])
get-result (http-request-binary {:hostname host :port port
:path (asset-path repo "pic1.png")
:method "GET"}
nil)
put-result (http-request-binary {:hostname host :port port
:path (asset-path repo "pic1.png")
:method "PUT"}
binary)]
(is (= 401 (:status get-result)))
(is (= 401 (:status put-result))))
(p/catch (fn [e]
(println "[db-worker-node-test] asset-auth error:" e)
(is false (str e))))
(p/finally (fn []
(if-let [stop! (:stop! @daemon)]
(-> (stop!) (p/finally done))
(done))))))))
(deftest db-worker-node-export-client-ops-db-binary
(async done
(let [daemon (atom nil)