Fase 5: serve and accept assets over HTTP on the daemon

Closes the gap the plan flagged as blocking the sync deletion: a web/mobile
client had no way to read or write asset bytes at all (only db-sync's now
end-of-life handlers touched platform/asset-*). Adds GET/PUT
/v1/assets/:repo/:name on db_worker_node.cljs, backed by three new
thread-api ops (asset-read-bytes/write-bytes/stat) that just call the
existing Node platform.cljs asset-*-bytes! functions -- reusing the same
JSON+transit /v1/invoke path already proven for export-db-binary, so no new
binary RPC mechanism was needed.

GET responses carry a real Content-Type (guessed from extension), a sha1
ETag, and immutable Cache-Control -- assets are content-addressed by
filename, so this is safe to cache forever once fetched. Auth accepts
?token= as a fallback for a direct <img src>/browser navigation, same
rationale as the SSE interim token, but as a genuine fallback alongside the
Authorization header rather than replacing it (an early version of this
made query-token exclusive with header auth, which 401'd every JS-driven
fetch -- caught before committing, real end-to-end curl testing against a
live daemon also caught a 500-instead-of-404 on missing assets and a
never-firing 304 from a hyphenated JS property access that needed `aget`
instead of `.-`).

Client side: frontend.handler.assets/resolve-asset-real-path-url and
<make-asset-url now resolve to the daemon's asset URL (with the server
token) instead of an unreachable file:// path when a web/mobile client has
a server configured (frontend.config/get-server-base-url) -- covers viewing
existing assets. frontend.fs/write-asset-file! gets the matching upload
path via a new frontend.persist-db.remote/upload-asset!, so pasting an
asset from a web/mobile client no longer silently lands in
frontend.fs.memory-fs and vanishes on reload.

Server-side routes and round-trip verified end-to-end against a real
daemon (curl) and with new automated tests (byte equality, content-type,
etag/304, 404, repo-mismatch, auth-required). The client-side URL
resolution changes could not be visually verified -- no browser in this
environment -- only reviewed and compiled clean.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

src/main/frontend/fs.cljs

@@ -8,6 +8,7 @@
             [frontend.fs.memory-fs :as memory-fs]
             [frontend.fs.node :as node]
             [frontend.fs.protocol :as protocol]
+            [frontend.persist-db.remote :as remote]
             [frontend.state :as state]
             [frontend.util :as util]
             [lambdaisland.glogi :as log]
@@ -115,15 +116,34 @@
                                                                   :user-agent (when js/navigator js/navigator.userAgent)
                                                                   :content-length (count content)}}])))))))
 
+(defn- remote-server-configured?
+  "True for a web/mobile client pointed at a self-hosted db-worker-node
+  server (Fase 3) -- it has no local filesystem to write assets to at all,
+  unlike Electron (writes straight to disk below) or a plain local browser
+  (falls through to memory-backend). Duplicated from the equivalent check in
+  frontend.handler.assets, which can't be required here without a cycle
+  (it already requires this ns)."
+  []
+  (and (not (util/electron?))
+       (some? (config/get-server-base-url))))
+
 (defn write-asset-file!
   [repo file-name data]
   (let [repo-dir (config/get-repo-dir repo)]
-    (if (util/electron?)
+    (cond
+      (util/electron?)
       (let [assets-dir (path/path-join repo-dir common-config/local-assets-dir)
             file-path (path/path-join assets-dir file-name)]
         ;; Use writeFileBytes directly instead of ipc/ipc (write-file!) because
         ;; binary data like ArrayBuffer can't be transit-serialized
         (js/window.apis.writeFileBytes file-path data))
+
+      (remote-server-configured?)
+      (remote/upload-asset! {:base-url (config/get-server-base-url)
+                             :auth-token (config/get-server-token)}
+                            repo file-name data)
+
+      :else
       (let [file-path (path/path-join common-config/local-assets-dir file-name)]
         (write-plain-text-file! repo repo-dir file-path data {:skip-transact? true
                                                               :skip-compare? true})))))

src/main/frontend/handler/assets.cljs

@@ -76,12 +76,52 @@
         (string/replace-first "/logseq__colon/" ":/"))
     url))
 
+(defn remote-server-configured?
+  "True for a web/mobile client pointed at a self-hosted db-worker-node
+  server (Fase 3) -- the counterpart to util/electron? for asset URL
+  resolution below, since such a client has no local filesystem to read
+  assets from at all (see docs/agent-guide Fase 5)."
+  []
+  (and (not (util/electron?))
+       (some? (config/get-server-base-url))))
+
+(defn- bare-asset-file-name
+  "Strips a leading assets/ (or ./assets/, ../assets/) prefix, if any --
+  the daemon's /v1/assets/:repo/:name route stores files flat per repo, so
+  it only wants the bare file name, not a path relative to the graph root."
+  [rpath]
+  (let [rpath (string/replace-first rpath #"^\.{1,2}/" "")
+        prefix (str common-config/local-assets-dir "/")]
+    (if (string/starts-with? rpath prefix)
+      (subs rpath (count prefix))
+      rpath)))
+
+(defn remote-asset-url
+  "Builds a directly-loadable URL (<img src>, browser navigation, plain
+  fetch) for an asset served by a remote db-worker-node daemon. The token
+  travels as a query param -- the same interim mechanism SSE used before
+  the Fase 4 WebSocket migration -- since a browser navigating straight to
+  this URL (rather than fetching it from JS) can't set an Authorization
+  header. See asset-get-request? in db_worker_node.cljs."
+  [repo rpath]
+  (let [base-url (string/replace (config/get-server-base-url) #"/$" "")
+        file-name (bare-asset-file-name rpath)
+        token (config/get-server-token)]
+    (cond-> (str base-url "/v1/assets/" (js/encodeURIComponent repo) "/" (js/encodeURIComponent file-name))
+      (seq token) (str "?token=" (js/encodeURIComponent token)))))
+
 (defn resolve-asset-real-path-url
   [repo rpath]
   (when-let [rpath (and (string? rpath)
                         (string/replace rpath #"^[.\/\\]+" ""))]
-    (if config/publishing?
+    (cond
+      config/publishing?
       (str "./" rpath)
+
+      (remote-server-configured?)
+      (remote-asset-url repo rpath)
+
+      :else
       (let [ret (let [rpath (if-not (string/starts-with? rpath common-config/local-assets-dir)
                               (path/path-join common-config/local-assets-dir rpath)
                               rpath)
@@ -176,6 +216,9 @@
             ;; fullpath will be encoded
             (path/prepend-protocol "assets:" (protect-windows-drive-in-assets-path full-path)))
 
+         (remote-server-configured?)
+         (remote-asset-url repo path)
+
          :else
          (p/let [binary (fs/read-file-raw repo-dir path {})
                  svg? (string/ends-with? path ".svg")

src/main/frontend/persist_db/remote.cljs

@@ -23,6 +23,14 @@
        "/v1/import-db-binary?repo="
        (js/encodeURIComponent repo)))
 
+(defn- asset-url
+  [base-url repo file-name]
+  (str (normalize-base-url base-url)
+       "/v1/assets/"
+       (js/encodeURIComponent repo)
+       "/"
+       (js/encodeURIComponent file-name)))
+
 (defn- events-url
   "EventSource has no header API, so the bearer token travels as a query
   param here instead of the `Authorization` header every other request
@@ -102,6 +110,26 @@
                                  :code (normalize-code (:code error))}
                           error (assoc :error error))))))))
 
+(defn upload-asset!
+  "PUTs raw asset bytes to a db-worker-node daemon's /v1/assets/:repo/:name
+  (see Fase 5 in docs/agent-guide). Used by frontend.fs/write-asset-file! for
+  a web/mobile client, which has no local filesystem to write assets to --
+  Electron keeps writing straight to disk via its own IPC instead."
+  [{:keys [base-url auth-token fetch-fn]} repo file-name data]
+  (p/let [{:keys [status body]} ((or fetch-fn default-fetch-json!)
+                                 {:method "PUT"
+                                  :url (asset-url base-url repo file-name)
+                                  :headers (binary-headers auth-token)
+                                  :body data})]
+    (if (<= 200 status 299)
+      true
+      (let [parsed (parse-response-body body)
+            error (:error parsed)]
+        (throw (ex-info (or (:message error) "asset upload failed")
+                        (cond-> {:status status
+                                 :code (normalize-code (:code error))}
+                          error (assoc :error error))))))))
+
 (defn- decode-event
   [{:keys [type payload]}]
   (let [decoded (when (some? payload)

src/main/frontend/worker/db_core.cljs

@@ -888,6 +888,25 @@
   [repo config-file files opts]
   (<import-file-graph! repo config-file files opts))
 
+;; Fase 5 (server-client migration): let a remote client read/write asset
+;; bytes over HTTP instead of a local filesystem it doesn't have -- see
+;; the /v1/assets/:repo/:name routes in db_worker_node.cljs, which call
+;; these via the regular JSON+transit /v1/invoke path (same as
+;; export-db-binary/import-db-binary; transit already round-trips
+;; Uint8Array, so no separate binary RPC is needed here).
+(def-thread-api :thread-api/asset-read-bytes
+  [repo file-name]
+  (-> (platform/asset-read-bytes! (platform/current) repo file-name)
+      (p/catch (fn [_error] nil))))
+
+(def-thread-api :thread-api/asset-write-bytes
+  [repo file-name payload]
+  (platform/asset-write-bytes! (platform/current) repo file-name payload))
+
+(def-thread-api :thread-api/asset-stat
+  [repo file-name]
+  (platform/asset-stat (platform/current) repo file-name))
+
 (comment
   (def-thread-api :general/dangerousRemoveAllDbs
     []

src/main/frontend/worker/db_worker_node.cljs

@@ -1,6 +1,7 @@
 (ns frontend.worker.db-worker-node
   "Node.js daemon entrypoint for db-worker."
-  (:require ["http" :as http]
+  (:require ["crypto" :as crypto]
+            ["http" :as http]
             ["ws" :as ws]
             [clojure.string :as string]
             [frontend.worker.db-core :as db-core]
@@ -384,17 +385,68 @@
   [^js req allowed-hosts]
   (contains? allowed-hosts (some-> req .-headers .-host)))
 
+(defn- asset-get-request?
+  [request-path method]
+  (and (= method "GET") (string/starts-with? request-path "/v1/assets/")))
+
 (defn- request-authorized?
-  "/v1/events (SSE) can't carry an Authorization header -- EventSource has no
+  "The Authorization header works for every route, including these two --
-  header API -- so it accepts the bearer token as a query param instead.
+  a plain JS fetch() can set it fine. It's only a *browser navigating
-  This is interim: the WebSocket migration (see docs/agent-guide) replaces
+  directly to a URL* that can't: EventSource (/v1/events, SSE, now a
-  it with Sec-WebSocket-Protocol, which doesn't leak into proxy/access logs
+  fallback since the Fase 4 WS migration) has no header API, and an
-  the way a query string does."
+  <img src>/direct navigation to an asset GET can't set custom headers
-  [^js req parsed-url request-path]
+  either. Both accept ?token= as a fallback in place of Authorization, never
-  (if (= request-path "/v1/events")
+  as a replacement for it."
-    (auth/authorized? (some->> (.get (.-searchParams parsed-url) "token")
+  [^js req parsed-url request-path method]
-                                (str "Bearer ")))
+  (or (auth/authorized? (.. req -headers -authorization))
-    (auth/authorized? (.. req -headers -authorization))))
+      (and (or (= request-path "/v1/events")
+               (asset-get-request? request-path method))
+           (auth/authorized? (some->> (.get (.-searchParams parsed-url) "token")
+                                       (str "Bearer "))))))
+
+(def ^:private asset-content-types
+  {"png" "image/png"
+   "jpg" "image/jpeg"
+   "jpeg" "image/jpeg"
+   "gif" "image/gif"
+   "webp" "image/webp"
+   "svg" "image/svg+xml"
+   "avif" "image/avif"
+   "bmp" "image/bmp"
+   "ico" "image/x-icon"
+   "pdf" "application/pdf"
+   "mp3" "audio/mpeg"
+   "wav" "audio/wav"
+   "ogg" "audio/ogg"
+   "mp4" "video/mp4"
+   "webm" "video/webm"
+   "mov" "video/quicktime"
+   "json" "application/json"
+   "txt" "text/plain"})
+
+(defn- guess-asset-content-type
+  [file-name]
+  (let [ext (some-> (re-find #"\.([A-Za-z0-9]+)$" (or file-name ""))
+                     second
+                     string/lower-case)]
+    (get asset-content-types ext "application/octet-stream")))
+
+(defn- parse-asset-path
+  "/v1/assets/:repo/:name -- both segments are URI-encoded by the client
+  (frontend.persist-db.remote), since a repo or asset file name could
+  contain characters that aren't legal in a URL path segment as-is."
+  [request-path]
+  (let [rest (subs request-path (count "/v1/assets/"))
+        slash-idx (string/index-of rest "/")]
+    (when (and slash-idx (pos? slash-idx) (< (inc slash-idx) (count rest)))
+      (try
+        {:repo (js/decodeURIComponent (subs rest 0 slash-idx))
+         :file-name (js/decodeURIComponent (subs rest (inc slash-idx)))}
+        (catch :default _ nil)))))
+
+(defn- asset-etag
+  [^js buf]
+  (str "\"" (-> (crypto/createHash "sha1") (.update buf) (.digest "hex")) "\""))
 
 (defn- ws-upgrade-handler
   [^js wss bound-repo port extra-allowed-hosts]
@@ -470,7 +522,7 @@
                         (send-json! res 400 {:ok false :error {:code :invalid-request
                                                                 :message "invalid login request body"}}))))
 
-         (not (request-authorized? req parsed-url request-path))
+         (not (request-authorized? req parsed-url request-path method))
          (send-json! res 401 {:ok false :error {:code :unauthorized
                                                  :message "missing or invalid credentials"}})
 
@@ -514,6 +566,51 @@
                               (send-json! res status payload))))))
            (send-text! res 405 "method-not-allowed"))
 
+         (and (string/starts-with? request-path "/v1/assets/") (= method "GET"))
+         (if-let [{:keys [repo file-name]} (parse-asset-path request-path)]
+           (let [method-kw :thread-api/asset-read-bytes
+                 method-str (normalize-method-str method-kw)
+                 args [repo file-name]]
+             (if-let [{:keys [status error]} (repo-error method-kw args bound-repo)]
+               (send-json! res status {:ok false :error error})
+               (-> (p/let [result (<invoke! proxy method-str method-kw args)
+                           bytes (ldb/read-transit-str result)]
+                     (if (some? bytes)
+                       (let [buf (js/Buffer.from bytes)
+                             etag (asset-etag buf)]
+                         (if (= etag (aget (.-headers req) "if-none-match"))
+                           (do
+                             (.writeHead res 304 (response-headers res #js {"ETag" etag
+                                                                            "Cache-Control" "public, max-age=31536000, immutable"}))
+                             (.end res))
+                           (do
+                             (.writeHead res 200 (response-headers
+                                                   res #js {"Content-Type" (guess-asset-content-type file-name)
+                                                            "Content-Length" (.-length buf)
+                                                            "ETag" etag
+                                                            "Cache-Control" "public, max-age=31536000, immutable"}))
+                             (.end res buf))))
+                       (send-json! res 404 {:ok false :error {:code :not-found :message "asset not found"}})))
+                   (p/catch (fn [error]
+                              (log-invoke-error! res error method-kw))))))
+           (send-json! res 400 {:ok false :error {:code :invalid-request :message "invalid asset path"}}))
+
+         (and (string/starts-with? request-path "/v1/assets/") (= method "PUT"))
+         (if-let [{:keys [repo file-name]} (parse-asset-path request-path)]
+           (-> (p/let [binary (<read-body-buffer req)
+                       method-kw :thread-api/asset-write-bytes
+                       method-str (normalize-method-str method-kw)
+                       args [repo file-name binary]]
+                 (if-let [{:keys [status error]} (repo-error method-kw args bound-repo)]
+                   (send-json! res status {:ok false :error error})
+                   (p/let [_ (let [{:keys [path lock]} @*lock-info]
+                               (db-lock/assert-lock-owner! path lock))
+                           _ (<invoke! proxy method-str method-kw args)]
+                     (send-json! res 200 {:ok true}))))
+               (p/catch (fn [error]
+                          (log-invoke-error! res error :thread-api/asset-write-bytes))))
+           (send-json! res 400 {:ok false :error {:code :invalid-request :message "invalid asset path"}}))
+
          (= request-path "/v1/invoke")
          (if (= method "POST")
            (->

src/test/frontend/worker/db_worker_node_test.cljs

@@ -195,6 +195,51 @@
   [^js client payload]
   (.send client (js/JSON.stringify (clj->js payload))))
 
+(defn- http-request-binary
+  "Like http-request, but keeps the raw response bytes instead of decoding
+  them as utf8 -- utf8-decoding arbitrary binary content (asset bytes) would
+  be lossy and non-invertible."
+  [opts body]
+  (p/create
+   (fn [resolve reject]
+     (let [req (.request http (clj->js opts)
+                         (fn [^js res]
+                           (let [chunks (array)]
+                             (.on res "data" (fn [chunk] (.push chunks chunk)))
+                             (.on res "end" (fn []
+                                              (resolve {:status (.-statusCode res)
+                                                        :headers (js->clj (.-headers res) :keywordize-keys true)
+                                                        :body-buffer (js/Buffer.concat chunks)}))))))
+           finish! (fn []
+                     (when body (.write req body))
+                     (.end req))]
+       (.on req "error" reject)
+       (finish!)))))
+
+(defn- asset-path
+  [repo file-name]
+  (str "/v1/assets/" (js/encodeURIComponent repo) "/" (js/encodeURIComponent file-name)))
+
+(defn- asset-put!
+  [host port repo file-name binary]
+  (p/let [headers (auth-headers! host port)]
+    (http-request-binary {:hostname host
+                          :port port
+                          :path (asset-path repo file-name)
+                          :method "PUT"
+                          :headers headers}
+                         binary)))
+
+(defn- asset-get
+  [host port repo file-name & [extra-headers]]
+  (p/let [headers (auth-headers! host port)]
+    (http-request-binary {:hostname host
+                          :port port
+                          :path (asset-path repo file-name)
+                          :method "GET"
+                          :headers (merge headers extra-headers)}
+                         nil)))
+
 (defn- lock-path
   [root-dir repo]
   (db-lock/lock-path root-dir repo))
@@ -1442,6 +1487,101 @@
                                 :else
                                 (done)))))))))
 
+(deftest db-worker-node-asset-put-get-round-trips-bytes-with-caching-headers
+  (async done
+         (let [daemon (atom nil)
+               data-dir (node-helper/create-tmp-dir "db-worker-asset-roundtrip")
+               repo (str "logseq_db_asset_roundtrip_" (subs (str (random-uuid)) 0 8))
+               binary (js/Buffer.from (js/Uint8Array. (into-array (range 256))))]
+           (-> (p/let [{:keys [host port stop!]} (start-daemon! {:root-dir data-dir :repo repo})
+                       _ (reset! daemon {:stop! stop!})
+                       _ (invoke host port "thread-api/create-or-open-db" [repo {}])
+                       put-result (asset-put! host port repo "pic1.png" binary)
+                       _ (is (= 200 (:status put-result)))
+                       get-result (asset-get host port repo "pic1.png")
+                       {:keys [status headers body-buffer]} get-result]
+                 (is (= 200 status))
+                 (is (= "image/png" (:content-type headers)))
+                 (is (string? (:etag headers)))
+                 (is (string/starts-with? (:cache-control headers) "public, max-age=31536000, immutable"))
+                 (is (= (.toString binary "base64") (.toString body-buffer "base64")))
+                 (p/let [conditional-result (asset-get host port repo "pic1.png"
+                                                        {"If-None-Match" (:etag headers)})]
+                   (is (= 304 (:status conditional-result)))))
+               (p/catch (fn [e]
+                          (println "[db-worker-node-test] asset-roundtrip error:" e)
+                          (is false (str e))))
+               (p/finally (fn []
+                            (if-let [stop! (:stop! @daemon)]
+                              (-> (stop!) (p/finally done))
+                              (done))))))))
+
+(deftest db-worker-node-asset-get-missing-file-returns-404
+  (async done
+         (let [daemon (atom nil)
+               data-dir (node-helper/create-tmp-dir "db-worker-asset-missing")
+               repo (str "logseq_db_asset_missing_" (subs (str (random-uuid)) 0 8))]
+           (-> (p/let [{:keys [host port stop!]} (start-daemon! {:root-dir data-dir :repo repo})
+                       _ (reset! daemon {:stop! stop!})
+                       _ (invoke host port "thread-api/create-or-open-db" [repo {}])
+                       {:keys [status]} (asset-get host port repo "does-not-exist.png")]
+                 (is (= 404 status)))
+               (p/catch (fn [e]
+                          (println "[db-worker-node-test] asset-missing error:" e)
+                          (is false (str e))))
+               (p/finally (fn []
+                            (if-let [stop! (:stop! @daemon)]
+                              (-> (stop!) (p/finally done))
+                              (done))))))))
+
+(deftest db-worker-node-asset-repo-mismatch-returns-409
+  (async done
+         (let [daemon (atom nil)
+               data-dir (node-helper/create-tmp-dir "db-worker-asset-repo-mismatch")
+               repo (str "logseq_db_asset_repo_mismatch_" (subs (str (random-uuid)) 0 8))
+               binary (js/Buffer.from #js [1 2 3])]
+           (-> (p/let [{:keys [host port stop!]} (start-daemon! {:root-dir data-dir :repo repo})
+                       _ (reset! daemon {:stop! stop!})
+                       _ (invoke host port "thread-api/create-or-open-db" [repo {}])
+                       get-result (asset-get host port "some_other_repo" "pic1.png")
+                       put-result (asset-put! host port "some_other_repo" "pic1.png" binary)]
+                 (is (= 409 (:status get-result)))
+                 (is (= 409 (:status put-result))))
+               (p/catch (fn [e]
+                          (println "[db-worker-node-test] asset-repo-mismatch error:" e)
+                          (is false (str e))))
+               (p/finally (fn []
+                            (if-let [stop! (:stop! @daemon)]
+                              (-> (stop!) (p/finally done))
+                              (done))))))))
+
+(deftest db-worker-node-asset-requires-auth
+  (async done
+         (let [daemon (atom nil)
+               data-dir (node-helper/create-tmp-dir "db-worker-asset-auth")
+               repo (str "logseq_db_asset_auth_" (subs (str (random-uuid)) 0 8))
+               binary (js/Buffer.from #js [1 2 3])]
+           (-> (p/let [{:keys [host port stop!]} (start-daemon! {:root-dir data-dir :repo repo})
+                       _ (reset! daemon {:stop! stop!})
+                       _ (invoke host port "thread-api/create-or-open-db" [repo {}])
+                       get-result (http-request-binary {:hostname host :port port
+                                                        :path (asset-path repo "pic1.png")
+                                                        :method "GET"}
+                                                       nil)
+                       put-result (http-request-binary {:hostname host :port port
+                                                        :path (asset-path repo "pic1.png")
+                                                        :method "PUT"}
+                                                       binary)]
+                 (is (= 401 (:status get-result)))
+                 (is (= 401 (:status put-result))))
+               (p/catch (fn [e]
+                          (println "[db-worker-node-test] asset-auth error:" e)
+                          (is false (str e))))
+               (p/finally (fn []
+                            (if-let [stop! (:stop! @daemon)]
+                              (-> (stop!) (p/finally done))
+                              (done))))))))
+
 (deftest db-worker-node-export-client-ops-db-binary
   (async done
          (let [daemon (atom nil)