diff --git a/src/main/frontend/fs.cljs b/src/main/frontend/fs.cljs index 8e0c0b7..c22ee2d 100644 --- a/src/main/frontend/fs.cljs +++ b/src/main/frontend/fs.cljs @@ -8,6 +8,7 @@ [frontend.fs.memory-fs :as memory-fs] [frontend.fs.node :as node] [frontend.fs.protocol :as protocol] + [frontend.persist-db.remote :as remote] [frontend.state :as state] [frontend.util :as util] [lambdaisland.glogi :as log] @@ -115,15 +116,34 @@ :user-agent (when js/navigator js/navigator.userAgent) :content-length (count content)}}]))))))) +(defn- remote-server-configured? + "True for a web/mobile client pointed at a self-hosted db-worker-node + server (Fase 3) -- it has no local filesystem to write assets to at all, + unlike Electron (writes straight to disk below) or a plain local browser + (falls through to memory-backend). Duplicated from the equivalent check in + frontend.handler.assets, which can't be required here without a cycle + (it already requires this ns)." + [] + (and (not (util/electron?)) + (some? (config/get-server-base-url)))) + (defn write-asset-file! [repo file-name data] (let [repo-dir (config/get-repo-dir repo)] - (if (util/electron?) + (cond + (util/electron?) (let [assets-dir (path/path-join repo-dir common-config/local-assets-dir) file-path (path/path-join assets-dir file-name)] ;; Use writeFileBytes directly instead of ipc/ipc (write-file!) because ;; binary data like ArrayBuffer can't be transit-serialized (js/window.apis.writeFileBytes file-path data)) + + (remote-server-configured?) + (remote/upload-asset! {:base-url (config/get-server-base-url) + :auth-token (config/get-server-token)} + repo file-name data) + + :else (let [file-path (path/path-join common-config/local-assets-dir file-name)] (write-plain-text-file! repo repo-dir file-path data {:skip-transact? true :skip-compare? true}))))) diff --git a/src/main/frontend/handler/assets.cljs b/src/main/frontend/handler/assets.cljs index a225701..0b588af 100644 --- a/src/main/frontend/handler/assets.cljs +++ b/src/main/frontend/handler/assets.cljs @@ -76,12 +76,52 @@ (string/replace-first "/logseq__colon/" ":/")) url)) +(defn remote-server-configured? + "True for a web/mobile client pointed at a self-hosted db-worker-node + server (Fase 3) -- the counterpart to util/electron? for asset URL + resolution below, since such a client has no local filesystem to read + assets from at all (see docs/agent-guide Fase 5)." + [] + (and (not (util/electron?)) + (some? (config/get-server-base-url)))) + +(defn- bare-asset-file-name + "Strips a leading assets/ (or ./assets/, ../assets/) prefix, if any -- + the daemon's /v1/assets/:repo/:name route stores files flat per repo, so + it only wants the bare file name, not a path relative to the graph root." + [rpath] + (let [rpath (string/replace-first rpath #"^\.{1,2}/" "") + prefix (str common-config/local-assets-dir "/")] + (if (string/starts-with? rpath prefix) + (subs rpath (count prefix)) + rpath))) + +(defn remote-asset-url + "Builds a directly-loadable URL (, browser navigation, plain + fetch) for an asset served by a remote db-worker-node daemon. The token + travels as a query param -- the same interim mechanism SSE used before + the Fase 4 WebSocket migration -- since a browser navigating straight to + this URL (rather than fetching it from JS) can't set an Authorization + header. See asset-get-request? in db_worker_node.cljs." + [repo rpath] + (let [base-url (string/replace (config/get-server-base-url) #"/$" "") + file-name (bare-asset-file-name rpath) + token (config/get-server-token)] + (cond-> (str base-url "/v1/assets/" (js/encodeURIComponent repo) "/" (js/encodeURIComponent file-name)) + (seq token) (str "?token=" (js/encodeURIComponent token))))) + (defn resolve-asset-real-path-url [repo rpath] (when-let [rpath (and (string? rpath) (string/replace rpath #"^[.\/\\]+" ""))] - (if config/publishing? + (cond + config/publishing? (str "./" rpath) + + (remote-server-configured?) + (remote-asset-url repo rpath) + + :else (let [ret (let [rpath (if-not (string/starts-with? rpath common-config/local-assets-dir) (path/path-join common-config/local-assets-dir rpath) rpath) @@ -176,6 +216,9 @@ ;; fullpath will be encoded (path/prepend-protocol "assets:" (protect-windows-drive-in-assets-path full-path))) + (remote-server-configured?) + (remote-asset-url repo path) + :else (p/let [binary (fs/read-file-raw repo-dir path {}) svg? (string/ends-with? path ".svg") diff --git a/src/main/frontend/persist_db/remote.cljs b/src/main/frontend/persist_db/remote.cljs index b893323..c6d0f47 100644 --- a/src/main/frontend/persist_db/remote.cljs +++ b/src/main/frontend/persist_db/remote.cljs @@ -23,6 +23,14 @@ "/v1/import-db-binary?repo=" (js/encodeURIComponent repo))) +(defn- asset-url + [base-url repo file-name] + (str (normalize-base-url base-url) + "/v1/assets/" + (js/encodeURIComponent repo) + "/" + (js/encodeURIComponent file-name))) + (defn- events-url "EventSource has no header API, so the bearer token travels as a query param here instead of the `Authorization` header every other request @@ -102,6 +110,26 @@ :code (normalize-code (:code error))} error (assoc :error error)))))))) +(defn upload-asset! + "PUTs raw asset bytes to a db-worker-node daemon's /v1/assets/:repo/:name + (see Fase 5 in docs/agent-guide). Used by frontend.fs/write-asset-file! for + a web/mobile client, which has no local filesystem to write assets to -- + Electron keeps writing straight to disk via its own IPC instead." + [{:keys [base-url auth-token fetch-fn]} repo file-name data] + (p/let [{:keys [status body]} ((or fetch-fn default-fetch-json!) + {:method "PUT" + :url (asset-url base-url repo file-name) + :headers (binary-headers auth-token) + :body data})] + (if (<= 200 status 299) + true + (let [parsed (parse-response-body body) + error (:error parsed)] + (throw (ex-info (or (:message error) "asset upload failed") + (cond-> {:status status + :code (normalize-code (:code error))} + error (assoc :error error)))))))) + (defn- decode-event [{:keys [type payload]}] (let [decoded (when (some? payload) diff --git a/src/main/frontend/worker/db_core.cljs b/src/main/frontend/worker/db_core.cljs index e49222f..5876ec4 100644 --- a/src/main/frontend/worker/db_core.cljs +++ b/src/main/frontend/worker/db_core.cljs @@ -888,6 +888,25 @@ [repo config-file files opts] ( (platform/asset-read-bytes! (platform/current) repo file-name) + (p/catch (fn [_error] nil)))) + +(def-thread-api :thread-api/asset-write-bytes + [repo file-name payload] + (platform/asset-write-bytes! (platform/current) repo file-name payload)) + +(def-thread-api :thread-api/asset-stat + [repo file-name] + (platform/asset-stat (platform/current) repo file-name)) + (comment (def-thread-api :general/dangerousRemoveAllDbs [] diff --git a/src/main/frontend/worker/db_worker_node.cljs b/src/main/frontend/worker/db_worker_node.cljs index 7b31b65..706413c 100644 --- a/src/main/frontend/worker/db_worker_node.cljs +++ b/src/main/frontend/worker/db_worker_node.cljs @@ -1,6 +1,7 @@ (ns frontend.worker.db-worker-node "Node.js daemon entrypoint for db-worker." - (:require ["http" :as http] + (:require ["crypto" :as crypto] + ["http" :as http] ["ws" :as ws] [clojure.string :as string] [frontend.worker.db-core :as db-core] @@ -384,17 +385,68 @@ [^js req allowed-hosts] (contains? allowed-hosts (some-> req .-headers .-host))) +(defn- asset-get-request? + [request-path method] + (and (= method "GET") (string/starts-with? request-path "/v1/assets/"))) + (defn- request-authorized? - "/v1/events (SSE) can't carry an Authorization header -- EventSource has no - header API -- so it accepts the bearer token as a query param instead. - This is interim: the WebSocket migration (see docs/agent-guide) replaces - it with Sec-WebSocket-Protocol, which doesn't leak into proxy/access logs - the way a query string does." - [^js req parsed-url request-path] - (if (= request-path "/v1/events") - (auth/authorized? (some->> (.get (.-searchParams parsed-url) "token") - (str "Bearer "))) - (auth/authorized? (.. req -headers -authorization)))) + "The Authorization header works for every route, including these two -- + a plain JS fetch() can set it fine. It's only a *browser navigating + directly to a URL* that can't: EventSource (/v1/events, SSE, now a + fallback since the Fase 4 WS migration) has no header API, and an + /direct navigation to an asset GET can't set custom headers + either. Both accept ?token= as a fallback in place of Authorization, never + as a replacement for it." + [^js req parsed-url request-path method] + (or (auth/authorized? (.. req -headers -authorization)) + (and (or (= request-path "/v1/events") + (asset-get-request? request-path method)) + (auth/authorized? (some->> (.get (.-searchParams parsed-url) "token") + (str "Bearer ")))))) + +(def ^:private asset-content-types + {"png" "image/png" + "jpg" "image/jpeg" + "jpeg" "image/jpeg" + "gif" "image/gif" + "webp" "image/webp" + "svg" "image/svg+xml" + "avif" "image/avif" + "bmp" "image/bmp" + "ico" "image/x-icon" + "pdf" "application/pdf" + "mp3" "audio/mpeg" + "wav" "audio/wav" + "ogg" "audio/ogg" + "mp4" "video/mp4" + "webm" "video/webm" + "mov" "video/quicktime" + "json" "application/json" + "txt" "text/plain"}) + +(defn- guess-asset-content-type + [file-name] + (let [ext (some-> (re-find #"\.([A-Za-z0-9]+)$" (or file-name "")) + second + string/lower-case)] + (get asset-content-types ext "application/octet-stream"))) + +(defn- parse-asset-path + "/v1/assets/:repo/:name -- both segments are URI-encoded by the client + (frontend.persist-db.remote), since a repo or asset file name could + contain characters that aren't legal in a URL path segment as-is." + [request-path] + (let [rest (subs request-path (count "/v1/assets/")) + slash-idx (string/index-of rest "/")] + (when (and slash-idx (pos? slash-idx) (< (inc slash-idx) (count rest))) + (try + {:repo (js/decodeURIComponent (subs rest 0 slash-idx)) + :file-name (js/decodeURIComponent (subs rest (inc slash-idx)))} + (catch :default _ nil))))) + +(defn- asset-etag + [^js buf] + (str "\"" (-> (crypto/createHash "sha1") (.update buf) (.digest "hex")) "\"")) (defn- ws-upgrade-handler [^js wss bound-repo port extra-allowed-hosts] @@ -470,7 +522,7 @@ (send-json! res 400 {:ok false :error {:code :invalid-request :message "invalid login request body"}})))) - (not (request-authorized? req parsed-url request-path)) + (not (request-authorized? req parsed-url request-path method)) (send-json! res 401 {:ok false :error {:code :unauthorized :message "missing or invalid credentials"}}) @@ -514,6 +566,51 @@ (send-json! res status payload)))))) (send-text! res 405 "method-not-allowed")) + (and (string/starts-with? request-path "/v1/assets/") (= method "GET")) + (if-let [{:keys [repo file-name]} (parse-asset-path request-path)] + (let [method-kw :thread-api/asset-read-bytes + method-str (normalize-method-str method-kw) + args [repo file-name]] + (if-let [{:keys [status error]} (repo-error method-kw args bound-repo)] + (send-json! res status {:ok false :error error}) + (-> (p/let [result ( (p/let [binary ( diff --git a/src/test/frontend/worker/db_worker_node_test.cljs b/src/test/frontend/worker/db_worker_node_test.cljs index 64b28db..e1c9d01 100644 --- a/src/test/frontend/worker/db_worker_node_test.cljs +++ b/src/test/frontend/worker/db_worker_node_test.cljs @@ -195,6 +195,51 @@ [^js client payload] (.send client (js/JSON.stringify (clj->js payload)))) +(defn- http-request-binary + "Like http-request, but keeps the raw response bytes instead of decoding + them as utf8 -- utf8-decoding arbitrary binary content (asset bytes) would + be lossy and non-invertible." + [opts body] + (p/create + (fn [resolve reject] + (let [req (.request http (clj->js opts) + (fn [^js res] + (let [chunks (array)] + (.on res "data" (fn [chunk] (.push chunks chunk))) + (.on res "end" (fn [] + (resolve {:status (.-statusCode res) + :headers (js->clj (.-headers res) :keywordize-keys true) + :body-buffer (js/Buffer.concat chunks)})))))) + finish! (fn [] + (when body (.write req body)) + (.end req))] + (.on req "error" reject) + (finish!))))) + +(defn- asset-path + [repo file-name] + (str "/v1/assets/" (js/encodeURIComponent repo) "/" (js/encodeURIComponent file-name))) + +(defn- asset-put! + [host port repo file-name binary] + (p/let [headers (auth-headers! host port)] + (http-request-binary {:hostname host + :port port + :path (asset-path repo file-name) + :method "PUT" + :headers headers} + binary))) + +(defn- asset-get + [host port repo file-name & [extra-headers]] + (p/let [headers (auth-headers! host port)] + (http-request-binary {:hostname host + :port port + :path (asset-path repo file-name) + :method "GET" + :headers (merge headers extra-headers)} + nil))) + (defn- lock-path [root-dir repo] (db-lock/lock-path root-dir repo)) @@ -1442,6 +1487,101 @@ :else (done))))))))) +(deftest db-worker-node-asset-put-get-round-trips-bytes-with-caching-headers + (async done + (let [daemon (atom nil) + data-dir (node-helper/create-tmp-dir "db-worker-asset-roundtrip") + repo (str "logseq_db_asset_roundtrip_" (subs (str (random-uuid)) 0 8)) + binary (js/Buffer.from (js/Uint8Array. (into-array (range 256))))] + (-> (p/let [{:keys [host port stop!]} (start-daemon! {:root-dir data-dir :repo repo}) + _ (reset! daemon {:stop! stop!}) + _ (invoke host port "thread-api/create-or-open-db" [repo {}]) + put-result (asset-put! host port repo "pic1.png" binary) + _ (is (= 200 (:status put-result))) + get-result (asset-get host port repo "pic1.png") + {:keys [status headers body-buffer]} get-result] + (is (= 200 status)) + (is (= "image/png" (:content-type headers))) + (is (string? (:etag headers))) + (is (string/starts-with? (:cache-control headers) "public, max-age=31536000, immutable")) + (is (= (.toString binary "base64") (.toString body-buffer "base64"))) + (p/let [conditional-result (asset-get host port repo "pic1.png" + {"If-None-Match" (:etag headers)})] + (is (= 304 (:status conditional-result))))) + (p/catch (fn [e] + (println "[db-worker-node-test] asset-roundtrip error:" e) + (is false (str e)))) + (p/finally (fn [] + (if-let [stop! (:stop! @daemon)] + (-> (stop!) (p/finally done)) + (done)))))))) + +(deftest db-worker-node-asset-get-missing-file-returns-404 + (async done + (let [daemon (atom nil) + data-dir (node-helper/create-tmp-dir "db-worker-asset-missing") + repo (str "logseq_db_asset_missing_" (subs (str (random-uuid)) 0 8))] + (-> (p/let [{:keys [host port stop!]} (start-daemon! {:root-dir data-dir :repo repo}) + _ (reset! daemon {:stop! stop!}) + _ (invoke host port "thread-api/create-or-open-db" [repo {}]) + {:keys [status]} (asset-get host port repo "does-not-exist.png")] + (is (= 404 status))) + (p/catch (fn [e] + (println "[db-worker-node-test] asset-missing error:" e) + (is false (str e)))) + (p/finally (fn [] + (if-let [stop! (:stop! @daemon)] + (-> (stop!) (p/finally done)) + (done)))))))) + +(deftest db-worker-node-asset-repo-mismatch-returns-409 + (async done + (let [daemon (atom nil) + data-dir (node-helper/create-tmp-dir "db-worker-asset-repo-mismatch") + repo (str "logseq_db_asset_repo_mismatch_" (subs (str (random-uuid)) 0 8)) + binary (js/Buffer.from #js [1 2 3])] + (-> (p/let [{:keys [host port stop!]} (start-daemon! {:root-dir data-dir :repo repo}) + _ (reset! daemon {:stop! stop!}) + _ (invoke host port "thread-api/create-or-open-db" [repo {}]) + get-result (asset-get host port "some_other_repo" "pic1.png") + put-result (asset-put! host port "some_other_repo" "pic1.png" binary)] + (is (= 409 (:status get-result))) + (is (= 409 (:status put-result)))) + (p/catch (fn [e] + (println "[db-worker-node-test] asset-repo-mismatch error:" e) + (is false (str e)))) + (p/finally (fn [] + (if-let [stop! (:stop! @daemon)] + (-> (stop!) (p/finally done)) + (done)))))))) + +(deftest db-worker-node-asset-requires-auth + (async done + (let [daemon (atom nil) + data-dir (node-helper/create-tmp-dir "db-worker-asset-auth") + repo (str "logseq_db_asset_auth_" (subs (str (random-uuid)) 0 8)) + binary (js/Buffer.from #js [1 2 3])] + (-> (p/let [{:keys [host port stop!]} (start-daemon! {:root-dir data-dir :repo repo}) + _ (reset! daemon {:stop! stop!}) + _ (invoke host port "thread-api/create-or-open-db" [repo {}]) + get-result (http-request-binary {:hostname host :port port + :path (asset-path repo "pic1.png") + :method "GET"} + nil) + put-result (http-request-binary {:hostname host :port port + :path (asset-path repo "pic1.png") + :method "PUT"} + binary)] + (is (= 401 (:status get-result))) + (is (= 401 (:status put-result)))) + (p/catch (fn [e] + (println "[db-worker-node-test] asset-auth error:" e) + (is false (str e)))) + (p/finally (fn [] + (if-let [stop! (:stop! @daemon)] + (-> (stop!) (p/finally done)) + (done)))))))) + (deftest db-worker-node-export-client-ops-db-binary (async done (let [daemon (atom nil)