dasemu/Synapse
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fase B6: temporal relevance decay and vector-based "similar pages"

Browse Source 
- combine-results now applies a bounded (never below 0.5x) exponential
  decay based on :block/updated-at, so more recently touched blocks/pages
  rank slightly higher for otherwise-equal matches.
- New search/similar-blocks reuses the existing zvec index: fetch a
  block's own already-indexed embedding (new :fetch capability on
  open-vector-index, via zvec's fetchSync) and re-query the index with
  it instead of embedding a fresh query string.
- Exposed as a :similar-pages render-resource (worker/handler/render-resource/similar.cljs),
  consistent with how linked/unlinked references are already wired, plus
  a raw :thread-api/search-similar-blocks for non-UI callers.
- New "Similar pages" section on the page view, above the existing
  linked/unlinked references.
This commit is contained in:
Darío Sevilla
2026-08-26 20:21:28 +02:00
parent d59aec181e
commit dadeca7165
9 changed files with 211 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/main/frontend/components/page.cljs
View File

@@ -15,6 +15,7 @@
[frontend.components.recycle :as recycle]
[frontend.components.reference :as reference]
[frontend.components.scheduled-deadlines :as scheduled]
[frontend.components.similar-pages :as similar-pages]
[frontend.config :as config]
[frontend.context.i18n :refer [t]]
[frontend.date :as date]
@@ -529,7 +530,15 @@
home?
class-page? property-page?)
[:div.fade-in.delay {:key "page-unlinked-references"}
(reference/unlinked-references (:block/uuid page) {:sidebar? sidebar?})])])]))
(reference/unlinked-references (:block/uuid page) {:sidebar? sidebar?})])
(when (and (entity/page? page)
(not block?)
(not sidebar?)
(not tag-dialog?)
(not home?)
(not class-page?) (not property-page?))
(similar-pages/similar-pages (:block/uuid page)))])]))
[:div.opacity-75 (t :page/not-found)])))
(defn- page-resource-key

19
src/main/frontend/components/similar_pages.cljs Normal file
View File

@@ -0,0 +1,19 @@
(ns frontend.components.similar-pages
"\"Similar pages\" section shown on a page, backed by the vector search
index's nearest neighbors to the page's own embedding."
(:require [frontend.components.block :as block]
[frontend.context.i18n :refer [t]]
[frontend.db.hooks :as db-hooks]
[io.factorhouse.hsx.core :as hsx]))
(hsx/defc similar-pages
[page-uuid]
(let [results (db-hooks/use-resource [:similar-pages page-uuid])]
(when (seq results)
[:div.similar-pages.fade-in.delay {:key "page-similar-pages"}
[:div.text-sm.font-medium.text-muted-foreground.mb-2 (t :page/similar-pages)]
[:div.flex.flex-col.gap-1
(for [{:block/keys [uuid] :as result} results]
^{:key (str uuid)}
[:div.similar-page-item
(block/page-cp {} result)])]])))

2
src/main/frontend/worker/handler/render_resource/engine.cljs
View File

@@ -6,6 +6,7 @@
[frontend.worker.handler.render-resource.common :as common]
[frontend.worker.handler.render-resource.property :as property]
[frontend.worker.handler.render-resource.query :as query]
[frontend.worker.handler.render-resource.similar :as similar]
[frontend.worker.handler.render-resource.view :as view]
[frontend.worker.state :as worker-state]))
@@ -14,6 +15,7 @@
property/resource-renderers
view/resource-renderers
query/resource-renderers
similar/resource-renderers
{:block-sync-conflicts
(common/renderer nil
(fn [_db resource-key _runtime]

22
src/main/frontend/worker/handler/render_resource/similar.cljs Normal file
View File

@@ -0,0 +1,22 @@
(ns frontend.worker.handler.render-resource.similar
"\"Similar to this\" renderer resource, backed by the vector search index."
(:require [frontend.worker.handler.render-resource.common :as common]
[frontend.worker.search :as search]
[frontend.worker.state :as worker-state]))
(def ^:private max-similar-pages 5)
(defn- similar-pages
[db resource-key {:keys [repo]}]
(let [block-uuid (second resource-key)]
(common/entity-by-uuid! db :block-uuid block-uuid)
(let [vector-index (worker-state/get-vector-index repo)
results (when vector-index
(search/similar-blocks (atom db) vector-index block-uuid
{:limit max-similar-pages}))]
[#{} (mapv #(select-keys % [:db/id :block/uuid :block/title
:block.temp/unique-title :page? :block/page])
(or results []))])))
(def resource-renderers
{:similar-pages (common/renderer 2 similar-pages)})

5
src/main/frontend/worker/handler/search.cljs
View File

@@ -286,6 +286,11 @@
[repo q option]
(<search-blocks repo q option))
(def-thread-api :thread-api/search-similar-blocks
[repo block-uuid option]
(when-let [conn (worker-state/get-datascript-conn repo)]
(search/similar-blocks conn (worker-state/get-vector-index repo) block-uuid option)))
(def-thread-api :thread-api/search-upsert-blocks
[repo blocks]
(when-let [db (get-search-db repo)]

15
src/main/frontend/worker/platform/node.cljs
View File

@@ -418,6 +418,19 @@
(or (nil? page)
(>= (count results) limit)))
(defn- fetch-zvec-doc
[collection ids]
(let [result (.fetchSync ^js collection (clj->js (vec ids)))]
(into {}
(keep (fn [id]
(when-let [doc (gobj/get result id)]
(let [vectors (gobj/get doc "vectors")
fields (gobj/get doc "fields")]
[id {:embedding (js->clj (gobj/get vectors zvec-vector-field))
:page (gobj/get fields zvec-page-field)
:vector-title (gobj/get fields zvec-title-field)}]))))
ids)))
(defn- open-vector-index
[{:keys [path dimension]}]
(p/let [_ (ensure-dir! (node-path/dirname path))]
@@ -449,6 +462,8 @@
:delete! (fn [ids]
(.deleteSync ^js @collection* (clj->js (vec ids)))
nil)
:fetch (fn [ids]
(fetch-zvec-doc @collection* ids))
:truncate! (fn []
(.destroySync ^js @collection*)
(reopen!)

64
src/main/frontend/worker/search.cljs
View File

@@ -158,6 +158,9 @@ DROP TRIGGER IF EXISTS blocks_au;
(def ^:private title-term-match-boost 0.002)
(def ^:private context-term-match-boost 0.0005)
(def ^:private max-vector-term-match-boost 0.005)
(def ^:private temporal-half-life-ms (* 60 24 60 60 1000)) ;; 60 days
(def ^:private min-temporal-decay 0.5) ;; never discount a strong match by more than half
(def ^:private max-similar-block-results 10)
(defn- query->terms
[q]
@@ -602,6 +605,7 @@ DROP TRIGGER IF EXISTS blocks_au;
'[:db/id
:block/uuid
:block/title
:block/updated-at
{:block/page [:block/uuid :block/name :block/title]}
{:block/parent [:db/id :block/uuid :block/title :logseq.property/built-in?
:logseq.property/hide? :logseq.property/deleted-at]}
@@ -743,6 +747,16 @@ DROP TRIGGER IF EXISTS blocks_au;
(min max-vector-term-match-boost score))
0))
(defn- temporal-decay
"Recently updated blocks are ranked slightly higher. Bounded at
`min-temporal-decay` so an old-but-strong match never gets crushed by a
recent-but-weak one, and unset `:block/updated-at` decays to no-op (1.0)."
[now updated-at]
(if (number? updated-at)
(let [age-ms (max 0 (- now updated-at))]
(max min-temporal-decay (js/Math.pow 0.5 (/ age-ms temporal-half-life-ms))))
1.0))
;; Combine and re-rank keyword and vector results
(defn combine-results
([db keyword-results]
@@ -753,6 +767,7 @@ DROP TRIGGER IF EXISTS blocks_au;
(let [keyword-results (vec (or keyword-results []))
vector-results (vec (or vector-results []))
use-rrf? (seq vector-results)
now (js/Date.now)
fused-score-by-id (when use-rrf?
(rrf-score-by-id [keyword-results vector-results]
[keyword-rrf-weight vector-rrf-weight]))
@@ -773,19 +788,20 @@ DROP TRIGGER IF EXISTS blocks_au;
vector-match-score (if use-rrf?
(vector-term-match-score q result block)
0)
combined-score (+ base-score
vector-match-score
(* source-score-tie-break-weight
(+ keyword-score vector-score))
(if-not use-rrf?
(cond
page?
0.02
(:block/tags block)
0.01
:else
0)
0))]
combined-score (* (+ base-score
vector-match-score
(* source-score-tie-break-weight
(+ keyword-score vector-score))
(if-not use-rrf?
(cond
page?
0.02
(:block/tags block)
0.01
:else
0)
0))
(temporal-decay now (:block/updated-at block)))]
(conj acc
(assoc result
search-result-block-key block
@@ -1022,6 +1038,28 @@ DROP TRIGGER IF EXISTS blocks_au;
:matched-count matched-count}
(take limit result))))))
(defn similar-blocks
"Blocks/pages whose already-indexed embedding is closest to the given
block's own embedding, i.e. \"similar to this\" rather than a fresh query."
[conn vector-index block-uuid option]
(when-let [fetch-fn (:fetch vector-index)]
(let [id (str block-uuid)
source (get (fetch-fn [id]) id)]
(when (seq (:embedding source))
(let [limit (min max-similar-block-results (or (:limit option) max-similar-block-results))
raw-result (vector-search-blocks vector-index
{:limit (inc limit)
:page (:page option)
:query-embedding (:embedding source)})
candidates (remove #(= (:id %) id) raw-result)
combined-result (combine-results @conn nil candidates)]
(->> combined-result
(common-util/distinct-by :id)
(keep #(search-result->block-result conn nil nil option %))
(remove #(= (:block/uuid %) block-uuid))
(take limit)
vec))))))
(defn upsert-vector-blocks!
[vector-index blocks]
(when-let [upsert-fn (:upsert! vector-index)]

1
src/resources/dicts/en.edn
View File

@@ -1142,6 +1142,7 @@
:page/scheduled-and-deadline "Scheduled and Deadline"
:page/search-index "search index"
:page/send-db-for-debugging "You can send it to help@logseq.com for debugging."
:page/similar-pages "Similar pages"
:page/something-went-wrong "Something went wrong"
:page/step "Step {1}"
:page/the-app "the app"

86
src/test/frontend/worker/search_test.cljs
View File

@@ -1245,6 +1245,92 @@
score (search-benchmark/score-results result [keyword-id] 1)]
(is (= 1 (:recall-at-1 score))))))))
(deftest combine-results-applies-temporal-decay-to-recency
(testing "temporal decay favors the more recently updated block among otherwise-equal candidates"
(let [old-id (test-uuid-string 950)
fresh-id (test-uuid-string 951)
page-id (test-uuid-string 952)
now (js/Date.now)
one-year-ms (* 365 24 60 60 1000)
blocks {old-id {:db/id 950
:block/uuid (uuid old-id)
:block/title "old block"
:block/updated-at (- now one-year-ms)
:block/page {:block/uuid (uuid page-id)}}
fresh-id {:db/id 951
:block/uuid (uuid fresh-id)
:block/title "fresh block"
:block/updated-at now
:block/page {:block/uuid (uuid page-id)}}}]
(with-redefs [d/pull-many (fn [_db _selector lookup-refs]
(mapv (fn [[_attr id]]
(get blocks (str id)))
lookup-refs))
ldb/hidden? (constantly false)
ldb/page? (constantly false)]
(let [result (search/combine-results
:db
[{:id old-id :keyword-score 1.0 :title "old block"}
{:id fresh-id :keyword-score 1.0 :title "fresh block"}])]
(is (= "fresh block" (:title (first result))))
(is (< (:combined-score (second result)) (:combined-score (first result)))))))))
(deftest combine-results-never-discounts-a-strong-hit-below-half
(testing "temporal decay is bounded so an ancient-but-exact match keeps most of its score"
(let [old-id (test-uuid-string 953)
blocks {old-id {:db/id 953
:block/uuid (uuid old-id)
:block/title "ancient but exact"
:block/updated-at 0}}]
(with-redefs [d/pull-many (fn [_db _selector lookup-refs]
(mapv (fn [[_attr id]]
(get blocks (str id)))
lookup-refs))
ldb/hidden? (constantly false)
ldb/page? (constantly false)]
(let [result (search/combine-results
:db
[{:id old-id :keyword-score 4.0 :title "ancient but exact"}])]
(is (>= (:combined-score (first result)) 2.0)))))))
(deftest similar-blocks-returns-nearest-neighbors-excluding-self
(testing "similar-blocks fetches the source embedding and re-queries the vector index, dropping the source itself"
(let [source-id (test-uuid-string 960)
neighbor-id (test-uuid-string 961)
page-id (test-uuid-string 962)
blocks {source-id {:db/id 960
:block/uuid (uuid source-id)
:block/title "source block"
:block/page {:block/uuid (uuid page-id)}}
neighbor-id {:db/id 961
:block/uuid (uuid neighbor-id)
:block/title "neighbor block"
:block/page {:block/uuid (uuid page-id)}}}
fetch-calls (atom [])
query-calls (atom [])
vector-index {:fetch (fn [ids]
(swap! fetch-calls conj ids)
(when (= ids [source-id])
{source-id {:embedding [0.1 0.2 0.3]}}))
:query (fn [embedding limit page]
(swap! query-calls conj {:embedding embedding :limit limit :page page})
[{:id source-id :page page-id :vector-score 1.0}
{:id neighbor-id :page page-id :vector-score 0.95}])}]
(with-redefs [d/entity (fn [_db [_attr id]] (get blocks (str id)))
d/pull-many (fn [_db _selector lookup-refs]
(mapv (fn [[_attr id]] (get blocks (str id))) lookup-refs))
ldb/hidden? (constantly false)
ldb/page? (constantly false)
ldb/built-in? (constantly false)]
(let [result (search/similar-blocks (atom :db) vector-index (uuid source-id) {})]
(is (= ["neighbor block"] (mapv :block/title result)))
(is (= 1 (count @fetch-calls)))
(is (= [0.1 0.2 0.3] (:embedding (first @query-calls)))))))))
(deftest similar-blocks-returns-nil-without-a-fetch-capable-vector-index
(testing "gracefully no-ops when the vector index has no :fetch capability (e.g. no index configured)"
(is (nil? (search/similar-blocks (atom :db) {} (uuid (test-uuid-string 970)) {})))))
(deftest search-result-keeps-page-title-when-alias-matches
(testing "alias matches annotate the page result without replacing its title"
(let [page-id #uuid "00000000-0000-0000-0000-000000000236"