Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 65 additions & 10 deletions crates/utopia-extract/src/open.rs
Original file line number Diff line number Diff line change
Expand Up @@ -149,15 +149,7 @@ its units, a percentage, an amount, a count, a title, a status. phrase is how th
what is measured or stated (\"area\", \"was completed\", \"joined\", \"占地面积\"); value is the \
literal as written. Wording that names or describes something is not a value: that something \
goes in \"e\" and is linked by a statement with an object. \"1,200 beekeepers joined\" is the \
thing \"beekeepers\" with \"joined\" = \"1,200\". Write every figure the passage states. In a \
table, a cell is a statement about its row's thing whose phrase is the column heading. When \
the column heading names one time (a date, a quarter, a period), it is when instead, the \
statement is about the thing the table's caption names (the company of a financial statement) \
and the phrase is the row label as written, path included (\"Operating expenses › Research and \
development\"); a heading that names a comparison between periods (a change from the quarter \
before, a change from a year ago) is not a time, it stays the phrase; a unit the caption or a \
heading gives (\"In millions\") is a qualifier on each such statement. Leave out cells whose \
column heading is not in the passage.\n\
thing \"beekeepers\" with \"joined\" = \"1,200\". Write every figure the passage states.{TABLE}\n\
A list is one statement per member, and so is a subject or object that joins several things \
(\"the city and the county funded the bridge\" is two statements).\n\
3. Nothing in a sentence is dropped. When more than two things take part, or the link carries \
Expand Down Expand Up @@ -214,6 +206,47 @@ $2 million on March 4, 2011.\" gives:\n\
\"n\": [],\n\
\"t\": []}";

/// 表格的读法,只在这一段里有表格时才进系统消息(见 [`has_table`]):没有表格的段落
/// 用不上它,而它占系统消息的一成不到
const OPEN_TABLE_RULES: &str = "\
In a \
table, a cell is a statement about its row's thing whose phrase is the column heading. When \
the column heading names one time (a date, a quarter, a period), it is when instead, the \
statement is about the thing the table's caption names (the company of a financial statement) \
and the phrase is the row label as written, path included (\"Operating expenses › Research and \
development\"); a heading that names a comparison between periods (a change from the quarter \
before, a change from a year ago) is not a time, it stays the phrase; a unit the caption or a \
heading gives (\"In millions\") is a qualifier on each such statement. Leave out cells whose \
column heading is not in the passage.";

/// 这一段里有没有表格:Markdown 的竖线、HTML 的单元格、制表符,或者一行里用成串空格
/// 隔开的三栏以上。宁可多认:认错了只是多带一段规则,和从前一样
fn has_table(passage: &str) -> bool {
let lower = passage.to_ascii_lowercase();
passage.contains('|')
|| passage.contains('\t')
|| lower.contains("<td")
|| lower.contains("<th")
|| lower.contains("<tr")
|| passage.lines().any(|l| {
l.trim()
.split(" ")
.filter(|c| !c.trim().is_empty())
.count()
>= 3
})
}

/// 这一段的系统消息:规则不变,只有表格的那几句按段落带或不带
fn system_for(passage: &str) -> String {
let table = if has_table(passage) {
format!(" {OPEN_TABLE_RULES}")
} else {
String::new()
};
OPEN_SYSTEM.replace("{TABLE}", &table)
}

/// 构造开放抽取的两条消息:常量系统消息 + `Document:` / 开头 / 已知实体 / `Passage:`。
///
/// 没有文档日期、没有类型与关系清单、没有属性——这些都不进提示词。开头与已知实体
Expand All @@ -233,7 +266,7 @@ pub fn build_open_messages(
vec![
ChatMessage {
role: "system".into(),
content: OPEN_SYSTEM.to_string(),
content: system_for(chunk_text),
},
ChatMessage {
role: "user".into(),
Expand Down Expand Up @@ -765,6 +798,28 @@ mod tests {

/// 提示词里没有文档日期、没有类型与关系清单、没有编号与句柄;有文件名、开头、
/// 已知名字与正文,并且按这个顺序
#[test]
fn the_table_rules_ride_only_with_a_passage_that_has_a_table() {
let plain = "The council met on March 4, 2011. It awarded the paving contract.";
let system = &build_open_messages("a.md", &[], None, plain)[0].content;
assert!(!system.contains("In a table"));
assert!(!system.contains("{TABLE}"));
assert!(system.contains("Write every figure the passage states.\n"));
for table in [
"| Item | Q3 2025 |\n|---|---|\n| Revenue | 12 |",
"<table><tr><td>Revenue</td><td>12</td></tr></table>",
"Revenue\t12\t14",
"Revenue 12 14\nCosts 7 8",
] {
let system = &build_open_messages("a.md", &[], None, table)[0].content;
assert!(
system.contains("Write every figure the passage states. In a table, a cell is"),
"{table}"
);
assert!(system.contains("column heading is not in the passage.\n"));
}
}

#[test]
fn the_prompt_carries_no_ontology_and_no_document_date() {
let msgs = build_open_messages(
Expand Down
38 changes: 38 additions & 0 deletions scripts/bench/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -864,3 +864,41 @@ bound / none / undecided。0044 的门槛:裁判精度不低于完整原型的
十条上下不再进人的队列,`votes.reason = "first_vote_none"` 查得到。
- 20 篇整轮(只有分批):合计 82.1 万 → 62.6 万,每篇 4.1 万 → 3.1 万;其中约一半来自分批,其余是
这一轮抽出的签名少、重问少。

### 第七次真跑:每篇只抽一遍,表格的规则只随表格带(2026-09-28,dev @ 7d73ff9,gemini-3.5-flash,20 篇)

**台子从前让每篇抽两遍。** 配了对话模型的库,文档解析完管线自己排抽取;`typed.mjs` 和 `timewords.mjs`
随后又对每篇调一次手动抽取,而手动抽取是强制全量(解雇在跑的任务、从头再抽)。20 篇 24 个分块,抽取调用
45 次。现在只补管线没排上的。这一节之前的每篇 token 都多算了将近一遍抽取,召回也可能受它影响(在跑的
那一遍被中途顶替):这一轮同句召回 20.1%,实体对召回 18.0%,只有 20 篇,要在 100 篇上再量。

| 调用 | 次数 | 每次输入 | 每次输出 | 每篇 |
|---|---|---|---|---|
| 短语对齐 | 38 | 3283 | 240 | 6695 |
| 抽取 | 26 | 2517 | 1152 | 4770 |
| 类别词对齐 | 24 | 2947 | 647 | 4314 |
| 提规则 | 14 | 4061 | 122 | 2929 |
| 时间解释 | 20 | 1770 | 141 | 1912 |
| 本体代理 | 5 | 4966 | 2021 | 1747 |
| 实体裁决 | 9 | 2504 | 503 | 1354 |
| 提问题 | 1 | 7603 | 1466 | 453 |

合计每篇 24175,输入占 85%。类型化 138,绑定 105 / 98 / 27,裁判精度 79.7%。

**抽取的系统消息约 2380 token,一篇 Re-DocRED 文档约 270。** 试了三处删减,每处单独量:31 个没有表格
的段落(这 20 篇的 24 个分块加时间台子的 7 篇),每种系统消息各抽三遍,回复用服务端的解析器读,对金标
的实体对和时间台子的期望打分:

| 系统消息 | 每次输入 | 金标实体对(三遍) | 同句(三遍) | 时间词记下 | 修复后只剩一条陈述的回复 |
|---|---|---|---|---|---|
| 原样 | 2488 | 27.0 / 27.6 / 28.6% | 42.6 / 44.6 / 47.0% | 28 / 28 / 27 | 0 / 93 |
| 不带表格的规则 | 2339 | 28.6 / 28.4 / 28.6% | 47.0 / 46.2 / 46.2% | 27 / 27 / 27 | 0 / 93 |
| 再去掉规则 6(引文的位置,规则 2 与 7 说过) | 2296 | 29.0 / 27.4 / 26.5% | 49.4 / 47.4 / 43.4% | 28 / 27 / 28 | 3 / 93 |
| 再去掉结尾的例子 | 2182 | 27.9 / 23.3 / 27.6% | 44.2 / 41.4 / 45.0% | 28 / 29 / 28 | 0 / 93 |

- 表格的规则只随有表格的段落带:没有表格的段落上分数不降,有表格的段落系统消息与从前逐字相同。落地。
- 规则 6 重复,却在管格式:去掉之后 93 次里 3 次括号写错,解析器修到最后一个完整对象,整段只剩一条
陈述。不去。
- 结尾的例子:三遍里一遍实体对掉到 23.3%,解析没有出错。分不清是波动还是损失,不去。
- 同一条系统消息、温度 0,两遍抽出的陈述(主语、短语、宾语)只有六成相同。逐条比对陈述量不出提示词的
改动,要用有标准答案的指标。
5 changes: 4 additions & 1 deletion scripts/bench/timewords.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,10 @@ async function run(n) {
if (!r.ok) throw new Error(`upload ${d.filename} -> ${r.status} ${(await r.text()).slice(0, 200)}`);
}
for (let i = 0; i < 60; i++) { await sleep(3000); if (q(`SELECT count(*) FROM documents WHERE kb_id='${KB}' AND status<>'ready'`) === "0") break; }
for (const d of (await api("GET", `/api/v1/kbs/${KB}/documents?limit=50`)).docs) await api("POST", `/api/v1/documents/${d.id}/extract`, {});
// 管线解析完自己排抽取;手动抽取是强制全量,只补没排上的,否则每篇抽两遍
for (const d of (await api("GET", `/api/v1/kbs/${KB}/documents?limit=50`)).docs) {
if (!["queued", "extracting", "done"].includes(d.graph_status)) await api("POST", `/api/v1/documents/${d.id}/extract`, {});
}
for (let i = 0; i < 240; i++) {
await sleep(5000);
const pending = q(`SELECT count(*) FROM jobs WHERE status IN ('queued','running') AND kind IN ('extract_document','resolve_time','process_document') AND (payload->>'kb_id'='${KB}' OR payload->>'document_id' IN (SELECT id::text FROM documents WHERE kb_id='${KB}'))`);
Expand Down
8 changes: 6 additions & 2 deletions scripts/bench/typed.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -176,8 +176,12 @@ async function extract(KB) {
// 温库第二批:只排这一批的文档,老文档的图不动
const mine = new Set(corpus.docs.map((d) => d.filename));
const docs = (await api("GET", `/api/v1/kbs/${KB}/documents?limit=500`)).docs.filter((d) => !INTO || mine.has(d.filename));
for (const d of docs) await api("POST", `/api/v1/documents/${d.id}/extract`, {});
log(`排队抽取 ${docs.length} 篇`);
// 配了对话模型的库,文档解析完管线自己排抽取。这里只补没排上的:手动抽取是强制全量
// (解雇在跑的任务、从头再抽),对每篇都调一次就是每篇抽两遍——2026-09-28 之前的每篇
// token 都多算了一遍抽取
const idle = docs.filter((d) => !["queued", "extracting", "done"].includes(d.graph_status));
for (const d of idle) await api("POST", `/api/v1/documents/${d.id}/extract`, {});
log(`排队抽取 ${idle.length} 篇(管线已排 ${docs.length - idle.length} 篇)`);
const live = `SELECT count(*) FROM chunks c JOIN documents d ON d.id=c.document_id WHERE d.kb_id='${KB}' AND c.superseded_at IS NULL`;
await until(() => {
const done = num(live.replace("count(*)", "count(c.extracted_at)"));
Expand Down