Repository navigation
Expand file tree
/
Copy pathconfig.sample.toml
More file actions
114 lines (102 loc) · 2.83 KB
/
Copy pathconfig.sample.toml
File metadata and controls
114 lines (102 loc) · 2.83 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
# localdataextractor sample config
[llm]
# Backend serving the model: "lmstudio" or "ollama"
provider = "lmstudio"
# Optional. Defaults per provider when omitted:
# lmstudio -> http://127.0.0.1:1234/v1
# ollama -> http://127.0.0.1:11434/v1
# base_url = "http://127.0.0.1:1234/v1"
# On Ollama, "glm-ocr" matches "glm-ocr:latest".
primary_model = "glm-ocr"
fallback_model = "glm-ocr"
timeout_seconds = 120
retries = 2
temperature = 0.1
enable_vlm_repair = true
# Text model for table repair; empty means primary_model.
# OCR models rewrite cell values - prefer e.g. "gemma3:4b".
repair_model = ""
# Skip LM table repair above this many rows: the model echoes
# every cell back, so large tables just exhaust the timeout.
repair_max_rows = 40
# Give up on repair for a file after this many consecutive
# failures instead of retrying every remaining table.
repair_failure_limit = 3
# Set if your local server requires an API key:
api_token = ""
[retry]
confidence_threshold = 95.0
max_route_attempts = 5
table_repair_attempts = 2
retry_backoff_seconds = 1.0
[processing]
max_workers = 2
include_globs = ["*", "**/*"]
exclude_globs = ["**/.DS_Store", "**/~$*"]
[ocr]
enabled = true
dpi = 300
deskew = true
clean = true
optimize = 0
language = "eng"
force_when_garbled = true
[tables]
high_effort = true
important_row_min = 2
strict_column_consistency = true
max_empty_ratio = 0.4
max_duplicate_row_ratio = 0.15
[table_extraction]
pdfplumber_lines = true
pdfplumber_text = true
pdfplumber_lines_strict = true
camelot_lattice = true
camelot_stream = true
bbox_overlap_threshold = 0.5
min_table_score = 40.0
[image_preprocess]
enabled = true
grayscale = true
deskew = true
denoise = true
binarize = true
binarize_method = "sauvola"
margin_trim = true
target_long_edge = 2400
[render]
image_dpi = 220
markdown_table_fallback = true
[routing]
fallback_order = ["markitdown", "docling", "ocr_docling", "tika"]
min_characters_for_good_extraction = 300
extraction_mode = "standard" # standard | glm_ocr | highest_accuracy
[glm_ocr]
enabled = true
model_name = "glm-ocr"
page_dpi = 300
max_pages = 50
timeout_seconds = 180
use_pypdfium2 = true
temperature_override = 0.0
table_retry_enabled = true
table_retry_dpi = 400
[tika]
enabled = false
tika_jar_path = ""
[pii]
# Screen extracted outputs (.md/.json) for PII and replace values
# with placeholders like [PERSON_1]. Source files stay untouched.
enabled = false
# Also run the local LLM to find person names and free-form
# addresses (regex covers structured PII on its own).
use_llm = true
# Text model for the PII pass; empty means llm.primary_model.
# OCR models follow NER instructions poorly - prefer e.g. "gemma3:4b".
llm_model = ""
categories = ["contact", "person", "financial", "government_id"]
# WARNING: the mapping file contains the original PII values.
write_mapping_file = false
llm_chunk_chars = 4000
[logging]
level = "INFO"