Skip to content
← Back

src/analysis/watchdog.ghul

1
@@IF.dotnet()
2
3
namespace Analysis is
4
use IO.Std
5
6
// Decides when the long-lived analyser should exit so the IDE can spawn a
7
// fresh process. Two independent triggers:
8
//
9
// - heap growth: the managed heap is sampled after every full compile.
10
// The first few readings are warm-up — the JIT, interned symbols and
11
// reflection-metadata caches are still filling — so the baseline is not
12
// fixed until that has settled: it is the high-water reading across the
13
// first warm_up_compiles compiles. Past that, the compiler retains state
14
// across compiles (ASTs, caches) that clear_symbols does not all
15
// reclaim, so a steady climb is the expected leak signature, and a
16
// recycle returns that memory to a fresh process.
17
//
18
// - instability: a burst of exceptions escaping command handlers. Isolated
19
// failures decay away; a sustained run of them means the analyser's
20
// state is corrupt and a fresh process is the only recovery.
21
class WATCHDOG is
22
// Recycle once the heap has grown past both thresholds relative to the
23
// baseline: a multiple of it, and an absolute floor so a small baseline
24
// can't trip a recycle on ordinary working-set noise.
25
heap_growth_factor: double static => 2.0D
26
heap_growth_floor_bytes: long static => 256L * 1024L * 1024L
27
28
// Consecutive failing operations needed to trip an instability recycle.
29
// request_restart() adds 2 per failure, a clean operation decays 1, so
30
// this is reached by roughly this many failures in an unbroken run.
31
instability_limit: int static => 20
32
33
// How often to echo a heap reading to the log (every Nth full compile).
34
heap_log_interval: int static => 32
35
36
// Once the baseline is fixed, the heap is sampled only every Nth full
37
// compile, not after every one — check_heap forces a full GC, too
38
// costly to run that often. Warm-up compiles (before the baseline is
39
// fixed) are always sampled: the baseline is their high-water reading.
40
heap_check_interval: int static => 32
41
42
// Full compiles to let pass before the baseline is fixed. The analyser's
43
// working set climbs over the first compiles as the JIT and caches fill
44
// — growth that is not a leak — so anchoring the baseline to the cold
45
// first reading would let ordinary warm-up trip a recycle.
46
warm_up_compiles: int static => 5
47
48
// When set, check_heap is skipped — the analyser does not sample the
49
// heap or force a GC after compiles. Driven by --no-analysis-heap-
50
// watchdog; intended for profiling, where the forced GC dominates.
51
heap_check_disabled: bool public
52
53
_restart_score: int
54
55
_have_baseline: bool
56
_baseline_heap_bytes: long
57
_warm_up_peak_heap_bytes: long
58
_full_compile_pending: bool
59
_compile_count: int
60
_compiles_since_heap_check: int
61
62
want_restart: bool
63
64
init() is si
65
66
// An exception escaped a command handler. Nudge the instability score;
67
// a sustained run of failures trips want_restart, isolated ones decay.
68
request_restart() is
69
_restart_score = _restart_score + 2
70
71
if _restart_score > instability_limit then
72
want_restart = true
73
fi
74
si
75
76
// A full project compile completed successfully. The heap reading is
77
// taken at the next operation boundary (on_operation_complete), once
78
// the in-flight response frame has been flushed.
79
note_full_compile() is
80
_full_compile_pending = true
81
_compile_count = _compile_count + 1
82
_compiles_since_heap_check = _compiles_since_heap_check + 1
83
si
84
85
// Called by the despatcher after every command, once the handler has
86
// returned and its response is on the wire. This is the only place a
87
// recycle is issued, so a recycle never truncates a response frame.
88
on_operation_complete(writer: IO.TextWriter) is
89
if want_restart then
90
recycle(writer, "instability")
91
fi
92
93
if _restart_score > 0 then
94
_restart_score = _restart_score - 1
95
fi
96
97
if _full_compile_pending then
98
_full_compile_pending = false
99
100
if !heap_check_disabled /\ should_check_heap() then
101
check_heap(writer)
102
fi
103
fi
104
si
105
106
// Whether the every-Nth-compile fallback should sample the heap now.
107
// Always during warm-up (the baseline needs every reading); afterwards
108
// once heap_check_interval compiles have passed since the last sample.
109
// An explicit #HEAPCHECK# resets that count (note_heap_sampled), so a
110
// recent IDE-driven check suppresses a redundant fallback sample.
111
should_check_heap() -> bool =>
112
!_have_baseline \/ _compiles_since_heap_check >= heap_check_interval
113
114
// Record that the heap was just sampled — restarts the fallback
115
// interval. Called whenever check_heap runs, so both an explicit
116
// #HEAPCHECK# and a fallback sample reset it.
117
note_heap_sampled() is
118
_compiles_since_heap_check = 0
119
si
120
121
// Sample the heap in response to an explicit IDE request — the VS Code
122
// extension sends one during a lull in editing, so the forced GC lands
123
// outside the latency path of an interactive request. The every-Nth-
124
// compile path above stays as a fallback for clients that never send
125
// it. Respects --no-analysis-heap-watchdog.
126
check_heap_on_request(writer: IO.TextWriter) is
127
if !heap_check_disabled then
128
check_heap(writer)
129
fi
130
si
131
132
check_heap(writer: IO.TextWriter) is
133
note_heap_sampled()
134
135
let heap = System.GC.get_total_memory(true)
136
137
note_heap(heap)
138
139
if is_heap_over_limit(heap) then
140
log_heap("heap", heap)
141
recycle(writer, "heap growth")
142
fi
143
si
144
145
// Fold one full-compile heap reading into the watchdog. Pure with
146
// respect to the system — the caller supplies the reading — so unit
147
// tests drive it directly. During warm-up it tracks the high-water
148
// reading and fixes the baseline to it once warm_up_compiles full
149
// compiles have completed.
150
note_heap(heap: long) is
151
if !_have_baseline then
152
if heap > _warm_up_peak_heap_bytes then
153
_warm_up_peak_heap_bytes = heap
154
fi
155
156
if _compile_count >= warm_up_compiles then
157
set_baseline(_warm_up_peak_heap_bytes)
158
fi
159
160
return
161
fi
162
163
if _compile_count % heap_log_interval == 0 then
164
log_heap("heap", heap)
165
fi
166
si
167
168
set_baseline(heap: long) is
169
_have_baseline = true
170
_baseline_heap_bytes = heap
171
172
log_heap("baseline", heap)
173
si
174
175
// The recycle decision for a heap reading. Pure — no sampling, no exit —
176
// so it can be exercised directly by unit tests. Both thresholds must be
177
// crossed: a multiple of the baseline and an absolute byte floor.
178
is_heap_over_limit(heap: long) -> bool =>
179
_have_baseline /\
180
heap - _baseline_heap_bytes > heap_growth_floor_bytes /\
181
cast double(heap) > cast double(_baseline_heap_bytes) * heap_growth_factor
182
183
log_heap(label: string, heap: long) is
184
Std.error.write_line(
185
"watchdog: {label} {heap / 1024L / 1024L} MB (baseline {_baseline_heap_bytes / 1024L / 1024L} MB, {_compile_count} compiles)"
186
)
187
Std.error.flush()
188
si
189
190
recycle(writer: IO.TextWriter, reason: string) is
191
Std.error.write_line("watchdog: recycling analyser ({reason})")
192
Std.error.flush()
193
194
Protocol.JSON_PROTOCOL.write_response(writer, Protocol.Response.RESTART())
195
196
Driver.COMPILER_ENVIRONMENT.exit(1)
197
si
198
si
199
si