Skip to content

Commit cfafc10

Browse files
heiskrCopilot
heiskr
and
Copilot
authored
Double-purge changed content keys to clear the origin shield (#62478)
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 11731c34-4a31-4175-ba46-4970237ba75d
1 parent f39067c commit cfafc10

2 files changed

Lines changed: 108 additions & 28 deletions

File tree

‎src/workflows/purge-fastly-changed-content.ts‎

Lines changed: 51 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,24 @@ const COMPARE_FILE_LIMIT = 300
3131
// giving up on it.
3232
const PURGE_MAX_RATE_LIMIT_RETRIES = 5
3333

34+
// Every key is purged twice because of Fastly shielding. A purge doesn't reach
35+
// every POP at the same instant, so a request arriving in between can repopulate
36+
// an already-purged edge node from the not-yet-purged shield, leaving the edge
37+
// holding pre-deploy content again. The second pass evicts that copy. Same
38+
// reasoning as the double purge in purge-fastly.ts; see the "Race conditions"
39+
// section of
40+
// https://www.fastly.com/documentation/guides/concepts/cache/purging#race-conditions
41+
const PURGE_PASSES = 2
42+
43+
// How long to wait before the second pass. It has to be long enough that any
44+
// re-populated edge copy already exists, otherwise the second purge runs too
45+
// early and the re-population happens after it. purge-fastly.ts uses the same
46+
// 20s for the same reason: Fastly suggests ~2s, but that has been too short in
47+
// practice. Unlike purge-fastly.ts we don't stagger keys within a pass, because
48+
// that spacing exists to keep whole-language purges from stampeding the backend
49+
// and we only purge the handful of pages that actually changed.
50+
const DELAY_BEFORE_SECOND_PURGE = 20 * 1000
51+
3452
// Jitter ceiling (ms) added to each backoff so retries that saw the same reset
3553
// timestamp don't wake in lockstep and re-burst.
3654
const PURGE_JITTER_MS = 150
@@ -248,30 +266,50 @@ async function hardPurgeKeyBatch(
248266
}
249267
}
250268

251-
// Hard-purge every key in batches of <= 256, one batch at a time. Collects
252-
// failures so one bad batch doesn't drop the rest, then throws at the end if any
253-
// failed so the workflow's failure alerting fires.
269+
// Hard-purge every key in batches of <= 256, one batch at a time, then do it all
270+
// again after a delay to clear anything the origin shield re-populated (see
271+
// PURGE_PASSES). Collects failures so one bad batch doesn't drop the rest, then
272+
// throws at the end if any failed so the workflow's failure alerting fires.
254273
export async function hardPurgeSurrogateKeys(
255274
keys: string[],
256275
fastlyToken: string,
257276
serviceId: string,
258277
rateLimitDelayFn: (response: Response, attempt: number) => number = rateLimitDelayMs,
278+
sleepFn: (ms: number) => Promise<void> = sleep,
259279
): Promise<void> {
260280
const batches = chunk(keys, MAX_KEYS_PER_PURGE)
261281
const errors: Error[] = []
262-
for (const [index, batch] of batches.entries()) {
263-
const label = `batch ${index + 1}/${batches.length} (${batch.length} key(s))`
264-
try {
265-
console.log(`Hard-purging ${label}...`)
266-
await hardPurgeKeyBatch(batch, fastlyToken, serviceId, rateLimitDelayFn)
267-
console.log(`Hard-purged ${label}.`)
268-
} catch (error) {
269-
console.error(error)
270-
errors.push(error instanceof Error ? error : new Error(String(error)))
282+
let attempts = 0
283+
284+
const purgeAllBatches = async (pass: number): Promise<void> => {
285+
for (const [index, batch] of batches.entries()) {
286+
const label =
287+
`pass ${pass}/${PURGE_PASSES}, batch ${index + 1}/${batches.length} ` +
288+
`(${batch.length} key(s))`
289+
attempts++
290+
try {
291+
console.log(`Hard-purging ${label}...`)
292+
await hardPurgeKeyBatch(batch, fastlyToken, serviceId, rateLimitDelayFn)
293+
console.log(`Hard-purged ${label}.`)
294+
} catch (error) {
295+
console.error(error)
296+
errors.push(error instanceof Error ? error : new Error(String(error)))
297+
}
298+
}
299+
}
300+
301+
for (let pass = 1; pass <= PURGE_PASSES; pass++) {
302+
// A failed first pass still gets a second one: the later attempt may well
303+
// succeed, and giving up here would guarantee stale content.
304+
if (pass > 1) {
305+
console.log(`Waiting ${DELAY_BEFORE_SECOND_PURGE}ms before pass ${pass}...`)
306+
await sleepFn(DELAY_BEFORE_SECOND_PURGE)
271307
}
308+
await purgeAllBatches(pass)
272309
}
310+
273311
if (errors.length) {
274-
throw new Error(`${errors.length} of ${batches.length} batch purge(s) failed`)
312+
throw new Error(`${errors.length} of ${attempts} batch purge(s) failed`)
275313
}
276314
}
277315

‎src/workflows/tests/purge-fastly-changed-content.ts‎

Lines changed: 57 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,9 @@ describe('chunk', () => {
154154
})
155155

156156
describe('hardPurgeSurrogateKeys', () => {
157+
// Skips the between-pass delay so tests don't wait 20 real seconds.
158+
const noSleep = async () => {}
159+
157160
// A minimal stand-in for a fetch Response, with a case-insensitive headers.get.
158161
function fakeResponse(
159162
status: number,
@@ -170,14 +173,16 @@ describe('hardPurgeSurrogateKeys', () => {
170173
}
171174
}
172175

173-
test('sends one hard batch purge with a surrogate_keys body (no soft header)', async () => {
176+
test('sends one hard batch purge per pass with a surrogate_keys body (no soft header)', async () => {
174177
fetchWithRetry.mockResolvedValue({ ok: true })
175178
await hardPurgeSurrogateKeys(
176179
['language:en,path:a.md', 'language:en,path:b.md'],
177180
'token-123',
178181
'svc-1',
182+
undefined,
183+
noSleep,
179184
)
180-
expect(fetchWithRetry).toHaveBeenCalledTimes(1)
185+
expect(fetchWithRetry).toHaveBeenCalledTimes(2)
181186
const [url, init] = fetchWithRetry.mock.calls[0]
182187
expect(url).toBe('https://api.fastly.com/service/svc-1/purge')
183188
expect(init.method).toBe('POST')
@@ -186,46 +191,83 @@ describe('hardPurgeSurrogateKeys', () => {
186191
expect(JSON.parse(init.body)).toEqual({
187192
surrogate_keys: ['language:en,path:a.md', 'language:en,path:b.md'],
188193
})
194+
// The second pass repeats the identical batch.
195+
expect(fetchWithRetry.mock.calls[1][1].body).toBe(init.body)
196+
})
197+
198+
test('waits between the two passes to let the shield re-populate first', async () => {
199+
fetchWithRetry.mockResolvedValue({ ok: true })
200+
const waits: number[] = []
201+
await hardPurgeSurrogateKeys(
202+
['language:en,path:a.md'],
203+
'tok',
204+
'svc',
205+
undefined,
206+
async (ms: number) => {
207+
waits.push(ms)
208+
},
209+
)
210+
expect(waits).toEqual([20_000])
189211
})
190212

191-
test('splits more than 256 keys into multiple batches', async () => {
213+
test('splits more than 256 keys into multiple batches, per pass', async () => {
192214
fetchWithRetry.mockResolvedValue({ ok: true })
193215
const keys = Array.from({ length: 257 }, (_unused, i) => `language:en,path:p${i}.md`)
194-
await hardPurgeSurrogateKeys(keys, 'tok', 'svc')
195-
expect(fetchWithRetry).toHaveBeenCalledTimes(2)
216+
await hardPurgeSurrogateKeys(keys, 'tok', 'svc', undefined, noSleep)
217+
// 2 batches x 2 passes.
218+
expect(fetchWithRetry).toHaveBeenCalledTimes(4)
196219
expect(JSON.parse(fetchWithRetry.mock.calls[0][1].body).surrogate_keys).toHaveLength(256)
197220
expect(JSON.parse(fetchWithRetry.mock.calls[1][1].body).surrogate_keys).toHaveLength(1)
221+
expect(JSON.parse(fetchWithRetry.mock.calls[2][1].body).surrogate_keys).toHaveLength(256)
222+
expect(JSON.parse(fetchWithRetry.mock.calls[3][1].body).surrogate_keys).toHaveLength(1)
198223
})
199224

200-
test('throws if any batch fails, after attempting all of them', async () => {
225+
test('throws if any batch fails, after attempting all of them in both passes', async () => {
201226
fetchWithRetry.mockResolvedValueOnce({ ok: true }).mockResolvedValueOnce({
202227
ok: false,
203228
status: 500,
204229
statusText: 'err',
205230
text: async () => 'boom',
206231
})
232+
fetchWithRetry.mockResolvedValue({ ok: true })
207233
const keys = Array.from({ length: 300 }, (_unused, i) => `language:en,path:p${i}.md`)
208-
await expect(hardPurgeSurrogateKeys(keys, 'tok', 'svc')).rejects.toThrow(
209-
/1 of 2 batch purge\(s\) failed/,
234+
await expect(hardPurgeSurrogateKeys(keys, 'tok', 'svc', undefined, noSleep)).rejects.toThrow(
235+
/1 of 4 batch purge\(s\) failed/,
210236
)
237+
expect(fetchWithRetry).toHaveBeenCalledTimes(4)
238+
})
239+
240+
test('still runs the second pass when the first one fails outright', async () => {
241+
fetchWithRetry
242+
.mockResolvedValueOnce({
243+
ok: false,
244+
status: 500,
245+
statusText: 'err',
246+
text: async () => 'boom',
247+
})
248+
.mockResolvedValue({ ok: true })
249+
await expect(
250+
hardPurgeSurrogateKeys(['language:en,path:a.md'], 'tok', 'svc', undefined, noSleep),
251+
).rejects.toThrow(/1 of 2 batch purge\(s\) failed/)
211252
expect(fetchWithRetry).toHaveBeenCalledTimes(2)
212253
})
213254

214255
test('retries a 429, honoring the hint, then succeeds', async () => {
215256
fetchWithRetry
216257
.mockResolvedValueOnce(fakeResponse(429, { headers: { 'retry-after': '0' } }))
217-
.mockResolvedValueOnce(fakeResponse(200, { ok: true }))
218-
await hardPurgeSurrogateKeys(['language:en,path:a.md'], 'tok', 'svc', () => 0)
219-
expect(fetchWithRetry).toHaveBeenCalledTimes(2)
258+
.mockResolvedValue(fakeResponse(200, { ok: true }))
259+
await hardPurgeSurrogateKeys(['language:en,path:a.md'], 'tok', 'svc', () => 0, noSleep)
260+
// 429 + retry on the first pass, then one call for the second pass.
261+
expect(fetchWithRetry).toHaveBeenCalledTimes(3)
220262
})
221263

222264
test('gives up after the retry budget and reports the batch as failed', async () => {
223265
fetchWithRetry.mockResolvedValue(fakeResponse(429, { headers: { 'retry-after': '0' } }))
224266
await expect(
225-
hardPurgeSurrogateKeys(['language:en,path:a.md'], 'tok', 'svc', () => 0),
226-
).rejects.toThrow(/1 of 1 batch purge\(s\) failed/)
227-
// Initial attempt + 5 retries.
228-
expect(fetchWithRetry).toHaveBeenCalledTimes(6)
267+
hardPurgeSurrogateKeys(['language:en,path:a.md'], 'tok', 'svc', () => 0, noSleep),
268+
).rejects.toThrow(/2 of 2 batch purge\(s\) failed/)
269+
// (Initial attempt + 5 retries) x 2 passes.
270+
expect(fetchWithRetry).toHaveBeenCalledTimes(12)
229271
})
230272
})
231273

0 commit comments

Comments
 (0)