mirror of
https://github.com/fluxerapp/fluxer.git
synced 2026-09-02 21:04:06 +03:00
perf(api): compile the phrase blocklist into one matcher (#2164)
This commit is contained in:
Hampus
@@ -10,7 +10,7 @@ import type {ValidationError} from '@fluxer/errors/src/domains/core/ValidationEr
|
||||
import type {Context, Env, Input, MiddlewareHandler, TypedResponse, ValidationTargets} from 'hono';
|
||||
import {getCookie} from 'hono/cookie';
|
||||
import type {ZodError, ZodTypeAny} from 'zod';
|
||||
import {parseJsonPreservingLargeIntegers} from './utils/LosslessJsonParser';
|
||||
import {readRequestJsonBody} from './utils/RequestJsonBody';
|
||||
import {initializeFluxerErrorMap} from './ZodErrorMap';
|
||||
|
||||
initializeFluxerErrorMap();
|
||||
@@ -200,12 +200,7 @@ export const Validator = <
|
||||
let value: unknown;
|
||||
switch (target) {
|
||||
case 'json':
|
||||
try {
|
||||
const raw = await c.req.text();
|
||||
value = raw.trim().length === 0 ? {} : parseJsonPreservingLargeIntegers(raw);
|
||||
} catch {
|
||||
value = {};
|
||||
}
|
||||
value = (await readRequestJsonBody(c.req)).value;
|
||||
break;
|
||||
case 'form': {
|
||||
const formData = await c.req.formData();
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
import {ContentBlockedError} from '@fluxer/errors/src/domains/content/ContentBlockedError';
|
||||
import {createMiddleware} from 'hono/factory';
|
||||
import {Logger} from '../Logger';
|
||||
import {readRequestJsonBody} from '../utils/RequestJsonBody';
|
||||
import {extractUrlCandidates} from '../utils/UrlNormalizer';
|
||||
import {phraseBlocklistCache} from './PhraseBlocklistCache';
|
||||
import {urlBlocklistCache} from './UrlBlocklistCache';
|
||||
@@ -136,13 +137,11 @@ const ContentFilterMiddleware = createMiddleware(async (ctx, next) => {
|
||||
if (!contentType.includes('application/json')) {
|
||||
return next();
|
||||
}
|
||||
let body: unknown;
|
||||
try {
|
||||
body = await ctx.req.json();
|
||||
} catch {
|
||||
const body = await readRequestJsonBody(ctx.req);
|
||||
if (!body.parsed) {
|
||||
return next();
|
||||
}
|
||||
const strings = extractStringValues(body);
|
||||
const strings = extractStringValues(body.value);
|
||||
if (strings.length === 0) {
|
||||
return next();
|
||||
}
|
||||
|
||||
@@ -5,14 +5,16 @@ import {AdminRepository} from '../admin/AdminRepository';
|
||||
import {BANNED_PHRASES_REFRESH_CHANNEL} from '../constants/ContentModeration';
|
||||
import {Logger} from '../Logger';
|
||||
import {buildPhraseMatchForms, canonicalizeStoredPhrase} from '../utils/PhraseBlocklistNormalization';
|
||||
import {SubstringMatcher} from '../utils/SubstringMatcher';
|
||||
|
||||
export class PhraseBlocklistCache {
|
||||
private rawPhrases: Array<string> = [];
|
||||
private rawPhraseSet = new Set<string>();
|
||||
private wordPhrases: Array<string> = [];
|
||||
private compactPhrases: Array<string> = [];
|
||||
private asciiWordPhrases: Array<string> = [];
|
||||
private asciiCompactPhrases: Array<string> = [];
|
||||
private rawMatcher: SubstringMatcher | null = null;
|
||||
private wordMatcher: SubstringMatcher | null = null;
|
||||
private compactMatcher: SubstringMatcher | null = null;
|
||||
private asciiWordMatcher: SubstringMatcher | null = null;
|
||||
private asciiCompactMatcher: SubstringMatcher | null = null;
|
||||
private isInitialized = false;
|
||||
private adminRepository = new AdminRepository();
|
||||
private kvClient: IKVProvider | null = null;
|
||||
@@ -81,22 +83,14 @@ export class PhraseBlocklistCache {
|
||||
if (this.rawPhrases.length === 0) return false;
|
||||
const forms = buildPhraseMatchForms(text);
|
||||
return (
|
||||
this.matchAny(forms.raw, this.rawPhrases) ||
|
||||
this.matchAny(forms.words, this.wordPhrases) ||
|
||||
this.matchAny(forms.compact, this.compactPhrases) ||
|
||||
this.matchAny(forms.asciiWords, this.asciiWordPhrases) ||
|
||||
this.matchAny(forms.asciiCompact, this.asciiCompactPhrases)
|
||||
this.rawMatcher?.test(forms.raw) === true ||
|
||||
this.wordMatcher?.test(forms.words) === true ||
|
||||
this.compactMatcher?.test(forms.compact) === true ||
|
||||
this.asciiWordMatcher?.test(forms.asciiWords) === true ||
|
||||
this.asciiCompactMatcher?.test(forms.asciiCompact) === true
|
||||
);
|
||||
}
|
||||
|
||||
private matchAny(text: string, phrases: Array<string>): boolean {
|
||||
if (!text || phrases.length === 0) return false;
|
||||
for (const phrase of phrases) {
|
||||
if (text.includes(phrase)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private rebuildMatchers(): void {
|
||||
const rawPhraseSet = new Set<string>();
|
||||
const wordPhraseSet = new Set<string>();
|
||||
@@ -115,10 +109,11 @@ export class PhraseBlocklistCache {
|
||||
}
|
||||
this.rawPhraseSet = rawPhraseSet;
|
||||
this.rawPhrases = Array.from(rawPhraseSet);
|
||||
this.wordPhrases = Array.from(wordPhraseSet);
|
||||
this.compactPhrases = Array.from(compactPhraseSet);
|
||||
this.asciiWordPhrases = Array.from(asciiWordPhraseSet);
|
||||
this.asciiCompactPhrases = Array.from(asciiCompactPhraseSet);
|
||||
this.rawMatcher = SubstringMatcher.fromPatterns(rawPhraseSet);
|
||||
this.wordMatcher = SubstringMatcher.fromPatterns(wordPhraseSet);
|
||||
this.compactMatcher = SubstringMatcher.fromPatterns(compactPhraseSet);
|
||||
this.asciiWordMatcher = SubstringMatcher.fromPatterns(asciiWordPhraseSet);
|
||||
this.asciiCompactMatcher = SubstringMatcher.fromPatterns(asciiCompactPhraseSet);
|
||||
}
|
||||
|
||||
isPhraseBanned(phrase: string): boolean {
|
||||
|
||||
@@ -0,0 +1,498 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import {transliterate} from 'transliteration';
|
||||
import {describe, expect, test} from 'vitest';
|
||||
import {buildPhraseMatchForms, canonicalizeStoredPhrase} from '../../utils/PhraseBlocklistNormalization';
|
||||
import {SubstringMatcher} from '../../utils/SubstringMatcher';
|
||||
import {PhraseBlocklistCache} from '../PhraseBlocklistCache';
|
||||
|
||||
const LEGACY_CONTROL_OR_FORMAT_REGEX = /[\p{Cc}\p{Cf}\uFFFE\uFFFF]/gu;
|
||||
const LEGACY_VARIATION_SELECTOR_REGEX = /(?:[\uFE00-\uFE0F]|[\u{E0100}-\u{E01EF}])/gu;
|
||||
const LEGACY_COMBINING_MARKS_REGEX = /\p{M}+/gu;
|
||||
const LEGACY_NON_ALPHANUMERIC_REGEX = /[^\p{L}\p{N}]+/gu;
|
||||
const LEGACY_ASCII_NON_ALPHANUMERIC_REGEX = /[^a-z0-9]+/g;
|
||||
const LEGACY_WHITESPACE_REGEX = /\s+/gu;
|
||||
const LEGACY_MIN_AGGRESSIVE_FORM_LENGTH = 3;
|
||||
|
||||
interface LegacyForms {
|
||||
raw: string;
|
||||
words: string;
|
||||
compact: string;
|
||||
asciiWords: string;
|
||||
asciiCompact: string;
|
||||
}
|
||||
|
||||
function legacyCollapseWhitespace(value: string): string {
|
||||
return value.replace(LEGACY_WHITESPACE_REGEX, ' ').trim();
|
||||
}
|
||||
|
||||
function legacyStripIgnorableCharacters(value: string): string {
|
||||
return value.replace(LEGACY_CONTROL_OR_FORMAT_REGEX, '').replace(LEGACY_VARIATION_SELECTOR_REGEX, '');
|
||||
}
|
||||
|
||||
function legacyMaybeKeepAggressiveForm(value: string): string {
|
||||
return value.length >= LEGACY_MIN_AGGRESSIVE_FORM_LENGTH ? value : '';
|
||||
}
|
||||
|
||||
function legacyBuildWordForm(value: string): string {
|
||||
return legacyCollapseWhitespace(
|
||||
value.normalize('NFKD').replace(LEGACY_COMBINING_MARKS_REGEX, '').replace(LEGACY_NON_ALPHANUMERIC_REGEX, ' '),
|
||||
);
|
||||
}
|
||||
|
||||
function legacyBuildAsciiWordForm(value: string): string {
|
||||
return legacyCollapseWhitespace(
|
||||
transliterate(value)
|
||||
.toLowerCase()
|
||||
.normalize('NFKD')
|
||||
.replace(LEGACY_COMBINING_MARKS_REGEX, '')
|
||||
.replace(LEGACY_ASCII_NON_ALPHANUMERIC_REGEX, ' '),
|
||||
);
|
||||
}
|
||||
|
||||
function legacyCanonicalize(value: string): string {
|
||||
return legacyStripIgnorableCharacters(value.normalize('NFKC')).toLowerCase().trim();
|
||||
}
|
||||
|
||||
function legacyBuildPhraseMatchForms(value: string): LegacyForms {
|
||||
const raw = legacyCanonicalize(value);
|
||||
const words = legacyMaybeKeepAggressiveForm(legacyBuildWordForm(raw));
|
||||
const compact = legacyMaybeKeepAggressiveForm(words.replace(LEGACY_WHITESPACE_REGEX, ''));
|
||||
const asciiWords = legacyMaybeKeepAggressiveForm(legacyBuildAsciiWordForm(raw));
|
||||
const asciiCompact = legacyMaybeKeepAggressiveForm(asciiWords.replace(LEGACY_WHITESPACE_REGEX, ''));
|
||||
return {raw, words, compact, asciiWords, asciiCompact};
|
||||
}
|
||||
|
||||
class LegacyPhraseBlocklist {
|
||||
private rawPhrases: Array<string> = [];
|
||||
private rawPhraseSet = new Set<string>();
|
||||
private wordPhrases: Array<string> = [];
|
||||
private compactPhrases: Array<string> = [];
|
||||
private asciiWordPhrases: Array<string> = [];
|
||||
private asciiCompactPhrases: Array<string> = [];
|
||||
|
||||
add(phrase: string): void {
|
||||
const canonical = legacyCanonicalize(phrase);
|
||||
if (!canonical || this.rawPhraseSet.has(canonical)) return;
|
||||
this.rawPhrases.push(canonical);
|
||||
this.rebuildMatchers();
|
||||
}
|
||||
|
||||
remove(phrase: string): void {
|
||||
const canonical = legacyCanonicalize(phrase);
|
||||
if (!canonical || !this.rawPhraseSet.has(canonical)) return;
|
||||
this.rawPhrases = this.rawPhrases.filter((item) => item !== canonical);
|
||||
this.rebuildMatchers();
|
||||
}
|
||||
|
||||
private rebuildMatchers(): void {
|
||||
const rawPhraseSet = new Set<string>();
|
||||
const wordPhraseSet = new Set<string>();
|
||||
const compactPhraseSet = new Set<string>();
|
||||
const asciiWordPhraseSet = new Set<string>();
|
||||
const asciiCompactPhraseSet = new Set<string>();
|
||||
for (const phrase of this.rawPhrases) {
|
||||
const canonical = legacyCanonicalize(phrase);
|
||||
if (!canonical) continue;
|
||||
const forms = legacyBuildPhraseMatchForms(canonical);
|
||||
rawPhraseSet.add(forms.raw);
|
||||
if (forms.words) wordPhraseSet.add(forms.words);
|
||||
if (forms.compact) compactPhraseSet.add(forms.compact);
|
||||
if (forms.asciiWords) asciiWordPhraseSet.add(forms.asciiWords);
|
||||
if (forms.asciiCompact) asciiCompactPhraseSet.add(forms.asciiCompact);
|
||||
}
|
||||
this.rawPhraseSet = rawPhraseSet;
|
||||
this.rawPhrases = Array.from(rawPhraseSet);
|
||||
this.wordPhrases = Array.from(wordPhraseSet);
|
||||
this.compactPhrases = Array.from(compactPhraseSet);
|
||||
this.asciiWordPhrases = Array.from(asciiWordPhraseSet);
|
||||
this.asciiCompactPhrases = Array.from(asciiCompactPhraseSet);
|
||||
}
|
||||
|
||||
private matchAny(text: string, phrases: Array<string>): boolean {
|
||||
if (!text || phrases.length === 0) return false;
|
||||
for (const phrase of phrases) {
|
||||
if (text.includes(phrase)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
containsBannedPhrase(text: string): boolean {
|
||||
if (this.rawPhrases.length === 0) return false;
|
||||
const forms = legacyBuildPhraseMatchForms(text);
|
||||
return (
|
||||
this.matchAny(forms.raw, this.rawPhrases) ||
|
||||
this.matchAny(forms.words, this.wordPhrases) ||
|
||||
this.matchAny(forms.compact, this.compactPhrases) ||
|
||||
this.matchAny(forms.asciiWords, this.asciiWordPhrases) ||
|
||||
this.matchAny(forms.asciiCompact, this.asciiCompactPhrases)
|
||||
);
|
||||
}
|
||||
|
||||
isPhraseBanned(phrase: string): boolean {
|
||||
const canonical = legacyCanonicalize(phrase);
|
||||
return !!canonical && this.rawPhraseSet.has(canonical);
|
||||
}
|
||||
|
||||
get size(): number {
|
||||
return this.rawPhraseSet.size;
|
||||
}
|
||||
}
|
||||
|
||||
function createRandom(seed: number): () => number {
|
||||
let state = seed >>> 0;
|
||||
return () => {
|
||||
state = (state + 0x6d2b79f5) >>> 0;
|
||||
let value = state;
|
||||
value = Math.imul(value ^ (value >>> 15), value | 1);
|
||||
value ^= value + Math.imul(value ^ (value >>> 7), value | 61);
|
||||
return ((value ^ (value >>> 14)) >>> 0) / 4294967296;
|
||||
};
|
||||
}
|
||||
|
||||
const CORPUS_ALPHABET: ReadonlyArray<string> = [
|
||||
'a',
|
||||
'b',
|
||||
'c',
|
||||
'n',
|
||||
'o',
|
||||
'r',
|
||||
't',
|
||||
'u',
|
||||
'0',
|
||||
'1',
|
||||
'4',
|
||||
'8',
|
||||
' ',
|
||||
' ',
|
||||
'\t',
|
||||
'\n',
|
||||
'.',
|
||||
'-',
|
||||
'_',
|
||||
'/',
|
||||
'\\',
|
||||
'+',
|
||||
'*',
|
||||
'?',
|
||||
'[',
|
||||
']',
|
||||
'(',
|
||||
')',
|
||||
'{',
|
||||
'}',
|
||||
'|',
|
||||
'^',
|
||||
'$',
|
||||
'\u200B',
|
||||
'\u200C',
|
||||
'\u200D',
|
||||
'\uFE0F',
|
||||
'\u0301',
|
||||
'\u0335',
|
||||
'\u3000',
|
||||
'\uFF55',
|
||||
'\uFF4E',
|
||||
'\u043E',
|
||||
'\u03BF',
|
||||
'\u00E9',
|
||||
'ß',
|
||||
'fi',
|
||||
'中',
|
||||
'א',
|
||||
'🔥',
|
||||
'😀',
|
||||
];
|
||||
|
||||
function randomString(random: () => number, maxUnits: number): string {
|
||||
const count = Math.floor(random() * (maxUnits + 1));
|
||||
let result = '';
|
||||
for (let index = 0; index < count; index++) {
|
||||
result += CORPUS_ALPHABET[Math.floor(random() * CORPUS_ALPHABET.length)]!;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
const TABLE_PHRASE_LISTS: ReadonlyArray<ReadonlyArray<string>> = [
|
||||
[],
|
||||
[''],
|
||||
['a'],
|
||||
['ab', 'b'],
|
||||
['abc', 'bc', 'c'],
|
||||
['unban tor', 'unban', 'ban'],
|
||||
['c++', 'c+', '++'],
|
||||
['a.b', 'a*b', 'a|b', 'a(b)c', 'a[b]c', 'a{2}', '^a$', 'a\\b', 'a?b'],
|
||||
['🔥', '🔥🔥'],
|
||||
['он', 'on', 'o'],
|
||||
['\u00E9', 'e', 'e\u0301'],
|
||||
['ｕｎ', 'un'],
|
||||
['aaa', 'aa', 'a'],
|
||||
['aaaa', 'aab', 'aba', 'baa'],
|
||||
['\u200B', 'x\u200By'],
|
||||
['tor', 'rot', 'ort'],
|
||||
];
|
||||
|
||||
const TABLE_TEXTS: ReadonlyArray<string> = [
|
||||
'',
|
||||
'a',
|
||||
'ab',
|
||||
'abc',
|
||||
'xabcx',
|
||||
'aa',
|
||||
'aaa',
|
||||
'aaaa',
|
||||
'banana',
|
||||
'please UNBAN TOR right now',
|
||||
'u n b a n t o r',
|
||||
'u.n-b_a_n t/o\\r',
|
||||
'ｕｎｂａｎ ｔｏｒ',
|
||||
'u\u200Bn\u200Bb\u200Ba\u200Bn t\u200Co\u200Dr',
|
||||
'u̵n̵b̵a̵n̵ t̵o̵r̵',
|
||||
'unban tоr',
|
||||
'unban tοr',
|
||||
'please keep tor banned',
|
||||
'ship c++ code',
|
||||
'compiler',
|
||||
'a\\b',
|
||||
'a?b',
|
||||
'^a$',
|
||||
'a{2}',
|
||||
'🔥🔥🔥',
|
||||
'x🔥y',
|
||||
'caf\u00E9',
|
||||
'cafe\u0301',
|
||||
'中中中',
|
||||
'אא',
|
||||
' ',
|
||||
'\t\n\t',
|
||||
'\u200B\u200C\u200D',
|
||||
];
|
||||
|
||||
function collectPhraseMismatches(phrases: ReadonlyArray<string>, texts: ReadonlyArray<string>): Array<string> {
|
||||
const legacy = new LegacyPhraseBlocklist();
|
||||
const current = new PhraseBlocklistCache();
|
||||
for (const phrase of phrases) {
|
||||
legacy.add(phrase);
|
||||
current.add(phrase);
|
||||
}
|
||||
const mismatches: Array<string> = [];
|
||||
if (legacy.size !== current.size) {
|
||||
mismatches.push(`size ${JSON.stringify(phrases)}: legacy=${legacy.size} current=${current.size}`);
|
||||
}
|
||||
for (const phrase of phrases) {
|
||||
const expected = legacy.isPhraseBanned(phrase);
|
||||
const actual = current.isPhraseBanned(phrase);
|
||||
if (expected !== actual) {
|
||||
mismatches.push(`isPhraseBanned ${JSON.stringify(phrase)}: legacy=${expected} current=${actual}`);
|
||||
}
|
||||
}
|
||||
for (const text of texts) {
|
||||
const expected = legacy.containsBannedPhrase(text);
|
||||
const actual = current.containsBannedPhrase(text);
|
||||
if (expected !== actual) {
|
||||
mismatches.push(
|
||||
`containsBannedPhrase ${JSON.stringify(phrases)} / ${JSON.stringify(text)}: legacy=${expected} current=${actual}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
return mismatches;
|
||||
}
|
||||
|
||||
describe('phrase blocklist matcher equivalence', () => {
|
||||
test('table-driven phrase lists match the legacy linear scan', () => {
|
||||
const mismatches: Array<string> = [];
|
||||
for (const phrases of TABLE_PHRASE_LISTS) {
|
||||
mismatches.push(...collectPhraseMismatches(phrases, TABLE_TEXTS));
|
||||
}
|
||||
expect(mismatches).toEqual([]);
|
||||
});
|
||||
|
||||
test('generated phrase lists and texts match the legacy linear scan', () => {
|
||||
const random = createRandom(0x5eed1234);
|
||||
const mismatches: Array<string> = [];
|
||||
for (let round = 0; round < 300; round++) {
|
||||
const phraseCount = 1 + Math.floor(random() * 8);
|
||||
const phrases: Array<string> = [];
|
||||
for (let index = 0; index < phraseCount; index++) {
|
||||
phrases.push(randomString(random, 6));
|
||||
}
|
||||
const texts: Array<string> = [];
|
||||
for (let index = 0; index < 8; index++) {
|
||||
texts.push(randomString(random, 40));
|
||||
}
|
||||
for (const phrase of phrases) {
|
||||
texts.push(phrase);
|
||||
texts.push(`${randomString(random, 5)}${phrase}${randomString(random, 5)}`);
|
||||
}
|
||||
mismatches.push(...collectPhraseMismatches(phrases, texts));
|
||||
}
|
||||
expect(mismatches).toEqual([]);
|
||||
});
|
||||
|
||||
test('removal keeps both implementations in step', () => {
|
||||
const random = createRandom(0x1234abcd);
|
||||
const mismatches: Array<string> = [];
|
||||
const legacy = new LegacyPhraseBlocklist();
|
||||
const current = new PhraseBlocklistCache();
|
||||
const phrases: Array<string> = [];
|
||||
for (let index = 0; index < 40; index++) {
|
||||
phrases.push(randomString(random, 6));
|
||||
}
|
||||
for (const phrase of phrases) {
|
||||
legacy.add(phrase);
|
||||
current.add(phrase);
|
||||
}
|
||||
for (let index = 0; index < phrases.length; index += 3) {
|
||||
legacy.remove(phrases[index]!);
|
||||
current.remove(phrases[index]!);
|
||||
}
|
||||
for (let index = 0; index < 200; index++) {
|
||||
const text = randomString(random, 30);
|
||||
const expected = legacy.containsBannedPhrase(text);
|
||||
const actual = current.containsBannedPhrase(text);
|
||||
if (expected !== actual) {
|
||||
mismatches.push(`${JSON.stringify(text)}: legacy=${expected} current=${actual}`);
|
||||
}
|
||||
}
|
||||
expect(mismatches).toEqual([]);
|
||||
});
|
||||
|
||||
test('a large blocklist stays exact', () => {
|
||||
const random = createRandom(0xfeedface);
|
||||
const legacy = new LegacyPhraseBlocklist();
|
||||
const current = new PhraseBlocklistCache();
|
||||
for (let index = 0; index < 300; index++) {
|
||||
const phrase = `${randomString(random, 4)}${index}`;
|
||||
legacy.add(phrase);
|
||||
current.add(phrase);
|
||||
}
|
||||
expect(current.size).toBe(legacy.size);
|
||||
const mismatches: Array<string> = [];
|
||||
for (let index = 0; index < 300; index++) {
|
||||
const text = randomString(random, 200);
|
||||
const expected = legacy.containsBannedPhrase(text);
|
||||
const actual = current.containsBannedPhrase(text);
|
||||
if (expected !== actual) {
|
||||
mismatches.push(`${JSON.stringify(text)}: legacy=${expected} current=${actual}`);
|
||||
}
|
||||
}
|
||||
expect(mismatches).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildPhraseMatchForms laziness', () => {
|
||||
test('lazy forms equal the eager forms for the whole corpus', () => {
|
||||
const random = createRandom(0xc0ffee11);
|
||||
const values: Array<string> = [...TABLE_TEXTS];
|
||||
for (let index = 0; index < 500; index++) {
|
||||
values.push(randomString(random, 40));
|
||||
}
|
||||
const mismatches: Array<string> = [];
|
||||
for (const value of values) {
|
||||
const expected = legacyBuildPhraseMatchForms(value);
|
||||
const forms = buildPhraseMatchForms(value);
|
||||
const actual = {
|
||||
raw: forms.raw,
|
||||
words: forms.words,
|
||||
compact: forms.compact,
|
||||
asciiWords: forms.asciiWords,
|
||||
asciiCompact: forms.asciiCompact,
|
||||
};
|
||||
if (JSON.stringify(expected) !== JSON.stringify(actual)) {
|
||||
mismatches.push(`${JSON.stringify(value)}: ${JSON.stringify(expected)} vs ${JSON.stringify(actual)}`);
|
||||
}
|
||||
}
|
||||
expect(mismatches).toEqual([]);
|
||||
});
|
||||
|
||||
test('reading a later form after an earlier one is still correct', () => {
|
||||
const value = 'unban tоr';
|
||||
const forms = buildPhraseMatchForms(value);
|
||||
const expected = legacyBuildPhraseMatchForms(value);
|
||||
expect(forms.asciiCompact).toBe(expected.asciiCompact);
|
||||
expect(forms.raw).toBe(expected.raw);
|
||||
expect(forms.words).toBe(expected.words);
|
||||
expect(forms.compact).toBe(expected.compact);
|
||||
expect(forms.asciiWords).toBe(expected.asciiWords);
|
||||
});
|
||||
|
||||
test('canonicalizeStoredPhrase is unchanged', () => {
|
||||
const random = createRandom(0x0badf00d);
|
||||
const mismatches: Array<string> = [];
|
||||
for (let index = 0; index < 500; index++) {
|
||||
const value = randomString(random, 30);
|
||||
if (canonicalizeStoredPhrase(value) !== legacyCanonicalize(value)) {
|
||||
mismatches.push(JSON.stringify(value));
|
||||
}
|
||||
}
|
||||
expect(mismatches).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('SubstringMatcher', () => {
|
||||
test('is null only for an empty pattern set', () => {
|
||||
expect(SubstringMatcher.fromPatterns([])).toBeNull();
|
||||
expect(SubstringMatcher.fromPatterns(new Set<string>())).toBeNull();
|
||||
expect(SubstringMatcher.fromPatterns([''])).not.toBeNull();
|
||||
});
|
||||
|
||||
test('agrees with String.includes on generated patterns and texts', () => {
|
||||
const random = createRandom(0x2468ace0);
|
||||
const mismatches: Array<string> = [];
|
||||
for (let round = 0; round < 500; round++) {
|
||||
const patternCount = 1 + Math.floor(random() * 6);
|
||||
const patterns: Array<string> = [];
|
||||
for (let index = 0; index < patternCount; index++) {
|
||||
patterns.push(randomString(random, 5));
|
||||
}
|
||||
const matcher = SubstringMatcher.fromPatterns(patterns);
|
||||
if (matcher === null) {
|
||||
mismatches.push(`unexpected null for ${JSON.stringify(patterns)}`);
|
||||
continue;
|
||||
}
|
||||
for (let index = 0; index < 10; index++) {
|
||||
const text = randomString(random, 30);
|
||||
const expected = text.length > 0 && patterns.some((pattern) => text.includes(pattern));
|
||||
const actual = matcher.test(text);
|
||||
if (expected !== actual) {
|
||||
mismatches.push(`${JSON.stringify(patterns)} / ${JSON.stringify(text)}: ${expected} vs ${actual}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
expect(mismatches).toEqual([]);
|
||||
});
|
||||
|
||||
test('an empty pattern matches any non-empty text', () => {
|
||||
const matcher = SubstringMatcher.fromPatterns(['', 'zzz']);
|
||||
expect(matcher).not.toBeNull();
|
||||
expect(matcher!.test('')).toBe(false);
|
||||
expect(matcher!.test('anything')).toBe(true);
|
||||
});
|
||||
|
||||
test('finds patterns that are suffixes of other pattern prefixes', () => {
|
||||
const matcher = SubstringMatcher.fromPatterns(['abcd', 'bc']);
|
||||
expect(matcher!.test('xabcx')).toBe(true);
|
||||
expect(matcher!.test('abd')).toBe(false);
|
||||
expect(matcher!.test('zabcdz')).toBe(true);
|
||||
});
|
||||
|
||||
test('scales to a very large pattern set without a compile limit', () => {
|
||||
const random = createRandom(0x13579bdf);
|
||||
const patterns: Array<string> = [];
|
||||
for (let index = 0; index < 20000; index++) {
|
||||
patterns.push(`${randomString(random, 4)}${index}`);
|
||||
}
|
||||
const matcher = SubstringMatcher.fromPatterns(patterns);
|
||||
expect(matcher).not.toBeNull();
|
||||
const mismatches: Array<string> = [];
|
||||
for (let index = 0; index < 100; index++) {
|
||||
const text = index % 2 === 0 ? randomString(random, 400) : `${randomString(random, 50)}${patterns[index]!}`;
|
||||
const expected = text.length > 0 && patterns.some((pattern) => text.includes(pattern));
|
||||
const actual = matcher!.test(text);
|
||||
if (expected !== actual) {
|
||||
mismatches.push(`${JSON.stringify(text)}: ${expected} vs ${actual}`);
|
||||
}
|
||||
}
|
||||
expect(mismatches).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -11,11 +11,11 @@ const WHITESPACE_REGEX = /\s+/gu;
|
||||
const MIN_AGGRESSIVE_FORM_LENGTH = 3;
|
||||
|
||||
interface PhraseMatchForms {
|
||||
raw: string;
|
||||
words: string;
|
||||
compact: string;
|
||||
asciiWords: string;
|
||||
asciiCompact: string;
|
||||
readonly raw: string;
|
||||
readonly words: string;
|
||||
readonly compact: string;
|
||||
readonly asciiWords: string;
|
||||
readonly asciiCompact: string;
|
||||
}
|
||||
|
||||
function collapseWhitespace(value: string): string {
|
||||
@@ -46,21 +46,50 @@ function buildAsciiWordFormFromCanonical(value: string): string {
|
||||
);
|
||||
}
|
||||
|
||||
class LazyPhraseMatchForms implements PhraseMatchForms {
|
||||
readonly raw: string;
|
||||
private wordsForm: string | null = null;
|
||||
private compactForm: string | null = null;
|
||||
private asciiWordsForm: string | null = null;
|
||||
private asciiCompactForm: string | null = null;
|
||||
|
||||
constructor(value: string) {
|
||||
this.raw = canonicalizeStoredPhrase(value);
|
||||
}
|
||||
|
||||
get words(): string {
|
||||
if (this.wordsForm === null) {
|
||||
this.wordsForm = maybeKeepAggressiveForm(buildWordFormFromCanonical(this.raw));
|
||||
}
|
||||
return this.wordsForm;
|
||||
}
|
||||
|
||||
get compact(): string {
|
||||
if (this.compactForm === null) {
|
||||
this.compactForm = maybeKeepAggressiveForm(this.words.replace(WHITESPACE_REGEX, ''));
|
||||
}
|
||||
return this.compactForm;
|
||||
}
|
||||
|
||||
get asciiWords(): string {
|
||||
if (this.asciiWordsForm === null) {
|
||||
this.asciiWordsForm = maybeKeepAggressiveForm(buildAsciiWordFormFromCanonical(this.raw));
|
||||
}
|
||||
return this.asciiWordsForm;
|
||||
}
|
||||
|
||||
get asciiCompact(): string {
|
||||
if (this.asciiCompactForm === null) {
|
||||
this.asciiCompactForm = maybeKeepAggressiveForm(this.asciiWords.replace(WHITESPACE_REGEX, ''));
|
||||
}
|
||||
return this.asciiCompactForm;
|
||||
}
|
||||
}
|
||||
|
||||
export function canonicalizeStoredPhrase(value: string): string {
|
||||
return stripIgnorableCharacters(value.normalize('NFKC')).toLowerCase().trim();
|
||||
}
|
||||
|
||||
export function buildPhraseMatchForms(value: string): PhraseMatchForms {
|
||||
const raw = canonicalizeStoredPhrase(value);
|
||||
const words = maybeKeepAggressiveForm(buildWordFormFromCanonical(raw));
|
||||
const compact = maybeKeepAggressiveForm(words.replace(WHITESPACE_REGEX, ''));
|
||||
const asciiWords = maybeKeepAggressiveForm(buildAsciiWordFormFromCanonical(raw));
|
||||
const asciiCompact = maybeKeepAggressiveForm(asciiWords.replace(WHITESPACE_REGEX, ''));
|
||||
return {
|
||||
raw,
|
||||
words,
|
||||
compact,
|
||||
asciiWords,
|
||||
asciiCompact,
|
||||
};
|
||||
return new LazyPhraseMatchForms(value);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
import type {HonoRequest} from 'hono';
|
||||
import {parseJsonPreservingLargeIntegers} from './LosslessJsonParser';
|
||||
|
||||
interface RequestJsonBody {
|
||||
parsed: boolean;
|
||||
value: unknown;
|
||||
}
|
||||
|
||||
const bodyCache = new WeakMap<Request, RequestJsonBody>();
|
||||
|
||||
export async function readRequestJsonBody(req: HonoRequest): Promise<RequestJsonBody> {
|
||||
const raw = req.raw;
|
||||
const cached = bodyCache.get(raw);
|
||||
if (cached) return cached;
|
||||
let body: RequestJsonBody;
|
||||
try {
|
||||
const text = await req.text();
|
||||
body = {parsed: true, value: text.trim().length === 0 ? {} : parseJsonPreservingLargeIntegers(text)};
|
||||
} catch {
|
||||
body = {parsed: false, value: {}};
|
||||
}
|
||||
bodyCache.set(raw, body);
|
||||
return body;
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
const ROOT_NODE = 0;
|
||||
const ASCII_ROOT_SIZE = 128;
|
||||
const NO_TRANSITION = -1;
|
||||
const EDGE_KEY_STRIDE = 65536;
|
||||
|
||||
export class SubstringMatcher {
|
||||
private readonly rowStart: Int32Array;
|
||||
private readonly edgeCodes: Int32Array;
|
||||
private readonly edgeTargets: Int32Array;
|
||||
private readonly failures: Int32Array;
|
||||
private readonly terminals: Uint8Array;
|
||||
private readonly asciiRoot: Int32Array;
|
||||
private readonly matchesEverything: boolean;
|
||||
|
||||
private constructor(
|
||||
rowStart: Int32Array,
|
||||
edgeCodes: Int32Array,
|
||||
edgeTargets: Int32Array,
|
||||
failures: Int32Array,
|
||||
terminals: Uint8Array,
|
||||
matchesEverything: boolean,
|
||||
) {
|
||||
this.rowStart = rowStart;
|
||||
this.edgeCodes = edgeCodes;
|
||||
this.edgeTargets = edgeTargets;
|
||||
this.failures = failures;
|
||||
this.terminals = terminals;
|
||||
this.matchesEverything = matchesEverything;
|
||||
this.asciiRoot = new Int32Array(ASCII_ROOT_SIZE);
|
||||
for (let edge = rowStart[ROOT_NODE]!; edge < rowStart[ROOT_NODE + 1]!; edge++) {
|
||||
const code = edgeCodes[edge]!;
|
||||
if (code < ASCII_ROOT_SIZE) {
|
||||
this.asciiRoot[code] = edgeTargets[edge]!;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static fromPatterns(patterns: Iterable<string>): SubstringMatcher | null {
|
||||
const children = new Map<number, number>();
|
||||
const edgeSources: Array<number> = [];
|
||||
const edgeLabels: Array<number> = [];
|
||||
const edgeChildren: Array<number> = [];
|
||||
const terminalNodes: Array<number> = [];
|
||||
let nodeCount = 1;
|
||||
let matchesEverything = false;
|
||||
let count = 0;
|
||||
for (const pattern of patterns) {
|
||||
count++;
|
||||
if (pattern.length === 0) {
|
||||
matchesEverything = true;
|
||||
continue;
|
||||
}
|
||||
let node = ROOT_NODE;
|
||||
for (let index = 0; index < pattern.length; index++) {
|
||||
const code = pattern.charCodeAt(index);
|
||||
const key = node * EDGE_KEY_STRIDE + code;
|
||||
let next = children.get(key);
|
||||
if (next === undefined) {
|
||||
next = nodeCount++;
|
||||
children.set(key, next);
|
||||
edgeSources.push(node);
|
||||
edgeLabels.push(code);
|
||||
edgeChildren.push(next);
|
||||
}
|
||||
node = next;
|
||||
}
|
||||
terminalNodes.push(node);
|
||||
}
|
||||
if (count === 0) {
|
||||
return null;
|
||||
}
|
||||
children.clear();
|
||||
const edgeCount = edgeSources.length;
|
||||
const rowStart = new Int32Array(nodeCount + 1);
|
||||
for (let edge = 0; edge < edgeCount; edge++) {
|
||||
rowStart[edgeSources[edge]! + 1]!++;
|
||||
}
|
||||
for (let node = 0; node < nodeCount; node++) {
|
||||
rowStart[node + 1]! += rowStart[node]!;
|
||||
}
|
||||
const cursor = rowStart.slice(0, nodeCount);
|
||||
const edgeCodes = new Int32Array(edgeCount);
|
||||
const edgeTargets = new Int32Array(edgeCount);
|
||||
for (let edge = 0; edge < edgeCount; edge++) {
|
||||
const source = edgeSources[edge]!;
|
||||
const slot = cursor[source]!;
|
||||
cursor[source] = slot + 1;
|
||||
edgeCodes[slot] = edgeLabels[edge]!;
|
||||
edgeTargets[slot] = edgeChildren[edge]!;
|
||||
}
|
||||
for (let node = 0; node < nodeCount; node++) {
|
||||
const start = rowStart[node]!;
|
||||
const end = rowStart[node + 1]!;
|
||||
for (let index = start + 1; index < end; index++) {
|
||||
const code = edgeCodes[index]!;
|
||||
const target = edgeTargets[index]!;
|
||||
let scan = index - 1;
|
||||
while (scan >= start && edgeCodes[scan]! > code) {
|
||||
edgeCodes[scan + 1] = edgeCodes[scan]!;
|
||||
edgeTargets[scan + 1] = edgeTargets[scan]!;
|
||||
scan--;
|
||||
}
|
||||
edgeCodes[scan + 1] = code;
|
||||
edgeTargets[scan + 1] = target;
|
||||
}
|
||||
}
|
||||
const failures = new Int32Array(nodeCount);
|
||||
const terminals = new Uint8Array(nodeCount);
|
||||
for (const node of terminalNodes) {
|
||||
terminals[node] = 1;
|
||||
}
|
||||
const queue = new Int32Array(nodeCount);
|
||||
let tail = 0;
|
||||
for (let edge = rowStart[ROOT_NODE]!; edge < rowStart[ROOT_NODE + 1]!; edge++) {
|
||||
queue[tail] = edgeTargets[edge]!;
|
||||
tail++;
|
||||
}
|
||||
for (let head = 0; head < tail; head++) {
|
||||
const node = queue[head]!;
|
||||
if (terminals[failures[node]!]! === 1) {
|
||||
terminals[node] = 1;
|
||||
}
|
||||
for (let edge = rowStart[node]!; edge < rowStart[node + 1]!; edge++) {
|
||||
const code = edgeCodes[edge]!;
|
||||
const child = edgeTargets[edge]!;
|
||||
let candidate = failures[node]!;
|
||||
let target = findTransition(rowStart, edgeCodes, edgeTargets, candidate, code);
|
||||
while (target === NO_TRANSITION && candidate !== ROOT_NODE) {
|
||||
candidate = failures[candidate]!;
|
||||
target = findTransition(rowStart, edgeCodes, edgeTargets, candidate, code);
|
||||
}
|
||||
failures[child] = target === NO_TRANSITION ? ROOT_NODE : target;
|
||||
queue[tail] = child;
|
||||
tail++;
|
||||
}
|
||||
}
|
||||
return new SubstringMatcher(rowStart, edgeCodes, edgeTargets, failures, terminals, matchesEverything);
|
||||
}
|
||||
|
||||
test(text: string): boolean {
|
||||
if (text.length === 0) {
|
||||
return false;
|
||||
}
|
||||
if (this.matchesEverything) {
|
||||
return true;
|
||||
}
|
||||
const rowStart = this.rowStart;
|
||||
const edgeCodes = this.edgeCodes;
|
||||
const edgeTargets = this.edgeTargets;
|
||||
const failures = this.failures;
|
||||
const terminals = this.terminals;
|
||||
const asciiRoot = this.asciiRoot;
|
||||
let node = ROOT_NODE;
|
||||
for (let index = 0; index < text.length; index++) {
|
||||
const code = text.charCodeAt(index);
|
||||
if (node === ROOT_NODE && code < ASCII_ROOT_SIZE) {
|
||||
node = asciiRoot[code]!;
|
||||
} else {
|
||||
let next = findTransition(rowStart, edgeCodes, edgeTargets, node, code);
|
||||
while (next === NO_TRANSITION && node !== ROOT_NODE) {
|
||||
node = failures[node]!;
|
||||
next = findTransition(rowStart, edgeCodes, edgeTargets, node, code);
|
||||
}
|
||||
node = next === NO_TRANSITION ? ROOT_NODE : next;
|
||||
}
|
||||
if (terminals[node]! === 1) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function findTransition(
|
||||
rowStart: Int32Array,
|
||||
edgeCodes: Int32Array,
|
||||
edgeTargets: Int32Array,
|
||||
node: number,
|
||||
code: number,
|
||||
): number {
|
||||
let low = rowStart[node]!;
|
||||
let high = rowStart[node + 1]! - 1;
|
||||
while (low <= high) {
|
||||
const middle = (low + high) >>> 1;
|
||||
const candidate = edgeCodes[middle]!;
|
||||
if (candidate === code) {
|
||||
return edgeTargets[middle]!;
|
||||
}
|
||||
if (candidate < code) {
|
||||
low = middle + 1;
|
||||
} else {
|
||||
high = middle - 1;
|
||||
}
|
||||
}
|
||||
return NO_TRANSITION;
|
||||
}
|
||||
Reference in New Issue
Block a user