Skip to content

Commit f16f4de

Browse files
andimarekandimarek
authored
Merge pull request #4461 from graphql-java/codex/backport-4440-to-25.x
Backport 25.1: Lexer cleanup
2 parents 7f25ee6 + 02e1368 commit f16f4de

10 files changed

Lines changed: 194 additions & 5 deletions

‎src/main/java/graphql/parser/Parser.java‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import graphql.parser.exceptions.ParseCancelledException;
1616
import graphql.parser.exceptions.ParseCancelledTooDeepException;
1717
import graphql.parser.exceptions.ParseCancelledTooManyCharsException;
18+
import graphql.parser.exceptions.ParseCancelledTooManyNumericLiteralCharactersException;
1819
import org.antlr.v4.runtime.BaseErrorListener;
1920
import org.antlr.v4.runtime.CharStreams;
2021
import org.antlr.v4.runtime.CodePointCharStream;
@@ -348,13 +349,25 @@ public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int
348349
private SafeTokenSource getSafeTokenSource(ParserEnvironment environment, ParserOptions parserOptions, MultiSourceReader multiSourceReader, GraphqlLexer lexer) {
349350
int maxTokens = parserOptions.getMaxTokens();
350351
int maxWhitespaceTokens = parserOptions.getMaxWhitespaceTokens();
352+
int maxNumericLiteralCharacters = parserOptions.getMaxNumericLiteralCharacters();
351353
BiConsumer<Integer, Token> onTooManyTokens = (maxTokenCount, token) -> throwIfTokenProblems(
352354
environment,
353355
token,
354356
maxTokenCount,
355357
multiSourceReader,
356358
ParseCancelledException.class);
357-
return new SafeTokenSource(lexer, maxTokens, maxWhitespaceTokens, onTooManyTokens);
359+
BiConsumer<Integer, Token> onTooManyNumericLiteralCharacters = (maxCharacters, token) -> {
360+
SourceLocation sourceLocation = AntlrHelper.createSourceLocation(multiSourceReader, token);
361+
throw new ParseCancelledTooManyNumericLiteralCharactersException(environment.getI18N(), sourceLocation, maxCharacters);
362+
};
363+
return new SafeTokenSource(
364+
lexer,
365+
maxTokens,
366+
maxWhitespaceTokens,
367+
maxNumericLiteralCharacters,
368+
onTooManyTokens,
369+
onTooManyNumericLiteralCharacters
370+
);
358371
}
359372

360373
private void setupParserListener(ParserEnvironment environment, MultiSourceReader multiSourceReader, GraphqlParser parser, GraphqlAntlrToLanguage toLanguage) {

‎src/main/java/graphql/parser/ParserOptions.java‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,16 @@ public class ParserOptions {
4343
*/
4444
public static final int MAX_WHITESPACE_TOKENS = 200_000;
4545

46+
/**
47+
* A numeric literal is represented by a single token, regardless of how many characters it contains. Converting
48+
* very large numeric literals into arbitrary precision numbers can consume excessive CPU and memory. To prevent
49+
* this for most users, graphql-java limits numeric literals to 100 characters.
50+
* <p>
51+
* If you want to allow more, then {@link #setDefaultParserOptions(ParserOptions)} allows you to change this
52+
* JVM wide.
53+
*/
54+
public static final int MAX_NUMERIC_LITERAL_CHARACTERS = 100;
55+
4656
/**
4757
* A graphql hacking vector is to send nonsensical queries that have lots of grammar rule depth to them which
4858
* can cause stack overflow exceptions during the query parsing. To prevent this for most users, graphql-java
@@ -61,6 +71,7 @@ public class ParserOptions {
6171
.maxCharacters(MAX_QUERY_CHARACTERS)
6272
.maxTokens(MAX_QUERY_TOKENS) // to prevent a billion laughs style attacks, we set a default for graphql-java
6373
.maxWhitespaceTokens(MAX_WHITESPACE_TOKENS)
74+
.maxNumericLiteralCharacters(MAX_NUMERIC_LITERAL_CHARACTERS)
6475
.maxRuleDepth(MAX_RULE_DEPTH)
6576
.redactTokenParserErrorMessages(false)
6677
.build();
@@ -73,6 +84,7 @@ public class ParserOptions {
7384
.maxCharacters(MAX_QUERY_CHARACTERS)
7485
.maxTokens(MAX_QUERY_TOKENS) // to prevent a billion laughs style attacks, we set a default for graphql-java
7586
.maxWhitespaceTokens(MAX_WHITESPACE_TOKENS)
87+
.maxNumericLiteralCharacters(MAX_NUMERIC_LITERAL_CHARACTERS)
7688
.maxRuleDepth(MAX_RULE_DEPTH)
7789
.redactTokenParserErrorMessages(false)
7890
.build();
@@ -85,6 +97,7 @@ public class ParserOptions {
8597
.maxCharacters(Integer.MAX_VALUE)
8698
.maxTokens(Integer.MAX_VALUE) // we are less worried about a billion laughs with SDL parsing since the call path is not facing attackers
8799
.maxWhitespaceTokens(Integer.MAX_VALUE)
100+
.maxNumericLiteralCharacters(MAX_NUMERIC_LITERAL_CHARACTERS)
88101
.maxRuleDepth(Integer.MAX_VALUE)
89102
.redactTokenParserErrorMessages(false)
90103
.build();
@@ -191,6 +204,7 @@ public static void setDefaultSdlParserOptions(ParserOptions options) {
191204
private final int maxCharacters;
192205
private final int maxTokens;
193206
private final int maxWhitespaceTokens;
207+
private final int maxNumericLiteralCharacters;
194208
private final int maxRuleDepth;
195209
private final boolean redactTokenParserErrorMessages;
196210
private final ParsingListener parsingListener;
@@ -203,6 +217,7 @@ private ParserOptions(Builder builder) {
203217
this.maxCharacters = builder.maxCharacters;
204218
this.maxTokens = builder.maxTokens;
205219
this.maxWhitespaceTokens = builder.maxWhitespaceTokens;
220+
this.maxNumericLiteralCharacters = builder.maxNumericLiteralCharacters;
206221
this.maxRuleDepth = builder.maxRuleDepth;
207222
this.redactTokenParserErrorMessages = builder.redactTokenParserErrorMessages;
208223
this.parsingListener = builder.parsingListener;
@@ -288,6 +303,17 @@ public int getMaxWhitespaceTokens() {
288303
return maxWhitespaceTokens;
289304
}
290305

306+
/**
307+
* A numeric literal is represented by a single token, regardless of how many characters it contains. Converting
308+
* very large numeric literals into arbitrary precision numbers can consume excessive CPU and memory. This limit
309+
* stops parsing before that conversion takes place.
310+
*
311+
* @return the maximum number of characters permitted in an integer or floating-point literal
312+
*/
313+
public int getMaxNumericLiteralCharacters() {
314+
return maxNumericLiteralCharacters;
315+
}
316+
291317
/**
292318
* A graphql hacking vector is to send nonsensical queries that have lots of rule depth to them which
293319
* can cause stack overflow exceptions during the query parsing. To prevent this you can set a value
@@ -333,6 +359,7 @@ public static class Builder {
333359
private int maxCharacters = MAX_QUERY_CHARACTERS;
334360
private int maxTokens = MAX_QUERY_TOKENS;
335361
private int maxWhitespaceTokens = MAX_WHITESPACE_TOKENS;
362+
private int maxNumericLiteralCharacters = MAX_NUMERIC_LITERAL_CHARACTERS;
336363
private int maxRuleDepth = MAX_RULE_DEPTH;
337364
private boolean redactTokenParserErrorMessages = false;
338365

@@ -346,6 +373,7 @@ public static class Builder {
346373
this.maxCharacters = parserOptions.maxCharacters;
347374
this.maxTokens = parserOptions.maxTokens;
348375
this.maxWhitespaceTokens = parserOptions.maxWhitespaceTokens;
376+
this.maxNumericLiteralCharacters = parserOptions.maxNumericLiteralCharacters;
349377
this.maxRuleDepth = parserOptions.maxRuleDepth;
350378
this.redactTokenParserErrorMessages = parserOptions.redactTokenParserErrorMessages;
351379
this.parsingListener = parserOptions.parsingListener;
@@ -386,6 +414,19 @@ public Builder maxWhitespaceTokens(int maxWhitespaceTokens) {
386414
return this;
387415
}
388416

417+
/**
418+
* Sets the maximum number of characters permitted in an integer or floating-point literal. Parsing is
419+
* cancelled before converting a larger literal into an arbitrary precision number.
420+
*
421+
* @param maxNumericLiteralCharacters the maximum number of characters permitted in a numeric literal
422+
*
423+
* @return this builder
424+
*/
425+
public Builder maxNumericLiteralCharacters(int maxNumericLiteralCharacters) {
426+
this.maxNumericLiteralCharacters = maxNumericLiteralCharacters;
427+
return this;
428+
}
429+
389430
public Builder maxRuleDepth(int maxRuleDepth) {
390431
this.maxRuleDepth = maxRuleDepth;
391432
return this;

‎src/main/java/graphql/parser/SafeTokenSource.java‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package graphql.parser;
22

33
import graphql.Internal;
4+
import graphql.parser.antlr.GraphqlLexer;
45
import org.antlr.v4.runtime.CharStream;
56
import org.antlr.v4.runtime.Token;
67
import org.antlr.v4.runtime.TokenFactory;
@@ -25,14 +26,20 @@ public class SafeTokenSource implements TokenSource {
2526
private final TokenSource lexer;
2627
private final int maxTokens;
2728
private final int maxWhitespaceTokens;
29+
private final int maxNumericLiteralCharacters;
2830
private final BiConsumer<Integer, Token> whenMaxTokensExceeded;
31+
private final BiConsumer<Integer, Token> whenMaxNumericLiteralCharactersExceeded;
2932
private final int channelCounts[];
3033

31-
public SafeTokenSource(TokenSource lexer, int maxTokens, int maxWhitespaceTokens, BiConsumer<Integer, Token> whenMaxTokensExceeded) {
34+
public SafeTokenSource(TokenSource lexer, int maxTokens, int maxWhitespaceTokens, int maxNumericLiteralCharacters,
35+
BiConsumer<Integer, Token> whenMaxTokensExceeded,
36+
BiConsumer<Integer, Token> whenMaxNumericLiteralCharactersExceeded) {
3237
this.lexer = lexer;
3338
this.maxTokens = maxTokens;
3439
this.maxWhitespaceTokens = maxWhitespaceTokens;
40+
this.maxNumericLiteralCharacters = maxNumericLiteralCharacters;
3541
this.whenMaxTokensExceeded = whenMaxTokensExceeded;
42+
this.whenMaxNumericLiteralCharactersExceeded = whenMaxNumericLiteralCharactersExceeded;
3643
// this could be a Map<int,int> however we want it to be faster as possible.
3744
// we only have 3 channels - but they are 0,2 and 3 so use 5 for safety - still faster than a map get/put
3845
// if we ever add another channel beyond 5 it will IOBEx during tests so future changes will be handled before release!
@@ -44,6 +51,7 @@ public SafeTokenSource(TokenSource lexer, int maxTokens, int maxWhitespaceTokens
4451
public Token nextToken() {
4552
Token token = lexer.nextToken();
4653
if (token != null) {
54+
callbackIfNumericLiteralTooLong(token);
4755
int channel = token.getChannel();
4856
int currentCount = ++channelCounts[channel];
4957
if (channel == Parser.CHANNEL_WHITESPACE) {
@@ -56,6 +64,18 @@ public Token nextToken() {
5664
return token;
5765
}
5866

67+
private void callbackIfNumericLiteralTooLong(Token token) {
68+
int tokenType = token.getType();
69+
if (tokenType != GraphqlLexer.IntValue && tokenType != GraphqlLexer.FloatValue) {
70+
return;
71+
}
72+
73+
int characterCount = token.getStopIndex() - token.getStartIndex() + 1;
74+
if (characterCount > maxNumericLiteralCharacters) {
75+
whenMaxNumericLiteralCharactersExceeded.accept(maxNumericLiteralCharacters, token);
76+
}
77+
}
78+
5979
private void callbackIfMaxExceeded(int maxCount, int currentCount, Token token) {
6080
if (currentCount > maxCount) {
6181
whenMaxTokensExceeded.accept(maxCount, token);

‎src/main/java/graphql/parser/exceptions/ParseCancelledTooManyNumericLiteralCharactersException.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
package graphql.parser.exceptions;
2+
3+
import graphql.Internal;
4+
import graphql.i18n.I18n;
5+
import graphql.language.SourceLocation;
6+
import graphql.parser.InvalidSyntaxException;
7+
import org.jspecify.annotations.NonNull;
8+
9+
@Internal
10+
public class ParseCancelledTooManyNumericLiteralCharactersException extends InvalidSyntaxException {
11+
12+
@Internal
13+
public ParseCancelledTooManyNumericLiteralCharactersException(@NonNull I18n i18N, @NonNull SourceLocation sourceLocation, int maxCharacters) {
14+
super(i18N.msg("ParseCancelled.tooManyNumericLiteralCharacters", maxCharacters),
15+
sourceLocation, null, null, null);
16+
}
17+
}

‎src/main/resources/i18n/Parsing.properties‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ InvalidSyntaxMoreTokens.full=Invalid syntax encountered. There are extra tokens
2222
ParseCancelled.full=More than {0} ''{1}'' tokens have been presented. To prevent Denial Of Service attacks, parsing has been cancelled.
2323
ParseCancelled.tooDeep=More than {0} deep ''{1}'' rules have been entered. To prevent Denial Of Service attacks, parsing has been cancelled.
2424
ParseCancelled.tooManyChars=More than {0} characters have been presented. To prevent Denial Of Service attacks, parsing has been cancelled.
25+
ParseCancelled.tooManyNumericLiteralCharacters=A numeric literal with more than {0} characters has been presented. To prevent Denial Of Service attacks, parsing has been cancelled.
2526
#
2627
InvalidUnicode.trailingLeadingSurrogate=Invalid unicode encountered. Trailing surrogate must be preceded with a leading surrogate. Offending token ''{0}'' at line {1} column {2}
2728
InvalidUnicode.leadingTrailingSurrogate=Invalid unicode encountered. Leading surrogate must be followed by a trailing surrogate. Offending token ''{0}'' at line {1} column {2}

‎src/main/resources/i18n/Parsing_de.properties‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ InvalidSyntaxMoreTokens.full=Es wurde eine ungültige Syntax festgestellt. Es gi
2222
ParseCancelled.full=Es wurden mehr als {0} ''{1}'' Token präsentiert. Um Denial-of-Service-Angriffe zu verhindern, wurde das Parsing abgebrochen.
2323
ParseCancelled.tooDeep=Es wurden mehr als {0} tief ''{1}'' Regeln ausgeführt. Um Denial-of-Service-Angriffe zu verhindern, wurde das Parsing abgebrochen.
2424
ParseCancelled.tooManyChars=Es wurden mehr als {0} Zeichen vorgelegt. Um Denial-of-Service-Angriffe zu verhindern, wurde das Parsing abgebrochen.
25+
ParseCancelled.tooManyNumericLiteralCharacters=Es wurde ein numerisches Literal mit mehr als {0} Zeichen vorgelegt. Um Denial-of-Service-Angriffe zu verhindern, wurde das Parsing abgebrochen.
2526
#
2627
InvalidUnicode.trailingLeadingSurrogate=Ungültiger Unicode gefunden. Trailing surrogate muss ein leading surrogate vorangestellt werden. Ungültiges Token ''{0}'' in Zeile {1} Spalte {2}
2728
InvalidUnicode.leadingTrailingSurrogate=Ungültiger Unicode gefunden. Auf ein leading surrogate muss ein trailing surrogate folgen. Ungültiges Token ''{0}'' in Zeile {1} Spalte {2}

‎src/main/resources/i18n/Parsing_nl.properties‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ InvalidSyntaxMoreTokens.full=Ongeldige syntaxis tegengekomen. Er zijn tokens in
2121
ParseCancelled.full=Meer dan {0} ''{1}'' tokens zijn gepresenteerd. Om een DDoS-aanval te voorkomen is het parsen gestopt.
2222
ParseCancelled.tooDeep=Meer dan {0} diep, ''{1}'' regels zijn uitgevoerd. Om een DDoS-aanval te voorkomen is het parsen gestopt.
2323
ParseCancelled.tooManyChars=Meer dan {0} tekens zijn voorgelegd. Om een DDoS-aanval te voorkomen is het parsen gestopt.
24+
ParseCancelled.tooManyNumericLiteralCharacters=Er is een numerieke literal met meer dan {0} tekens aangeboden. Om een DDoS-aanval te voorkomen is het parsen gestopt.
2425
#
2526
InvalidUnicode.trailingLeadingSurrogate=Ongeldige Unicode tegengekomen. Trailing surrogate moet vooropgaan aan een leading surrogate. Ongeldige token ''{0}'' op lijn {1} kolom {2}
2627
InvalidUnicode.leadingTrailingSurrogate=Ongeldige Unicode tegengekomen. Leading surrogate moet voorafgaan aan een trailing surrogate. Ongeldige token ''{0}'' op lijn {1} kolom {2}

‎src/test/groovy/graphql/parser/ParserOptionsTest.groovy‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ class ParserOptionsTest extends Specification {
2626
defaultOptions.getMaxCharacters() == ONE_MB
2727
defaultOptions.getMaxTokens() == 15_000
2828
defaultOptions.getMaxWhitespaceTokens() == 200_000
29+
defaultOptions.getMaxNumericLiteralCharacters() == 100
2930
defaultOptions.isCaptureSourceLocation()
3031
defaultOptions.isCaptureLineComments()
3132
!defaultOptions.isCaptureIgnoredChars()
@@ -34,6 +35,7 @@ class ParserOptionsTest extends Specification {
3435

3536
defaultOperationOptions.getMaxTokens() == 15_000
3637
defaultOperationOptions.getMaxWhitespaceTokens() == 200_000
38+
defaultOperationOptions.getMaxNumericLiteralCharacters() == 100
3739
defaultOperationOptions.isCaptureSourceLocation()
3840
!defaultOperationOptions.isCaptureLineComments()
3941
!defaultOperationOptions.isCaptureIgnoredChars()
@@ -43,6 +45,7 @@ class ParserOptionsTest extends Specification {
4345
defaultSdlOptions.getMaxCharacters() == Integer.MAX_VALUE
4446
defaultSdlOptions.getMaxTokens() == Integer.MAX_VALUE
4547
defaultSdlOptions.getMaxWhitespaceTokens() == Integer.MAX_VALUE
48+
defaultSdlOptions.getMaxNumericLiteralCharacters() == 100
4649
defaultSdlOptions.isCaptureSourceLocation()
4750
defaultSdlOptions.isCaptureLineComments()
4851
!defaultSdlOptions.isCaptureIgnoredChars()
@@ -61,6 +64,7 @@ class ParserOptionsTest extends Specification {
6164
it.captureIgnoredChars(true)
6265
.captureLineComments(true)
6366
.maxCharacters(1_000_000)
67+
.maxNumericLiteralCharacters(200)
6468
.maxWhitespaceTokens(300_000)
6569
})
6670
def newDefaultSDlOptions = defaultSdlOptions.transform(
@@ -84,6 +88,7 @@ class ParserOptionsTest extends Specification {
8488
currentDefaultOptions.getMaxCharacters() == ONE_MB
8589
currentDefaultOptions.getMaxTokens() == 15_000
8690
currentDefaultOptions.getMaxWhitespaceTokens() == 200_000
91+
currentDefaultOptions.getMaxNumericLiteralCharacters() == 100
8792
currentDefaultOptions.isCaptureSourceLocation()
8893
currentDefaultOptions.isCaptureLineComments()
8994
currentDefaultOptions.isCaptureIgnoredChars()
@@ -93,6 +98,7 @@ class ParserOptionsTest extends Specification {
9398
currentDefaultOperationOptions.getMaxCharacters() == 1_000_000
9499
currentDefaultOperationOptions.getMaxTokens() == 15_000
95100
currentDefaultOperationOptions.getMaxWhitespaceTokens() == 300_000
101+
currentDefaultOperationOptions.getMaxNumericLiteralCharacters() == 200
96102
currentDefaultOperationOptions.isCaptureSourceLocation()
97103
currentDefaultOperationOptions.isCaptureLineComments()
98104
currentDefaultOperationOptions.isCaptureIgnoredChars()
@@ -102,6 +108,7 @@ class ParserOptionsTest extends Specification {
102108
currentDefaultSdlOptions.getMaxCharacters() == Integer.MAX_VALUE
103109
currentDefaultSdlOptions.getMaxTokens() == Integer.MAX_VALUE
104110
currentDefaultSdlOptions.getMaxWhitespaceTokens() == 300_000
111+
currentDefaultSdlOptions.getMaxNumericLiteralCharacters() == 100
105112
currentDefaultSdlOptions.isCaptureSourceLocation()
106113
currentDefaultSdlOptions.isCaptureLineComments()
107114
currentDefaultSdlOptions.isCaptureIgnoredChars()

0 commit comments

Comments
 (0)