diff --git a/gson/src/main/java/com/google/gson/stream/JsonReader.java b/gson/src/main/java/com/google/gson/stream/JsonReader.java index 9b691397..13f3b36f 100644 --- a/gson/src/main/java/com/google/gson/stream/JsonReader.java +++ b/gson/src/main/java/com/google/gson/stream/JsonReader.java @@ -33,8 +33,8 @@ import java.util.List; * Within JSON objects, name/value pairs are represented by a single token. * *

Parsing JSON

- * To create a recursive descent parser your own JSON streams, first create an - * entry point method that creates a {@code JsonReader}. + * To create a recursive descent parser for your own JSON streams, first create + * an entry point method that creates a {@code JsonReader}. * *

Next, create handler methods for each structure in your JSON text. You'll * need a method for each object type and for each array type. @@ -87,7 +87,11 @@ import java.util.List; * * public List readJsonStream(InputStream in) throws IOException { * JsonReader reader = new JsonReader(new InputStreamReader(in, "UTF-8")); - * return readMessagesArray(reader); + * try { + * return readMessagesArray(reader); + * } finally { + * reader.close(); + * } * } * * public List readMessagesArray(JsonReader reader) throws IOException { @@ -189,6 +193,9 @@ public class JsonReader implements Closeable { /** The only non-execute prefix this parser permits */ private static final char[] NON_EXECUTE_PREFIX = ")]}'\n".toCharArray(); + private static final String TRUE = "true"; + private static final String FALSE = "false"; + private final StringPool stringPool = new StringPool(); /** The input JSON. */ @@ -200,6 +207,8 @@ public class JsonReader implements Closeable { /** * Use a manual buffer to easily read and unread upcoming characters, and * also so we can create strings without an intermediate StringBuilder. + * We decode literals directly out of this buffer, so it must be at least as + * long as the longest token that can be reported as a number. */ private final char[] buffer = new char[1024]; private int pos = 0; @@ -216,27 +225,22 @@ public class JsonReader implements Closeable { push(JsonScope.EMPTY_DOCUMENT); } - /** - * True if we've already read the next token. If we have, the string value - * for that token will be assigned to {@code value} if such a string value - * exists. And the token type will be assigned to {@code token} if the token - * type is known. The token type may be null for literals, since we derive - * that lazily. - */ - private boolean hasToken; - /** * The type of the next token to be returned by {@link #peek} and {@link - * #advance}, or {@code null} if it is unknown and must first be derived - * from {@code value}. This value is undefined if {@code hasToken} is false. + * #advance}. If null, peek() will assign a value. */ private JsonToken token; /** The text of the next name. */ private String name; - /** The text of the next literal value. */ + /* + * For the next literal value, we may have the text value, or the position + * and length in the buffer. + */ private String value; + private int valuePos; + private int valueLength; /** True if we're currently handling a skipValue() call. */ private boolean skipping = false; @@ -327,7 +331,7 @@ public class JsonReader implements Closeable { * Consumes {@code expected}. */ private void expect(JsonToken expected) throws IOException { - quickPeek(); + peek(); if (token != expected) { throw new IllegalStateException("Expected " + expected + " but was " + peek()); } @@ -338,7 +342,7 @@ public class JsonReader implements Closeable { * Returns true if the current array or object has another element. */ public boolean hasNext() throws IOException { - quickPeek(); + peek(); return token != JsonToken.END_OBJECT && token != JsonToken.END_ARRAY; } @@ -346,22 +350,7 @@ public class JsonReader implements Closeable { * Returns the type of the next token without consuming it. */ public JsonToken peek() throws IOException { - quickPeek(); - - if (token == null) { - decodeLiteral(); - } - - return token; - } - - /** - * Ensures that a token is ready. After this call either {@code token} or - * {@code value} will be non-null. To ensure {@code token} has a definitive - * value, use {@link #peek()} - */ - private JsonToken quickPeek() throws IOException { - if (hasToken) { + if (token != null) { return token; } @@ -372,8 +361,9 @@ public class JsonReader implements Closeable { } replaceTop(JsonScope.NONEMPTY_DOCUMENT); JsonToken firstToken = nextValue(); - if (!lenient && firstToken != JsonToken.BEGIN_ARRAY && firstToken != JsonToken.BEGIN_OBJECT) { - syntaxError("Expected JSON document to start with '[' or '{'"); + if (!lenient && token != JsonToken.BEGIN_ARRAY && token != JsonToken.BEGIN_OBJECT) { + throw new IOException( + "Expected JSON document to start with '[' or '{' but was " + token); } return firstToken; case EMPTY_ARRAY: @@ -394,8 +384,7 @@ public class JsonReader implements Closeable { } throw syntaxError("Expected EOF"); } catch (EOFException e) { - hasToken = true; // TODO: avoid throwing here? - return token = JsonToken.END_DOCUMENT; + return token = JsonToken.END_DOCUMENT; // TODO: avoid throwing here? } case CLOSED: throw new IllegalStateException("JsonReader is closed"); @@ -430,10 +419,9 @@ public class JsonReader implements Closeable { * Advances the cursor in the JSON stream to the next token. */ private JsonToken advance() throws IOException { - quickPeek(); + peek(); JsonToken result = token; - hasToken = false; token = null; value = null; name = null; @@ -448,7 +436,7 @@ public class JsonReader implements Closeable { * name. */ public String nextName() throws IOException { - quickPeek(); + peek(); if (token != JsonToken.NAME) { throw new IllegalStateException("Expected a name but was " + peek()); } @@ -467,7 +455,7 @@ public class JsonReader implements Closeable { */ public String nextString() throws IOException { peek(); - if (value == null || (token != JsonToken.STRING && token != JsonToken.NUMBER)) { + if (token != JsonToken.STRING && token != JsonToken.NUMBER) { throw new IllegalStateException("Expected a string but was " + peek()); } @@ -484,20 +472,12 @@ public class JsonReader implements Closeable { * this reader is closed. */ public boolean nextBoolean() throws IOException { - quickPeek(); - if (value == null || token == JsonToken.STRING) { - throw new IllegalStateException("Expected a boolean but was " + peek()); - } - - boolean result; - if (value.equalsIgnoreCase("true")) { - result = true; - } else if (value.equalsIgnoreCase("false")) { - result = false; - } else { - throw new IllegalStateException("Not a boolean: " + value); + peek(); + if (token != JsonToken.BOOLEAN) { + throw new IllegalStateException("Expected a boolean but was " + token); } + boolean result = (value == TRUE); advance(); return result; } @@ -510,13 +490,9 @@ public class JsonReader implements Closeable { * reader is closed. */ public void nextNull() throws IOException { - quickPeek(); - if (value == null || token == JsonToken.STRING) { - throw new IllegalStateException("Expected null but was " + peek()); - } - - if (!value.equalsIgnoreCase("null")) { - throw new IllegalStateException("Not a null: " + value); + peek(); + if (token != JsonToken.NULL) { + throw new IllegalStateException("Expected null but was " + token); } advance(); @@ -525,26 +501,25 @@ public class JsonReader implements Closeable { /** * Returns the {@link JsonToken#NUMBER double} value of the next token, * consuming it. If the next token is a string, this method will attempt to - * parse it as a double. + * parse it as a double using {@link Double#parseDouble(String)}. * * @throws IllegalStateException if the next token is not a literal value. * @throws NumberFormatException if the next literal value cannot be parsed * as a double, or is non-finite. */ public double nextDouble() throws IOException { - quickPeek(); - if (value == null) { - throw new IllegalStateException("Expected a double but was " + peek()); + peek(); + if (token != JsonToken.STRING && token != JsonToken.NUMBER) { + throw new IllegalStateException("Expected a double but was " + token); } double result = Double.parseDouble(value); if ((result >= 1.0d && value.startsWith("0"))) { - throw new NumberFormatException("JSON forbids octal prefixes: " + value); + throw new MalformedJsonException("JSON forbids octal prefixes: " + value); } - if (!lenient && (Double.isNaN(result) || Double.isInfinite(result))) { - throw new NumberFormatException("JSON forbids NaN and infinities: " + value); + throw new MalformedJsonException("JSON forbids NaN and infinities: " + value); } advance(); @@ -562,9 +537,9 @@ public class JsonReader implements Closeable { * as a number, or exactly represented as a long. */ public long nextLong() throws IOException { - quickPeek(); - if (value == null) { - throw new IllegalStateException("Expected a long but was " + peek()); + peek(); + if (token != JsonToken.STRING && token != JsonToken.NUMBER) { + throw new IllegalStateException("Expected a long but was " + token); } long result; @@ -579,7 +554,7 @@ public class JsonReader implements Closeable { } if (result >= 1L && value.startsWith("0")) { - throw new NumberFormatException("JSON forbids octal prefixes: " + value); + throw new MalformedJsonException("JSON forbids octal prefixes: " + value); } advance(); @@ -597,9 +572,9 @@ public class JsonReader implements Closeable { * as a number, or exactly represented as an int. */ public int nextInt() throws IOException { - quickPeek(); - if (value == null) { - throw new IllegalStateException("Expected an int but was " + peek()); + peek(); + if (token != JsonToken.STRING && token != JsonToken.NUMBER) { + throw new IllegalStateException("Expected an int but was " + token); } int result; @@ -614,7 +589,7 @@ public class JsonReader implements Closeable { } if (result >= 1L && value.startsWith("0")) { - throw new NumberFormatException("JSON forbids octal prefixes: " + value); + throw new MalformedJsonException("JSON forbids octal prefixes: " + value); } advance(); @@ -625,7 +600,6 @@ public class JsonReader implements Closeable { * Closes this JSON reader and the underlying {@link Reader}. */ public void close() throws IOException { - hasToken = false; value = null; token = null; stack.clear(); @@ -683,7 +657,6 @@ public class JsonReader implements Closeable { switch (nextNonWhitespace()) { case ']': pop(); - hasToken = true; return token = JsonToken.END_ARRAY; case ';': checkLenient(); // fall-through @@ -698,7 +671,6 @@ public class JsonReader implements Closeable { case ']': if (firstElement) { pop(); - hasToken = true; return token = JsonToken.END_ARRAY; } // fall-through to handle ",]" @@ -707,7 +679,6 @@ public class JsonReader implements Closeable { /* In lenient mode, a 0-length literal means 'null' */ checkLenient(); pos--; - hasToken = true; value = "null"; return token = JsonToken.NULL; default: @@ -728,7 +699,6 @@ public class JsonReader implements Closeable { switch (nextNonWhitespace()) { case '}': pop(); - hasToken = true; return token = JsonToken.END_OBJECT; default: pos--; @@ -737,7 +707,6 @@ public class JsonReader implements Closeable { switch (nextNonWhitespace()) { case '}': pop(); - hasToken = true; return token = JsonToken.END_OBJECT; case ';': case ',': @@ -758,14 +727,13 @@ public class JsonReader implements Closeable { default: checkLenient(); pos--; - name = nextLiteral(); + name = nextLiteral(false); if (name.length() == 0) { throw syntaxError("Expected name"); } } replaceTop(JsonScope.DANGLING_NAME); - hasToken = true; return token = JsonToken.NAME; } @@ -797,19 +765,16 @@ public class JsonReader implements Closeable { switch (c) { case '{': push(JsonScope.EMPTY_OBJECT); - hasToken = true; return token = JsonToken.BEGIN_OBJECT; case '[': push(JsonScope.EMPTY_ARRAY); - hasToken = true; return token = JsonToken.BEGIN_ARRAY; case '\'': checkLenient(); // fall-through case '"': value = nextString((char) c); - hasToken = true; return token = JsonToken.STRING; default: @@ -1016,25 +981,30 @@ public class JsonReader implements Closeable { } /** - * Returns the string up to but not including any delimiter characters. This + * Reads the value up to but not including any delimiter characters. This * does not consume the delimiter character. + * + * @param assignOffsetsOnly true for this method to only set the valuePos + * and valueLength fields and return a null result. This only works if + * the literal is short; a string is returned otherwise. */ @SuppressWarnings("fallthrough") - private String nextLiteral() throws IOException { + private String nextLiteral(boolean assignOffsetsOnly) throws IOException { StringBuilder builder = null; - do { - /* the index of the first character not yet appended to the builder. */ - int start = pos; - while (pos < limit) { - int c = buffer[pos++]; - switch (c) { + valuePos = -1; + valueLength = 0; + int i = 0; + + findNonLiteralCharacter: + while (true) { + for (; pos + i < limit; i++) { + switch (buffer[pos + i]) { case '/': case '\\': case ';': case '#': case '=': - checkLenient(); // fall-through - + checkLenient(); // fall-through case '{': case '}': case '[': @@ -1046,25 +1016,52 @@ public class JsonReader implements Closeable { case '\f': case '\r': case '\n': - pos--; - if (skipping) { - return "skipped!"; - } else if (builder == null) { - return stringPool.get(buffer, start, pos - start); - } else { - builder.append(buffer, start, pos - start); - return builder.toString(); - } + break findNonLiteralCharacter; } } + /* + * Attempt to load the entire literal into the buffer at once. If + * we run out of input, add a non-literal character at the end so + * that decoding doesn't need to do bounds checks. + */ + if (i < buffer.length) { + if (fillBuffer(i + 1)) { + continue; + } else { + buffer[limit] = '\0'; + break; + } + } + + // use a StringBuilder when the value is too long. It must be an unquoted string. if (builder == null) { builder = new StringBuilder(); } - builder.append(buffer, start, pos - start); - } while (fillBuffer(1)); + builder.append(buffer, pos, i); + valueLength += i; + pos += i; + i = 0; + if (!fillBuffer(1)) { + break; + } + } - return builder.toString(); + String result; + if (assignOffsetsOnly && builder == null) { + valuePos = pos; + result = null; + } else if (skipping) { + result = "skipped!"; + } else if (builder == null) { + result = stringPool.get(buffer, pos, i); + } else { + builder.append(buffer, pos, i); + result = builder.toString(); + } + valueLength += i; + pos += i; + return result; } @Override public String toString() { @@ -1122,32 +1119,103 @@ public class JsonReader implements Closeable { * Reads a null, boolean, numeric or unquoted string literal value. */ private JsonToken readLiteral() throws IOException { - String literal = nextLiteral(); - if (literal.length() == 0) { + value = nextLiteral(true); + if (valueLength == 0) { throw syntaxError("Expected literal value"); } - value = literal; - hasToken = true; - return token = null; // use decodeLiteral() to get the token type + token = decodeLiteral(); + if (token == JsonToken.STRING) { + checkLenient(); + } + return token; } /** * Assigns {@code nextToken} based on the value of {@code nextValue}. */ - private void decodeLiteral() throws IOException { - if (value.equalsIgnoreCase("null")) { - token = JsonToken.NULL; - } else if (value.equalsIgnoreCase("true") || value.equalsIgnoreCase("false")) { - token = JsonToken.BOOLEAN; + private JsonToken decodeLiteral() throws IOException { + if (valuePos == -1) { + // it was too long to fit in the buffer so it can only be a string + return JsonToken.STRING; + } else if (valueLength == 4 + && ('n' == buffer[valuePos ] || 'N' == buffer[valuePos ]) + && ('u' == buffer[valuePos + 1] || 'U' == buffer[valuePos + 1]) + && ('l' == buffer[valuePos + 2] || 'L' == buffer[valuePos + 2]) + && ('l' == buffer[valuePos + 3] || 'L' == buffer[valuePos + 3])) { + value = "null"; + return JsonToken.NULL; + } else if (valueLength == 4 + && ('t' == buffer[valuePos ] || 'T' == buffer[valuePos ]) + && ('r' == buffer[valuePos + 1] || 'R' == buffer[valuePos + 1]) + && ('u' == buffer[valuePos + 2] || 'U' == buffer[valuePos + 2]) + && ('e' == buffer[valuePos + 3] || 'E' == buffer[valuePos + 3])) { + value = TRUE; + return JsonToken.BOOLEAN; + } else if (valueLength == 5 + && ('f' == buffer[valuePos ] || 'F' == buffer[valuePos ]) + && ('a' == buffer[valuePos + 1] || 'A' == buffer[valuePos + 1]) + && ('l' == buffer[valuePos + 2] || 'L' == buffer[valuePos + 2]) + && ('s' == buffer[valuePos + 3] || 'S' == buffer[valuePos + 3]) + && ('e' == buffer[valuePos + 4] || 'E' == buffer[valuePos + 4])) { + value = FALSE; + return JsonToken.BOOLEAN; } else { - try { - Double.parseDouble(value); // this work could potentially be cached - token = JsonToken.NUMBER; - } catch (NumberFormatException ignored) { - // this must be an unquoted string - checkLenient(); - token = JsonToken.STRING; + value = stringPool.get(buffer, valuePos, valueLength); + return decodeNumber(buffer, valuePos, valueLength); + } + } + + /** + * Determine whether the characters is a JSON number. Numbers are of the + * form -12.34e+56. Fractional and exponential parts are optional. Leading + * zeroes are not allowed in the value or exponential part, but are allowed + * in the fraction. + */ + private JsonToken decodeNumber(char[] chars, int offset, int length) { + int i = offset; + int c = chars[i]; + + if (c == '-') { + c = chars[++i]; + } + + if (c == '0') { + c = chars[++i]; + } else if (c >= '1' && c <= '9') { + c = chars[++i]; + while (c >= '0' && c <= '9') { + c = chars[++i]; } + } else { + return JsonToken.STRING; + } + + if (c == '.') { + c = chars[++i]; + while (c >= '0' && c <= '9') { + c = chars[++i]; + } + } + + if (c == 'e' || c == 'E') { + c = chars[++i]; + if (c == '+' || c == '-') { + c = chars[++i]; + } + if (c >= '0' && c <= '9') { + c = chars[++i]; + while (c >= '0' && c <= '9') { + c = chars[++i]; + } + } else { + return JsonToken.STRING; + } + } + + if (i == offset + length) { + return JsonToken.NUMBER; + } else { + return JsonToken.STRING; } } @@ -1172,7 +1240,7 @@ public class JsonReader implements Closeable { static { JsonReaderInternalAccess.INSTANCE = new JsonReaderInternalAccess() { @Override public void promoteNameToValue(JsonReader reader) throws IOException { - reader.quickPeek(); + reader.peek(); if (reader.token != JsonToken.NAME) { throw new IllegalStateException("Expected a name but was " + reader.peek()); } diff --git a/gson/src/test/java/com/google/gson/stream/JsonReaderTest.java b/gson/src/test/java/com/google/gson/stream/JsonReaderTest.java index fd296657..22242fa5 100644 --- a/gson/src/test/java/com/google/gson/stream/JsonReaderTest.java +++ b/gson/src/test/java/com/google/gson/stream/JsonReaderTest.java @@ -201,7 +201,7 @@ public final class JsonReaderTest extends TestCase { try { reader.nextDouble(); fail(); - } catch (NumberFormatException expected) { + } catch (MalformedJsonException expected) { } } @@ -265,20 +265,25 @@ public final class JsonReaderTest extends TestCase { String json = "[01]"; JsonReader reader = new JsonReader(new StringReader(json)); reader.beginArray(); + try { + reader.peek(); + fail(); + } catch (MalformedJsonException expected) { + } try { reader.nextInt(); fail(); - } catch (NumberFormatException expected) { + } catch (MalformedJsonException expected) { } try { reader.nextLong(); fail(); - } catch (NumberFormatException expected) { + } catch (MalformedJsonException expected) { } try { reader.nextDouble(); fail(); - } catch (NumberFormatException expected) { + } catch (MalformedJsonException expected) { } assertEquals("01", reader.nextString()); reader.endArray();