diff --git a/gson/src/main/java/com/google/gson/stream/JsonReader.java b/gson/src/main/java/com/google/gson/stream/JsonReader.java
index 9b691397..13f3b36f 100644
--- a/gson/src/main/java/com/google/gson/stream/JsonReader.java
+++ b/gson/src/main/java/com/google/gson/stream/JsonReader.java
@@ -33,8 +33,8 @@ import java.util.List;
* Within JSON objects, name/value pairs are represented by a single token.
*
*
Parsing JSON
- * To create a recursive descent parser your own JSON streams, first create an
- * entry point method that creates a {@code JsonReader}.
+ * To create a recursive descent parser for your own JSON streams, first create
+ * an entry point method that creates a {@code JsonReader}.
*
* Next, create handler methods for each structure in your JSON text. You'll
* need a method for each object type and for each array type.
@@ -87,7 +87,11 @@ import java.util.List;
*
* public List readJsonStream(InputStream in) throws IOException {
* JsonReader reader = new JsonReader(new InputStreamReader(in, "UTF-8"));
- * return readMessagesArray(reader);
+ * try {
+ * return readMessagesArray(reader);
+ * } finally {
+ * reader.close();
+ * }
* }
*
* public List readMessagesArray(JsonReader reader) throws IOException {
@@ -189,6 +193,9 @@ public class JsonReader implements Closeable {
/** The only non-execute prefix this parser permits */
private static final char[] NON_EXECUTE_PREFIX = ")]}'\n".toCharArray();
+ private static final String TRUE = "true";
+ private static final String FALSE = "false";
+
private final StringPool stringPool = new StringPool();
/** The input JSON. */
@@ -200,6 +207,8 @@ public class JsonReader implements Closeable {
/**
* Use a manual buffer to easily read and unread upcoming characters, and
* also so we can create strings without an intermediate StringBuilder.
+ * We decode literals directly out of this buffer, so it must be at least as
+ * long as the longest token that can be reported as a number.
*/
private final char[] buffer = new char[1024];
private int pos = 0;
@@ -216,27 +225,22 @@ public class JsonReader implements Closeable {
push(JsonScope.EMPTY_DOCUMENT);
}
- /**
- * True if we've already read the next token. If we have, the string value
- * for that token will be assigned to {@code value} if such a string value
- * exists. And the token type will be assigned to {@code token} if the token
- * type is known. The token type may be null for literals, since we derive
- * that lazily.
- */
- private boolean hasToken;
-
/**
* The type of the next token to be returned by {@link #peek} and {@link
- * #advance}, or {@code null} if it is unknown and must first be derived
- * from {@code value}. This value is undefined if {@code hasToken} is false.
+ * #advance}. If null, peek() will assign a value.
*/
private JsonToken token;
/** The text of the next name. */
private String name;
- /** The text of the next literal value. */
+ /*
+ * For the next literal value, we may have the text value, or the position
+ * and length in the buffer.
+ */
private String value;
+ private int valuePos;
+ private int valueLength;
/** True if we're currently handling a skipValue() call. */
private boolean skipping = false;
@@ -327,7 +331,7 @@ public class JsonReader implements Closeable {
* Consumes {@code expected}.
*/
private void expect(JsonToken expected) throws IOException {
- quickPeek();
+ peek();
if (token != expected) {
throw new IllegalStateException("Expected " + expected + " but was " + peek());
}
@@ -338,7 +342,7 @@ public class JsonReader implements Closeable {
* Returns true if the current array or object has another element.
*/
public boolean hasNext() throws IOException {
- quickPeek();
+ peek();
return token != JsonToken.END_OBJECT && token != JsonToken.END_ARRAY;
}
@@ -346,22 +350,7 @@ public class JsonReader implements Closeable {
* Returns the type of the next token without consuming it.
*/
public JsonToken peek() throws IOException {
- quickPeek();
-
- if (token == null) {
- decodeLiteral();
- }
-
- return token;
- }
-
- /**
- * Ensures that a token is ready. After this call either {@code token} or
- * {@code value} will be non-null. To ensure {@code token} has a definitive
- * value, use {@link #peek()}
- */
- private JsonToken quickPeek() throws IOException {
- if (hasToken) {
+ if (token != null) {
return token;
}
@@ -372,8 +361,9 @@ public class JsonReader implements Closeable {
}
replaceTop(JsonScope.NONEMPTY_DOCUMENT);
JsonToken firstToken = nextValue();
- if (!lenient && firstToken != JsonToken.BEGIN_ARRAY && firstToken != JsonToken.BEGIN_OBJECT) {
- syntaxError("Expected JSON document to start with '[' or '{'");
+ if (!lenient && token != JsonToken.BEGIN_ARRAY && token != JsonToken.BEGIN_OBJECT) {
+ throw new IOException(
+ "Expected JSON document to start with '[' or '{' but was " + token);
}
return firstToken;
case EMPTY_ARRAY:
@@ -394,8 +384,7 @@ public class JsonReader implements Closeable {
}
throw syntaxError("Expected EOF");
} catch (EOFException e) {
- hasToken = true; // TODO: avoid throwing here?
- return token = JsonToken.END_DOCUMENT;
+ return token = JsonToken.END_DOCUMENT; // TODO: avoid throwing here?
}
case CLOSED:
throw new IllegalStateException("JsonReader is closed");
@@ -430,10 +419,9 @@ public class JsonReader implements Closeable {
* Advances the cursor in the JSON stream to the next token.
*/
private JsonToken advance() throws IOException {
- quickPeek();
+ peek();
JsonToken result = token;
- hasToken = false;
token = null;
value = null;
name = null;
@@ -448,7 +436,7 @@ public class JsonReader implements Closeable {
* name.
*/
public String nextName() throws IOException {
- quickPeek();
+ peek();
if (token != JsonToken.NAME) {
throw new IllegalStateException("Expected a name but was " + peek());
}
@@ -467,7 +455,7 @@ public class JsonReader implements Closeable {
*/
public String nextString() throws IOException {
peek();
- if (value == null || (token != JsonToken.STRING && token != JsonToken.NUMBER)) {
+ if (token != JsonToken.STRING && token != JsonToken.NUMBER) {
throw new IllegalStateException("Expected a string but was " + peek());
}
@@ -484,20 +472,12 @@ public class JsonReader implements Closeable {
* this reader is closed.
*/
public boolean nextBoolean() throws IOException {
- quickPeek();
- if (value == null || token == JsonToken.STRING) {
- throw new IllegalStateException("Expected a boolean but was " + peek());
- }
-
- boolean result;
- if (value.equalsIgnoreCase("true")) {
- result = true;
- } else if (value.equalsIgnoreCase("false")) {
- result = false;
- } else {
- throw new IllegalStateException("Not a boolean: " + value);
+ peek();
+ if (token != JsonToken.BOOLEAN) {
+ throw new IllegalStateException("Expected a boolean but was " + token);
}
+ boolean result = (value == TRUE);
advance();
return result;
}
@@ -510,13 +490,9 @@ public class JsonReader implements Closeable {
* reader is closed.
*/
public void nextNull() throws IOException {
- quickPeek();
- if (value == null || token == JsonToken.STRING) {
- throw new IllegalStateException("Expected null but was " + peek());
- }
-
- if (!value.equalsIgnoreCase("null")) {
- throw new IllegalStateException("Not a null: " + value);
+ peek();
+ if (token != JsonToken.NULL) {
+ throw new IllegalStateException("Expected null but was " + token);
}
advance();
@@ -525,26 +501,25 @@ public class JsonReader implements Closeable {
/**
* Returns the {@link JsonToken#NUMBER double} value of the next token,
* consuming it. If the next token is a string, this method will attempt to
- * parse it as a double.
+ * parse it as a double using {@link Double#parseDouble(String)}.
*
* @throws IllegalStateException if the next token is not a literal value.
* @throws NumberFormatException if the next literal value cannot be parsed
* as a double, or is non-finite.
*/
public double nextDouble() throws IOException {
- quickPeek();
- if (value == null) {
- throw new IllegalStateException("Expected a double but was " + peek());
+ peek();
+ if (token != JsonToken.STRING && token != JsonToken.NUMBER) {
+ throw new IllegalStateException("Expected a double but was " + token);
}
double result = Double.parseDouble(value);
if ((result >= 1.0d && value.startsWith("0"))) {
- throw new NumberFormatException("JSON forbids octal prefixes: " + value);
+ throw new MalformedJsonException("JSON forbids octal prefixes: " + value);
}
-
if (!lenient && (Double.isNaN(result) || Double.isInfinite(result))) {
- throw new NumberFormatException("JSON forbids NaN and infinities: " + value);
+ throw new MalformedJsonException("JSON forbids NaN and infinities: " + value);
}
advance();
@@ -562,9 +537,9 @@ public class JsonReader implements Closeable {
* as a number, or exactly represented as a long.
*/
public long nextLong() throws IOException {
- quickPeek();
- if (value == null) {
- throw new IllegalStateException("Expected a long but was " + peek());
+ peek();
+ if (token != JsonToken.STRING && token != JsonToken.NUMBER) {
+ throw new IllegalStateException("Expected a long but was " + token);
}
long result;
@@ -579,7 +554,7 @@ public class JsonReader implements Closeable {
}
if (result >= 1L && value.startsWith("0")) {
- throw new NumberFormatException("JSON forbids octal prefixes: " + value);
+ throw new MalformedJsonException("JSON forbids octal prefixes: " + value);
}
advance();
@@ -597,9 +572,9 @@ public class JsonReader implements Closeable {
* as a number, or exactly represented as an int.
*/
public int nextInt() throws IOException {
- quickPeek();
- if (value == null) {
- throw new IllegalStateException("Expected an int but was " + peek());
+ peek();
+ if (token != JsonToken.STRING && token != JsonToken.NUMBER) {
+ throw new IllegalStateException("Expected an int but was " + token);
}
int result;
@@ -614,7 +589,7 @@ public class JsonReader implements Closeable {
}
if (result >= 1L && value.startsWith("0")) {
- throw new NumberFormatException("JSON forbids octal prefixes: " + value);
+ throw new MalformedJsonException("JSON forbids octal prefixes: " + value);
}
advance();
@@ -625,7 +600,6 @@ public class JsonReader implements Closeable {
* Closes this JSON reader and the underlying {@link Reader}.
*/
public void close() throws IOException {
- hasToken = false;
value = null;
token = null;
stack.clear();
@@ -683,7 +657,6 @@ public class JsonReader implements Closeable {
switch (nextNonWhitespace()) {
case ']':
pop();
- hasToken = true;
return token = JsonToken.END_ARRAY;
case ';':
checkLenient(); // fall-through
@@ -698,7 +671,6 @@ public class JsonReader implements Closeable {
case ']':
if (firstElement) {
pop();
- hasToken = true;
return token = JsonToken.END_ARRAY;
}
// fall-through to handle ",]"
@@ -707,7 +679,6 @@ public class JsonReader implements Closeable {
/* In lenient mode, a 0-length literal means 'null' */
checkLenient();
pos--;
- hasToken = true;
value = "null";
return token = JsonToken.NULL;
default:
@@ -728,7 +699,6 @@ public class JsonReader implements Closeable {
switch (nextNonWhitespace()) {
case '}':
pop();
- hasToken = true;
return token = JsonToken.END_OBJECT;
default:
pos--;
@@ -737,7 +707,6 @@ public class JsonReader implements Closeable {
switch (nextNonWhitespace()) {
case '}':
pop();
- hasToken = true;
return token = JsonToken.END_OBJECT;
case ';':
case ',':
@@ -758,14 +727,13 @@ public class JsonReader implements Closeable {
default:
checkLenient();
pos--;
- name = nextLiteral();
+ name = nextLiteral(false);
if (name.length() == 0) {
throw syntaxError("Expected name");
}
}
replaceTop(JsonScope.DANGLING_NAME);
- hasToken = true;
return token = JsonToken.NAME;
}
@@ -797,19 +765,16 @@ public class JsonReader implements Closeable {
switch (c) {
case '{':
push(JsonScope.EMPTY_OBJECT);
- hasToken = true;
return token = JsonToken.BEGIN_OBJECT;
case '[':
push(JsonScope.EMPTY_ARRAY);
- hasToken = true;
return token = JsonToken.BEGIN_ARRAY;
case '\'':
checkLenient(); // fall-through
case '"':
value = nextString((char) c);
- hasToken = true;
return token = JsonToken.STRING;
default:
@@ -1016,25 +981,30 @@ public class JsonReader implements Closeable {
}
/**
- * Returns the string up to but not including any delimiter characters. This
+ * Reads the value up to but not including any delimiter characters. This
* does not consume the delimiter character.
+ *
+ * @param assignOffsetsOnly true for this method to only set the valuePos
+ * and valueLength fields and return a null result. This only works if
+ * the literal is short; a string is returned otherwise.
*/
@SuppressWarnings("fallthrough")
- private String nextLiteral() throws IOException {
+ private String nextLiteral(boolean assignOffsetsOnly) throws IOException {
StringBuilder builder = null;
- do {
- /* the index of the first character not yet appended to the builder. */
- int start = pos;
- while (pos < limit) {
- int c = buffer[pos++];
- switch (c) {
+ valuePos = -1;
+ valueLength = 0;
+ int i = 0;
+
+ findNonLiteralCharacter:
+ while (true) {
+ for (; pos + i < limit; i++) {
+ switch (buffer[pos + i]) {
case '/':
case '\\':
case ';':
case '#':
case '=':
- checkLenient(); // fall-through
-
+ checkLenient(); // fall-through
case '{':
case '}':
case '[':
@@ -1046,25 +1016,52 @@ public class JsonReader implements Closeable {
case '\f':
case '\r':
case '\n':
- pos--;
- if (skipping) {
- return "skipped!";
- } else if (builder == null) {
- return stringPool.get(buffer, start, pos - start);
- } else {
- builder.append(buffer, start, pos - start);
- return builder.toString();
- }
+ break findNonLiteralCharacter;
}
}
+ /*
+ * Attempt to load the entire literal into the buffer at once. If
+ * we run out of input, add a non-literal character at the end so
+ * that decoding doesn't need to do bounds checks.
+ */
+ if (i < buffer.length) {
+ if (fillBuffer(i + 1)) {
+ continue;
+ } else {
+ buffer[limit] = '\0';
+ break;
+ }
+ }
+
+ // use a StringBuilder when the value is too long. It must be an unquoted string.
if (builder == null) {
builder = new StringBuilder();
}
- builder.append(buffer, start, pos - start);
- } while (fillBuffer(1));
+ builder.append(buffer, pos, i);
+ valueLength += i;
+ pos += i;
+ i = 0;
+ if (!fillBuffer(1)) {
+ break;
+ }
+ }
- return builder.toString();
+ String result;
+ if (assignOffsetsOnly && builder == null) {
+ valuePos = pos;
+ result = null;
+ } else if (skipping) {
+ result = "skipped!";
+ } else if (builder == null) {
+ result = stringPool.get(buffer, pos, i);
+ } else {
+ builder.append(buffer, pos, i);
+ result = builder.toString();
+ }
+ valueLength += i;
+ pos += i;
+ return result;
}
@Override public String toString() {
@@ -1122,32 +1119,103 @@ public class JsonReader implements Closeable {
* Reads a null, boolean, numeric or unquoted string literal value.
*/
private JsonToken readLiteral() throws IOException {
- String literal = nextLiteral();
- if (literal.length() == 0) {
+ value = nextLiteral(true);
+ if (valueLength == 0) {
throw syntaxError("Expected literal value");
}
- value = literal;
- hasToken = true;
- return token = null; // use decodeLiteral() to get the token type
+ token = decodeLiteral();
+ if (token == JsonToken.STRING) {
+ checkLenient();
+ }
+ return token;
}
/**
* Assigns {@code nextToken} based on the value of {@code nextValue}.
*/
- private void decodeLiteral() throws IOException {
- if (value.equalsIgnoreCase("null")) {
- token = JsonToken.NULL;
- } else if (value.equalsIgnoreCase("true") || value.equalsIgnoreCase("false")) {
- token = JsonToken.BOOLEAN;
+ private JsonToken decodeLiteral() throws IOException {
+ if (valuePos == -1) {
+ // it was too long to fit in the buffer so it can only be a string
+ return JsonToken.STRING;
+ } else if (valueLength == 4
+ && ('n' == buffer[valuePos ] || 'N' == buffer[valuePos ])
+ && ('u' == buffer[valuePos + 1] || 'U' == buffer[valuePos + 1])
+ && ('l' == buffer[valuePos + 2] || 'L' == buffer[valuePos + 2])
+ && ('l' == buffer[valuePos + 3] || 'L' == buffer[valuePos + 3])) {
+ value = "null";
+ return JsonToken.NULL;
+ } else if (valueLength == 4
+ && ('t' == buffer[valuePos ] || 'T' == buffer[valuePos ])
+ && ('r' == buffer[valuePos + 1] || 'R' == buffer[valuePos + 1])
+ && ('u' == buffer[valuePos + 2] || 'U' == buffer[valuePos + 2])
+ && ('e' == buffer[valuePos + 3] || 'E' == buffer[valuePos + 3])) {
+ value = TRUE;
+ return JsonToken.BOOLEAN;
+ } else if (valueLength == 5
+ && ('f' == buffer[valuePos ] || 'F' == buffer[valuePos ])
+ && ('a' == buffer[valuePos + 1] || 'A' == buffer[valuePos + 1])
+ && ('l' == buffer[valuePos + 2] || 'L' == buffer[valuePos + 2])
+ && ('s' == buffer[valuePos + 3] || 'S' == buffer[valuePos + 3])
+ && ('e' == buffer[valuePos + 4] || 'E' == buffer[valuePos + 4])) {
+ value = FALSE;
+ return JsonToken.BOOLEAN;
} else {
- try {
- Double.parseDouble(value); // this work could potentially be cached
- token = JsonToken.NUMBER;
- } catch (NumberFormatException ignored) {
- // this must be an unquoted string
- checkLenient();
- token = JsonToken.STRING;
+ value = stringPool.get(buffer, valuePos, valueLength);
+ return decodeNumber(buffer, valuePos, valueLength);
+ }
+ }
+
+ /**
+ * Determine whether the characters is a JSON number. Numbers are of the
+ * form -12.34e+56. Fractional and exponential parts are optional. Leading
+ * zeroes are not allowed in the value or exponential part, but are allowed
+ * in the fraction.
+ */
+ private JsonToken decodeNumber(char[] chars, int offset, int length) {
+ int i = offset;
+ int c = chars[i];
+
+ if (c == '-') {
+ c = chars[++i];
+ }
+
+ if (c == '0') {
+ c = chars[++i];
+ } else if (c >= '1' && c <= '9') {
+ c = chars[++i];
+ while (c >= '0' && c <= '9') {
+ c = chars[++i];
}
+ } else {
+ return JsonToken.STRING;
+ }
+
+ if (c == '.') {
+ c = chars[++i];
+ while (c >= '0' && c <= '9') {
+ c = chars[++i];
+ }
+ }
+
+ if (c == 'e' || c == 'E') {
+ c = chars[++i];
+ if (c == '+' || c == '-') {
+ c = chars[++i];
+ }
+ if (c >= '0' && c <= '9') {
+ c = chars[++i];
+ while (c >= '0' && c <= '9') {
+ c = chars[++i];
+ }
+ } else {
+ return JsonToken.STRING;
+ }
+ }
+
+ if (i == offset + length) {
+ return JsonToken.NUMBER;
+ } else {
+ return JsonToken.STRING;
}
}
@@ -1172,7 +1240,7 @@ public class JsonReader implements Closeable {
static {
JsonReaderInternalAccess.INSTANCE = new JsonReaderInternalAccess() {
@Override public void promoteNameToValue(JsonReader reader) throws IOException {
- reader.quickPeek();
+ reader.peek();
if (reader.token != JsonToken.NAME) {
throw new IllegalStateException("Expected a name but was " + reader.peek());
}
diff --git a/gson/src/test/java/com/google/gson/stream/JsonReaderTest.java b/gson/src/test/java/com/google/gson/stream/JsonReaderTest.java
index fd296657..22242fa5 100644
--- a/gson/src/test/java/com/google/gson/stream/JsonReaderTest.java
+++ b/gson/src/test/java/com/google/gson/stream/JsonReaderTest.java
@@ -201,7 +201,7 @@ public final class JsonReaderTest extends TestCase {
try {
reader.nextDouble();
fail();
- } catch (NumberFormatException expected) {
+ } catch (MalformedJsonException expected) {
}
}
@@ -265,20 +265,25 @@ public final class JsonReaderTest extends TestCase {
String json = "[01]";
JsonReader reader = new JsonReader(new StringReader(json));
reader.beginArray();
+ try {
+ reader.peek();
+ fail();
+ } catch (MalformedJsonException expected) {
+ }
try {
reader.nextInt();
fail();
- } catch (NumberFormatException expected) {
+ } catch (MalformedJsonException expected) {
}
try {
reader.nextLong();
fail();
- } catch (NumberFormatException expected) {
+ } catch (MalformedJsonException expected) {
}
try {
reader.nextDouble();
fail();
- } catch (NumberFormatException expected) {
+ } catch (MalformedJsonException expected) {
}
assertEquals("01", reader.nextString());
reader.endArray();