Johannes/gson-comments
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Decode JSON literal types eagerly and with our own decoder. Previously we relied on Double.parseDouble() to decode tokens. Since that method is expensive, we deferred calling it unless absolutely necessary. Now we decode the literal type immediately. For efficiency we decode the token right out of the char buffer. This makes things more complicated but it saves many calls to charAt(). It also opens up the possibility to deferring string creation.

Browse Source
This commit is contained in:
Jesse Wilson 2011-11-27 16:50:45 +00:00
parent 852cd72059
commit ec42d600af
2 changed files with 203 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

320
gson/src/main/java/com/google/gson/stream/JsonReader.java
View File

@ -33,8 +33,8 @@ import java.util.List;
* Within JSON objects, name/value pairs are represented by a single token.
*
* <h3>Parsing JSON</h3>
* To create a recursive descent parser your own JSON streams, first create an
* entry point method that creates a {@code JsonReader}.
* To create a recursive descent parser for your own JSON streams, first create
* an entry point method that creates a {@code JsonReader}.
*
* <p>Next, create handler methods for each structure in your JSON text. You'll
* need a method for each object type and for each array type.
@ -87,7 +87,11 @@ import java.util.List;
*
* public List<Message> readJsonStream(InputStream in) throws IOException {
* JsonReader reader = new JsonReader(new InputStreamReader(in, "UTF-8"));
* return readMessagesArray(reader);
* try {
* return readMessagesArray(reader);
* } finally {
* reader.close();
* }
* }
*
* public List<Message> readMessagesArray(JsonReader reader) throws IOException {
@ -189,6 +193,9 @@ public class JsonReader implements Closeable {
/** The only non-execute prefix this parser permits */
private static final char[] NON_EXECUTE_PREFIX = ")]}'\n".toCharArray();
private static final String TRUE = "true";
private static final String FALSE = "false";
private final StringPool stringPool = new StringPool();
/** The input JSON. */
@ -200,6 +207,8 @@ public class JsonReader implements Closeable {
/**
* Use a manual buffer to easily read and unread upcoming characters, and
* also so we can create strings without an intermediate StringBuilder.
* We decode literals directly out of this buffer, so it must be at least as
* long as the longest token that can be reported as a number.
*/
private final char[] buffer = new char[1024];
private int pos = 0;
@ -216,27 +225,22 @@ public class JsonReader implements Closeable {
push(JsonScope.EMPTY_DOCUMENT);
}
/**
* True if we've already read the next token. If we have, the string value
* for that token will be assigned to {@code value} if such a string value
* exists. And the token type will be assigned to {@code token} if the token
* type is known. The token type may be null for literals, since we derive
* that lazily.
*/
private boolean hasToken;
/**
* The type of the next token to be returned by {@link #peek} and {@link
* #advance}, or {@code null} if it is unknown and must first be derived
* from {@code value}. This value is undefined if {@code hasToken} is false.
* #advance}. If null, peek() will assign a value.
*/
private JsonToken token;
/** The text of the next name. */
private String name;
/** The text of the next literal value. */
/*
* For the next literal value, we may have the text value, or the position
* and length in the buffer.
*/
private String value;
private int valuePos;
private int valueLength;
/** True if we're currently handling a skipValue() call. */
private boolean skipping = false;
@ -327,7 +331,7 @@ public class JsonReader implements Closeable {
* Consumes {@code expected}.
*/
private void expect(JsonToken expected) throws IOException {
quickPeek();
peek();
if (token != expected) {
throw new IllegalStateException("Expected " + expected + " but was " + peek());
}
@ -338,7 +342,7 @@ public class JsonReader implements Closeable {
* Returns true if the current array or object has another element.
*/
public boolean hasNext() throws IOException {
quickPeek();
peek();
return token != JsonToken.END_OBJECT && token != JsonToken.END_ARRAY;
}
@ -346,22 +350,7 @@ public class JsonReader implements Closeable {
* Returns the type of the next token without consuming it.
*/
public JsonToken peek() throws IOException {
quickPeek();
if (token == null) {
decodeLiteral();
}
return token;
}
/**
* Ensures that a token is ready. After this call either {@code token} or
* {@code value} will be non-null. To ensure {@code token} has a definitive
* value, use {@link #peek()}
*/
private JsonToken quickPeek() throws IOException {
if (hasToken) {
if (token != null) {
return token;
}
@ -372,8 +361,9 @@ public class JsonReader implements Closeable {
}
replaceTop(JsonScope.NONEMPTY_DOCUMENT);
JsonToken firstToken = nextValue();
if (!lenient && firstToken != JsonToken.BEGIN_ARRAY && firstToken != JsonToken.BEGIN_OBJECT) {
syntaxError("Expected JSON document to start with '[' or '{'");
if (!lenient && token != JsonToken.BEGIN_ARRAY && token != JsonToken.BEGIN_OBJECT) {
throw new IOException(
"Expected JSON document to start with '[' or '{' but was " + token);
}
return firstToken;
case EMPTY_ARRAY:
@ -394,8 +384,7 @@ public class JsonReader implements Closeable {
}
throw syntaxError("Expected EOF");
} catch (EOFException e) {
hasToken = true; // TODO: avoid throwing here?
return token = JsonToken.END_DOCUMENT;
return token = JsonToken.END_DOCUMENT; // TODO: avoid throwing here?
}
case CLOSED:
throw new IllegalStateException("JsonReader is closed");
@ -430,10 +419,9 @@ public class JsonReader implements Closeable {
* Advances the cursor in the JSON stream to the next token.
*/
private JsonToken advance() throws IOException {
quickPeek();
peek();
JsonToken result = token;
hasToken = false;
token = null;
value = null;
name = null;
@ -448,7 +436,7 @@ public class JsonReader implements Closeable {
* name.
*/
public String nextName() throws IOException {
quickPeek();
peek();
if (token != JsonToken.NAME) {
throw new IllegalStateException("Expected a name but was " + peek());
}
@ -467,7 +455,7 @@ public class JsonReader implements Closeable {
*/
public String nextString() throws IOException {
peek();
if (value == null || (token != JsonToken.STRING && token != JsonToken.NUMBER)) {
if (token != JsonToken.STRING && token != JsonToken.NUMBER) {
throw new IllegalStateException("Expected a string but was " + peek());
}
@ -484,20 +472,12 @@ public class JsonReader implements Closeable {
* this reader is closed.
*/
public boolean nextBoolean() throws IOException {
quickPeek();
if (value == null || token == JsonToken.STRING) {
throw new IllegalStateException("Expected a boolean but was " + peek());
}
boolean result;
if (value.equalsIgnoreCase("true")) {
result = true;
} else if (value.equalsIgnoreCase("false")) {
result = false;
} else {
throw new IllegalStateException("Not a boolean: " + value);
peek();
if (token != JsonToken.BOOLEAN) {
throw new IllegalStateException("Expected a boolean but was " + token);
}
boolean result = (value == TRUE);
advance();
return result;
}
@ -510,13 +490,9 @@ public class JsonReader implements Closeable {
* reader is closed.
*/
public void nextNull() throws IOException {
quickPeek();
if (value == null || token == JsonToken.STRING) {
throw new IllegalStateException("Expected null but was " + peek());
}
if (!value.equalsIgnoreCase("null")) {
throw new IllegalStateException("Not a null: " + value);
peek();
if (token != JsonToken.NULL) {
throw new IllegalStateException("Expected null but was " + token);
}
advance();
@ -525,26 +501,25 @@ public class JsonReader implements Closeable {
/**
* Returns the {@link JsonToken#NUMBER double} value of the next token,
* consuming it. If the next token is a string, this method will attempt to
* parse it as a double.
* parse it as a double using {@link Double#parseDouble(String)}.
*
* @throws IllegalStateException if the next token is not a literal value.
* @throws NumberFormatException if the next literal value cannot be parsed
* as a double, or is non-finite.
*/
public double nextDouble() throws IOException {
quickPeek();
if (value == null) {
throw new IllegalStateException("Expected a double but was " + peek());
peek();
if (token != JsonToken.STRING && token != JsonToken.NUMBER) {
throw new IllegalStateException("Expected a double but was " + token);
}
double result = Double.parseDouble(value);
if ((result >= 1.0d && value.startsWith("0"))) {
throw new NumberFormatException("JSON forbids octal prefixes: " + value);
throw new MalformedJsonException("JSON forbids octal prefixes: " + value);
}
if (!lenient && (Double.isNaN(result) || Double.isInfinite(result))) {
throw new NumberFormatException("JSON forbids NaN and infinities: " + value);
throw new MalformedJsonException("JSON forbids NaN and infinities: " + value);
}
advance();
@ -562,9 +537,9 @@ public class JsonReader implements Closeable {
* as a number, or exactly represented as a long.
*/
public long nextLong() throws IOException {
quickPeek();
if (value == null) {
throw new IllegalStateException("Expected a long but was " + peek());
peek();
if (token != JsonToken.STRING && token != JsonToken.NUMBER) {
throw new IllegalStateException("Expected a long but was " + token);
}
long result;
@ -579,7 +554,7 @@ public class JsonReader implements Closeable {
}
if (result >= 1L && value.startsWith("0")) {
throw new NumberFormatException("JSON forbids octal prefixes: " + value);
throw new MalformedJsonException("JSON forbids octal prefixes: " + value);
}
advance();
@ -597,9 +572,9 @@ public class JsonReader implements Closeable {
* as a number, or exactly represented as an int.
*/
public int nextInt() throws IOException {
quickPeek();
if (value == null) {
throw new IllegalStateException("Expected an int but was " + peek());
peek();
if (token != JsonToken.STRING && token != JsonToken.NUMBER) {
throw new IllegalStateException("Expected an int but was " + token);
}
int result;
@ -614,7 +589,7 @@ public class JsonReader implements Closeable {
}
if (result >= 1L && value.startsWith("0")) {
throw new NumberFormatException("JSON forbids octal prefixes: " + value);
throw new MalformedJsonException("JSON forbids octal prefixes: " + value);
}
advance();
@ -625,7 +600,6 @@ public class JsonReader implements Closeable {
* Closes this JSON reader and the underlying {@link Reader}.
*/
public void close() throws IOException {
hasToken = false;
value = null;
token = null;
stack.clear();
@ -683,7 +657,6 @@ public class JsonReader implements Closeable {
switch (nextNonWhitespace()) {
case ']':
pop();
hasToken = true;
return token = JsonToken.END_ARRAY;
case ';':
checkLenient(); // fall-through
@ -698,7 +671,6 @@ public class JsonReader implements Closeable {
case ']':
if (firstElement) {
pop();
hasToken = true;
return token = JsonToken.END_ARRAY;
}
// fall-through to handle ",]"
@ -707,7 +679,6 @@ public class JsonReader implements Closeable {
/* In lenient mode, a 0-length literal means 'null' */
checkLenient();
pos--;
hasToken = true;
value = "null";
return token = JsonToken.NULL;
default:
@ -728,7 +699,6 @@ public class JsonReader implements Closeable {
switch (nextNonWhitespace()) {
case '}':
pop();
hasToken = true;
return token = JsonToken.END_OBJECT;
default:
pos--;
@ -737,7 +707,6 @@ public class JsonReader implements Closeable {
switch (nextNonWhitespace()) {
case '}':
pop();
hasToken = true;
return token = JsonToken.END_OBJECT;
case ';':
case ',':
@ -758,14 +727,13 @@ public class JsonReader implements Closeable {
default:
checkLenient();
pos--;
name = nextLiteral();
name = nextLiteral(false);
if (name.length() == 0) {
throw syntaxError("Expected name");
}
}
replaceTop(JsonScope.DANGLING_NAME);
hasToken = true;
return token = JsonToken.NAME;
}
@ -797,19 +765,16 @@ public class JsonReader implements Closeable {
switch (c) {
case '{':
push(JsonScope.EMPTY_OBJECT);
hasToken = true;
return token = JsonToken.BEGIN_OBJECT;
case '[':
push(JsonScope.EMPTY_ARRAY);
hasToken = true;
return token = JsonToken.BEGIN_ARRAY;
case '\'':
checkLenient(); // fall-through
case '"':
value = nextString((char) c);
hasToken = true;
return token = JsonToken.STRING;
default:
@ -1016,25 +981,30 @@ public class JsonReader implements Closeable {
}
/**
* Returns the string up to but not including any delimiter characters. This
* Reads the value up to but not including any delimiter characters. This
* does not consume the delimiter character.
*
* @param assignOffsetsOnly true for this method to only set the valuePos
* and valueLength fields and return a null result. This only works if
* the literal is short; a string is returned otherwise.
*/
@SuppressWarnings("fallthrough")
private String nextLiteral() throws IOException {
private String nextLiteral(boolean assignOffsetsOnly) throws IOException {
StringBuilder builder = null;
do {
/* the index of the first character not yet appended to the builder. */
int start = pos;
while (pos < limit) {
int c = buffer[pos++];
switch (c) {
valuePos = -1;
valueLength = 0;
int i = 0;
findNonLiteralCharacter:
while (true) {
for (; pos + i < limit; i++) {
switch (buffer[pos + i]) {
case '/':
case '\\':
case ';':
case '#':
case '=':
checkLenient(); // fall-through
checkLenient(); // fall-through
case '{':
case '}':
case '[':
@ -1046,25 +1016,52 @@ public class JsonReader implements Closeable {
case '\f':
case '\r':
case '\n':
pos--;
if (skipping) {
return "skipped!";
} else if (builder == null) {
return stringPool.get(buffer, start, pos - start);
} else {
builder.append(buffer, start, pos - start);
return builder.toString();
}
break findNonLiteralCharacter;
}
}
/*
* Attempt to load the entire literal into the buffer at once. If
* we run out of input, add a non-literal character at the end so
* that decoding doesn't need to do bounds checks.
*/
if (i < buffer.length) {
if (fillBuffer(i + 1)) {
continue;
} else {
buffer[limit] = '\0';
break;
}
}
// use a StringBuilder when the value is too long. It must be an unquoted string.
if (builder == null) {
builder = new StringBuilder();
}
builder.append(buffer, start, pos - start);
} while (fillBuffer(1));
builder.append(buffer, pos, i);
valueLength += i;
pos += i;
i = 0;
if (!fillBuffer(1)) {
break;
}
}
return builder.toString();
String result;
if (assignOffsetsOnly && builder == null) {
valuePos = pos;
result = null;
} else if (skipping) {
result = "skipped!";
} else if (builder == null) {
result = stringPool.get(buffer, pos, i);
} else {
builder.append(buffer, pos, i);
result = builder.toString();
}
valueLength += i;
pos += i;
return result;
}
@Override public String toString() {
@ -1122,32 +1119,103 @@ public class JsonReader implements Closeable {
* Reads a null, boolean, numeric or unquoted string literal value.
*/
private JsonToken readLiteral() throws IOException {
String literal = nextLiteral();
if (literal.length() == 0) {
value = nextLiteral(true);
if (valueLength == 0) {
throw syntaxError("Expected literal value");
}
value = literal;
hasToken = true;
return token = null; // use decodeLiteral() to get the token type
token = decodeLiteral();
if (token == JsonToken.STRING) {
checkLenient();
}
return token;
}
/**
* Assigns {@code nextToken} based on the value of {@code nextValue}.
*/
private void decodeLiteral() throws IOException {
if (value.equalsIgnoreCase("null")) {
token = JsonToken.NULL;
} else if (value.equalsIgnoreCase("true") || value.equalsIgnoreCase("false")) {
token = JsonToken.BOOLEAN;
private JsonToken decodeLiteral() throws IOException {
if (valuePos == -1) {
// it was too long to fit in the buffer so it can only be a string
return JsonToken.STRING;
} else if (valueLength == 4
&& ('n' == buffer[valuePos ] || 'N' == buffer[valuePos ])
&& ('u' == buffer[valuePos + 1] || 'U' == buffer[valuePos + 1])
&& ('l' == buffer[valuePos + 2] || 'L' == buffer[valuePos + 2])
&& ('l' == buffer[valuePos + 3] || 'L' == buffer[valuePos + 3])) {
value = "null";
return JsonToken.NULL;
} else if (valueLength == 4
&& ('t' == buffer[valuePos ] || 'T' == buffer[valuePos ])
&& ('r' == buffer[valuePos + 1] || 'R' == buffer[valuePos + 1])
&& ('u' == buffer[valuePos + 2] || 'U' == buffer[valuePos + 2])
&& ('e' == buffer[valuePos + 3] || 'E' == buffer[valuePos + 3])) {
value = TRUE;
return JsonToken.BOOLEAN;
} else if (valueLength == 5
&& ('f' == buffer[valuePos ] || 'F' == buffer[valuePos ])
&& ('a' == buffer[valuePos + 1] || 'A' == buffer[valuePos + 1])
&& ('l' == buffer[valuePos + 2] || 'L' == buffer[valuePos + 2])
&& ('s' == buffer[valuePos + 3] || 'S' == buffer[valuePos + 3])
&& ('e' == buffer[valuePos + 4] || 'E' == buffer[valuePos + 4])) {
value = FALSE;
return JsonToken.BOOLEAN;
} else {
try {
Double.parseDouble(value); // this work could potentially be cached
token = JsonToken.NUMBER;
} catch (NumberFormatException ignored) {
// this must be an unquoted string
checkLenient();
token = JsonToken.STRING;
value = stringPool.get(buffer, valuePos, valueLength);
return decodeNumber(buffer, valuePos, valueLength);
}
}
/**
* Determine whether the characters is a JSON number. Numbers are of the
* form -12.34e+56. Fractional and exponential parts are optional. Leading
* zeroes are not allowed in the value or exponential part, but are allowed
* in the fraction.
*/
private JsonToken decodeNumber(char[] chars, int offset, int length) {
int i = offset;
int c = chars[i];
if (c == '-') {
c = chars[++i];
}
if (c == '0') {
c = chars[++i];
} else if (c >= '1' && c <= '9') {
c = chars[++i];
while (c >= '0' && c <= '9') {
c = chars[++i];
}
} else {
return JsonToken.STRING;
}
if (c == '.') {
c = chars[++i];
while (c >= '0' && c <= '9') {
c = chars[++i];
}
}
if (c == 'e' || c == 'E') {
c = chars[++i];
if (c == '+' || c == '-') {
c = chars[++i];
}
if (c >= '0' && c <= '9') {
c = chars[++i];
while (c >= '0' && c <= '9') {
c = chars[++i];
}
} else {
return JsonToken.STRING;
}
}
if (i == offset + length) {
return JsonToken.NUMBER;
} else {
return JsonToken.STRING;
}
}
@ -1172,7 +1240,7 @@ public class JsonReader implements Closeable {
static {
JsonReaderInternalAccess.INSTANCE = new JsonReaderInternalAccess() {
@Override public void promoteNameToValue(JsonReader reader) throws IOException {
reader.quickPeek();
reader.peek();
if (reader.token != JsonToken.NAME) {
throw new IllegalStateException("Expected a name but was " + reader.peek());
}

13
gson/src/test/java/com/google/gson/stream/JsonReaderTest.java
View File

@ -201,7 +201,7 @@ public final class JsonReaderTest extends TestCase {
try {
reader.nextDouble();
fail();
} catch (NumberFormatException expected) {
} catch (MalformedJsonException expected) {
}
}
@ -265,20 +265,25 @@ public final class JsonReaderTest extends TestCase {
String json = "[01]";
JsonReader reader = new JsonReader(new StringReader(json));
reader.beginArray();
try {
reader.peek();
fail();
} catch (MalformedJsonException expected) {
}
try {
reader.nextInt();
fail();
} catch (NumberFormatException expected) {
} catch (MalformedJsonException expected) {
}
try {
reader.nextLong();
fail();
} catch (NumberFormatException expected) {
} catch (MalformedJsonException expected) {
}
try {
reader.nextDouble();
fail();
} catch (NumberFormatException expected) {
} catch (MalformedJsonException expected) {
}
assertEquals("01", reader.nextString());
reader.endArray();