Make Object and JsonElement deserialization iterative (#1912)

* Make Object and JsonElement deserialization iterative

Often when Object and JsonElement are deserialized the format of the JSON
data is unknown and it might come from an untrusted source. To avoid a
StackOverflowError from maliciously crafted JSON, deserialize Object and
JsonElement iteratively instead of recursively.

Concept based on 51fd2faab7
But implementation is not based on it.

* Improve imports grouping

* Address review feedback
This commit is contained in:
Marcono1234 2022-06-23 02:42:19 +02:00 committed by GitHub
parent d2aee6502b
commit 2d01d6a20f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 381 additions and 91 deletions

View File

@ -17,8 +17,8 @@
package com.google.gson.internal.bind;
import com.google.gson.Gson;
import com.google.gson.ToNumberStrategy;
import com.google.gson.ToNumberPolicy;
import com.google.gson.ToNumberStrategy;
import com.google.gson.TypeAdapter;
import com.google.gson.TypeAdapterFactory;
import com.google.gson.internal.LinkedTreeMap;
@ -26,9 +26,10 @@ import com.google.gson.reflect.TypeToken;
import com.google.gson.stream.JsonReader;
import com.google.gson.stream.JsonToken;
import com.google.gson.stream.JsonWriter;
import java.io.IOException;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Deque;
import java.util.List;
import java.util.Map;
@ -70,42 +71,98 @@ public final class ObjectTypeAdapter extends TypeAdapter<Object> {
}
}
/**
* Tries to begin reading a JSON array or JSON object, returning {@code null} if
* the next element is neither of those.
*/
private Object tryBeginNesting(JsonReader in, JsonToken peeked) throws IOException {
switch (peeked) {
case BEGIN_ARRAY:
in.beginArray();
return new ArrayList<>();
case BEGIN_OBJECT:
in.beginObject();
return new LinkedTreeMap<>();
default:
return null;
}
}
/** Reads an {@code Object} which cannot have any nested elements */
private Object readTerminal(JsonReader in, JsonToken peeked) throws IOException {
switch (peeked) {
case STRING:
return in.nextString();
case NUMBER:
return toNumberStrategy.readNumber(in);
case BOOLEAN:
return in.nextBoolean();
case NULL:
in.nextNull();
return null;
default:
// When read(JsonReader) is called with JsonReader in invalid state
throw new IllegalStateException("Unexpected token: " + peeked);
}
}
@Override public Object read(JsonReader in) throws IOException {
JsonToken token = in.peek();
switch (token) {
case BEGIN_ARRAY:
List<Object> list = new ArrayList<>();
in.beginArray();
// Either List or Map
Object current;
JsonToken peeked = in.peek();
current = tryBeginNesting(in, peeked);
if (current == null) {
return readTerminal(in, peeked);
}
Deque<Object> stack = new ArrayDeque<>();
while (true) {
while (in.hasNext()) {
list.add(read(in));
String name = null;
// Name is only used for JSON object members
if (current instanceof Map) {
name = in.nextName();
}
peeked = in.peek();
Object value = tryBeginNesting(in, peeked);
boolean isNesting = value != null;
if (value == null) {
value = readTerminal(in, peeked);
}
if (current instanceof List) {
@SuppressWarnings("unchecked")
List<Object> list = (List<Object>) current;
list.add(value);
} else {
@SuppressWarnings("unchecked")
Map<String, Object> map = (Map<String, Object>) current;
map.put(name, value);
}
if (isNesting) {
stack.addLast(current);
current = value;
}
}
in.endArray();
return list;
case BEGIN_OBJECT:
Map<String, Object> map = new LinkedTreeMap<>();
in.beginObject();
while (in.hasNext()) {
map.put(in.nextName(), read(in));
// End current element
if (current instanceof List) {
in.endArray();
} else {
in.endObject();
}
in.endObject();
return map;
case STRING:
return in.nextString();
case NUMBER:
return toNumberStrategy.readNumber(in);
case BOOLEAN:
return in.nextBoolean();
case NULL:
in.nextNull();
return null;
default:
throw new IllegalStateException();
if (stack.isEmpty()) {
return current;
} else {
// Continue with enclosing element
current = stack.removeLast();
}
}
}

View File

@ -16,32 +16,6 @@
package com.google.gson.internal.bind;
import java.io.IOException;
import java.lang.reflect.AccessibleObject;
import java.lang.reflect.Field;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.net.InetAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.security.AccessController;
import java.security.PrivilegedAction;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.Calendar;
import java.util.Currency;
import java.util.GregorianCalendar;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.StringTokenizer;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicIntegerArray;
import com.google.gson.Gson;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
@ -58,6 +32,33 @@ import com.google.gson.reflect.TypeToken;
import com.google.gson.stream.JsonReader;
import com.google.gson.stream.JsonToken;
import com.google.gson.stream.JsonWriter;
import java.io.IOException;
import java.lang.reflect.AccessibleObject;
import java.lang.reflect.Field;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.net.InetAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.security.AccessController;
import java.security.PrivilegedAction;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.Calendar;
import java.util.Currency;
import java.util.Deque;
import java.util.GregorianCalendar;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.StringTokenizer;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicIntegerArray;
/**
* Type adapters for basic types.
@ -695,44 +696,99 @@ public final class TypeAdapters {
public static final TypeAdapterFactory LOCALE_FACTORY = newFactory(Locale.class, LOCALE);
public static final TypeAdapter<JsonElement> JSON_ELEMENT = new TypeAdapter<JsonElement>() {
/**
* Tries to begin reading a JSON array or JSON object, returning {@code null} if
* the next element is neither of those.
*/
private JsonElement tryBeginNesting(JsonReader in, JsonToken peeked) throws IOException {
switch (peeked) {
case BEGIN_ARRAY:
in.beginArray();
return new JsonArray();
case BEGIN_OBJECT:
in.beginObject();
return new JsonObject();
default:
return null;
}
}
/** Reads a {@link JsonElement} which cannot have any nested elements */
private JsonElement readTerminal(JsonReader in, JsonToken peeked) throws IOException {
switch (peeked) {
case STRING:
return new JsonPrimitive(in.nextString());
case NUMBER:
String number = in.nextString();
return new JsonPrimitive(new LazilyParsedNumber(number));
case BOOLEAN:
return new JsonPrimitive(in.nextBoolean());
case NULL:
in.nextNull();
return JsonNull.INSTANCE;
default:
// When read(JsonReader) is called with JsonReader in invalid state
throw new IllegalStateException("Unexpected token: " + peeked);
}
}
@Override public JsonElement read(JsonReader in) throws IOException {
if (in instanceof JsonTreeReader) {
return ((JsonTreeReader) in).nextJsonElement();
}
switch (in.peek()) {
case STRING:
return new JsonPrimitive(in.nextString());
case NUMBER:
String number = in.nextString();
return new JsonPrimitive(new LazilyParsedNumber(number));
case BOOLEAN:
return new JsonPrimitive(in.nextBoolean());
case NULL:
in.nextNull();
return JsonNull.INSTANCE;
case BEGIN_ARRAY:
JsonArray array = new JsonArray();
in.beginArray();
// Either JsonArray or JsonObject
JsonElement current;
JsonToken peeked = in.peek();
current = tryBeginNesting(in, peeked);
if (current == null) {
return readTerminal(in, peeked);
}
Deque<JsonElement> stack = new ArrayDeque<>();
while (true) {
while (in.hasNext()) {
array.add(read(in));
String name = null;
// Name is only used for JSON object members
if (current instanceof JsonObject) {
name = in.nextName();
}
peeked = in.peek();
JsonElement value = tryBeginNesting(in, peeked);
boolean isNesting = value != null;
if (value == null) {
value = readTerminal(in, peeked);
}
if (current instanceof JsonArray) {
((JsonArray) current).add(value);
} else {
((JsonObject) current).add(name, value);
}
if (isNesting) {
stack.addLast(current);
current = value;
}
}
in.endArray();
return array;
case BEGIN_OBJECT:
JsonObject object = new JsonObject();
in.beginObject();
while (in.hasNext()) {
object.add(in.nextName(), read(in));
// End current element
if (current instanceof JsonArray) {
in.endArray();
} else {
in.endObject();
}
if (stack.isEmpty()) {
return current;
} else {
// Continue with enclosing element
current = stack.removeLast();
}
in.endObject();
return object;
case END_DOCUMENT:
case NAME:
case END_OBJECT:
case END_ARRAY:
default:
throw new IllegalArgumentException();
}
}
@ -803,7 +859,7 @@ public final class TypeAdapters {
T constant = (T)(constantField.get(null));
String name = constant.name();
String toStringVal = constant.toString();
SerializedName annotation = constantField.getAnnotation(SerializedName.class);
if (annotation != null) {
name = annotation.value();

View File

@ -0,0 +1,41 @@
package com.google.gson;
import static org.junit.Assert.assertEquals;
import java.io.IOException;
import java.util.Arrays;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameter;
import org.junit.runners.Parameterized.Parameters;
@RunWith(Parameterized.class)
public class JsonParserParameterizedTest {
@Parameters
public static Iterable<String> data() {
return Arrays.asList(
"[]",
"{}",
"null",
"1.0",
"true",
"\"string\"",
"[true,1.0,null,{},2.0,{\"a\":[false]},[3.0,\"test\"],4.0]",
"{\"\":1.0,\"a\":true,\"b\":null,\"c\":[],\"d\":{\"a1\":2.0,\"b2\":[true,{\"a3\":3.0}]},\"e\":[{\"f\":4.0},\"test\"]}"
);
}
private final TypeAdapter<JsonElement> adapter = new Gson().getAdapter(JsonElement.class);
@Parameter
public String json;
@Test
public void testParse() throws IOException {
JsonElement deserialized = JsonParser.parseString(json);
String actualSerialized = adapter.toJson(deserialized);
// Serialized JsonElement should be the same as original JSON
assertEquals(json, actualSerialized);
}
}

View File

@ -18,8 +18,8 @@ package com.google.gson;
import java.io.CharArrayReader;
import java.io.CharArrayWriter;
import java.io.IOException;
import java.io.StringReader;
import junit.framework.TestCase;
import com.google.gson.common.TestTypes.BagOfPrimitives;
@ -90,6 +90,54 @@ public class JsonParserTest extends TestCase {
assertEquals("stringValue", array.get(2).getAsString());
}
private static String repeat(String s, int times) {
StringBuilder stringBuilder = new StringBuilder(s.length() * times);
for (int i = 0; i < times; i++) {
stringBuilder.append(s);
}
return stringBuilder.toString();
}
/** Deeply nested JSON arrays should not cause {@link StackOverflowError} */
public void testParseDeeplyNestedArrays() throws IOException {
int times = 10000;
// [[[ ... ]]]
String json = repeat("[", times) + repeat("]", times);
int actualTimes = 0;
JsonArray current = JsonParser.parseString(json).getAsJsonArray();
while (true) {
actualTimes++;
if (current.isEmpty()) {
break;
}
assertEquals(1, current.size());
current = current.get(0).getAsJsonArray();
}
assertEquals(times, actualTimes);
}
/** Deeply nested JSON objects should not cause {@link StackOverflowError} */
public void testParseDeeplyNestedObjects() throws IOException {
int times = 10000;
// {"a":{"a": ... {"a":null} ... }}
String json = repeat("{\"a\":", times) + "null" + repeat("}", times);
int actualTimes = 0;
JsonObject current = JsonParser.parseString(json).getAsJsonObject();
while (true) {
assertEquals(1, current.size());
actualTimes++;
JsonElement next = current.get("a");
if (next.isJsonNull()) {
break;
} else {
current = next.getAsJsonObject();
}
}
assertEquals(times, actualTimes);
}
public void testParseReader() {
StringReader reader = new StringReader("{a:10,b:'c'}");
JsonElement e = JsonParser.parseReader(reader);

View File

@ -0,0 +1,41 @@
package com.google.gson;
import static org.junit.Assert.assertEquals;
import java.io.IOException;
import java.util.Arrays;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameter;
import org.junit.runners.Parameterized.Parameters;
@RunWith(Parameterized.class)
public class ObjectTypeAdapterParameterizedTest {
@Parameters
public static Iterable<String> data() {
return Arrays.asList(
"[]",
"{}",
"null",
"1.0",
"true",
"\"string\"",
"[true,1.0,null,{},2.0,{\"a\":[false]},[3.0,\"test\"],4.0]",
"{\"\":1.0,\"a\":true,\"b\":null,\"c\":[],\"d\":{\"a1\":2.0,\"b2\":[true,{\"a3\":3.0}]},\"e\":[{\"f\":4.0},\"test\"]}"
);
}
private final TypeAdapter<Object> adapter = new Gson().getAdapter(Object.class);
@Parameter
public String json;
@Test
public void testReadWrite() throws IOException {
Object deserialized = adapter.fromJson(json);
String actualSerialized = adapter.toJson(deserialized);
// Serialized Object should be the same as original JSON
assertEquals(json, actualSerialized);
}
}

View File

@ -16,9 +16,11 @@
package com.google.gson;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import junit.framework.TestCase;
@ -38,7 +40,7 @@ public final class ObjectTypeAdapterTest extends TestCase {
Object object = new RuntimeType();
assertEquals("{'a':5,'b':[1,2,null]}", adapter.toJson(object).replace("\"", "'"));
}
public void testSerializeNullValue() throws Exception {
Map<String, Object> map = new LinkedHashMap<>();
map.put("a", null);
@ -55,6 +57,51 @@ public final class ObjectTypeAdapterTest extends TestCase {
assertEquals("{}", adapter.toJson(new Object()));
}
private static String repeat(String s, int times) {
StringBuilder stringBuilder = new StringBuilder(s.length() * times);
for (int i = 0; i < times; i++) {
stringBuilder.append(s);
}
return stringBuilder.toString();
}
/** Deeply nested JSON arrays should not cause {@link StackOverflowError} */
@SuppressWarnings("unchecked")
public void testDeserializeDeeplyNestedArrays() throws IOException {
int times = 10000;
// [[[ ... ]]]
String json = repeat("[", times) + repeat("]", times);
int actualTimes = 0;
List<List<?>> current = (List<List<?>>) adapter.fromJson(json);
while (true) {
actualTimes++;
if (current.isEmpty()) {
break;
}
assertEquals(1, current.size());
current = (List<List<?>>) current.get(0);
}
assertEquals(times, actualTimes);
}
/** Deeply nested JSON objects should not cause {@link StackOverflowError} */
@SuppressWarnings("unchecked")
public void testDeserializeDeeplyNestedObjects() throws IOException {
int times = 10000;
// {"a":{"a": ... {"a":null} ... }}
String json = repeat("{\"a\":", times) + "null" + repeat("}", times);
int actualTimes = 0;
Map<String, Map<?, ?>> current = (Map<String, Map<?, ?>>) adapter.fromJson(json);
while (current != null) {
assertEquals(1, current.size());
actualTimes++;
current = (Map<String, Map<?, ?>>) current.get("a");
}
assertEquals(times, actualTimes);
}
@SuppressWarnings("unused")
private class RuntimeType {
Object a = 5;