Johannes
/
gson-comments
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Updated JsonParser to use Token to match strings instead of productions. This 

enables Gson to handle much larger strings (~10s of MB) than previously
possible (<100kb). This also reduces memory and stack requirements, and
increases Gson performance as well.
This commit is contained in:
Inderjeet Singh 2008-10-10 21:52:02 +00:00
parent 1abf693b70
commit d74ecbfe82
9 changed files with 506 additions and 176 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
gson/src/main/java/com/google/gson/Escaper.java
View File

@ -36,7 +36,7 @@ import java.util.Set;
*/
class Escaper {
private static final char[] HEX_CHARS = {
static final char[] HEX_CHARS = {
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'
};

139
gson/src/main/java/com/google/gson/JsonParser.java
View File

@ -18,21 +18,21 @@ final class JsonParser implements JsonParserConstants {
final public JsonElement parse() throws ParseException {
JsonElement json = null;
switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
case 17:
case 20:
json = JsonObject();
break;
case 22:
case 25:
json = JsonArray();
break;
case DIGITS:
case SINGLE_QUOTE_LITERAL:
case QUOTE:
case 24:
case 25:
case 26:
case DOUBLE_QUOTE_LITERAL:
case 27:
case 28:
case 29:
json = JsonPrimitive();
break;
case 19:
case 22:
json = JsonNull();
break;
default:
@ -46,24 +46,24 @@ final class JsonParser implements JsonParserConstants {
final private JsonObject JsonObject() throws ParseException {
JsonObject o = new JsonObject();
jj_consume_token(17);
jj_consume_token(20);
switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
case SINGLE_QUOTE_LITERAL:
case QUOTE:
case DOUBLE_QUOTE_LITERAL:
Members(o);
break;
default:
jj_la1[1] = jj_gen;
;
}
jj_consume_token(18);
jj_consume_token(21);
{if (true) return o;}
throw new Error("Missing return statement in function");
}
final private JsonNull JsonNull() throws ParseException {
JsonNull json = new JsonNull();
jj_consume_token(19);
jj_consume_token(22);
{if (true) return json;}
throw new Error("Missing return statement in function");
}
@ -71,8 +71,8 @@ final class JsonParser implements JsonParserConstants {
final private void Members(JsonObject o) throws ParseException {
Pair(o);
switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
case 20:
jj_consume_token(20);
case 23:
jj_consume_token(23);
Members(o);
break;
default:
@ -85,31 +85,31 @@ final class JsonParser implements JsonParserConstants {
JsonPrimitive property;
JsonElement value;
property = JsonString();
jj_consume_token(21);
jj_consume_token(24);
value = JsonValue();
o.add(property.getAsString(), value);
}
final private JsonArray JsonArray() throws ParseException {
JsonArray array = new JsonArray();
jj_consume_token(22);
jj_consume_token(25);
switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
case DIGITS:
case SINGLE_QUOTE_LITERAL:
case QUOTE:
case 17:
case 19:
case DOUBLE_QUOTE_LITERAL:
case 20:
case 22:
case 24:
case 25:
case 26:
case 27:
case 28:
case 29:
Elements(array);
break;
default:
jj_la1[3] = jj_gen;
;
}
jj_consume_token(23);
jj_consume_token(26);
array.reverse();
{if (true) return array;}
throw new Error("Missing return statement in function");
@ -119,8 +119,8 @@ final class JsonParser implements JsonParserConstants {
JsonElement element;
element = JsonValue();
switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
case 20:
jj_consume_token(20);
case 23:
jj_consume_token(23);
Elements(array);
break;
default:
@ -134,29 +134,29 @@ final class JsonParser implements JsonParserConstants {
JsonElement o = null;
switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
case SINGLE_QUOTE_LITERAL:
case QUOTE:
case DOUBLE_QUOTE_LITERAL:
o = JsonString();
break;
case DIGITS:
case 26:
case 29:
o = JsonNumber();
break;
case 17:
case 20:
o = JsonObject();
break;
case 22:
case 25:
o = JsonArray();
break;
case 24:
jj_consume_token(24);
case 27:
jj_consume_token(27);
o = new JsonPrimitive(true);
break;
case 25:
jj_consume_token(25);
case 28:
jj_consume_token(28);
o = new JsonPrimitive(false);
break;
case 19:
jj_consume_token(19);
case 22:
jj_consume_token(22);
break;
default:
jj_la1[5] = jj_gen;
@ -171,21 +171,21 @@ final class JsonParser implements JsonParserConstants {
JsonPrimitive value;
switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
case SINGLE_QUOTE_LITERAL:
case QUOTE:
case DOUBLE_QUOTE_LITERAL:
value = JsonString();
{if (true) return value;}
break;
case DIGITS:
case 26:
case 29:
value = JsonNumber();
{if (true) return value;}
break;
case 24:
jj_consume_token(24);
case 27:
jj_consume_token(27);
{if (true) return new JsonPrimitive(true);}
break;
case 25:
jj_consume_token(25);
case 28:
jj_consume_token(28);
{if (true) return new JsonPrimitive(false);}
break;
default:
@ -202,7 +202,7 @@ final class JsonParser implements JsonParserConstants {
exppart = null;
intpart = JsonInt();
switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
case 27:
case 30:
fracpart = JsonFrac();
break;
default:
@ -241,8 +241,8 @@ final class JsonParser implements JsonParserConstants {
String digits;
boolean negative = false;
switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
case 26:
jj_consume_token(26);
case 29:
jj_consume_token(29);
negative = true;
break;
default:
@ -258,7 +258,7 @@ final class JsonParser implements JsonParserConstants {
final private String JsonFrac() throws ParseException {
String digits;
jj_consume_token(27);
jj_consume_token(30);
digits = Digits();
{if (true) return "." + digits;}
throw new Error("Missing return statement in function");
@ -281,34 +281,21 @@ final class JsonParser implements JsonParserConstants {
}
final private JsonPrimitive JsonString() throws ParseException {
StringBuffer strbuf = new StringBuffer(); Token t;
Token t;
switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
case SINGLE_QUOTE_LITERAL:
t = jj_consume_token(SINGLE_QUOTE_LITERAL);
String value = t.image;
String valueWithInQuotes = value.substring(1, value.length()-1);
{if (true) return new JsonPrimitive(valueWithInQuotes);}
break;
case QUOTE:
jj_consume_token(QUOTE);
switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
case CHAR:
case CNTRL_ESC:
case HEX_ESC:
Chars(strbuf);
break;
default:
jj_la1[10] = jj_gen;
;
}
jj_consume_token(ENDQUOTE);
{if (true) return new JsonPrimitive(strbuf.toString());}
case DOUBLE_QUOTE_LITERAL:
t = jj_consume_token(DOUBLE_QUOTE_LITERAL);
break;
default:
jj_la1[11] = jj_gen;
jj_la1[10] = jj_gen;
jj_consume_token(-1);
throw new ParseException();
}
String value = StringUnmarshaller.unmarshall(t.image);
{if (true) return new JsonPrimitive(value);}
throw new Error("Missing return statement in function");
}
@ -322,7 +309,7 @@ final class JsonParser implements JsonParserConstants {
Chars(strbuf);
break;
default:
jj_la1[12] = jj_gen;
jj_la1[11] = jj_gen;
;
}
strbuf.insert(0, c);
@ -341,7 +328,7 @@ final class JsonParser implements JsonParserConstants {
t = jj_consume_token(HEX_ESC);
break;
default:
jj_la1[13] = jj_gen;
jj_la1[12] = jj_gen;
jj_consume_token(-1);
throw new ParseException();
}
@ -374,13 +361,13 @@ final class JsonParser implements JsonParserConstants {
public Token token, jj_nt;
private int jj_ntk;
private int jj_gen;
final private int[] jj_la1 = new int[14];
final private int[] jj_la1 = new int[13];
static private int[] jj_la1_0;
static {
jj_la1_0();
}
private static void jj_la1_0() {
jj_la1_0 = new int[] {0x74a0340,0x300,0x100000,0x74a0340,0x100000,0x74a0340,0x7000340,0x8000000,0x20,0x4000000,0x13000,0x300,0x13000,0x13000,};
jj_la1_0 = new int[] {0x3a500c40,0xc00,0x800000,0x3a500c40,0x800000,0x3a500c40,0x38000c40,0x40000000,0x20,0x20000000,0xc00,0x98000,0x98000,};
}
public JsonParser(java.io.InputStream stream) {
@ -392,7 +379,7 @@ final class JsonParser implements JsonParserConstants {
token = new Token();
jj_ntk = -1;
jj_gen = 0;
for (int i = 0; i < 14; i++) jj_la1[i] = -1;
for (int i = 0; i < 13; i++) jj_la1[i] = -1;
}
public void ReInit(java.io.InputStream stream) {
@ -404,7 +391,7 @@ final class JsonParser implements JsonParserConstants {
token = new Token();
jj_ntk = -1;
jj_gen = 0;
for (int i = 0; i < 14; i++) jj_la1[i] = -1;
for (int i = 0; i < 13; i++) jj_la1[i] = -1;
}
public JsonParser(java.io.Reader stream) {
@ -413,7 +400,7 @@ final class JsonParser implements JsonParserConstants {
token = new Token();
jj_ntk = -1;
jj_gen = 0;
for (int i = 0; i < 14; i++) jj_la1[i] = -1;
for (int i = 0; i < 13; i++) jj_la1[i] = -1;
}
public void ReInit(java.io.Reader stream) {
@ -422,7 +409,7 @@ final class JsonParser implements JsonParserConstants {
token = new Token();
jj_ntk = -1;
jj_gen = 0;
for (int i = 0; i < 14; i++) jj_la1[i] = -1;
for (int i = 0; i < 13; i++) jj_la1[i] = -1;
}
public JsonParser(JsonParserTokenManager tm) {
@ -430,7 +417,7 @@ final class JsonParser implements JsonParserConstants {
token = new Token();
jj_ntk = -1;
jj_gen = 0;
for (int i = 0; i < 14; i++) jj_la1[i] = -1;
for (int i = 0; i < 13; i++) jj_la1[i] = -1;
}
public void ReInit(JsonParserTokenManager tm) {
@ -438,7 +425,7 @@ final class JsonParser implements JsonParserConstants {
token = new Token();
jj_ntk = -1;
jj_gen = 0;
for (int i = 0; i < 14; i++) jj_la1[i] = -1;
for (int i = 0; i < 13; i++) jj_la1[i] = -1;
}
final private Token jj_consume_token(int kind) throws ParseException {
@ -485,15 +472,15 @@ final class JsonParser implements JsonParserConstants {
public ParseException generateParseException() {
jj_expentries.removeAllElements();
boolean[] la1tokens = new boolean[28];
for (int i = 0; i < 28; i++) {
boolean[] la1tokens = new boolean[31];
for (int i = 0; i < 31; i++) {
la1tokens[i] = false;
}
if (jj_kind >= 0) {
la1tokens[jj_kind] = true;
jj_kind = -1;
}
for (int i = 0; i < 14; i++) {
for (int i = 0; i < 13; i++) {
if (jj_la1[i] == jj_gen) {
for (int j = 0; j < 32; j++) {
if ((jj_la1_0[i] & (1<<j)) != 0) {
@ -502,7 +489,7 @@ final class JsonParser implements JsonParserConstants {
}
}
}
for (int i = 0; i < 28; i++) {
for (int i = 0; i < 31; i++) {
if (la1tokens[i]) {
jj_expentry = new int[1];
jj_expentry[0] = i;

22
gson/src/main/java/com/google/gson/JsonParserConstants.java
View File

@ -7,14 +7,17 @@ interface JsonParserConstants {
int EOF = 0;
int E = 5;
int DIGITS = 6;
int ESCAPE_CHAR = 7;
int SINGLE_QUOTE_LITERAL = 8;
int QUOTE = 9;
int ENDQUOTE = 11;
int CHAR = 12;
int CNTRL_ESC = 13;
int HEX = 15;
int HEX_ESC = 16;
int HEX_CHAR = 7;
int UNICODE_CHAR = 8;
int ESCAPE_CHAR = 9;
int SINGLE_QUOTE_LITERAL = 10;
int DOUBLE_QUOTE_LITERAL = 11;
int QUOTE = 12;
int ENDQUOTE = 14;
int CHAR = 15;
int CNTRL_ESC = 16;
int HEX = 18;
int HEX_ESC = 19;
int DEFAULT = 0;
int STRING_STATE = 1;
@ -29,8 +32,11 @@ interface JsonParserConstants {
"\"\\r\"",
"<E>",
"<DIGITS>",
"<HEX_CHAR>",
"<UNICODE_CHAR>",
"<ESCAPE_CHAR>",
"<SINGLE_QUOTE_LITERAL>",
"<DOUBLE_QUOTE_LITERAL>",
"\"\\\"\"",
"\"\\\\\"",
"<ENDQUOTE>",

335
gson/src/main/java/com/google/gson/JsonParserTokenManager.java
View File

@ -80,8 +80,8 @@ private final int jjMoveNfa_3(int startState, int curPos)
jjstateSet[jjnewStateCnt++] = 3;
break;
case 3:
if ((0x3ff000000000000L & l) != 0L && kind > 16)
kind = 16;
if ((0x3ff000000000000L & l) != 0L && kind > 19)
kind = 19;
break;
default : break;
}
@ -107,8 +107,8 @@ private final int jjMoveNfa_3(int startState, int curPos)
jjstateSet[jjnewStateCnt++] = 3;
break;
case 3:
if ((0x7e0000007eL & l) != 0L && kind > 16)
kind = 16;
if ((0x7e0000007eL & l) != 0L && kind > 19)
kind = 19;
break;
default : break;
}
@ -146,6 +146,10 @@ private final int jjStopStringLiteralDfa_0(int pos, long active0)
{
switch (pos)
{
case 0:
if ((active0 & 0x1000L) != 0L)
return 31;
return -1;
default :
return -1;
}
@ -173,29 +177,29 @@ private final int jjMoveStringLiteralDfa0_0()
switch(curChar)
{
case 34:
return jjStopAtPos(0, 9);
return jjStartNfaWithStates_0(0, 12, 31);
case 44:
return jjStopAtPos(0, 20);
case 45:
return jjStopAtPos(0, 26);
case 46:
return jjStopAtPos(0, 27);
case 58:
return jjStopAtPos(0, 21);
case 91:
return jjStopAtPos(0, 22);
case 93:
return jjStopAtPos(0, 23);
case 45:
return jjStopAtPos(0, 29);
case 46:
return jjStopAtPos(0, 30);
case 58:
return jjStopAtPos(0, 24);
case 91:
return jjStopAtPos(0, 25);
case 93:
return jjStopAtPos(0, 26);
case 102:
return jjMoveStringLiteralDfa1_0(0x2000000L);
return jjMoveStringLiteralDfa1_0(0x10000000L);
case 110:
return jjMoveStringLiteralDfa1_0(0x80000L);
return jjMoveStringLiteralDfa1_0(0x400000L);
case 116:
return jjMoveStringLiteralDfa1_0(0x1000000L);
return jjMoveStringLiteralDfa1_0(0x8000000L);
case 123:
return jjStopAtPos(0, 17);
return jjStopAtPos(0, 20);
case 125:
return jjStopAtPos(0, 18);
return jjStopAtPos(0, 21);
default :
return jjMoveNfa_0(0, 0);
}
@ -210,11 +214,11 @@ private final int jjMoveStringLiteralDfa1_0(long active0)
switch(curChar)
{
case 97:
return jjMoveStringLiteralDfa2_0(active0, 0x2000000L);
return jjMoveStringLiteralDfa2_0(active0, 0x10000000L);
case 114:
return jjMoveStringLiteralDfa2_0(active0, 0x1000000L);
return jjMoveStringLiteralDfa2_0(active0, 0x8000000L);
case 117:
return jjMoveStringLiteralDfa2_0(active0, 0x80000L);
return jjMoveStringLiteralDfa2_0(active0, 0x400000L);
default :
break;
}
@ -232,9 +236,9 @@ private final int jjMoveStringLiteralDfa2_0(long old0, long active0)
switch(curChar)
{
case 108:
return jjMoveStringLiteralDfa3_0(active0, 0x2080000L);
return jjMoveStringLiteralDfa3_0(active0, 0x10400000L);
case 117:
return jjMoveStringLiteralDfa3_0(active0, 0x1000000L);
return jjMoveStringLiteralDfa3_0(active0, 0x8000000L);
default :
break;
}
@ -252,15 +256,15 @@ private final int jjMoveStringLiteralDfa3_0(long old0, long active0)
switch(curChar)
{
case 101:
if ((active0 & 0x1000000L) != 0L)
return jjStopAtPos(3, 24);
if ((active0 & 0x8000000L) != 0L)
return jjStopAtPos(3, 27);
break;
case 108:
if ((active0 & 0x80000L) != 0L)
return jjStopAtPos(3, 19);
if ((active0 & 0x400000L) != 0L)
return jjStopAtPos(3, 22);
break;
case 115:
return jjMoveStringLiteralDfa4_0(active0, 0x2000000L);
return jjMoveStringLiteralDfa4_0(active0, 0x10000000L);
default :
break;
}
@ -278,8 +282,8 @@ private final int jjMoveStringLiteralDfa4_0(long old0, long active0)
switch(curChar)
{
case 101:
if ((active0 & 0x2000000L) != 0L)
return jjStopAtPos(4, 25);
if ((active0 & 0x10000000L) != 0L)
return jjStopAtPos(4, 28);
break;
default :
break;
@ -296,7 +300,7 @@ private final int jjMoveNfa_0(int startState, int curPos)
{
int[] nextStates;
int startsAt = 0;
jjnewStateCnt = 8;
jjnewStateCnt = 31;
int i = 1;
jjstateSet[0] = startState;
int j, kind = 0x7fffffff;
@ -311,6 +315,15 @@ private final int jjMoveNfa_0(int startState, int curPos)
{
switch(jjstateSet[--i])
{
case 31:
if ((0xfffffffbffffdbffL & l) != 0L)
jjCheckNAddStates(0, 3);
else if (curChar == 34)
{
if (kind > 11)
kind = 11;
}
break;
case 0:
if ((0x3ff000000000000L & l) != 0L)
{
@ -318,8 +331,10 @@ private final int jjMoveNfa_0(int startState, int curPos)
kind = 6;
jjCheckNAdd(2);
}
else if (curChar == 34)
jjCheckNAddStates(0, 3);
else if (curChar == 39)
jjCheckNAddStates(0, 2);
jjCheckNAddStates(4, 7);
break;
case 1:
if ((0x280000000000L & l) != 0L && kind > 5)
@ -332,22 +347,86 @@ private final int jjMoveNfa_0(int startState, int curPos)
kind = 6;
jjCheckNAdd(2);
break;
case 3:
if (curChar == 39)
jjCheckNAddStates(0, 2);
break;
case 4:
if ((0xffffff7fffffdbffL & l) != 0L)
jjCheckNAddStates(0, 2);
if ((0x3ff000000000000L & l) != 0L)
jjstateSet[jjnewStateCnt++] = 5;
break;
case 5:
if ((0x3ff000000000000L & l) != 0L)
jjstateSet[jjnewStateCnt++] = 6;
break;
case 6:
if ((0x8400000000L & l) != 0L)
jjCheckNAddStates(0, 2);
if ((0x3ff000000000000L & l) != 0L)
jjstateSet[jjnewStateCnt++] = 7;
break;
case 7:
if (curChar == 39 && kind > 8)
if ((0x3ff000000000000L & l) != 0L && kind > 8)
kind = 8;
break;
case 9:
if (curChar == 39)
jjCheckNAddStates(4, 7);
break;
case 10:
if ((0xffffff7fffffdbffL & l) != 0L)
jjCheckNAddStates(4, 7);
break;
case 12:
if ((0x8400000000L & l) != 0L)
jjCheckNAddStates(4, 7);
break;
case 14:
if ((0x3ff000000000000L & l) != 0L)
jjstateSet[jjnewStateCnt++] = 15;
break;
case 15:
if ((0x3ff000000000000L & l) != 0L)
jjstateSet[jjnewStateCnt++] = 16;
break;
case 16:
if ((0x3ff000000000000L & l) != 0L)
jjstateSet[jjnewStateCnt++] = 17;
break;
case 17:
if ((0x3ff000000000000L & l) != 0L)
jjCheckNAddStates(4, 7);
break;
case 19:
if (curChar == 39 && kind > 10)
kind = 10;
break;
case 20:
if (curChar == 34)
jjCheckNAddStates(0, 3);
break;
case 21:
if ((0xfffffffbffffdbffL & l) != 0L)
jjCheckNAddStates(0, 3);
break;
case 23:
if ((0x8400000000L & l) != 0L)
jjCheckNAddStates(0, 3);
break;
case 25:
if ((0x3ff000000000000L & l) != 0L)
jjstateSet[jjnewStateCnt++] = 26;
break;
case 26:
if ((0x3ff000000000000L & l) != 0L)
jjstateSet[jjnewStateCnt++] = 27;
break;
case 27:
if ((0x3ff000000000000L & l) != 0L)
jjstateSet[jjnewStateCnt++] = 28;
break;
case 28:
if ((0x3ff000000000000L & l) != 0L)
jjCheckNAddStates(0, 3);
break;
case 30:
if (curChar == 34 && kind > 11)
kind = 11;
break;
default : break;
}
} while(i != startsAt);
@ -359,24 +438,119 @@ private final int jjMoveNfa_0(int startState, int curPos)
{
switch(jjstateSet[--i])
{
case 31:
if ((0xffffffffefffffffL & l) != 0L)
jjCheckNAddStates(0, 3);
else if (curChar == 92)
jjstateSet[jjnewStateCnt++] = 24;
if (curChar == 92)
jjstateSet[jjnewStateCnt++] = 23;
break;
case 0:
if ((0x2000000020L & l) == 0L)
break;
if (kind > 5)
kind = 5;
jjstateSet[jjnewStateCnt++] = 1;
if ((0x2000000020L & l) != 0L)
{
if (kind > 5)
kind = 5;
jjstateSet[jjnewStateCnt++] = 1;
}
else if (curChar == 92)
jjstateSet[jjnewStateCnt++] = 3;
break;
case 3:
if (curChar == 117)
jjstateSet[jjnewStateCnt++] = 4;
break;
case 4:
if ((0xffffffffefffffffL & l) != 0L)
jjCheckNAddStates(0, 2);
if ((0x7e0000007eL & l) != 0L)
jjstateSet[jjnewStateCnt++] = 5;
break;
case 5:
if (curChar == 92)
if ((0x7e0000007eL & l) != 0L)
jjstateSet[jjnewStateCnt++] = 6;
break;
case 6:
if ((0x7e0000007eL & l) != 0L)
jjstateSet[jjnewStateCnt++] = 7;
break;
case 7:
if ((0x7e0000007eL & l) != 0L && kind > 8)
kind = 8;
break;
case 8:
if (curChar == 92)
jjstateSet[jjnewStateCnt++] = 3;
break;
case 10:
if ((0xffffffffefffffffL & l) != 0L)
jjCheckNAddStates(4, 7);
break;
case 11:
if (curChar == 92)
jjstateSet[jjnewStateCnt++] = 12;
break;
case 12:
if ((0x14404410000000L & l) != 0L)
jjCheckNAddStates(0, 2);
jjCheckNAddStates(4, 7);
break;
case 13:
if (curChar == 117)
jjstateSet[jjnewStateCnt++] = 14;
break;
case 14:
if ((0x7e0000007eL & l) != 0L)
jjstateSet[jjnewStateCnt++] = 15;
break;
case 15:
if ((0x7e0000007eL & l) != 0L)
jjstateSet[jjnewStateCnt++] = 16;
break;
case 16:
if ((0x7e0000007eL & l) != 0L)
jjstateSet[jjnewStateCnt++] = 17;
break;
case 17:
if ((0x7e0000007eL & l) != 0L)
jjCheckNAddStates(4, 7);
break;
case 18:
if (curChar == 92)
jjstateSet[jjnewStateCnt++] = 13;
break;
case 21:
if ((0xffffffffefffffffL & l) != 0L)
jjCheckNAddStates(0, 3);
break;
case 22:
if (curChar == 92)
jjstateSet[jjnewStateCnt++] = 23;
break;
case 23:
if ((0x14404410000000L & l) != 0L)
jjCheckNAddStates(0, 3);
break;
case 24:
if (curChar == 117)
jjstateSet[jjnewStateCnt++] = 25;
break;
case 25:
if ((0x7e0000007eL & l) != 0L)
jjstateSet[jjnewStateCnt++] = 26;
break;
case 26:
if ((0x7e0000007eL & l) != 0L)
jjstateSet[jjnewStateCnt++] = 27;
break;
case 27:
if ((0x7e0000007eL & l) != 0L)
jjstateSet[jjnewStateCnt++] = 28;
break;
case 28:
if ((0x7e0000007eL & l) != 0L)
jjCheckNAddStates(0, 3);
break;
case 29:
if (curChar == 92)
jjstateSet[jjnewStateCnt++] = 24;
break;
default : break;
}
@ -393,9 +567,14 @@ private final int jjMoveNfa_0(int startState, int curPos)
{
switch(jjstateSet[--i])
{
case 4:
case 31:
case 21:
if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjAddStates(0, 2);
jjCheckNAddStates(0, 3);
break;
case 10:
if (jjCanMove_0(hiByte, i1, i2, l1, l2))
jjAddStates(4, 7);
break;
default : break;
}
@ -408,7 +587,7 @@ private final int jjMoveNfa_0(int startState, int curPos)
kind = 0x7fffffff;
}
++curPos;
if ((i = jjnewStateCnt) == (startsAt = 8 - (jjnewStateCnt = startsAt)))
if ((i = jjnewStateCnt) == (startsAt = 31 - (jjnewStateCnt = startsAt)))
return curPos;
try { curChar = input_stream.readChar(); }
catch(java.io.IOException e) { return curPos; }
@ -439,7 +618,7 @@ private final int jjMoveStringLiteralDfa0_2()
switch(curChar)
{
case 117:
return jjStopAtPos(0, 14);
return jjStopAtPos(0, 17);
default :
return jjMoveNfa_2(0, 0);
}
@ -465,7 +644,7 @@ private final int jjMoveNfa_2(int startState, int curPos)
{
case 0:
if ((0x800400000000L & l) != 0L)
kind = 13;
kind = 16;
break;
default : break;
}
@ -480,7 +659,7 @@ private final int jjMoveNfa_2(int startState, int curPos)
{
case 0:
if ((0x14404410000000L & l) != 0L)
kind = 13;
kind = 16;
break;
default : break;
}
@ -539,7 +718,7 @@ private final int jjMoveStringLiteralDfa0_1()
switch(curChar)
{
case 92:
return jjStopAtPos(0, 10);
return jjStopAtPos(0, 13);
default :
return jjMoveNfa_1(0, 0);
}
@ -566,18 +745,18 @@ private final int jjMoveNfa_1(int startState, int curPos)
case 0:
if ((0xfffffffbffffffffL & l) != 0L)
{
if (kind > 12)
kind = 12;
if (kind > 15)
kind = 15;
}
else if (curChar == 34)
{
if (kind > 11)
kind = 11;
if (kind > 14)
kind = 14;
}
break;
case 1:
if ((0xfffffffbffffffffL & l) != 0L)
kind = 12;
kind = 15;
break;
default : break;
}
@ -592,7 +771,7 @@ private final int jjMoveNfa_1(int startState, int curPos)
{
case 0:
if ((0xffffffffefffffffL & l) != 0L)
kind = 12;
kind = 15;
break;
default : break;
}
@ -610,8 +789,8 @@ private final int jjMoveNfa_1(int startState, int curPos)
switch(jjstateSet[--i])
{
case 0:
if (jjCanMove_0(hiByte, i1, i2, l1, l2) && kind > 12)
kind = 12;
if (jjCanMove_0(hiByte, i1, i2, l1, l2) && kind > 15)
kind = 15;
break;
default : break;
}
@ -631,7 +810,7 @@ private final int jjMoveNfa_1(int startState, int curPos)
}
}
static final int[] jjnextStates = {
4, 5, 7,
21, 22, 29, 30, 10, 11, 18, 19,
};
private static final boolean jjCanMove_0(int hiByte, int i1, int i2, long l1, long l2)
{
@ -646,9 +825,9 @@ private static final boolean jjCanMove_0(int hiByte, int i1, int i2, long l1, lo
}
}
public static final String[] jjstrLiteralImages = {
"", null, null, null, null, null, null, null, null, "\42", null, null, null,
null, null, null, null, "\173", "\175", "\156\165\154\154", "\54", "\72", "\133",
"\135", "\164\162\165\145", "\146\141\154\163\145", "\55", "\56", };
"", null, null, null, null, null, null, null, null, null, null, null, "\42",
null, null, null, null, null, null, null, "\173", "\175", "\156\165\154\154", "\54",
"\72", "\133", "\135", "\164\162\165\145", "\146\141\154\163\145", "\55", "\56", };
public static final String[] lexStateNames = {
"DEFAULT",
"STRING_STATE",
@ -656,21 +835,21 @@ public static final String[] lexStateNames = {
"HEX_STATE",
};
public static final int[] jjnewLexState = {
-1, -1, -1, -1, -1, -1, -1, -1, -1, 1, 2, 0, -1, 1, 3, -1, 1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, 2, 0, -1, 1, 3, -1, 1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1,
};
static final long[] jjtoToken = {
0xfff3b61L,
0x7ff9dd61L,
};
static final long[] jjtoSkip = {
0x1eL,
};
static final long[] jjtoMore = {
0x4400L,
0x22000L,
};
protected SimpleCharStream input_stream;
private final int[] jjrounds = new int[8];
private final int[] jjstateSet = new int[16];
private final int[] jjrounds = new int[31];
private final int[] jjstateSet = new int[62];
protected char curChar;
public JsonParserTokenManager(SimpleCharStream stream){
if (SimpleCharStream.staticFlag)
@ -692,7 +871,7 @@ private final void ReInitRounds()
{
int i;
jjround = 0x80000001;
for (i = 8; i-- > 0;)
for (i = 31; i-- > 0;)
jjrounds[i] = 0x80000000;
}
public void ReInit(SimpleCharStream stream, int lexState)

71
gson/src/main/java/com/google/gson/StringUnmarshaller.java Normal file
View File

@ -0,0 +1,71 @@
package com.google.gson;
final class StringUnmarshaller {
static String unmarshall(String str) {
str = str.substring(1, str.length()-1);
int len = str.length();
StringBuilder sb = new StringBuilder(len);
int i = 0;
while (i < len) {
char c = str.charAt(i);
++i;
if (c == '\\') {
char c1 = str.charAt(i);
++i;
if (c1 == 'u') { // This is a unicode escape
// TODO(inder): Handle the case where code points are of size bigger than 4
int codePoint = getCodePoint(str, i);
sb.appendCodePoint(codePoint);
i += 4;
} else {
char escapedChar = getEscapedChar(str, c1);
sb.append(escapedChar);
}
} else {
sb.append(c);
}
}
return sb.toString();
}
private static int getCodePoint(String str, int i) {
// int codePoint = Character.codePointAt(str, i);
String s = str.substring(i, i+4);
int codePoint = Integer.parseInt(s, 16);
return codePoint;
}
private static char getEscapedChar(String str, char c) {
char ch;
switch (c) {
case 'n':
ch = '\n';
break;
case 'b':
ch = '\b';
break;
case 'f':
ch = '\f';
break;
case 't':
ch = '\t';
break;
case 'r':
ch = '\r';
break;
case '\"':
ch = '\"';
break;
case '\'':
ch = '\'';
break;
case '\\':
ch = '\\';
break;
default:
throw new IllegalStateException("Unexpected character: " + c + " in " + str);
}
return ch;
}
}

17
gson/src/main/javacc/JsonParser.jj
View File

@ -39,8 +39,11 @@ SKIP : { " " | "\t" | "\n" | "\r" }
TOKEN : {
<E : ["e","E"](["+","-"])?>
| <DIGITS : (["0"-"9"])+>
| <#HEX_CHAR : ["a"-"f","A"-"F","0"-"9"]>
| <UNICODE_CHAR : "\\u" <HEX_CHAR><HEX_CHAR><HEX_CHAR><HEX_CHAR> >
| <#ESCAPE_CHAR: "\\" ["n","t","b","r","f","\\","'","\""] >
| <SINGLE_QUOTE_LITERAL: "\'" ( (~["\'","\\","\n","\r"]) | <ESCAPE_CHAR>)* "\'" >
| <SINGLE_QUOTE_LITERAL: "\'" ( (~["\'","\\","\n","\r"]) | <ESCAPE_CHAR> | <UNICODE_CHAR>)* "\'" >
| <DOUBLE_QUOTE_LITERAL: "\"" ( (~["\"","\\","\n","\r"]) | <ESCAPE_CHAR> | <UNICODE_CHAR>)* "\"" >
| <QUOTE : "\""> : STRING_STATE
}
<STRING_STATE> MORE : { "\\" : ESC_STATE }
@ -216,15 +219,11 @@ private String Digits() :
}
private JsonPrimitive JsonString() :
{ StringBuffer strbuf = new StringBuffer(); Token t; }
{ Token t; }
{
t=<SINGLE_QUOTE_LITERAL> {
String value = t.image;
String valueWithInQuotes = value.substring(1, value.length()-1);
return new JsonPrimitive(valueWithInQuotes);
}
| <QUOTE> [ Chars(strbuf) ] <ENDQUOTE> {
return new JsonPrimitive(strbuf.toString());
(t=<SINGLE_QUOTE_LITERAL> | t=<DOUBLE_QUOTE_LITERAL>) {
String value = StringDemarshaller.unmarshall(t.image);
return new JsonPrimitive(value);
}
}

55
gson/src/test/java/com/google/gson/StringUnmarshallerTest.java Normal file
View File

@ -0,0 +1,55 @@
package com.google.gson;
import junit.framework.TestCase;
/**
* Unit tests for {@link StringUnmarshaller}.
*
* @author Inderjeet Singh
*/
public class StringUnmarshallerTest extends TestCase {
public void testCtrlN() {
assertEquals("a\nb", StringUnmarshaller.unmarshall("'a\\nb'"));
}
public void testCtrlR() {
assertEquals("a\rb", StringUnmarshaller.unmarshall("'a\\rb'"));
}
public void testCtrlT() {
assertEquals("\tb", StringUnmarshaller.unmarshall("'\\tb'"));
}
public void testBackSpace() {
assertEquals("\b foo", StringUnmarshaller.unmarshall("'\\b foo'"));
}
public void testFormFeed() {
assertEquals("\f bar", StringUnmarshaller.unmarshall("'\\f bar'"));
}
public void testSingleQuote() {
assertEquals("a'b", StringUnmarshaller.unmarshall("'a'b'"));
}
public void testSingleQuoteEscaped() {
assertEquals("a'b", StringUnmarshaller.unmarshall("'a\\'b'"));
}
public void testDoubleQuote() {
assertEquals("a\"b", StringUnmarshaller.unmarshall("'a\"b'"));
}
public void testDoubleQuoteEscaped() {
assertEquals("a\"b", StringUnmarshaller.unmarshall("'a\\\"b'"));
}
public void testBackslash() {
assertEquals("a\\b", StringUnmarshaller.unmarshall("'a\\\\b'"));
}
public void testUnicodeString() {
assertEquals("\u03a9b", StringUnmarshaller.unmarshall("'\\u03a9b'"));
}
}

6
gson/src/test/java/com/google/gson/functional/InternationalizationTest.java
View File

@ -65,9 +65,7 @@ public class InternationalizationTest extends TestCase {
}
public void testStringsWithUnicodeChineseCharactersEscapedDeserialization() throws Exception {
String json = "\"" + "\\u597d\\u597d\\u597d" + "\"";
String actual = gson.fromJson(json, String.class);
String expected = "\u597d\u597d\u597d";
assertEquals(expected, actual);
String actual = gson.fromJson("'\\u597d\\u597d\\u597d'", String.class);
assertEquals("\u597d\u597d\u597d", actual);
}
}

35
gson/src/test/java/com/google/gson/functional/StringTest.java
View File

@ -37,6 +37,41 @@ public class StringTest extends TestCase {
assertEquals(valueWithQuotes, gson.fromJson(jsonRepresentation, String.class));
}
public void testEscapedCtrlNInStringSerialization() throws Exception {
String value = "a\nb";
String json = gson.toJson(value);
assertEquals("\"a\\nb\"", json);
}
public void testEscapedCtrlNInStringDeserialization() throws Exception {
String json = "'a\\nb'";
String actual = gson.fromJson(json, String.class);
assertEquals("a\nb", actual);
}
public void testEscapedCtrlRInStringSerialization() throws Exception {
String value = "a\rb";
String json = gson.toJson(value);
assertEquals("\"a\\rb\"", json);
}
public void testEscapedCtrlRInStringDeserialization() throws Exception {
String json = "'a\\rb'";
String actual = gson.fromJson(json, String.class);
assertEquals("a\rb", actual);
}
public void testEscapedBackslashInStringSerialization() throws Exception {
String value = "a\\b";
String json = gson.toJson(value);
assertEquals("\"a\\\\b\"", json);
}
public void testEscapedBackslashInStringDeserialization() throws Exception {
String actual = gson.fromJson("'a\\\\b'", String.class);
assertEquals("a\\b", actual);
}
public void testSingleQuoteInStringDeserialization() throws Exception {
String value = "beforeQuote'afterQuote";
String actual = gson.fromJson("\"" + value + "\"", String.class);