FGCOM/jansson
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor decoder input stream

Browse Source 
- Add a new field position to the json_error_t structure. This is the
  position in bytes from the beginning of the input.

- Keep track of line, column and input position in the stream level.
  Previously, only line was tracked, and it was in the lexer level, so
  this info was not available for UTF-8 decoding errors.

- While at it, refactor tests so that no separate "stripped" tests are
  required. json_process is now able to strip whitespace from its
  input, and the "valid" and "invalid" test suites now use this to
  test both non-stripped and stripped input.

Closes GH-9.
This commit is contained in:
Petri Lehtinen
2011-02-22 10:47:02 +02:00
parent e54ea1f7c9
commit 5df7b79397
245 changed files with 379 additions and 465 deletions
Download Patch File Download Diff File Expand all files Collapse all files

201
src/load.c
View File

@@ -20,6 +20,10 @@
#include "strbuffer.h"
#include "utf.h"
#define STREAM_STATE_OK 0
#define STREAM_STATE_EOF -1
#define STREAM_STATE_ERROR -2
#define TOKEN_INVALID -1
#define TOKEN_EOF 0
#define TOKEN_STRING 256
@@ -29,27 +33,26 @@
#define TOKEN_FALSE 260
#define TOKEN_NULL 261
/* read one byte from stream, return EOF on end of file */
/* Read one byte from stream, convert to unsigned char, then int, and
return. return EOF on end of file. This corresponds to the
behaviour of fgetc(). */
typedef int (*get_func)(void *data);
/* return non-zero if end of file has been reached */
typedef int (*eof_func)(void *data);
typedef struct {
get_func get;
eof_func eof;
void *data;
int stream_pos;
char buffer[5];
int buffer_pos;
int state;
int line;
int column, last_column;
size_t position;
} stream_t;
typedef struct {
stream_t stream;
strbuffer_t saved_text;
int token;
int line, column;
union {
char *string;
json_int_t integer;
@@ -57,6 +60,8 @@ typedef struct {
} value;
} lex_t;
#define stream_to_lex(stream) container_of(stream, lex_t, stream)
/*** error reporting ***/
@@ -67,6 +72,7 @@ static void error_set(json_error_t *error, const lex_t *lex,
char msg_text[JSON_ERROR_TEXT_LENGTH];
int line = -1, col = -1;
size_t pos = 0;
const char *result = msg_text;
if(!error)
@@ -81,7 +87,9 @@ static void error_set(json_error_t *error, const lex_t *lex,
const char *saved_text = strbuffer_value(&lex->saved_text);
char msg_with_context[JSON_ERROR_TEXT_LENGTH];
line = lex->line;
line = lex->stream.line;
col = lex->stream.column;
pos = lex->stream.position;
if(saved_text && saved_text[0])
{
@@ -93,41 +101,57 @@ static void error_set(json_error_t *error, const lex_t *lex,
}
else
{
snprintf(msg_with_context, JSON_ERROR_TEXT_LENGTH,
"%s near end of file", msg_text);
result = msg_with_context;
if(lex->stream.state == STREAM_STATE_ERROR) {
/* No context for UTF-8 decoding errors */
result = msg_text;
}
else {
snprintf(msg_with_context, JSON_ERROR_TEXT_LENGTH,
"%s near end of file", msg_text);
result = msg_with_context;
}
}
}
jsonp_error_set(error, line, col, "%s", result);
jsonp_error_set(error, line, col, pos, "%s", result);
}
/*** lexical analyzer ***/
static void
stream_init(stream_t *stream, get_func get, eof_func eof, void *data)
stream_init(stream_t *stream, get_func get, void *data)
{
stream->get = get;
stream->eof = eof;
stream->data = data;
stream->stream_pos = 0;
stream->buffer[0] = '\0';
stream->buffer_pos = 0;
stream->state = STREAM_STATE_OK;
stream->line = 1;
stream->column = 0;
stream->position = 0;
}
static char stream_get(stream_t *stream, json_error_t *error)
static int stream_get(stream_t *stream, json_error_t *error)
{
char c;
int c;
if(stream->state != STREAM_STATE_OK)
return stream->state;
if(!stream->buffer[stream->buffer_pos])
{
stream->buffer[0] = stream->get(stream->data);
c = stream->get(stream->data);
if(c == EOF) {
stream->state = STREAM_STATE_EOF;
return STREAM_STATE_EOF;
}
stream->buffer[0] = c;
stream->buffer_pos = 0;
c = stream->buffer[0];
if((unsigned char)c >= 0x80 && c != (char)EOF)
if(0x80 <= c && c <= 0xFF)
{
/* multi-byte UTF-8 sequence */
int i, count;
@@ -144,30 +168,47 @@ static char stream_get(stream_t *stream, json_error_t *error)
if(!utf8_check_full(stream->buffer, count, NULL))
goto out;
stream->stream_pos += count;
stream->buffer[count] = '\0';
}
else {
else
stream->buffer[1] = '\0';
stream->stream_pos++;
}
}
return stream->buffer[stream->buffer_pos++];
c = stream->buffer[stream->buffer_pos++];
stream->position++;
if(c == '\n') {
stream->line++;
stream->last_column = stream->column;
stream->column = 0;
}
else if(utf8_check_first(c)) {
/* track the Unicode character column, so increment only if
this is the first character of a UTF-8 sequence */
stream->column++;
}
return c;
out:
error_set(error, NULL, "unable to decode byte 0x%x at position %d",
(unsigned char)c, stream->stream_pos);
stream->buffer[0] = EOF;
stream->buffer[1] = '\0';
stream->buffer_pos = 1;
return EOF;
stream->state = STREAM_STATE_ERROR;
error_set(error, stream_to_lex(stream), "unable to decode byte 0x%x", c);
return STREAM_STATE_ERROR;
}
static void stream_unget(stream_t *stream, char c)
static void stream_unget(stream_t *stream, int c)
{
if(c == STREAM_STATE_EOF || c == STREAM_STATE_ERROR)
return;
stream->position--;
if(c == '\n') {
stream->line--;
stream->column = stream->last_column;
}
else if(utf8_check_first(c))
stream->column--;
assert(stream->buffer_pos > 0);
stream->buffer_pos--;
assert(stream->buffer[stream->buffer_pos] == c);
@@ -179,29 +220,32 @@ static int lex_get(lex_t *lex, json_error_t *error)
return stream_get(&lex->stream, error);
}
static int lex_eof(lex_t *lex)
{
return lex->stream.eof(lex->stream.data);
}
static void lex_save(lex_t *lex, char c)
static void lex_save(lex_t *lex, int c)
{
strbuffer_append_byte(&lex->saved_text, c);
}
static int lex_get_save(lex_t *lex, json_error_t *error)
{
char c = stream_get(&lex->stream, error);
lex_save(lex, c);
int c = stream_get(&lex->stream, error);
if(c != STREAM_STATE_EOF && c != STREAM_STATE_ERROR)
lex_save(lex, c);
return c;
}
static void lex_unget_unsave(lex_t *lex, char c)
static void lex_unget(lex_t *lex, int c)
{
char d;
stream_unget(&lex->stream, c);
d = strbuffer_pop(&lex->saved_text);
assert(c == d);
}
static void lex_unget_unsave(lex_t *lex, int c)
{
if(c != STREAM_STATE_EOF && c != STREAM_STATE_ERROR) {
char d;
stream_unget(&lex->stream, c);
d = strbuffer_pop(&lex->saved_text);
assert(c == d);
}
}
static void lex_save_cached(lex_t *lex)
@@ -210,6 +254,7 @@ static void lex_save_cached(lex_t *lex)
{
lex_save(lex, lex->stream.buffer[lex->stream.buffer_pos]);
lex->stream.buffer_pos++;
lex->stream.position++;
}
}
@@ -239,7 +284,7 @@ static int32_t decode_unicode_escape(const char *str)
static void lex_scan_string(lex_t *lex, json_error_t *error)
{
char c;
int c;
const char *p;
char *t;
int i;
@@ -250,14 +295,15 @@ static void lex_scan_string(lex_t *lex, json_error_t *error)
c = lex_get_save(lex, error);
while(c != '"') {
if(c == (char)EOF) {
lex_unget_unsave(lex, c);
if(lex_eof(lex))
error_set(error, lex, "premature end of input");
if(c == STREAM_STATE_ERROR)
goto out;
else if(c == STREAM_STATE_EOF) {
error_set(error, lex, "premature end of input");
goto out;
}
else if((unsigned char)c <= 0x1F) {
else if(0 <= c && c <= 0x1F) {
/* control character */
lex_unget_unsave(lex, c);
if(c == '\n')
@@ -273,7 +319,6 @@ static void lex_scan_string(lex_t *lex, json_error_t *error)
c = lex_get_save(lex, error);
for(i = 0; i < 4; i++) {
if(!isxdigit(c)) {
lex_unget_unsave(lex, c);
error_set(error, lex, "invalid escape");
goto out;
}
@@ -284,7 +329,6 @@ static void lex_scan_string(lex_t *lex, json_error_t *error)
c == 'f' || c == 'n' || c == 'r' || c == 't')
c = lex_get_save(lex, error);
else {
lex_unget_unsave(lex, c);
error_set(error, lex, "invalid escape");
goto out;
}
@@ -399,7 +443,7 @@ out:
#define json_strtoint strtol
#endif
static int lex_scan_number(lex_t *lex, char c, json_error_t *error)
static int lex_scan_number(lex_t *lex, int c, json_error_t *error)
{
const char *saved_text;
char *end;
@@ -423,8 +467,8 @@ static int lex_scan_number(lex_t *lex, char c, json_error_t *error)
c = lex_get_save(lex, error);
}
else {
lex_unget_unsave(lex, c);
goto out;
lex_unget_unsave(lex, c);
goto out;
}
if(c != '.' && c != 'E' && c != 'e') {
@@ -453,8 +497,10 @@ static int lex_scan_number(lex_t *lex, char c, json_error_t *error)
if(c == '.') {
c = lex_get(lex, error);
if(!isdigit(c))
if(!isdigit(c)) {
lex_unget(lex, c);
goto out;
}
lex_save(lex, c);
c = lex_get_save(lex, error);
@@ -498,7 +544,7 @@ out:
static int lex_scan(lex_t *lex, json_error_t *error)
{
char c;
int c;
strbuffer_clear(&lex->saved_text);
@@ -509,18 +555,15 @@ static int lex_scan(lex_t *lex, json_error_t *error)
c = lex_get(lex, error);
while(c == ' ' || c == '\t' || c == '\n' || c == '\r')
{
if(c == '\n')
lex->line++;
c = lex_get(lex, error);
if(c == STREAM_STATE_EOF) {
lex->token = TOKEN_EOF;
goto out;
}
if(c == (char)EOF) {
if(lex_eof(lex))
lex->token = TOKEN_EOF;
else
lex->token = TOKEN_INVALID;
if(c == STREAM_STATE_ERROR) {
lex->token = TOKEN_INVALID;
goto out;
}
@@ -580,15 +623,13 @@ static char *lex_steal_string(lex_t *lex)
return result;
}
static int lex_init(lex_t *lex, get_func get, eof_func eof, void *data)
static int lex_init(lex_t *lex, get_func get, void *data)
{
stream_init(&lex->stream, get, eof, data);
stream_init(&lex->stream, get, data);
if(strbuffer_init(&lex->saved_text))
return -1;
lex->token = TOKEN_INVALID;
lex->line = 1;
return 0;
}
@@ -791,16 +832,10 @@ static int string_get(void *data)
else
{
stream->pos++;
return c;
return (unsigned char)c;
}
}
static int string_eof(void *data)
{
string_data_t *stream = (string_data_t *)data;
return (stream->data[stream->pos] == '\0');
}
json_t *json_loads(const char *string, size_t flags, json_error_t *error)
{
lex_t lex;
@@ -809,7 +844,7 @@ json_t *json_loads(const char *string, size_t flags, json_error_t *error)
(void)flags; /* unused */
if(lex_init(&lex, string_get, string_eof, (void *)&stream_data))
if(lex_init(&lex, string_get, (void *)&stream_data))
return NULL;
jsonp_error_init(error, "<string>");
@@ -837,7 +872,7 @@ json_t *json_loadf(FILE *input, size_t flags, json_error_t *error)
json_t *result;
(void)flags; /* unused */
if(lex_init(&lex, (get_func)fgetc, (eof_func)feof, input))
if(lex_init(&lex, (get_func)fgetc, input))
return NULL;
if(input == stdin)