jansson 2.10

Add JSON_EMBED encoding flag

CHANGES

@@ -1,3 +1,28 @@
+Version 2.10
+============
+
+Released 2017-03-02
+
+* New features:
+
+  - Add JSON_EMBED encoding flag allowing arrays and objects to be encoded
+    into existing streams (#329).
+
+  - Add `json_dumpb()` function for dumping to a pre-allocated buffer (#328).
+
+  - Add `json_dumpfd()` and `json_loadfd()` functions for dumping to streaming
+    file descriptors (#328).
+
+  - Add support for parsing buffers larger than 2GB (#309).
+
+* Build:
+
+  - Fix CMake build when LONG_LONG_INT is defined as "" (#321)
+
+* Other:
+
+  - Internal code cleanup (#311, #314)
+
 Version 2.9
 ===========
 

CMakeLists.txt

@@ -86,10 +86,10 @@ endif (WIN32)
 # set (JANSSON_VERSION "2.3.1")
 # set (JANSSON_SOVERSION 2)
 
-set(JANSSON_DISPLAY_VERSION "2.9")
+set(JANSSON_DISPLAY_VERSION "2.10")
 
 # This is what is required to match the same numbers as automake's
-set(JANSSON_VERSION "4.9.0")
+set(JANSSON_VERSION "4.10.0")
 set(JANSSON_SOVERSION 4)
 
 # for CheckFunctionKeywords
@@ -250,7 +250,7 @@ endif ()
 # detect what to use for the 64 bit type.
 # Note: I will prefer long long if I can get it, as that is what the automake system aimed for.
 if (NOT DEFINED JSON_INT_T)
-   if (HAVE_LONG_LONG_INT AND (${LONG_LONG_INT} EQUAL 8))
+   if (HAVE_LONG_LONG_INT AND ((${LONG_LONG_INT}) EQUAL 8))
       set (JSON_INT_T "long long")
    elseif (HAVE_INT64_T)
       set (JSON_INT_T int64_t)

configure.ac

@@ -1,5 +1,5 @@
 AC_PREREQ([2.60])
-AC_INIT([jansson], [2.9], [petri@digip.org])
+AC_INIT([jansson], [2.10], [petri@digip.org])
 
 AC_CONFIG_AUX_DIR([.])
 AM_INIT_AUTOMAKE([1.10 foreign])

doc/apiref.rst

@@ -940,6 +940,13 @@ can be ORed together to obtain *flags*.
 
    .. versionadded:: 2.7
 
+``JSON_EMBED``
+   If this flag is used, the opening and closing characters of the top-level
+   array ('[', ']') or object ('{', '}') are omitted during encoding. This
+   flag is useful when concatenating multiple arrays or objects into a stream.
+
+   .. versionadded:: 2.10
+
 These functions output UTF-8:
 
 .. function:: char *json_dumps(const json_t *json, size_t flags)
@@ -948,6 +955,28 @@ These functions output UTF-8:
    error. *flags* is described above. The return value must be freed
    by the caller using :func:`free()`.
 
+.. function:: size_t json_dumpb(const json_t *json, char *buffer, size_t size, size_t flags)
+
+   Writes the JSON representation of *json* to the *buffer* of
+   *size* bytes. Returns the number of bytes that would be written
+   or 0 on error. *flags* is described above. *buffer* is not
+   null-terminated.
+
+   This function never writes more than *size* bytes. If the return
+   value is greater than *size*, the contents of the *buffer* are
+   undefined. This behavior enables you to specify a NULL *buffer*
+   to determine the length of the encoding. For example::
+
+       size_t size = json_dumpb(json, NULL, 0, 0);
+       if (size == 0)
+           return -1;
+
+       char *buf = alloca(size);
+
+       size = json_dumpb(json, buf, size, 0);
+
+   .. versionadded:: 2.10
+
 .. function:: int json_dumpf(const json_t *json, FILE *output, size_t flags)
 
    Write the JSON representation of *json* to the stream *output*.
@@ -956,6 +985,23 @@ These functions output UTF-8:
    *output*. In this case, the output is undefined and most likely not
    valid JSON.
 
+.. function:: int json_dumpfd(const json_t *json, int output, size_t flags)
+
+   Write the JSON representation of *json* to the stream *output*.
+   *flags* is described above. Returns 0 on success and -1 on error.
+   If an error occurs, something may have already been written to
+   *output*. In this case, the output is undefined and most likely not
+   valid JSON.
+
+   It is important to note that this function can only succeed on stream
+   file descriptors (such as SOCK_STREAM). Using this function on a
+   non-stream file descriptor will result in undefined behavior. For
+   non-stream file descriptors, see instead :func:`json_dumpb()`.
+
+   This function requires POSIX and fails on all non-POSIX systems.
+
+   .. versionadded:: 2.10
+
 .. function:: int json_dump_file(const json_t *json, const char *path, size_t flags)
 
    Write the JSON representation of *json* to the file *path*. If
@@ -1111,7 +1157,7 @@ If no error or position information is needed, you can pass *NULL*.
    above.
 
    This function will start reading the input from whatever position
-   the input file was, without attempting to seek first. If an error
+   the input file was in, without attempting to seek first. If an error
    occurs, the file position will be left indeterminate. On success,
    the file position will be at EOF, unless ``JSON_DISABLE_EOF_CHECK``
    flag was used. In this case, the file position will be at the first
@@ -1120,6 +1166,35 @@ If no error or position information is needed, you can pass *NULL*.
    multiple times, if the input consists of consecutive JSON texts,
    possibly separated by whitespace.
 
+.. function:: json_t *json_loadfd(int input, size_t flags, json_error_t *error)
+
+   .. refcounting:: new
+
+   Decodes the JSON text in stream *input* and returns the array or
+   object it contains, or *NULL* on error, in which case *error* is
+   filled with information about the error. *flags* is described
+   above.
+
+   This function will start reading the input from whatever position
+   the input file descriptor was in, without attempting to seek first.
+   If an error occurs, the file position will be left indeterminate.
+   On success, the file position will be at EOF, unless
+   ``JSON_DISABLE_EOF_CHECK`` flag was used. In this case, the file
+   descriptor's position will be at the first character after the last
+   ``]`` or ``}`` in the JSON input. This allows calling
+   :func:`json_loadfd()` on the same file descriptor multiple times,
+   if the input consists of consecutive JSON texts, possibly separated
+   by whitespace.
+
+   It is important to note that this function can only succeed on stream
+   file descriptors (such as SOCK_STREAM). Using this function on a
+   non-stream file descriptor will result in undefined behavior. For
+   non-stream file descriptors, see instead :func:`json_loadb()`.
+
+   This function requires POSIX and fails on all non-POSIX systems.
+
+   .. versionadded:: 2.10
+
 .. function:: json_t *json_load_file(const char *path, size_t flags, json_error_t *error)
 
    .. refcounting:: new

doc/conf.py

@@ -48,7 +48,7 @@ copyright = u'2009-2016, Petri Lehtinen'
 # built documents.
 #
 # The short X.Y version.
-version = '2.9-dev'
+version = '2.10'
 # The full version, including alpha/beta/rc tags.
 release = version
 

src/Makefile.am

@@ -24,4 +24,4 @@ libjansson_la_SOURCES = \
 libjansson_la_LDFLAGS = \
 	-no-undefined \
 	-export-symbols-regex '^json_' \
-	-version-info 13:0:9
+	-version-info 14:0:10

src/dump.c

@@ -9,13 +9,17 @@
 #define _GNU_SOURCE
 #endif
 
+#include "jansson_private.h"
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <assert.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
 
 #include "jansson.h"
-#include "jansson_private.h"
 #include "strbuffer.h"
 #include "utf.h"
 
@@ -25,11 +29,28 @@
 #define FLAGS_TO_INDENT(f)      ((f) & 0x1F)
 #define FLAGS_TO_PRECISION(f)   (((f) >> 11) & 0x1F)
 
+struct buffer {
+    const size_t size;
+    size_t used;
+    char *data;
+};
+
 static int dump_to_strbuffer(const char *buffer, size_t size, void *data)
 {
     return strbuffer_append_bytes((strbuffer_t *)data, buffer, size);
 }
 
+static int dump_to_buffer(const char *buffer, size_t size, void *data)
+{
+    struct buffer *buf = (struct buffer *)data;
+
+    if(buf->used + size <= buf->size)
+        memcpy(&buf->data[buf->used], buffer, size);
+
+    buf->used += size;
+    return 0;
+}
+
 static int dump_to_file(const char *buffer, size_t size, void *data)
 {
     FILE *dest = (FILE *)data;
@@ -38,6 +59,16 @@ static int dump_to_file(const char *buffer, size_t size, void *data)
     return 0;
 }
 
+static int dump_to_fd(const char *buffer, size_t size, void *data)
+{
+    int *dest = (int *)data;
+#ifdef HAVE_UNISTD_H
+    if(write(*dest, buffer, size) == (ssize_t)size)
+        return 0;
+#endif
+    return -1;
+}
+
 /* 32 spaces (the maximum indentation size) */
 static const char whitespace[] = "                                ";
 
@@ -168,6 +199,10 @@ static int compare_keys(const void *key1, const void *key2)
 static int do_dump(const json_t *json, size_t flags, int depth,
                    json_dump_callback_t dump, void *data)
 {
+    int embed = flags & JSON_EMBED;
+
+    flags &= ~JSON_EMBED;
+
     if(!json)
         return -1;
 
@@ -227,11 +262,11 @@ static int do_dump(const json_t *json, size_t flags, int depth,
 
             n = json_array_size(json);
 
-            if(dump("[", 1, data))
+            if(!embed && dump("[", 1, data))
                 goto array_error;
             if(n == 0) {
                 array->visited = 0;
-                return dump("]", 1, data);
+                return embed ? 0 : dump("]", 1, data);
             }
             if(dump_indent(flags, depth + 1, 0, dump, data))
                 goto array_error;
@@ -255,7 +290,7 @@ static int do_dump(const json_t *json, size_t flags, int depth,
             }
 
             array->visited = 0;
-            return dump("]", 1, data);
+            return embed ? 0 : dump("]", 1, data);
 
         array_error:
             array->visited = 0;
@@ -286,11 +321,11 @@ static int do_dump(const json_t *json, size_t flags, int depth,
 
             iter = json_object_iter((json_t *)json);
 
-            if(dump("{", 1, data))
+            if(!embed && dump("{", 1, data))
                 goto object_error;
             if(!iter) {
                 object->visited = 0;
-                return dump("}", 1, data);
+                return embed ? 0 : dump("}", 1, data);
             }
             if(dump_indent(flags, depth + 1, 0, dump, data))
                 goto object_error;
@@ -386,7 +421,7 @@ static int do_dump(const json_t *json, size_t flags, int depth,
             }
 
             object->visited = 0;
-            return dump("}", 1, data);
+            return embed ? 0 : dump("}", 1, data);
 
         object_error:
             object->visited = 0;
@@ -416,11 +451,26 @@ char *json_dumps(const json_t *json, size_t flags)
     return result;
 }
 
+size_t json_dumpb(const json_t *json, char *buffer, size_t size, size_t flags)
+{
+    struct buffer buf = { size, 0, buffer };
+
+    if(json_dump_callback(json, dump_to_buffer, (void *)&buf, flags))
+        return 0;
+
+    return buf.used;
+}
+
 int json_dumpf(const json_t *json, FILE *output, size_t flags)
 {
     return json_dump_callback(json, dump_to_file, (void *)output, flags);
 }
 
+int json_dumpfd(const json_t *json, int output, size_t flags)
+{
+    return json_dump_callback(json, dump_to_fd, (void *)&output, flags);
+}
+
 int json_dump_file(const json_t *json, const char *path, size_t flags)
 {
     int result;

src/hashtable_seed.c

@@ -168,12 +168,12 @@ static uint32_t generate_seed() {
     int done = 0;
 
 #if !defined(_WIN32) && defined(USE_URANDOM)
-    if (!done && seed_from_urandom(&seed) == 0)
+    if (seed_from_urandom(&seed) == 0)
         done = 1;
 #endif
 
 #if defined(_WIN32) && defined(USE_WINDOWS_CRYPTOAPI)
-    if (!done && seed_from_windows_cryptoapi(&seed) == 0)
+    if (seed_from_windows_cryptoapi(&seed) == 0)
         done = 1;
 #endif
 

src/jansson.def

@@ -48,12 +48,15 @@ EXPORTS
     json_object_key_to_iter
     json_object_seed
     json_dumps
+    json_dumpb
     json_dumpf
+    json_dumpfd
     json_dump_file
     json_dump_callback
     json_loads
     json_loadb
     json_loadf
+    json_loadfd
     json_load_file
     json_load_callback
     json_equal

src/jansson.h

@@ -21,11 +21,11 @@ extern "C" {
 /* version */
 
 #define JANSSON_MAJOR_VERSION  2
-#define JANSSON_MINOR_VERSION  9
+#define JANSSON_MINOR_VERSION  10
 #define JANSSON_MICRO_VERSION  0
 
 /* Micro version is omitted if it's 0 */
-#define JANSSON_VERSION  "2.9"
+#define JANSSON_VERSION  "2.10"
 
 /* Version as a 3-byte hex number, e.g. 0x010201 == 1.2.1. Use this
    for numeric comparisons, e.g. #if JANSSON_VERSION_HEX >= ... */
@@ -273,6 +273,7 @@ typedef size_t (*json_load_callback_t)(void *buffer, size_t buflen, void *data);
 json_t *json_loads(const char *input, size_t flags, json_error_t *error);
 json_t *json_loadb(const char *buffer, size_t buflen, size_t flags, json_error_t *error);
 json_t *json_loadf(FILE *input, size_t flags, json_error_t *error);
+json_t *json_loadfd(int input, size_t flags, json_error_t *error);
 json_t *json_load_file(const char *path, size_t flags, json_error_t *error);
 json_t *json_load_callback(json_load_callback_t callback, void *data, size_t flags, json_error_t *error);
 
@@ -288,11 +289,14 @@ json_t *json_load_callback(json_load_callback_t callback, void *data, size_t fla
 #define JSON_ENCODE_ANY         0x200
 #define JSON_ESCAPE_SLASH       0x400
 #define JSON_REAL_PRECISION(n)  (((n) & 0x1F) << 11)
+#define JSON_EMBED              0x10000
 
 typedef int (*json_dump_callback_t)(const char *buffer, size_t size, void *data);
 
 char *json_dumps(const json_t *json, size_t flags);
+size_t json_dumpb(const json_t *json, char *buffer, size_t size, size_t flags);
 int json_dumpf(const json_t *json, FILE *output, size_t flags);
+int json_dumpfd(const json_t *json, int output, size_t flags);
 int json_dump_file(const json_t *json, const char *path, size_t flags);
 int json_dump_callback(const json_t *json, json_dump_callback_t callback, void *data, size_t flags);
 

src/jansson_private.h

@@ -8,6 +8,7 @@
 #ifndef JANSSON_PRIVATE_H
 #define JANSSON_PRIVATE_H
 
+#include "jansson_private_config.h"
 #include <stddef.h>
 #include "jansson.h"
 #include "hashtable.h"

src/load.c

@@ -9,15 +9,19 @@
 #define _GNU_SOURCE
 #endif
 
+#include "jansson_private.h"
+
 #include <errno.h>
 #include <limits.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <assert.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
 
 #include "jansson.h"
-#include "jansson_private.h"
 #include "strbuffer.h"
 #include "utf.h"
 
@@ -340,7 +344,7 @@ static void lex_scan_string(lex_t *lex, json_error_t *error)
             /* control character */
             lex_unget_unsave(lex, c);
             if(c == '\n')
-                error_set(error, lex, "unexpected newline", c);
+                error_set(error, lex, "unexpected newline");
             else
                 error_set(error, lex, "control character 0x%x", c);
             goto out;
@@ -914,7 +918,7 @@ static json_t *parse_json(lex_t *lex, size_t flags, json_error_t *error)
 typedef struct
 {
     const char *data;
-    int pos;
+    size_t pos;
 } string_data_t;
 
 static int string_get(void *data)
@@ -1028,6 +1032,45 @@ json_t *json_loadf(FILE *input, size_t flags, json_error_t *error)
     return result;
 }
 
+static int fd_get_func(int *fd)
+{
+    uint8_t c;
+#ifdef HAVE_UNISTD_H
+    if (read(*fd, &c, 1) == 1)
+        return c;
+#endif
+    return EOF;
+}
+
+json_t *json_loadfd(int input, size_t flags, json_error_t *error)
+{
+    lex_t lex;
+    const char *source;
+    json_t *result;
+
+#ifdef HAVE_UNISTD_H
+    if(input == STDIN_FILENO)
+        source = "<stdin>";
+    else
+#endif
+        source = "<stream>";
+
+    jsonp_error_init(error, source);
+
+    if (input < 0) {
+        error_set(error, NULL, "wrong arguments");
+        return NULL;
+    }
+
+    if(lex_init(&lex, (get_func)fd_get_func, flags, &input))
+        return NULL;
+
+    result = parse_json(&lex, flags, error);
+
+    lex_close(&lex);
+    return result;
+}
+
 json_t *json_load_file(const char *path, size_t flags, json_error_t *error)
 {
     json_t *result;

test/suites/api/test_dump.c

@@ -5,8 +5,13 @@
  * it under the terms of the MIT license. See LICENSE for details.
  */
 
+#include "jansson_private_config.h"
+
 #include <jansson.h>
 #include <string.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
 #include "util.h"
 
 static int encode_null_callback(const char *buffer, size_t size, void *data)
@@ -22,9 +27,17 @@ static void encode_null()
     if(json_dumps(NULL, JSON_ENCODE_ANY) != NULL)
         fail("json_dumps didn't fail for NULL");
 
+    if(json_dumpb(NULL, NULL, 0, JSON_ENCODE_ANY) != 0)
+        fail("json_dumps didn't fail for NULL");
+
     if(json_dumpf(NULL, stderr, JSON_ENCODE_ANY) != -1)
         fail("json_dumpf didn't fail for NULL");
 
+#ifdef HAVE_UNISTD_H
+    if(json_dumpfd(NULL, STDERR_FILENO, JSON_ENCODE_ANY) != -1)
+        fail("json_dumpfd didn't fail for NULL");
+#endif
+
     /* Don't test json_dump_file to avoid creating a file */
 
     if(json_dump_callback(NULL, encode_null_callback, NULL, JSON_ENCODE_ANY) != -1)
@@ -124,14 +137,15 @@ static void encode_other_than_array_or_object()
      * succeed if the JSON_ENCODE_ANY flag is used */
 
     json_t *json;
-    FILE *fp = NULL;
     char *result;
 
     json = json_string("foo");
     if(json_dumps(json, 0) != NULL)
         fail("json_dumps encoded a string!");
-    if(json_dumpf(json, fp, 0) == 0)
+    if(json_dumpf(json, NULL, 0) == 0)
         fail("json_dumpf encoded a string!");
+    if(json_dumpfd(json, -1, 0) == 0)
+        fail("json_dumpfd encoded a string!");
 
     result = json_dumps(json, JSON_ENCODE_ANY);
     if(!result || strcmp(result, "\"foo\"") != 0)
@@ -143,8 +157,10 @@ static void encode_other_than_array_or_object()
     json = json_integer(42);
     if(json_dumps(json, 0) != NULL)
         fail("json_dumps encoded an integer!");
-    if(json_dumpf(json, fp, 0) == 0)
+    if(json_dumpf(json, NULL, 0) == 0)
         fail("json_dumpf encoded an integer!");
+    if(json_dumpfd(json, -1, 0) == 0)
+        fail("json_dumpfd encoded an integer!");
 
     result = json_dumps(json, JSON_ENCODE_ANY);
     if(!result || strcmp(result, "42") != 0)
@@ -212,6 +228,89 @@ static void dump_file()
     remove("json_dump_file.json");
 }
 
+static void dumpb()
+{
+    char buf[2];
+    json_t *obj;
+    size_t size;
+
+    obj = json_object();
+
+    size = json_dumpb(obj, buf, sizeof(buf), 0);
+    if(size != 2 || strncmp(buf, "{}", 2))
+      fail("json_dumpb failed");
+
+    json_decref(obj);
+    obj = json_pack("{s:s}", "foo", "bar");
+
+    size = json_dumpb(obj, buf, sizeof(buf), JSON_COMPACT);
+    if(size != 13)
+      fail("json_dumpb size check failed");
+
+    json_decref(obj);
+}
+
+static void dumpfd()
+{
+#ifdef HAVE_UNISTD_H
+    int fds[2] = {-1, -1};
+    json_t *a, *b;
+
+    if(pipe(fds))
+        fail("pipe() failed");
+
+    a = json_pack("{s:s}", "foo", "bar");
+
+    if(json_dumpfd(a, fds[1], 0))
+        fail("json_dumpfd() failed");
+    close(fds[1]);
+
+    b = json_loadfd(fds[0], 0, NULL);
+    if (!b)
+        fail("json_loadfd() failed");
+    close(fds[0]);
+
+    if (!json_equal(a, b))
+        fail("json_equal() failed for fd test");
+
+    json_decref(a);
+    json_decref(b);
+#endif
+}
+
+static void embed()
+{
+    static const char *plains[] = {
+        "{\"bar\":[],\"foo\":{}}",
+        "[[],{}]",
+        "{}",
+        "[]",
+        NULL
+    };
+
+    size_t i;
+
+    for(i = 0; plains[i]; i++) {
+        const char *plain = plains[i];
+        json_t *parse = NULL;
+        char *embed = NULL;
+        size_t psize = 0;
+        size_t esize = 0;
+
+        psize = strlen(plain) - 2;
+        embed = calloc(1, psize);
+        parse = json_loads(plain, 0, NULL);
+        esize = json_dumpb(parse, embed, psize,
+                           JSON_COMPACT | JSON_SORT_KEYS | JSON_EMBED);
+        json_decref(parse);
+        if(esize != psize)
+            fail("json_dumpb(JSON_EMBED) returned an invalid size");
+        if(strncmp(plain + 1, embed, esize) != 0)
+            fail("json_dumps(JSON_EMBED) returned an invalid value");
+        free(embed);
+    }
+}
+
 static void run_tests()
 {
     encode_null();
@@ -221,4 +320,7 @@ static void run_tests()
     escape_slashes();
     encode_nul_byte();
     dump_file();
+    dumpb();
+    dumpfd();
+    embed();
 }

test/suites/api/test_load.c

@@ -180,9 +180,13 @@ static void load_wrong_args()
     if (json)
         fail("json_loadf should return NULL if the first argument is NULL");
 
+    json = json_loadfd(-1, 0, &error);
+    if (json)
+        fail("json_loadfd should return NULL if the first argument is < 0");
+
     json = json_load_file(NULL, 0, &error);
     if (json)
-        fail("json_loadf should return NULL if the first argument is NULL");
+        fail("json_load_file should return NULL if the first argument is NULL");
 }
 
 static void position()