doc edits

Author: petrelharp

c/examples/json_struct_metadata.c

@@ -5,7 +5,7 @@
 #include <tskit.h>
 
 // these are properties of the ``json+struct`` codec, documented in tskit
-#define JSON_STRUCT_CODEC_HEADER_SIZE 21
+#define JSON_STRUCT_HEADER_SIZE 21
 
 const uint8_t json_struct_codec_magic[4] = { 'J', 'B', 'L', 'B' };
 const uint8_t json_struct_codec_version = 1;
@@ -50,7 +50,7 @@ json_struct_codec_get_components(uint8_t *metadata, tsk_size_t metadata_length,
     if (metadata == NULL || json == NULL || json_length == NULL || binary == NULL
         || binary_length == NULL)
         errx(EXIT_FAILURE, "bad parameter value.");
-    if (metadata_length < JSON_STRUCT_CODEC_HEADER_SIZE)
+    if (metadata_length < JSON_STRUCT_HEADER_SIZE)
         errx(EXIT_FAILURE, "metadata truncated.");
     if (memcmp(metadata, json_struct_codec_magic, sizeof(json_struct_codec_magic)) != 0)
         errx(EXIT_FAILURE, "bad magic bytes.");
@@ -61,33 +61,30 @@ json_struct_codec_get_components(uint8_t *metadata, tsk_size_t metadata_length,
 
     uint64_t json_length_u64 = load_u64_le(metadata + 5);
     uint64_t binary_length_u64 = load_u64_le(metadata + 13);
-    if (json_length_u64 > UINT64_MAX - (uint64_t) JSON_STRUCT_CODEC_HEADER_SIZE)
+    if (json_length_u64 > UINT64_MAX - (uint64_t) JSON_STRUCT_HEADER_SIZE)
         errx(EXIT_FAILURE, "invalid length.");
 
     // determine the number of padding bytes and do more safety checks
-    uint64_t header_and_json_length
-        = (uint64_t) JSON_STRUCT_CODEC_HEADER_SIZE + json_length_u64;
-    uint64_t padding_length = (8 - (header_and_json_length & 0x07)) % 8;
-    uint64_t header_and_json_and_padding_length
-        = header_and_json_length + padding_length;
-    if (binary_length_u64 > UINT64_MAX - header_and_json_and_padding_length)
+    uint64_t length = (uint64_t) JSON_STRUCT_HEADER_SIZE + json_length_u64;
+    uint64_t padding_length = (8 - (length & 0x07)) % 8;
+    length += padding_length;
+    if (binary_length_u64 > UINT64_MAX - length)
         errx(EXIT_FAILURE, "invalid length.");
 
-    uint64_t total_length = header_and_json_and_padding_length + binary_length_u64;
-    if ((uint64_t) metadata_length != total_length)
+    length += binary_length_u64;
+    if ((uint64_t) metadata_length != length)
         errx(EXIT_FAILURE, "unexpected size.");
 
-    uint8_t *padding_start = metadata + JSON_STRUCT_CODEC_HEADER_SIZE + json_length_u64;
-    for (uint64_t padding_index = 0; padding_index < padding_length; ++padding_index)
-        if (*(padding_start + padding_index) != 0)
+    uint8_t *padding_start = metadata + JSON_STRUCT_HEADER_SIZE + json_length_u64;
+    for (uint64_t j = 0; j < padding_length; ++j)
+        if (*(padding_start + j) != 0)
             errx(EXIT_FAILURE, "padding bytes are nonzero.");
 
     // the structure of the codec data seems valid; return components
-    *json = metadata + JSON_STRUCT_CODEC_HEADER_SIZE;
+    *json = metadata + JSON_STRUCT_HEADER_SIZE;
     *json_length = (tsk_size_t) json_length_u64;
 
-    *binary
-        = metadata + JSON_STRUCT_CODEC_HEADER_SIZE + json_length_u64 + padding_length;
+    *binary = metadata + JSON_STRUCT_HEADER_SIZE + json_length_u64 + padding_length;
     *binary_length = (tsk_size_t) binary_length_u64;
 }
 
@@ -99,7 +96,7 @@ json_struct_codec_create_buffer(const uint8_t *json, tsk_size_t json_length,
     tsk_size_t *buffer_length)
 {
     // figure out the total length of the codec's data and allocate the buffer for it
-    tsk_size_t header_length = JSON_STRUCT_CODEC_HEADER_SIZE;
+    tsk_size_t header_length = JSON_STRUCT_HEADER_SIZE;
     tsk_size_t padding_length = (8 - ((header_length + json_length) & 0x07)) % 8;
     tsk_size_t total_length
         = header_length + json_length + padding_length + binary_length;

docs/c-api.rst

@@ -959,27 +959,27 @@ parse metadata using an external JSON library, and for
 struct-encoded metadata the values can be directly unpacked.
 Examples of both can be found in 
 `the SLiM code <https://messerlab.github.com/slim/>`_.
-(In Python, tskit automatically decodes both JSON and binary
-metadata and provides it as Python-data-typed metadata,
-just as for other codecs.)
 
-The :ref:`"json+struct" <sec_metadata_codecs_jsonstruct>`_
+The :ref:`"json+struct" <sec_metadata_codecs_jsonstruct>`
 metadata codec is a little less straightforward to use,
 so we provide here an example of how to write to it
-and read from it in C. See :ref:`sec_metadata_codes_jsonstruct` 
+and read from it in C. See :ref:`sec_metadata_codecs_jsonstruct` 
 for details of how the metadata is encoded.
+(In Python, tskit automatically decodes both JSON and binary
+metadata and provides it as Python-data-typed metadata,
+just as for other codecs.)
 
 The structure of this example is as follows:
 
 1. Values specific to the metadata's header (e.g., the magic bytes `JBLB`).
-2. Functions that encode/decode `uint_64t`, used to store the lengths
+2. Functions that encode/decode `uint64_t`, used to store the lengths
     of the two components in the header.
 3. A method to "read" the metadata: really, to get pointers to the
     json and struct components.
-4. A method to write the metadata, again just given pointers to
+4. A method to "write" the metadata, again just given pointers to
     and lengths of the two components.
 5. The program itself just round-trips a very simple chunk of metadata,
-    consisting of the JSON "`{"a": 1}`" and some binary `uint_8t` bytes ("`1234`").
+    consisting of the JSON "`{"a": 1}`" and some binary `uint8_t` bytes ("`1234`").
 
 .. literalinclude:: ../c/examples/json_struct_metadata.c
     :language: c
@@ -993,3 +993,16 @@ into two buffers (to then be decoded, for instance).
 However, that would double the memory footprint,
 and since this codec is intended for large metadata,
 we did not use that approach in this example.
+
+Along the same lines, it is worth noting that this example does make a copy of
+the JSON and binary data when writing, in ``json_struct_codec_create_buffer()``,
+which doubles the memory footprint at that point, and adds the
+overhead of copying the data.  A more efficient approach would be to calculate
+the buffer length needed for the codec’s data, allocate the buffer with that
+length, and then generate the necessary JSON and binary metadata directly into
+that buffer.  This would require the metadata-generating code to be more
+closely entwined with the code for handling the json+struct codec header and
+padding bytes, and so we have chosen not to adopt that approach here, for
+pedagogical purposes; but if your use of this codec will involve large
+metadata, such an approach is recommended.
+

docs/metadata.md

@@ -527,7 +527,7 @@ of `B`, `H`, `I`, `L` or `Q` which have the same meaning as in the numeric
 types above. `L` is the default. As an example:
 
 ```
-{"type": "array", {"items": {"type":"number", "binaryFormat":"h"}}, "arrayLengthFormat":"B"}
+{"type": "array", "items": {"type":"number", "binaryFormat":"h"}, "arrayLengthFormat":"B"}
 ```
 
 Will result in an array of 2 byte integers, prepended by a single-byte array-length.
@@ -555,6 +555,73 @@ As a special case under the `struct` codec, the top-level type of metadata can b
 union of `object` and `null`. Set `"type": ["object", "null"]`. Properties should
 be defined as normal, and will be ignored if the metadata is `None`.
 
+(sec_metadata_codecs_jsonstruct)=
+
+### `json+struct`
+
+An additional codec provides the ability to store *both* JSON and binary-encoded data.
+This is provided for the case where we want to store some arbitrary metadata
+(as JSON) along with a relatively large amount of data (as binary, for efficiency).
+For instance, we might want to record a raster map of the sampled area
+along with a few pieces of generic information (e.g., the name of the area).
+
+The metadata schema for "json+struct" metadata basically just specifies both
+a JSON metadata schema and a struct metadata schema.
+Each entry in the metadata is encoded with either the JSON or the struct codec.
+Here is a simple example:
+
+```{code-cell}
+schema = {
+    "codec": "json+struct",
+    "json": {
+        "type": "object",
+        "properties": {
+            "label": {"type": "string"},
+            "id": {"type": "number"},
+        },
+        "required": ["label"],
+    },
+    "struct": {
+        "type": "object",
+        "properties": {
+            "values": {
+                "type": "array",
+                "arrayLengthFormat": "B",
+                "items": {"type": "number", "binaryFormat": "i"},
+            }, 
+        },
+    },
+}
+ms = tskit.MetadataSchema(schema)
+row = {"label": "alpha", "id": 7, "values": [5, 10, 2, 12]}
+encoded = ms.validate_and_encode_row(row)
+print("Encoded:", encoded)
+print("Decoded:", ms.decode_row(encoded))
+```
+
+This encodes two things in JSON: a label and an ID number,
+and then an array of integers in binary (using the ``struct`` codec).
+If the array of integers is large, this could result in
+much better performance.
+
+
+#### Binary representation
+
+The underlying structure of the JSON+struct codec is as follows.
+(If you're not writing out data in this format,
+you don't need to worry about this.)
+(1) some magic bytes;
+(2) a version number;
+(3) the length of the JSON in bytes;
+(4) the length of the binary (struct) data in bytes;
+(5) the JSON data;
+(6) zero-ed "padding" bytes to bring the start of the binary section
+ into 8-byte alignment;
+(7) the binary data.
+The structure of the binary data is specified using the "struct" portion
+of the metadata schema.
+
+
 (sec_metadata_schema_examples)=
 
 ## Schema examples