From 339f9d90d1fe001978d15e1c007a3861a7145453 Mon Sep 17 00:00:00 2001 From: David Robillard Date: Sat, 2 Dec 2023 15:53:25 -0500 Subject: [WIP] Add support for converting literals to canonical form --- NEWS | 1 + doc/man/serd-pipe.1 | 14 +- include/serd/canon.h | 47 +++++++ include/serd/serd.h | 1 + meson.build | 2 + src/canon.c | 220 ++++++++++++++++++++++++++++++ src/string_utils.h | 8 +- test/extra/canon/bad-boolean.ttl | 5 + test/extra/canon/bad-decimal-leading.ttl | 4 + test/extra/canon/bad-decimal-trailing.ttl | 4 + test/extra/canon/bad-empty-boolean.ttl | 5 + test/extra/canon/bad-integer-leading.ttl | 4 + test/extra/canon/bad-integer-trailing.ttl | 4 + test/extra/canon/bad-lang-long.ttl | 3 + test/extra/canon/manifest.ttl | 65 +++++++++ test/extra/canon/test-canon.nt | 70 ++++++++++ test/extra/canon/test-canon.ttl | 76 +++++++++++ test/meson.build | 9 ++ test/test_canon.c | 103 ++++++++++++++ tools/serd-pipe.c | 25 ++-- 20 files changed, 657 insertions(+), 13 deletions(-) create mode 100644 include/serd/canon.h create mode 100644 src/canon.c create mode 100644 test/extra/canon/bad-boolean.ttl create mode 100644 test/extra/canon/bad-decimal-leading.ttl create mode 100644 test/extra/canon/bad-decimal-trailing.ttl create mode 100644 test/extra/canon/bad-empty-boolean.ttl create mode 100644 test/extra/canon/bad-integer-leading.ttl create mode 100644 test/extra/canon/bad-integer-trailing.ttl create mode 100644 test/extra/canon/bad-lang-long.ttl create mode 100644 test/extra/canon/manifest.ttl create mode 100644 test/extra/canon/test-canon.nt create mode 100644 test/extra/canon/test-canon.ttl create mode 100644 test/test_canon.c diff --git a/NEWS b/NEWS index b82e70a9..a222c99b 100644 --- a/NEWS +++ b/NEWS @@ -3,6 +3,7 @@ serd (1.1.1) unstable; urgency=medium * Add SerdBuffer for mutable buffers to keep SerdChunk const-correct * Add SerdWorld for shared library state * Add extensible logging API + * Add support for converting literals to canonical form * Add support for parsing variables * Add support for writing terse output with minimal newlines * Add support for xsd:float and xsd:double literals diff --git a/doc/man/serd-pipe.1 b/doc/man/serd-pipe.1 index 793737f9..d731f0b4 100644 --- a/doc/man/serd-pipe.1 +++ b/doc/man/serd-pipe.1 @@ -8,7 +8,7 @@ .Nd read and write RDF data .Sh SYNOPSIS .Nm serd-pipe -.Op Fl afhlqtvx +.Op Fl Cafhlqtvx .Op Fl B Ar base .Op Fl b Ar bytes .Op Fl c Ar prefix @@ -68,6 +68,18 @@ When the input is a file, the URI of the file is automatically used as the base URI. This option can be used to override that, or to provide a base URI for input from stdin or a string. +.It Fl C +Convert literals to canonical form. +Literals with supported XSD datatypes will be parsed and rewritten canonically. +Invalid literals will cause an error. +All numeric datatypes are supported, as well as +.Vt boolean , +.Vt duration , +.Vt datetime , +.Vt time , +.Vt hexBinary , +and +.Vt base64Binary . .It Fl a Write ASCII output. If this is enabled, all non-ASCII characters will be escaped, even if the output syntax allows them to be written in UTF-8. diff --git a/include/serd/canon.h b/include/serd/canon.h new file mode 100644 index 00000000..862a4db0 --- /dev/null +++ b/include/serd/canon.h @@ -0,0 +1,47 @@ +// Copyright 2011-2022 David Robillard +// SPDX-License-Identifier: ISC + +#ifndef SERD_CANON_H +#define SERD_CANON_H + +#include "serd/attributes.h" +#include "serd/sink.h" +#include "serd/world.h" +#include "zix/attributes.h" + +#include + +SERD_BEGIN_DECLS + +/** + @defgroup serd_canon Canon + @ingroup serd_streaming + @{ +*/ + +/// Flags that control canonical node transformation +typedef enum { + SERD_CANON_LAX = 1U << 0U, ///< Tolerate and pass through invalid input +} SerdCanonFlag; + +/// Bitwise OR of SerdCanonFlag values +typedef uint32_t SerdCanonFlags; + +/** + Return a new sink that transforms literals to canonical form where possible. + + The returned sink acts like `target` in all respects, except literal nodes + in statements may be modified from the original. +*/ +SERD_API SerdSink* ZIX_ALLOCATED +serd_canon_new(const SerdWorld* ZIX_NONNULL world, + const SerdSink* ZIX_NONNULL target, + SerdCanonFlags flags); + +/** + @} +*/ + +SERD_END_DECLS + +#endif // SERD_CANON_H diff --git a/include/serd/serd.h b/include/serd/serd.h index 88be5daa..d264192f 100644 --- a/include/serd/serd.h +++ b/include/serd/serd.h @@ -69,6 +69,7 @@ @{ */ +#include "serd/canon.h" #include "serd/env.h" #include "serd/event.h" #include "serd/sink.h" diff --git a/meson.build b/meson.build index 7bd2e560..82dc839b 100644 --- a/meson.build +++ b/meson.build @@ -129,6 +129,7 @@ include_dirs = include_directories('include') c_headers = files( 'include/serd/attributes.h', 'include/serd/buffer.h', + 'include/serd/canon.h', 'include/serd/caret.h', 'include/serd/env.h', 'include/serd/event.h', @@ -158,6 +159,7 @@ sources = files( 'src/block_dumper.c', 'src/buffer.c', 'src/byte_source.c', + 'src/canon.c', 'src/caret.c', 'src/env.c', 'src/input_stream.c', diff --git a/src/canon.c b/src/canon.c new file mode 100644 index 00000000..c0ce8ef4 --- /dev/null +++ b/src/canon.c @@ -0,0 +1,220 @@ +// Copyright 2019-2022 David Robillard +// SPDX-License-Identifier: ISC + +#include "caret.h" // IWYU pragma: keep +#include "memory.h" +#include "namespaces.h" +#include "node.h" +#include "statement.h" // IWYU pragma: keep +#include "string_utils.h" + +#include "exess/exess.h" +#include "serd/canon.h" +#include "serd/caret.h" +#include "serd/event.h" +#include "serd/log.h" +#include "serd/memory.h" +#include "serd/node.h" +#include "serd/sink.h" +#include "serd/statement.h" +#include "serd/status.h" +#include "serd/string_view.h" +#include "serd/world.h" +#include "zix/attributes.h" + +#include +#include +#include +#include + +typedef struct { + const SerdWorld* world; + const SerdSink* target; + SerdCanonFlags flags; +} SerdCanonData; + +static ExessResult +build_typed(SerdAllocator* const ZIX_NONNULL allocator, + SerdNode** const out, + const SerdNode* const ZIX_NONNULL node, + const SerdNode* const ZIX_NONNULL datatype) +{ + *out = NULL; + + const char* str = serd_node_string(node); + const char* datatype_uri = serd_node_string(datatype); + ExessResult r = {EXESS_SUCCESS, 0}; + + if (!strcmp(datatype_uri, NS_RDF "langString")) { + *out = + serd_node_new(allocator, serd_a_string_view(serd_node_string_view(node))); + return r; + } + + const ExessDatatype value_type = exess_datatype_from_uri(datatype_uri); + if (value_type == EXESS_NOTHING) { + return r; + } + + // Measure canonical form to know how much space to allocate for node + if ((r = exess_write_canonical(str, value_type, 0, NULL)).status) { + return r; + } + + // Allocate node + const size_t datatype_uri_len = serd_node_length(datatype); + const size_t datatype_size = serd_node_total_size(datatype); + const size_t len = serd_node_pad_length(r.count); + const size_t total_len = sizeof(SerdNode) + len + datatype_size; + SerdNode* const result = serd_node_malloc(allocator, total_len); + if (!result) { + r.status = EXESS_NO_SPACE; + return r; + } + + result->length = r.count; + result->flags = SERD_HAS_DATATYPE; + result->type = SERD_LITERAL; + + // Write canonical form directly into node + exess_write_canonical(str, value_type, r.count + 1, serd_node_buffer(result)); + + SerdNode* const datatype_node = result + 1 + (len / sizeof(SerdNode)); + char* const datatype_buf = serd_node_buffer(datatype_node); + + datatype_node->length = datatype_uri_len; + datatype_node->type = SERD_URI; + memcpy(datatype_buf, datatype_uri, datatype_uri_len + 1); + + *out = result; + return r; +} + +static ExessResult +build_tagged(SerdAllocator* const ZIX_NONNULL allocator, + SerdNode** const out, + const SerdNode* const ZIX_NONNULL node, + const SerdNode* const ZIX_NONNULL language) +{ +#define MAX_LANG_LEN 48 // RFC5646 requires 35, RFC4646 recommends 42 + + const size_t node_len = serd_node_length(node); + const char* const lang = serd_node_string(language); + const size_t lang_len = serd_node_length(language); + if (lang_len > MAX_LANG_LEN) { + const ExessResult r = {EXESS_NO_SPACE, node_len}; + return r; + } + + // Convert language tag to lower-case + char canonical_lang[MAX_LANG_LEN] = {0}; + for (size_t i = 0U; i < lang_len; ++i) { + canonical_lang[i] = serd_to_lower(lang[i]); + } + + // Make a new literal that is otherwise identical + *out = + serd_node_new(allocator, + serd_a_literal(serd_node_string_view(node), + serd_node_flags(node), + serd_substring(canonical_lang, lang_len))); + + const ExessResult r = {EXESS_SUCCESS, node_len}; + return r; + +#undef MAX_LANG_LEN +} + +static SerdStatus +serd_canon_on_statement(SerdCanonData* const data, + const SerdStatementFlags flags, + const SerdStatement* const statement) +{ + SerdAllocator* const allocator = serd_world_allocator(data->world); + const SerdNode* const object = serd_statement_object(statement); + const SerdNode* const datatype = serd_node_datatype(object); + const SerdNode* const language = serd_node_language(object); + if (!datatype && !language) { + return serd_sink_write_statement(data->target, flags, statement); + } + + SerdNode* normo = NULL; + const ExessResult r = datatype + ? build_typed(allocator, &normo, object, datatype) + : build_tagged(allocator, &normo, object, language); + + if (r.status) { + SerdCaret caret = {NULL, 0U, 0U}; + const bool lax = (data->flags & SERD_CANON_LAX); + + if (statement->caret) { + // Adjust column to point at the error within the literal + caret.document = statement->caret->document; + caret.line = statement->caret->line; + caret.col = statement->caret->col + 1 + (unsigned)r.count; + } + + serd_logf_at(data->world, + lax ? SERD_LOG_LEVEL_WARNING : SERD_LOG_LEVEL_ERROR, + statement->caret ? &caret : NULL, + "invalid literal (%s)", + exess_strerror(r.status)); + + if (!lax) { + return r.status == EXESS_NO_SPACE ? SERD_BAD_ALLOC : SERD_BAD_LITERAL; + } + } + + if (!normo) { + return serd_sink_write_statement(data->target, flags, statement); + } + + const SerdStatus st = serd_sink_write(data->target, + flags, + statement->nodes[0], + statement->nodes[1], + normo, + statement->nodes[3]); + serd_node_free(allocator, normo); + return st; +} + +static SerdStatus +serd_canon_on_event(void* const handle, const SerdEvent* const event) +{ + SerdCanonData* const data = (SerdCanonData*)handle; + + return (event->type == SERD_STATEMENT) + ? serd_canon_on_statement( + data, event->statement.flags, event->statement.statement) + : serd_sink_write_event(data->target, event); +} + +SerdSink* +serd_canon_new(const SerdWorld* const world, + const SerdSink* const target, + const SerdCanonFlags flags) +{ + assert(world); + assert(target); + + SerdCanonData* const data = + (SerdCanonData*)serd_wcalloc(world, 1, sizeof(SerdCanonData)); + + if (!data) { + return NULL; + } + + data->world = world; + data->target = target; + data->flags = flags; + + SerdSink* const sink = + serd_sink_new(serd_world_allocator(world), data, serd_canon_on_event, free); + + if (!sink) { + serd_wfree(world, data); + } + + return sink; +} diff --git a/src/string_utils.h b/src/string_utils.h index 2517b270..3337f012 100644 --- a/src/string_utils.h +++ b/src/string_utils.h @@ -67,7 +67,7 @@ is_utf8_continuation(const uint8_t c) } static inline bool -is_space(const char c) +is_space(const int c) { switch (c) { case ' ': @@ -102,16 +102,16 @@ hex_digit_value(const uint8_t c) } static inline char -serd_to_upper(const char c) +serd_to_lower(const char c) { - return (char)((c >= 'a' && c <= 'z') ? c - 32 : c); + return (char)((c >= 'A' && c <= 'Z') ? c + 32 : c); } static inline int serd_strncasecmp(const char* s1, const char* s2, size_t n) { for (; n > 0 && *s2; s1++, s2++, --n) { - if (serd_to_upper(*s1) != serd_to_upper(*s2)) { + if (serd_to_lower(*s1) != serd_to_lower(*s2)) { return (*s1 < *s2) ? -1 : +1; } } diff --git a/test/extra/canon/bad-boolean.ttl b/test/extra/canon/bad-boolean.ttl new file mode 100644 index 00000000..c4fc3eb5 --- /dev/null +++ b/test/extra/canon/bad-boolean.ttl @@ -0,0 +1,5 @@ +@base . +@prefix xsd: . + +[] " ja "^^xsd:boolean . + diff --git a/test/extra/canon/bad-decimal-leading.ttl b/test/extra/canon/bad-decimal-leading.ttl new file mode 100644 index 00000000..0d18eac7 --- /dev/null +++ b/test/extra/canon/bad-decimal-leading.ttl @@ -0,0 +1,4 @@ +@base . +@prefix xsd: . + +[] " junk 1234.5678 "^^xsd:decimal . diff --git a/test/extra/canon/bad-decimal-trailing.ttl b/test/extra/canon/bad-decimal-trailing.ttl new file mode 100644 index 00000000..10882ef5 --- /dev/null +++ b/test/extra/canon/bad-decimal-trailing.ttl @@ -0,0 +1,4 @@ +@base . +@prefix xsd: . + +[] " 1234.5678 junk "^^xsd:decimal . diff --git a/test/extra/canon/bad-empty-boolean.ttl b/test/extra/canon/bad-empty-boolean.ttl new file mode 100644 index 00000000..9a390c46 --- /dev/null +++ b/test/extra/canon/bad-empty-boolean.ttl @@ -0,0 +1,5 @@ +@base . +@prefix xsd: . + +[] ""^^xsd:boolean . + diff --git a/test/extra/canon/bad-integer-leading.ttl b/test/extra/canon/bad-integer-leading.ttl new file mode 100644 index 00000000..80c1a6af --- /dev/null +++ b/test/extra/canon/bad-integer-leading.ttl @@ -0,0 +1,4 @@ +@base . +@prefix xsd: . + +[] " junk 987654321 "^^xsd:integer . diff --git a/test/extra/canon/bad-integer-trailing.ttl b/test/extra/canon/bad-integer-trailing.ttl new file mode 100644 index 00000000..a94a9ec4 --- /dev/null +++ b/test/extra/canon/bad-integer-trailing.ttl @@ -0,0 +1,4 @@ +@base . +@prefix xsd: . + +[] " 987654321 junk "^^xsd:integer . diff --git a/test/extra/canon/bad-lang-long.ttl b/test/extra/canon/bad-lang-long.ttl new file mode 100644 index 00000000..f84df07f --- /dev/null +++ b/test/extra/canon/bad-lang-long.ttl @@ -0,0 +1,3 @@ +@base . + +[] "hello"@ridiculously-long-lang-tag-beyond-even-RFC4646-recommendation . diff --git a/test/extra/canon/manifest.ttl b/test/extra/canon/manifest.ttl new file mode 100644 index 00000000..143928ee --- /dev/null +++ b/test/extra/canon/manifest.ttl @@ -0,0 +1,65 @@ +@prefix mf: . +@prefix rdfs: . +@prefix rdft: . + +<> + a mf:Manifest ; + rdfs:comment "Serd canonical literal writing test suite" ; + mf:entries ( + <#bad-boolean> + <#bad-decimal-leading> + <#bad-decimal-trailing> + <#bad-empty-boolean> + <#bad-integer-leading> + <#bad-integer-trailing> + <#bad-lang-long> + <#test-canon> + ) . + +<#bad-boolean> + a rdft:TestTurtleNegativeEval ; + rdfs:comment "Invalid xsd::boolean syntax" ; + mf:action ; + mf:name "bad-boolean" . + +<#bad-decimal-leading> + a rdft:TestTurtleNegativeEval ; + rdfs:comment "Invalid xsd::decimal syntax (leading garbage)" ; + mf:action ; + mf:name "bad-decimal-leading" . + +<#bad-decimal-trailing> + a rdft:TestTurtleNegativeEval ; + rdfs:comment "Invalid xsd::decimal syntax (trailing garbage)" ; + mf:action ; + mf:name "bad-decimal-trailing" . + +<#bad-empty-boolean> + a rdft:TestTurtleNegativeEval ; + rdfs:comment "Invalid xsd::boolean syntax (no value)" ; + mf:action ; + mf:name "bad-empty-boolean" . + +<#bad-integer-leading> + a rdft:TestTurtleNegativeEval ; + rdfs:comment "Invalid xsd::integer syntax (leading garbage)" ; + mf:action ; + mf:name "bad-integer-leading" . + +<#bad-integer-trailing> + a rdft:TestTurtleNegativeEval ; + rdfs:comment "Invalid xsd::integer syntax (trailing garbage)" ; + mf:action ; + mf:name "bad-integer-trailing" . + +<#bad-lang-long> + a rdft:TestTurtleNegativeEval ; + rdfs:comment "Overly long language tag" ; + mf:action ; + mf:name "bad-lang-long" . + +<#test-canon> + a rdft:TestTurtleEval ; + mf:action ; + mf:name "test-canon" ; + mf:result . diff --git a/test/extra/canon/test-canon.nt b/test/extra/canon/test-canon.nt new file mode 100644 index 00000000..ff492890 --- /dev/null +++ b/test/extra/canon/test-canon.nt @@ -0,0 +1,70 @@ +_:b1 "false"^^ . +_:b1 "false"^^ . +_:b1 "true"^^ . +_:b1 "true"^^ . +_:b1 "1.0E2"^^ . +_:b1 "-1.0E2"^^ . +_:b1 "1.0E3"^^ . +_:b1 "-1.0E3"^^ . +_:b1 "9223372036854775807"^^ . +_:b1 "-9223372036854775808"^^ . +_:b1 "2147483647"^^ . +_:b1 "-2147483648"^^ . +_:b1 "32767"^^ . +_:b1 "-32768"^^ . +_:b1 "127"^^ . +_:b1 "-128"^^ . +_:b1 "1"^^ . +_:b1 "18446744073709551615"^^ . +_:b1 "1"^^ . +_:b1 "4294967295"^^ . +_:b1 "1"^^ . +_:b1 "65535"^^ . +_:b1 "1"^^ . +_:b1 "255"^^ . +_:b1 "0.0"^^ . +_:b1 "0.0"^^ . +_:b1 "-0.0"^^ . +_:b1 "36893488147419103232.0"^^ . +_:b1 "36893488147419103232.0"^^ . +_:b1 "36893488147419103232.0"^^ . +_:b1 "36893488147419103232.0"^^ . +_:b1 "36893488147419103232.0"^^ . +_:b1 "36893488147419103232.0"^^ . +_:b1 "36893488147419103232.123"^^ . +_:b1 "-36893488147419103232.0"^^ . +_:b1 "-36893488147419103232.0"^^ . +_:b1 "-36893488147419103232.0"^^ . +_:b1 "-36893488147419103232.0"^^ . +_:b1 "-36893488147419103232.123"^^ . +_:b1 "0.123"^^ . +_:b1 "0.123"^^ . +_:b1 "0.123"^^ . +_:b1 "0.123"^^ . +_:b1 "-0.123"^^ . +_:b1 "-0.123"^^ . +_:b1 "36893488147419103232"^^ . +_:b1 "36893488147419103232"^^ . +_:b1 "36893488147419103232"^^ . +_:b1 "36893488147419103232"^^ . +_:b1 "-36893488147419103232"^^ . +_:b1 "-36893488147419103232"^^ . +_:b1 "0"^^ . +_:b1 "-36893488147419103232"^^ . +_:b1 "-1"^^ . +_:b1 "-36893488147419103232"^^ . +_:b1 "0"^^ . +_:b1 "36893488147419103232"^^ . +_:b1 "1"^^ . +_:b1 "36893488147419103232"^^ . +_:b1 "no language tag" . +_:b1 "english"@en-ca . +_:b1 "P1Y6M"^^ . +_:b1 "12:15:01Z"^^ . +_:b1 "2004-04-12Z"^^ . +_:b1 "A1B7F080"^^ . +_:b1 "Zm9vYmF="^^ . +_:b1 "untyped" . +_:b1 . +_:b1 "notxsd"^^ . +_:b1 "unsupported"^^ . diff --git a/test/extra/canon/test-canon.ttl b/test/extra/canon/test-canon.ttl new file mode 100644 index 00000000..0d0b4682 --- /dev/null +++ b/test/extra/canon/test-canon.ttl @@ -0,0 +1,76 @@ +@base . +@prefix rdf: . +@prefix xsd: . + +[ + " false "^^xsd:boolean , + " 0 "^^xsd:boolean , + " true "^^xsd:boolean , + " 1 "^^xsd:boolean ; + " +0100.0 "^^xsd:float , + " -0100.0 "^^xsd:float , + " +01000.0 "^^xsd:double , + " -01000.0 "^^xsd:double ; + " +09223372036854775807 "^^xsd:long , + " -09223372036854775808 "^^xsd:long , + " +02147483647 "^^xsd:int , + " -02147483648 "^^xsd:int , + " +032767 "^^xsd:short , + " -032768 "^^xsd:short , + " +0127 "^^xsd:byte , + " -0128 "^^xsd:byte , + " 01 "^^xsd:unsignedLong , + " 018446744073709551615 "^^xsd:unsignedLong , + " 01 "^^xsd:unsignedInt , + " 04294967295 "^^xsd:unsignedInt , + " 01 "^^xsd:unsignedShort , + " 065535 "^^xsd:unsignedShort , + " 01 "^^xsd:unsignedByte , + " 0255 "^^xsd:unsignedByte ; + " 00 "^^xsd:decimal , + " +0 "^^xsd:decimal , + " -0 "^^xsd:decimal , + " 36893488147419103232 "^^xsd:decimal , + " 0036893488147419103232 "^^xsd:decimal , + " +36893488147419103232 "^^xsd:decimal , + " +0036893488147419103232 "^^xsd:decimal , + " +0036893488147419103232. "^^xsd:decimal , + " +0036893488147419103232.00 "^^xsd:decimal , + " +0036893488147419103232.12300 "^^xsd:decimal , + " -36893488147419103232 "^^xsd:decimal , + " -0036893488147419103232 "^^xsd:decimal , + " -0036893488147419103232. "^^xsd:decimal , + " -0036893488147419103232.00 "^^xsd:decimal , + " -0036893488147419103232.12300 "^^xsd:decimal , + " 00.12300 "^^xsd:decimal , + " .12300 "^^xsd:decimal , + " +.12300 "^^xsd:decimal , + " +00.12300 "^^xsd:decimal , + " -.12300 "^^xsd:decimal , + " -00.12300 "^^xsd:decimal ; + " 36893488147419103232 "^^xsd:integer , + " 0036893488147419103232 "^^xsd:integer , + " +36893488147419103232 "^^xsd:integer , + " +0036893488147419103232 "^^xsd:integer , + " -36893488147419103232 "^^xsd:integer , + " -0036893488147419103232 "^^xsd:integer , + " 00 "^^xsd:nonPositiveInteger , + " -036893488147419103232 "^^xsd:nonPositiveInteger , + " -01 "^^xsd:negativeInteger , + " -036893488147419103232 "^^xsd:negativeInteger , + " 00 "^^xsd:nonNegativeInteger , + " 036893488147419103232 "^^xsd:nonNegativeInteger , + " +01 "^^xsd:positiveInteger , + " 036893488147419103232 "^^xsd:positiveInteger ; + "no language tag"^^rdf:langString ; + "english"@EN-CA ; +