Add support for parsing variables

This adds a reader flag and serdi option for extending a syntax with support
for SPARQL-like variables, for storing things like patterns or simple queries.

19 files changed, 186 insertions, 14 deletions

diff --git a/NEWS b/NEWS
index ee2f04e1..d1fa387d 100644
--- a/NEWS
+++ b/NEWS
@@ -7,6 +7,7 @@ serd (1.0.1) unstable;
   * Add model for storing statements in memory
   * Add option for writing terse output without newlines
   * Add support for basic literal normalisation
+  * Add support for parsing variables
   * Add support for validation
   * Add support for writing terse collections
   * Add support for xsd:float and xsd:double literals
@@ -22,7 +23,7 @@ serd (1.0.1) unstable;
   * Use a fixed-size reader stack
   * Use char* for strings in public API
 
- -- David Robillard <d@drobilla.net>  Sat, 19 Jan 2019 12:31:12 +0000
+ -- David Robillard <d@drobilla.net>  Thu, 19 Dec 2019 19:47:28 +0000
 
 serd (0.30.6) stable;
 
diff --git a/doc/serdi.1 b/doc/serdi.1
index 01600700..23b5cd55 100644
--- a/doc/serdi.1
+++ b/doc/serdi.1
@@ -108,6 +108,11 @@ Write terser output without newlines.
 .BR \-v
 Display version information and exit.
 
+.TP
+.BR \-x
+Support parsing variable nodes.
+Variables can be written in SPARQL style, for example \*(lq?var\*(rq or \*(lq$var\*(rq.
+
 .SH AUTHOR
 Serdi was written by David Robillard <d@drobilla.net>
 
diff --git a/serd/serd.h b/serd/serd.h
index 2574a592..794fe285 100644
--- a/serd/serd.h
+++ b/serd/serd.h
@@ -188,7 +188,16 @@ typedef enum {
 	   is meaningful only within this serialisation.  @see [RDF 1.1
 	   Turtle](http://www.w3.org/TR/turtle/#grammar-production-BLANK_NODE_LABEL)
 	*/
-	SERD_BLANK = 4
+	SERD_BLANK = 4,
+
+	/**
+	   A variable node
+
+	   Value is a variable name without any syntactic prefix, like "name",
+	   which is meaningful only within this serialisation.  @see [SPARQL 1.1
+	   Query Language](https://www.w3.org/TR/sparql11-query/#rVar)
+	*/
+	SERD_VARIABLE = 5
 } SerdNodeType;
 
 /// Flags indicating certain string properties relevant to serialisation
@@ -258,7 +267,8 @@ typedef struct {
 
 /// Reader options
 typedef enum {
-	SERD_READ_LAX = 1 << 0  ///< Tolerate invalid input where possible
+	SERD_READ_LAX       = 1 << 0, ///< Tolerate invalid input where possible
+	SERD_READ_VARIABLES = 1 << 1  ///< Support variable nodes
 } SerdReaderFlag;
 
 /// Bitwise OR of SerdReaderFlag values
diff --git a/src/env.c b/src/env.c
index 5cbb6a43..3abbc857 100644
--- a/src/env.c
+++ b/src/env.c
@@ -371,6 +371,7 @@ serd_env_expand(const SerdEnv* env, const SerdNode* node)
 		case SERD_CURIE:
 			return expand_curie(env, node);
 		case SERD_BLANK:
+		case SERD_VARIABLE:
 			return serd_node_copy(node);
 		}
 	}
diff --git a/src/n3.c b/src/n3.c
index 4fe0363c..39cd515f 100644
--- a/src/n3.c
+++ b/src/n3.c
@@ -866,10 +866,50 @@ read_literal(SerdReader* reader, SerdNode** dest, bool* ate_dot)
 }
 
 static SerdStatus
+read_VARNAME(SerdReader* reader, SerdNode** dest)
+{
+	// Simplified from SPARQL: VARNAME ::= (PN_CHARS_U | [0-9])+
+	SerdNode*  n  = *dest;
+	SerdStatus st = SERD_SUCCESS;
+	int        c  = 0;
+	peek_byte(reader);
+	while ((c = peek_byte(reader))) {
+		if (is_digit(c) || c == '_') {
+			push_byte(reader, n, eat_byte_safe(reader, c));
+		} else if ((st = read_PN_CHARS(reader, n))) {
+			st = st > SERD_FAILURE ? st : SERD_SUCCESS;
+			break;
+		}
+	}
+
+	return st;
+}
+
+static SerdStatus
+read_Var(SerdReader* reader, SerdNode** dest)
+{
+	if (!(reader->flags & SERD_READ_VARIABLES)) {
+		return r_err(reader, SERD_ERR_BAD_SYNTAX,
+		             "syntax does not support variables\n");
+	} else if (!(*dest = push_node(reader, SERD_VARIABLE, "", 0))) {
+		return SERD_ERR_OVERFLOW;
+	}
+
+	assert(peek_byte(reader) == '$' || peek_byte(reader) == '?');
+	serd_byte_source_advance(&reader->source);
+
+	return read_VARNAME(reader, dest);
+}
+
+static SerdStatus
 read_verb(SerdReader* reader, SerdNode** dest)
 {
 	const size_t orig_stack_size = reader->stack.size;
-	if (peek_byte(reader) == '<') {
+
+	switch (peek_byte(reader)) {
+	case '$': case '?':
+		return read_Var(reader, dest);
+	case '<':
 		return read_IRIREF(reader, dest);
 	}
 
@@ -1030,7 +1070,13 @@ read_object(SerdReader* reader, ReadContext* ctx, bool emit, bool* ate_dot)
 	const int  c      = peek_byte(reader);
 	if (!fancy_syntax(reader)) {
 		switch (c) {
-		case '"': case ':': case '<': case '_': break;
+		case '"': case ':': case '<': case '_':
+			break;
+		case '$': case '?':
+			if (reader->flags & SERD_READ_VARIABLES) {
+				break;
+			}
+			break;
 		default:
 			return r_err(reader, SERD_ERR_BAD_SYNTAX,
 			             "expected: ':', '<', or '_'\n");
@@ -1040,6 +1086,9 @@ read_object(SerdReader* reader, ReadContext* ctx, bool emit, bool* ate_dot)
 	switch (c) {
 	case EOF: case ')':
 		return r_err(reader, SERD_ERR_BAD_SYNTAX, "expected object\n");
+	case '$': case '?':
+		ret = read_Var(reader, &o);
+		break;
 	case '[':
 		simple = false;
 		ret = read_anon(reader, *ctx, false, &o);
@@ -1241,6 +1290,9 @@ read_subject(SerdReader* reader, ReadContext ctx, SerdNode** dest, int* s_type)
 	SerdStatus st      = SERD_SUCCESS;
 	bool       ate_dot = false;
 	switch ((*s_type = peek_byte(reader))) {
+	case '$': case '?':
+		st = read_Var(reader, dest);
+		break;
 	case '[':
 		read_anon(reader, ctx, true, dest);
 		break;
@@ -1558,11 +1610,21 @@ read_nquadsDoc(SerdReader* reader)
 			return SERD_ERR_BAD_SYNTAX;
 		}
 
-		// subject predicate object
 		if ((st = read_subject(reader, ctx, &ctx.subject, &s_type)) ||
-		    !read_ws_star(reader) ||
-		    (st = read_IRIREF(reader, &ctx.predicate)) ||
-		    !read_ws_star(reader) ||
+		    !read_ws_star(reader)) {
+			return st;
+		}
+
+		switch (peek_byte(reader)) {
+		case '$': case '?':
+			st = read_Var(reader, &ctx.predicate);
+			break;
+		case '<':
+			st = read_IRIREF(reader, &ctx.predicate);
+			break;
+		}
+
+		if (st || !read_ws_star(reader) ||
 		    (st = read_object(reader, &ctx, false, &ate_dot))) {
 			return st;
 		}
diff --git a/src/node.c b/src/node.c
index b33a03e9..66964556 100644
--- a/src/node.c
+++ b/src/node.c
@@ -156,7 +156,8 @@ serd_new_simple_node(SerdNodeType type, const char* str, const size_t len)
 {
 	if (!str) {
 		return NULL;
-	} else if (type != SERD_BLANK && type != SERD_CURIE && type != SERD_URI) {
+	} else if (type != SERD_BLANK && type != SERD_CURIE && type != SERD_URI &&
+	           type != SERD_VARIABLE) {
 		return NULL;
 	}
 
diff --git a/src/reader.c b/src/reader.c
index 668cd38d..3dd2f465 100644
--- a/src/reader.c
+++ b/src/reader.c
@@ -172,6 +172,7 @@ serd_reader_new(SerdWorld*      world,
 	me->sink          = sink;
 	me->stack         = serd_stack_new(stack_size, serd_node_align);
 	me->syntax        = syntax;
+	me->flags         = flags;
 	me->next_id       = 1;
 	me->strict        = !(flags & SERD_READ_LAX);
 
diff --git a/src/reader.h b/src/reader.h
index 21dc2e04..a5e1f3f0 100644
--- a/src/reader.h
+++ b/src/reader.h
@@ -60,6 +60,7 @@ struct SerdReaderImpl {
 	SerdByteSource    source;
 	SerdStack         stack;
 	SerdSyntax        syntax;
+	SerdReaderFlags   flags;
 	unsigned          next_id;
 	uint8_t*          buf;
 	char*             bprefix;
diff --git a/src/serdi.c b/src/serdi.c
index 7f4880fd..1339e593 100644
--- a/src/serdi.c
+++ b/src/serdi.c
@@ -72,6 +72,7 @@ print_usage(const char* name, bool error)
 	fprintf(os, "  -s INPUT     Parse INPUT as string.\n");
 	fprintf(os, "  -t           Write terser output without newlines.\n");
 	fprintf(os, "  -v           Display version information and exit.\n");
+	fprintf(os, "  -x           Support parsing variable nodes like `?x'.\n");
 	return error ? 1 : 0;
 }
 
@@ -228,6 +229,8 @@ main(int argc, char** argv)
 				return missing_arg(argv[0], 'r');
 			}
 			root_uri = argv[a];
+		} else if (argv[a][1] == 'x') {
+			reader_flags |= SERD_READ_VARIABLES;
 		} else {
 			SERDI_ERRORF("invalid option -- '%s'\n", argv[a] + 1);
 			return print_usage(argv[0], true);
diff --git a/src/writer.c b/src/writer.c
index acb63d7d..d3326647 100644
--- a/src/writer.c
+++ b/src/writer.c
@@ -773,6 +773,18 @@ write_blank(SerdWriter* const        writer,
 }
 
 SERD_WARN_UNUSED_RESULT static SerdStatus
+write_variable(SerdWriter* const writer, const SerdNode* node)
+{
+	SerdStatus st = SERD_SUCCESS;
+
+	TRY(st, esink("?", 1, writer));
+	TRY(st, esink(serd_node_string(node), node->n_bytes, writer));
+
+	writer->last_sep = SEP_NONE;
+	return st;
+}
+
+SERD_WARN_UNUSED_RESULT static SerdStatus
 write_node(SerdWriter*        writer,
            const SerdNode*    node,
            const SerdField    field,
@@ -787,6 +799,8 @@ write_node(SerdWriter*        writer,
 		return write_curie(writer, node, field, flags);
 	case SERD_BLANK:
 		return write_blank(writer, node, field, flags);
+	case SERD_VARIABLE:
+		return write_variable(writer, node);
 	}
 
 	return SERD_ERR_INTERNAL;
diff --git a/tests/bad/bad-var.ttl b/tests/bad/bad-var.ttl
new file mode 100644
index 00000000..29b5b008
--- /dev/null
+++ b/tests/bad/bad-var.ttl
@@ -0,0 +1,2 @@
+?s
+	<http://example.org/p> <http://example.org/o> .
diff --git a/tests/bad/manifest.ttl b/tests/bad/manifest.ttl
index e8ecb396..1bcf6611 100644
--- a/tests/bad/manifest.ttl
+++ b/tests/bad/manifest.ttl
@@ -70,6 +70,7 @@
 		<#bad-string>
 		<#bad-subject>
 		<#bad-uri-escape>
+		<#bad-var>
 		<#bad-verb>
 		<#invalid-char-in-local>
 		<#invalid-char-in-prefix>
@@ -394,6 +395,11 @@
 	mf:name "bad-uri-escape" ;
 	mf:action <bad-uri-escape.ttl> .
 
+<#bad-var>
+	rdf:type rdft:TestTurtleNegativeSyntax ;
+	mf:name "bad-var" ;
+	mf:action <bad-var.ttl> .
+
 <#bad-verb>
 	rdf:type rdft:TestTurtleNegativeSyntax ;
 	mf:name "bad-verb" ;
diff --git a/tests/overflow_test.c b/tests/overflow_test.c
index 918f1cc4..847884de 100644
--- a/tests/overflow_test.c
+++ b/tests/overflow_test.c
@@ -24,8 +24,8 @@
 static SerdStatus
 test(SerdWorld* world, SerdSink* sink, const char* str, size_t stack_size)
 {
-	SerdReader* reader =
-		serd_reader_new(world, SERD_TURTLE, 0, sink, stack_size);
+	SerdReader* reader = serd_reader_new(
+	        world, SERD_TURTLE, SERD_READ_VARIABLES, sink, stack_size);
 
 	serd_reader_start_string(reader, str, NULL);
 	const SerdStatus st = serd_reader_read_document(reader);
@@ -54,6 +54,7 @@ main(void)
 	                      {":s :p \"literal\"", sizes + 264},
 	                      {":s :p \"verb\"", sizes + 263},
 	                      {":s :p _:blank .", sizes + 276},
+	                      {":s :p ?o .", sizes + 295},
 	                      {":s :p true .", sizes + 295},
 	                      {":s :p true .", sizes + 329},
 	                      {":s :p \"\"@en .", sizes + 302},
diff --git a/tests/pattern/bad-pattern.nq b/tests/pattern/bad-pattern.nq
new file mode 100644
index 00000000..a7e0c994
--- /dev/null
+++ b/tests/pattern/bad-pattern.nq
@@ -0,0 +1 @@
+<http://example.org/s> ?pµ <http://example.org/o> .
diff --git a/tests/pattern/bad-pattern.ttl b/tests/pattern/bad-pattern.ttl
new file mode 100644
index 00000000..5f3dbfdd
--- /dev/null
+++ b/tests/pattern/bad-pattern.ttl
@@ -0,0 +1,2 @@
+?sµ
+	<http://example.org/p> <http://example.org/o> .
diff --git a/tests/pattern/manifest.ttl b/tests/pattern/manifest.ttl
new file mode 100644
index 00000000..a179a64d
--- /dev/null
+++ b/tests/pattern/manifest.ttl
@@ -0,0 +1,35 @@
+@prefix mf: <http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#> .
+@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
+@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
+@prefix rdft: <http://www.w3.org/ns/rdftest#> .
+
+<>
+	rdf:type mf:Manifest ;
+	rdfs:comment "Serd pattern syntax test cases" ;
+	mf:entries (
+		<#bad-pattern-nq>
+		<#bad-pattern-ttl>
+		<#test-pattern-nt>
+		<#test-pattern-ttl>
+	) .
+
+<#bad-pattern-nq>
+	rdf:type rdft:TestNQuadsNegativeSyntax ;
+	mf:name "bad-pattern-nq" ;
+	mf:action <bad-pattern.nq> .
+
+<#bad-pattern-ttl>
+	rdf:type rdft:TestTurtleNegativeSyntax ;
+	mf:name "bad-pattern" ;
+	mf:action <bad-pattern.ttl> .
+
+<#test-pattern-nt>
+	rdf:type rdft:TestNTriplesPositiveSyntax ;
+	mf:name "test-pattern-nt" ;
+	mf:action <test-pattern.nt> .
+
+<#test-pattern-ttl>
+	rdf:type rdft:TestTurtleEval ;
+	mf:name "test-pattern" ;
+	mf:action <test-pattern.ttl> ;
+	mf:result <test-pattern.nt> .
diff --git a/tests/pattern/test-pattern.nt b/tests/pattern/test-pattern.nt
new file mode 100644
index 00000000..ddfe6d3c
--- /dev/null
+++ b/tests/pattern/test-pattern.nt
@@ -0,0 +1,6 @@
+?s <http://example.org/p1> <http://example.org/o1> .
+<http://example.org/s> ?p <http://example.org/o1> .
+<http://example.org/s> <http://example.org/p1> ?o .
+<http://example.org/s> <http://example.org/p2> _:b1 .
+_:b1 ?2p <http://example.org/o2> .
+_:b1 <http://example.org/p3> ?_o .
diff --git a/tests/pattern/test-pattern.ttl b/tests/pattern/test-pattern.ttl
new file mode 100644
index 00000000..3742e5ed
--- /dev/null
+++ b/tests/pattern/test-pattern.ttl
@@ -0,0 +1,10 @@
+?s
+	<http://example.org/p1> <http://example.org/o1> .
+
+<http://example.org/s>
+	?p <http://example.org/o1> ;
+	<http://example.org/p1> ?o ;
+	<http://example.org/p2> [
+		?2p <http://example.org/o2> ;
+		<http://example.org/p3> ?_o
+	] .
diff --git a/wscript b/wscript
index 404e0d4c..8a00a12f 100644
--- a/wscript
+++ b/wscript
@@ -679,8 +679,17 @@ def test(tst):
     import tempfile
 
     # Create test output directories
-    for i in ['bad', 'good', 'lax', 'normalise', 'terse', 'multifile',
-              'TurtleTests', 'NTriplesTests', 'NQuadsTests', 'TriGTests']:
+    for i in ['bad',
+              'good',
+              'lax',
+              'normalise',
+              'pattern',
+              'terse',
+              'multifile',
+              'TurtleTests',
+              'NTriplesTests',
+              'NQuadsTests',
+              'TriGTests']:
         try:
             test_dir = os.path.join('tests', i)
             os.makedirs(test_dir)
@@ -807,6 +816,7 @@ def test(tst):
     test_suite(tst, serd_base + 'bad/', 'bad', None, 'Turtle')
     test_suite(tst, serd_base + 'lax/', 'lax', None, 'Turtle', ['-l'])
     test_suite(tst, serd_base + 'lax/', 'lax', None, 'Turtle')
+    test_suite(tst, serd_base + 'pattern/', 'pattern', None, 'Turtle', ['-x'])
     test_suite(tst, serd_base + 'normalise/', 'normalise', None, 'Turtle',
                ['-n'])
     test_suite(tst, serd_base + 'terse/', 'terse', None, 'Turtle', ['-t'],