Add statement filter sink and serd-filter tool

16 files changed, 449 insertions, 17 deletions

diff --git a/test/extra/filter/input.ttl b/test/extra/filter/input.ttl
new file mode 100644
index 00000000..59aa67f7
--- /dev/null
+++ b/test/extra/filter/input.ttl
@@ -0,0 +1,9 @@
+@prefix eg: <http://example.org/> .
+
+eg:s1
+	eg:p1 eg:o1 ;
+	eg:p2 eg:o2 .
+
+eg:s2
+	eg:p1 eg:o1 ;
+	eg:p2 eg:o2 .
diff --git a/test/extra/filter/manifest.ttl b/test/extra/filter/manifest.ttl
new file mode 100644
index 00000000..6ac5cec8
--- /dev/null
+++ b/test/extra/filter/manifest.ttl
@@ -0,0 +1,48 @@
+@prefix checks: <http://drobilla.net/ns/serd/checks#> .
+@prefix mf: <http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#> .
+@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
+@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
+@prefix rdft: <http://www.w3.org/ns/rdftest#> .
+@prefix serd: <http://drobilla.net/ns/serd#> .
+
+serd:TestFilterPositive
+	a rdfs:Class ;
+	rdfs:label "Positive Filtering" ;
+	rdfs:subClassOf rdft:Test .
+
+serd:patternFile
+	a rdf:Property ;
+	rdfs:label "pattern file" .
+
+rdft:Test
+	rdfs:subClassOf mf:ManifestEntry .
+
+<>
+	a mf:Manifest ;
+	rdfs:comment "Serd statement filtering test suite" ;
+	mf:entries (
+		<#o1>
+		<#p1>
+		<#s1>
+	) .
+
+<#o1>
+	a serd:TestFilterPositive ;
+	serd:patternFile <o1.pattern.nt> ;
+	mf:action <input.ttl> ;
+	mf:name "o1" ;
+	mf:result <o1.result.nt> .
+
+<#p1>
+	a serd:TestFilterPositive ;
+	serd:patternFile <p1.pattern.nt> ;
+	mf:action <input.ttl> ;
+	mf:name "p1" ;
+	mf:result <p1.result.nt> .
+
+<#s1>
+	a serd:TestFilterPositive ;
+	serd:patternFile <s1.pattern.nt> ;
+	mf:action <input.ttl> ;
+	mf:name "s1" ;
+	mf:result <s1.result.nt> .
diff --git a/test/extra/filter/meson.build b/test/extra/filter/meson.build
new file mode 100644
index 00000000..f03c7130
--- /dev/null
+++ b/test/extra/filter/meson.build
@@ -0,0 +1,17 @@
+base_uri = 'http://drobilla.net/sw/serd/test/filter/'
+
+test(
+  'filter',
+  run_filter_suite,
+  args: common_script_args + [
+    '--pipe',
+    serd_pipe,
+    '--filter',
+    serd_filter,
+    files('manifest.ttl'),
+    base_uri,
+  ],
+  env: test_env,
+  suite: ['suite', 'extra'],
+  timeout: 240,
+)
diff --git a/test/extra/filter/o1.pattern.nt b/test/extra/filter/o1.pattern.nt
new file mode 100644
index 00000000..41932fd7
--- /dev/null
+++ b/test/extra/filter/o1.pattern.nt
@@ -0,0 +1 @@
+?s ?p <http://example.org/o1> .
diff --git a/test/extra/filter/o1.result.nt b/test/extra/filter/o1.result.nt
new file mode 100644
index 00000000..e7b1e759
--- /dev/null
+++ b/test/extra/filter/o1.result.nt
@@ -0,0 +1,2 @@
+<http://example.org/s1> <http://example.org/p1> <http://example.org/o1> .
+<http://example.org/s2> <http://example.org/p1> <http://example.org/o1> .
diff --git a/test/extra/filter/p1.pattern.nt b/test/extra/filter/p1.pattern.nt
new file mode 100644
index 00000000..fca20e94
--- /dev/null
+++ b/test/extra/filter/p1.pattern.nt
@@ -0,0 +1 @@
+?s <http://example.org/p1> ?o .
diff --git a/test/extra/filter/p1.result.nt b/test/extra/filter/p1.result.nt
new file mode 100644
index 00000000..e7b1e759
--- /dev/null
+++ b/test/extra/filter/p1.result.nt
@@ -0,0 +1,2 @@
+<http://example.org/s1> <http://example.org/p1> <http://example.org/o1> .
+<http://example.org/s2> <http://example.org/p1> <http://example.org/o1> .
diff --git a/test/extra/filter/s1.pattern.nt b/test/extra/filter/s1.pattern.nt
new file mode 100644
index 00000000..f5b87db1
--- /dev/null
+++ b/test/extra/filter/s1.pattern.nt
@@ -0,0 +1 @@
+<http://example.org/s1> ?p ?o .
diff --git a/test/extra/filter/s1.result.nt b/test/extra/filter/s1.result.nt
new file mode 100644
index 00000000..023faf42
--- /dev/null
+++ b/test/extra/filter/s1.result.nt
@@ -0,0 +1,2 @@
+<http://example.org/s1> <http://example.org/p1> <http://example.org/o1> .
+<http://example.org/s1> <http://example.org/p2> <http://example.org/o2> .
diff --git a/test/meson.build b/test/meson.build
index c186b6e4..6ca0e38b 100644
--- a/test/meson.build
+++ b/test/meson.build
@@ -1,6 +1,7 @@
 # Copyright 2020-2023 David Robillard <d@drobilla.net>
 # SPDX-License-Identifier: 0BSD OR ISC
 
+run_filter_suite = find_program('run_filter_suite.py')
 run_suite = find_program('run_suite.py')
 wrapper = meson.get_external_property('exe_wrapper', '')
 
@@ -15,10 +16,12 @@ plot_script_paths = [
 simple_script_paths = [
   '../scripts/check_formatting.py',
   'serd_test_util/__init__.py',
+  'run_filter_suite.py',
   'run_suite.py',
   'test_base.py',
   'test_empty.py',
   'test_multifile.py',
+  'test_patterns.py',
   'test_quiet.py',
   'test_stdin.py',
   'test_write_error.py',
@@ -127,6 +130,7 @@ unit_tests = [
   'canon',
   'caret',
   'env',
+  'filter',
   'free_null',
   'log',
   'node',
@@ -177,6 +181,25 @@ if wrapper != ''
 endif
 
 simple_command_tests = {
+  'filter': {
+    'bad': [
+      ['-B', 'unknown'],
+      ['-F', '', '-G', ''],
+      ['-F', '?s ?p ?o . ?q ?r ?s .', '-s', ''],
+      ['-F', '?s ?p ?o .\n?q ?r ?s .\n', '-s', ''],
+      ['-F', 'bad_pattern', '-s', ''],
+      ['-F'],
+      ['-G', '?s ?p ?o . ?q ?r ?s .', '-s', ''],
+      ['-G', 'bad_pattern', '-s', ''],
+      ['-G'],
+      ['-f', '/no/such/file.nt', '-'],
+      ['-z'],
+    ],
+    'good': [
+      ['-V'],
+      ['-h'],
+    ],
+  },
   'pipe': {
     'bad': [
       ['-B', 'nonuriorpath'],
@@ -206,6 +229,24 @@ simple_command_tests = {
   },
 }
 
+foreach tool, tests : simple_command_tests
+  tool_var_name = 'serd_' + tool
+  if is_variable(tool_var_name)
+    foreach kind, cases : tests
+      foreach args : cases
+        test(
+          ' '.join(args).substring(1).underscorify(),
+          get_variable(tool_var_name),
+          args: args,
+          env: test_env,
+          should_fail: kind == 'bad',
+          suite: ['tools', tool, 'options'],
+        )
+      endforeach
+    endforeach
+  endif
+endforeach
+
 if is_variable('serd_pipe')
   pipe_script_args = common_script_args + ['--tool', serd_pipe]
   serd_ttl = files('../serd.ttl')[0]
@@ -218,19 +259,6 @@ if is_variable('serd_pipe')
 
   cmd_suite = ['tools', 'pipe', 'options']
 
-  foreach kind, cases : simple_command_tests['pipe']
-    foreach args : cases
-      test(
-        ' '.join(args).substring(1).underscorify(),
-        serd_pipe,
-        args: args,
-        env: test_env,
-        should_fail: kind == 'bad',
-        suite: cmd_suite,
-      )
-    endforeach
-  endforeach
-
   # Base URI options
 
   test(
@@ -405,6 +433,62 @@ if is_variable('serd_pipe')
   endif
 endif
 
+# Test specifics to serd-filter
+if is_variable('serd_filter')
+  tool = serd_filter
+  filter_script_args = common_script_args + ['--tool', serd_filter]
+
+  # Command line options
+
+  test(
+    'garbage_pattern',
+    tool,
+    args: ['junk', serd_ttl],
+    env: test_env,
+    should_fail: true,
+    suite: ['tools', 'filter', 'options'],
+  )
+  test(
+    'multiple_patterns',
+    tool,
+    args: ['?s ?p ?o .\n?t ?u ?v .\n', serd_ttl],
+    env: test_env,
+    should_fail: true,
+    suite: ['tools', 'filter', 'output'],
+  )
+  test(
+    'missing_output',
+    tool,
+    args: ['-o', '/does/not/exist.ttl', '?s ?p ?o .', serd_ttl],
+    env: test_env,
+    should_fail: true,
+    suite: ['tools', 'filter', 'output'],
+  )
+
+  # Different input sources
+
+  test(
+    'filter_dir',
+    tool,
+    args: ['?s ?p ?o .', serd_src_root],
+    env: test_env,
+    should_fail: true,
+    suite: ['tools', 'filter', 'input'],
+  )
+
+  # Filtering
+
+  test(
+    'patterns',
+    files('test_patterns.py'),
+    args: filter_script_args,
+    env: test_env,
+    suite: ['tools'],
+  )
+
+  subdir('extra/filter')
+endif
+
 ###########################
 # Data-Driven Test Suites #
 ###########################
diff --git a/test/run_filter_suite.py b/test/run_filter_suite.py
new file mode 100755
index 00000000..a1134538
--- /dev/null
+++ b/test/run_filter_suite.py
@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+
+# Copyright 2022-2023 David Robillard <d@drobilla.net>
+# SPDX-License-Identifier: ISC
+
+"""Run the RDF-based test suite for serd-filter."""
+
+# pylint: disable=duplicate-code
+
+import argparse
+import os
+import shlex
+import subprocess
+import sys
+import tempfile
+
+import serd_test_util as util
+
+NS_MF = "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#"
+NS_RDFT = "http://www.w3.org/ns/rdftest#"
+NS_SERD = "http://drobilla.net/ns/serd#"
+
+
+def run_entry(entry, filter_command, out_dir, suite_dir):
+    """Run a single test entry from the manifest."""
+
+    pattern_path = util.file_path(suite_dir, entry[NS_SERD + "patternFile"][0])
+    in_path = util.file_path(suite_dir, entry[NS_MF + "action"][0])
+    good_path = util.file_path(suite_dir, entry[NS_MF + "result"][0])
+    out_path = os.path.join(out_dir, os.path.basename(good_path))
+
+    # Run the command to write the output file
+    options = ["-f", pattern_path, "-o", out_path]
+    command = filter_command + options + [in_path]
+    subprocess.run(command, check=True)
+
+    # Check that the filtered output matches the expected result
+    return util.file_equals(good_path, out_path)
+
+
+def run_suite(manifest_path, base_uri, filter_command, pipe_command, out_dir):
+    """Run all tests in the manifest."""
+
+    # Load manifest model
+    suite_dir = os.path.dirname(manifest_path)
+    load_command = pipe_command + ["-B", base_uri]
+    filter_command = filter_command + ["-B", base_uri]
+    model, instances = util.load_rdf(load_command, manifest_path)
+
+    # Run all filter tests in the test suite
+    results = util.Results()
+    for klass, instances in instances.items():
+        if klass != "http://drobilla.net/ns/serd#TestFilterPositive":
+            continue
+
+        for instance in instances:
+            try:
+                entry = model[instance]
+                results.check(
+                    run_entry(entry, filter_command, out_dir, suite_dir)
+                )
+
+            except subprocess.CalledProcessError as exception:
+                if exception.stderr is not None:
+                    sys.stderr.write(exception.stderr.decode("utf-8"))
+
+                results.check(False, str(exception))
+
+    return util.print_result_summary(results)
+
+
+def main():
+    """Run the filter test suite via the command line tools."""
+
+    parser = argparse.ArgumentParser(
+        usage="%(prog)s [OPTION]... MANIFEST BASE_URI -- [ARG]...",
+        description=__doc__,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+
+    parser.add_argument(
+        "--pipe", default="tools/serd-pipe", help="pipe executable"
+    )
+
+    parser.add_argument(
+        "--filter", default="tools/serd-filter", help="filter executable"
+    )
+
+    parser.add_argument("--wrapper", default="", help="executable wrapper")
+    parser.add_argument("manifest", help="test suite manifest.ttl file")
+    parser.add_argument("base_uri", help="base URI for tests")
+
+    args = parser.parse_args(sys.argv[1:])
+    wrapper_prefix = shlex.split(args.wrapper)
+    filter_command = wrapper_prefix + [args.filter]
+    pipe_command = wrapper_prefix + [args.pipe]
+
+    with tempfile.TemporaryDirectory() as test_out_dir:
+        return run_suite(
+            args.manifest,
+            args.base_uri,
+            filter_command,
+            pipe_command,
+            test_out_dir,
+        )
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/test/run_suite.py b/test/run_suite.py
index 52a418ef..463e40c4 100755
--- a/test/run_suite.py
+++ b/test/run_suite.py
@@ -125,7 +125,6 @@ def run_suite(args, command, out_dir):
 
                 # Run test and record result
                 passed = run_entry(args, entry, command, out_dir, top)
-                results.check(passed)
 
                 # Write test report entry
                 if args.report:
@@ -137,13 +136,13 @@ def run_suite(args, command, out_dir):
                 if exception.stderr is not None:
                     sys.stderr.write(exception.stderr)
 
-                results.check(False, str(exception) + "\n")
+            results.check(passed)
 
     return util.print_result_summary(results)
 
 
 def main():
-    """Run the command line tool."""
+    """Run the test suite via the command line tool."""
 
     parser = argparse.ArgumentParser(
         usage="%(prog)s [OPTION]... MANIFEST BASE_URI -- [ARG]...",
diff --git a/test/serd_test_util/__init__.py b/test/serd_test_util/__init__.py
index 5f0e0033..a75bb2ae 100644
--- a/test/serd_test_util/__init__.py
+++ b/test/serd_test_util/__init__.py
@@ -182,3 +182,16 @@ def lines_equal(from_lines, to_lines, from_filename, to_filename):
         same = False
 
     return same
+
+
+def file_equals(patha, pathb):
+    """Return true if the file at patha is the same as the file at pathb."""
+
+    for path in (patha, pathb):
+        if not os.access(path, os.F_OK):
+            error("missing file {}\n".format(path))
+            return False
+
+    with open(patha, "r", encoding="utf-8") as fa:
+        with open(pathb, "r", encoding="utf-8") as fb:
+            return lines_equal(fa.readlines(), fb.readlines(), patha, pathb)
diff --git a/test/test_filter.c b/test/test_filter.c
new file mode 100644
index 00000000..652adb48
--- /dev/null
+++ b/test/test_filter.c
@@ -0,0 +1,61 @@
+// Copyright 2021 David Robillard <d@drobilla.net>
+// SPDX-License-Identifier: ISC
+
+#undef NDEBUG
+
+#include "failing_allocator.h"
+
+#include "serd/filter.h"
+#include "serd/node.h"
+#include "serd/nodes.h"
+#include "serd/sink.h"
+#include "serd/world.h"
+
+#include <assert.h>
+#include <stdbool.h>
+#include <stddef.h>
+
+static void
+test_new_failed_alloc(void)
+{
+  const SerdNodeArgs s_args = serd_a_uri_string("http://example.org/s");
+  const SerdNodeArgs p_args = serd_a_uri_string("http://example.org/p");
+  const SerdNodeArgs o_args = serd_a_uri_string("http://example.org/o");
+  const SerdNodeArgs g_args = serd_a_uri_string("http://example.org/g");
+
+  SerdFailingAllocator allocator = serd_failing_allocator();
+
+  SerdWorld* const world = serd_world_new(&allocator.base);
+  SerdNodes* const nodes = serd_nodes_new(&allocator.base);
+
+  const SerdNode* const s = serd_nodes_get(nodes, s_args);
+  const SerdNode* const p = serd_nodes_get(nodes, p_args);
+  const SerdNode* const o = serd_nodes_get(nodes, o_args);
+  const SerdNode* const g = serd_nodes_get(nodes, g_args);
+
+  SerdSink*    target = serd_sink_new(&allocator.base, NULL, NULL, NULL);
+  const size_t n_setup_allocs = allocator.n_allocations;
+
+  // Successfully allocate a filter to count the number of allocations
+  SerdSink* filter = serd_filter_new(world, target, s, p, o, g, true);
+  assert(filter);
+
+  // Test that each allocation failing is handled gracefully
+  const size_t n_new_allocs = allocator.n_allocations - n_setup_allocs;
+  for (size_t i = 0; i < n_new_allocs; ++i) {
+    allocator.n_remaining = i;
+    assert(!serd_filter_new(world, target, s, p, o, g, true));
+  }
+
+  serd_sink_free(filter);
+  serd_sink_free(target);
+  serd_nodes_free(nodes);
+  serd_world_free(world);
+}
+
+int
+main(void)
+{
+  test_new_failed_alloc();
+  return 0;
+}
diff --git a/test/test_patterns.py b/test/test_patterns.py
new file mode 100755
index 00000000..50571a92
--- /dev/null
+++ b/test/test_patterns.py
@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+
+# Copyright 2021-2023 David Robillard <d@drobilla.net>
+# SPDX-License-Identifier: ISC
+
+"""Test filtering statements inclusively and exclusively."""
+
+import serd_test_util as util
+
+DOCS = {
+    "ntriples": """
+<http://example.org/s> <http://example.org/p> <http://example.org/o> .
+<http://example.org/N> <http://example.org/I> <http://example.org/L> .
+""",
+    "nquads": """
+<urn:example:s> <urn:example:p> <urn:example:o> <urn:example:g> .
+<urn:example:N> <urn:example:U> <urn:example:L> <urn:example:L> .
+""",
+}
+
+args = util.wrapper_args(__doc__)
+
+
+def check_pattern(syntax, pattern, expected_inclusive, expected_exclusive):
+    """Run a check with an exclusive pattern."""
+
+    command = [args.tool, "-I", syntax, pattern]
+    inclusive = util.command_output(args.wrapper, command, DOCS[syntax])
+    assert inclusive == expected_inclusive
+
+    command = [args.tool, "-I", syntax, "-v", pattern]
+    exclusive = util.command_output(args.wrapper, command, DOCS[syntax])
+    assert exclusive == expected_exclusive
+
+
+check_pattern(
+    "ntriples",
+    "?s <http://example.org/p> <http://example.org/o> .",
+    "<http://example.org/s> <http://example.org/p> <http://example.org/o> .\n",
+    "<http://example.org/N> <http://example.org/I> <http://example.org/L> .\n",
+)
+
+check_pattern(
+    "ntriples",
+    "<http://example.org/s> ?p <http://example.org/o> .",
+    "<http://example.org/s> <http://example.org/p> <http://example.org/o> .\n",
+    "<http://example.org/N> <http://example.org/I> <http://example.org/L> .\n",
+)
+
+check_pattern(
+    "ntriples",
+    "<http://example.org/s> <http://example.org/p> ?o .",
+    "<http://example.org/s> <http://example.org/p> <http://example.org/o> .\n",
+    "<http://example.org/N> <http://example.org/I> <http://example.org/L> .\n",
+)
+
+check_pattern(
+    "nquads",
+    "?s <urn:example:p> <urn:example:o> <urn:example:g> .",
+    "<urn:example:s> <urn:example:p> <urn:example:o> <urn:example:g> .\n",
+    "<urn:example:N> <urn:example:U> <urn:example:L> <urn:example:L> .\n",
+)
+
+check_pattern(
+    "nquads",
+    "<urn:example:s> ?p <urn:example:o> <urn:example:g> .",
+    "<urn:example:s> <urn:example:p> <urn:example:o> <urn:example:g> .\n",
+    "<urn:example:N> <urn:example:U> <urn:example:L> <urn:example:L> .\n",
+)
+
+check_pattern(
+    "nquads",
+    "<urn:example:s> <urn:example:p> ?o <urn:example:g> .",
+    "<urn:example:s> <urn:example:p> <urn:example:o> <urn:example:g> .\n",
+    "<urn:example:N> <urn:example:U> <urn:example:L> <urn:example:L> .\n",
+)
+
+check_pattern(
+    "nquads",
+    "<urn:example:s> <urn:example:p> <urn:example:o> ?g .",
+    "<urn:example:s> <urn:example:p> <urn:example:o> <urn:example:g> .\n",
+    "<urn:example:N> <urn:example:U> <urn:example:L> <urn:example:L> .\n",
+)
diff --git a/test/test_string.c b/test/test_string.c
index 5205cc9c..2d16936a 100644
--- a/test/test_string.c
+++ b/test/test_string.c
@@ -14,7 +14,7 @@ test_strerror(void)
 {
   const char* msg = serd_strerror(SERD_SUCCESS);
   assert(!strcmp(msg, "Success"));
-  for (int i = SERD_FAILURE; i <= SERD_BAD_LITERAL; ++i) {
+  for (int i = SERD_FAILURE; i <= SERD_BAD_PATTERN; ++i) {
     msg = serd_strerror((SerdStatus)i);
     assert(strcmp(msg, "Success"));
   }