From 207d78fafb99d26363e6e239c653482694d32510 Mon Sep 17 00:00:00 2001 From: David Robillard Date: Tue, 14 Jul 2020 00:55:52 +0200 Subject: WIP: Generate Sphinx documentation --- bindings/python/_static/custom.css | 31 ++ bindings/python/_static/serd.svg | 135 ++++++++ bindings/python/conf.py | 91 ++++++ bindings/python/index.rst | 11 + bindings/python/overview.rst | 616 +++++++++++++++++++++++++++++++++++++ bindings/python/reference.rst | 8 + wscript | 15 + 7 files changed, 907 insertions(+) create mode 100644 bindings/python/_static/custom.css create mode 100644 bindings/python/_static/serd.svg create mode 100644 bindings/python/conf.py create mode 100644 bindings/python/index.rst create mode 100644 bindings/python/overview.rst create mode 100644 bindings/python/reference.rst diff --git a/bindings/python/_static/custom.css b/bindings/python/_static/custom.css new file mode 100644 index 00000000..37807922 --- /dev/null +++ b/bindings/python/_static/custom.css @@ -0,0 +1,31 @@ +div.document { + margin : 0 +} + +div.body { + margin-top : 2em +} + +div.sphinxsidebarwrapper { + background : #EEE +} + +div.sphinxsidebarwrapper p.blurb { + text-align : center +} + +img.logo { + width : 6em +} + +.class { + padding-top : 1.5em +} + +.function { + padding-top : 1.5em +} + +.method { + padding-top : 0.75em +} diff --git a/bindings/python/_static/serd.svg b/bindings/python/_static/serd.svg new file mode 100644 index 00000000..6682c2e2 --- /dev/null +++ b/bindings/python/_static/serd.svg @@ -0,0 +1,135 @@ + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/bindings/python/conf.py b/bindings/python/conf.py new file mode 100644 index 00000000..3f9acac1 --- /dev/null +++ b/bindings/python/conf.py @@ -0,0 +1,91 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +sys.path.insert(0, os.path.abspath('../../build/bindings/python')) + + + +from unittest.mock import Mock as MagicMock + +class Mock(MagicMock): + @classmethod + def __getattr__(cls, name): + return MagicMock() + +MOCK_MODULES = ['cython', 'libc.stdint'] +sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES) + + +# -- Project information ----------------------------------------------------- + +project = 'Serd' +copyright = '2020, David Robillard' +author = 'David Robillard' + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.doctest', + 'sphinx.ext.napoleon', + 'sphinx.ext.viewcode', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + + +html_theme_options = {'body_max_width': '80em'} + +html_sidebars = { + '**': [ + 'about.html', + 'localtoc.html', + 'donate.html', + ] +} + +html_theme_options = { + 'description': 'A lightweight library for working with RDF data.', + 'donate_url': 'http://drobilla.net/pages/donate.html', + 'github_repo': 'serd', + 'github_user': 'drobilla', + 'logo': 'serd.svg', + 'logo_name': True, + 'logo_text_align': 'center', + 'page_width': '80em - 15em', + 'sidebar_width': '15em', +} diff --git a/bindings/python/index.rst b/bindings/python/index.rst new file mode 100644 index 00000000..d939ed49 --- /dev/null +++ b/bindings/python/index.rst @@ -0,0 +1,11 @@ +######################### +Serd Python Documentation +######################### + +.. toctree:: + + overview + reference + +:ref:`genindex` +:ref:`modindex` diff --git a/bindings/python/overview.rst b/bindings/python/overview.rst new file mode 100644 index 00000000..582c3762 --- /dev/null +++ b/bindings/python/overview.rst @@ -0,0 +1,616 @@ +.. testsetup:: * + + import serd + +======== +Overview +======== + +Serd is a lightweight C library for working with RDF data. This is the +documentation for its Python bindings, which also serves as a gentle +introduction to the basics of RDF. + +Serd is designed for high-performance or resource-constrained applications, and +makes it possible to work with very large documents quickly and/or using +minimal memory. In particular, it is dramatically faster than `rdflib +`_, though it is less fully-featured +and not pure Python. + +Nodes +===== + +Nodes are the basic building blocks of data. Nodes are essentially strings: + +>>> print(serd.uri("http://example.org/something")) +http://example.org/something + +>>> print(serd.string("hello")) +hello + +>>> print(serd.decimal(1234)) +1234.0 + +>>> len(serd.string("hello")) +5 + +However, nodes also have a :meth:`~serd.Node.type`, and optionally either a +:meth:`~serd.Node.datatype` or :meth:`~serd.Node.language`. + +Representation +-------------- + +The string content of a node as shown above can be ambiguous. For example, it +is impossible to tell a URI from a string literal using only their string +contents. The :meth:`~serd.Node.to_syntax` method returns a complete +representation of a node, in the `Turtle `_ +syntax by default: + +>>> print(serd.uri("http://example.org/something").to_syntax()) + + +>>> print(serd.string("hello").to_syntax()) +"hello" + +>>> print(serd.decimal(1234).to_syntax()) +1234.0 + +Note that the representation of a node in some syntax *may* be the same as the +``str()`` contents which are printed, but this is usually not the case. For +example, as shown above, URIs and strings are quoted differently in Turtle. + +A different syntax can be used by specifying one explicitly: + +>>> print(serd.decimal(1234).to_syntax(serd.Syntax.NTRIPLES)) +"1234.0"^^ + +An identical node can be recreated from such a string using the +:meth:`~serd.Node.from_syntax` method: + +>>> node = serd.decimal(1234) +>>> copy = serd.Node.from_syntax(node.to_syntax()) # Don't actually do this +>>> print(copy) +1234.0 + +Alternatively, the ``repr()`` builtin will return the Python construction +representation: + +>>> repr(serd.decimal(1234)) +'serd.typed_literal("1234.0", "http://www.w3.org/2001/XMLSchema#decimal")' + +Any node can be round-tripped to and from a string using these methods. That +is, for any node `n`, both:: + + serd.Node.from_syntax(n.to_syntax()) + +and:: + + eval(repr(n)) + +produce an equivalent node. Using the `to_syntax()` method is generally +recommended, since it uses standard syntax. + +Primitives +---------- + +For convenience, nodes can be constructed from Python primitives by simply +passing a value to the constructor: + +>>> repr(serd.Node(True)) +'serd.boolean(True)' +>>> repr(serd.Node("hello")) +'serd.string("hello")' +>>> repr(serd.Node(1234)) +'serd.typed_literal("1234", "http://www.w3.org/2001/XMLSchema#integer")' +>>> repr(serd.Node(12.34)) +'serd.typed_literal("1.234E1", "http://www.w3.org/2001/XMLSchema#double")' + +Note that it is not possible to construct every type of node this way, and care +should be taken to not accidentally construct a string literal where a URI is +desired. + +Fundamental Constructors +------------------------ + +As the above examples suggest, several node constructors are just convenience +wrappers for more fundamental ones. All node constructors reduce to one of the +following: + + * :func:`serd.plain_literal` - A string with optional language, like + `"hallo"@de` in Turtle. + + * :func:`serd.typed_literal` - A string with optional datatype, like + ``"1.2E9"^^xsd:float`` in Turtle. + + * :func:`serd.blank` - A blank node, like "b42", which would be ``_:b42`` in + Turtle. + + * :func:`serd.curie` - A compact URI, like "eg:name". + + * :func:`serd.uri` - A URI, like "http://example.org", which would be + ```` in Turtle. + +Convenience Constructors +------------------------ + + * :func:`serd.string` - A string literal with no language or datatype. + * :func:`serd.decimal` - An `xsd:decimal + `_ like "123.45". + * :func:`serd.double` - An `xsd:double + `_ like "1.2345E2". + * :func:`serd.float` - An `xsd:float + `_ like "1.2345E2". + * :func:`serd.integer` - An `xsd:integer + `_ like "1234567". + * :func:`serd.boolean` - An `xsd:boolean + `_ like "true" or "false". + * :func:`serd.blob` - An `xsd:base64Binary + `_ like "aGVsbG8=". + * :func:`serd.resolved_uri` - A URI resolved against a base like "http://example.org/rel". + * :func:`serd.file_uri` - A file URI like "file:///doc.ttl". + * :func:`serd.relative_uri` - A relative URI reference like "foo/bar". + +Namespaces +========== + +It is common to use many URIs that share a common prefix. The +:class:`~serd.Namespace` utility class can be used to make code more readable +and make mistakes less likely: + +>>> eg = serd.Namespace("http://example.org/") +>>> print(eg.thing) +http://example.org/thing + +.. testsetup:: * + + eg = serd.Namespace("http://example.org/") + +Dictionary syntax can also be used: + +>>> print(eg["thing"]) +http://example.org/thing + +For convenience, namespaces also act like strings in many cases: + +>>> print(eg) +http://example.org/ +>>> print(eg + "stringeyName") +http://example.org/stringeyName + +Note that this class is just a simple syntactic convenience, it does not +"remember" names and there is no corresponding C API. + +Statements +========== + +A :class:`~serd.Statement` is a tuple of either 3 or 4 nodes: the subject, +predicate, object, and optional graph. Statements declare that a subject has +some property. The predicate identifies the property, and the object is its +value. + +A statement is a bit like a very simple machine-readable sentence. The +"subject" and "object" are as in natural language, and the predicate is like +the verb, but more general. For example, we could make a statement in English +about your intrepid author: + + drobilla has the first name "David" + +We can break this statement into 3 pieces like so: + +.. list-table:: + :header-rows: 1 + + * - Subject + - Predicate + - Object + * - drobilla + - has the first name + - "David" + +To make a :class:`~serd.Statement` out of this, we need to define some URIs. In +RDF, the subject and predicate must be *resources* with an identifier (for +example, neither can be a string). Conventionally, predicate names do not +start with "has" or similar words, since that would be redundant in this +context. So, we assume that ``http://example.org/drobilla`` is the URI for +drobilla, and ``http://example.org/firstName`` has been defined somewhere to be +a property with the appropriate meaning, and can make an equivalent +:class:`~serd.Statement`: + +>>> print(serd.Statement(eg.drobilla, eg.firstName, serd.string("David"))) + "David" + +If you find this terminology confusing, it may help to think in terms of +dictionaries instead. For example, the above can be thought of as equivalent +to:: + + drobilla[firstName] = "David" + +or:: + + drobilla.firstName = "David" + +Accessing Fields +---------------- + +Statement fields can be accessed via named methods or array indexing: + +>>> statement = serd.Statement(eg.s, eg.p, eg.o, eg.g) +>>> print(statement.subject()) +http://example.org/s +>>> print(statement[serd.Field.SUBJECT]) +http://example.org/s +>>> print(statement[0]) +http://example.org/s + +Graph +----- + +The graph field can be used as a context to distinguish otherwise identical +statements. For example, it is often set to the URI of the document that the +statement was loaded from: + +>>> print(serd.Statement(eg.s, eg.p, eg.o, serd.uri("file:///doc.ttl"))) + + +The graph field is always accessible, but may be ``None``: + + >>> triple = serd.Statement(eg.s, eg.p, eg.o) + >>> print(triple.graph()) + None + >>> quad = serd.Statement(eg.s, eg.p, eg.o, eg.g) + >>> print(quad.graph()) + http://example.org/g + +World +===== + +So far, we have only used nodes and statements, which are simple independent +objects. Higher-level facilities in serd require a :class:`~serd.World` which +represents the global library state. + +A program typically uses just one world, which can be constructed with no +arguments:: + + world = serd.World() + +.. testsetup:: * + + world = serd.World() + +Note that the world is not a database, it only manages a small amount of +library state for things like configuration and logging. + +All "global" state is handle explicitly via the world. Serd does not contain +any static mutable data, making it suitable for use in modules or plugins. If +multiple worlds *are* used in a single program, they must never be mixed: +objects "inside" one world can not be used with objects inside another. + +Generating Blanks +----------------- + +Blank nodes, or simply "blanks", are used for resources that do not have URIs. +Unlike URIs, they are not global identifiers, and only have meaning within +their local context (for example, a document). The world provides a method for +automatically generating unique blank identifiers: + +>>> print(repr(world.get_blank())) +serd.blank("b1") +>>> print(repr(world.get_blank())) +serd.blank("b2") + +Model +===== + +A :class:`~serd.Model` is an indexed set of statements. A model can be used to +store any set of data, from a few statements (for example, a protocol message), +to an entire document, to a database with millions of statements. + +A model can be constructed and statements inserted manually using the +:meth:`~serd.Model.insert` method. Tuple syntax is supported as a shorthand +for creating statements: + +>>> model = serd.Model(world) +>>> model.insert((eg.s, eg.p, eg.o1)) +>>> model.insert((eg.s, eg.p, eg.o2)) +>>> model.insert((eg.t, eg.p, eg.o3)) + +.. testsetup:: model_manual + + import serd + eg = serd.Namespace("http://example.org/") + world = serd.World() + model = serd.Model(world) + model.insert((eg.s, eg.p, eg.o1)) + model.insert((eg.s, eg.p, eg.o2)) + model.insert((eg.t, eg.p, eg.o3)) + +Iterating over the model yields every statement: + +>>> for s in model: print(s) + + + + +Familiar Pythonic collection operations work as you would expect: + +>>> print(len(model)) +3 +>>> print((eg.s, eg.p, eg.o4) in model) +False +>>> model += (eg.s, eg.p, eg.o4) +>>> print((eg.s, eg.p, eg.o4) in model) +True + +Pattern Matching +---------------- + +The :meth:`~serd.Model.ask` method can be used to check if a statement is in a +model: + +>>> print(model.ask(eg.s, eg.p, eg.o1)) +True +>>> print(model.ask(eg.s, eg.p, eg.s)) +False + +This method is more powerful than the ``in`` statement because it also does +pattern matching. To check for a pattern, use `None` as a wildcard: + +>>> print(model.ask(eg.s, None, None)) +True +>>> print(model.ask(eg.unknown, None, None)) +False + +The :meth:`~serd.Model.count` method works similarly, but instead returns the +number of statements that match the pattern: + +>>> print(model.count(eg.s, None, None)) +3 +>>> print(model.count(eg.unknown, None, None)) +0 + +Getting Values +-------------- + +Sometimes you are only interested in a single node, and it is cumbersome to +first search for a statement and then get the node from it. The +:meth:`~serd.Model.get` method provides a more convenient way to do this. To +get a value, specify a triple pattern where exactly one field is ``None``. If +a statement matches, then the node that "fills" the wildcard will be returned: + +>>> print(model.get(eg.t, eg.p, None)) +http://example.org/o3 + +If multiple statements match the pattern, then the matching node from an +arbitrary statement is returned. It is an error to specify more than one +wildcard, excluding the graph. + +Erasing Statements +------------------ + +>>> model2 = model.copy() +>>> for s in model2: print(s) + + + + + +Individual statements can be erased by value, again with tuple syntax supported +for convenience: + +>>> model2.erase((eg.s, eg.p, eg.o1)) +>>> for s in model2: print(s) + + + + +Many statements can be erased at once by erasing a range: + +>>> model2.erase(model2.range((eg.s, None, None))) +>>> for s in model2: print(s) + + +Saving Documents +---------------- + +Serd provides simple methods to save an entire model to a file or string, which +are similar to functions in the standard Python ``json`` module. + +A model can be saved to a file with the :meth:`~serd.World.dump` method: + +.. doctest:: + :options: +NORMALIZE_WHITESPACE + + >>> world.dump(model, "out.ttl") + >>> print(open("out.ttl", "r").read()) + + , + , + . + + + . + + +Similarly, a model can be written as a string with the :meth:`serd.World.dumps` +method: + +.. doctest:: + :options: +ELLIPSIS + + >>> print(world.dumps(model)) + + ... + +Loading Documents +----------------- + +There are also simple methods to load an entire model, again loosely following +the standard Python ``json`` module. + +A model can be loaded from a file with the :meth:`~serd.World.load` method: + +>>> model3 = world.load("out.ttl") +>>> print(model3 == model) +True + +By default, the syntax type is determined by the file extension, and only +:attr:`serd.ModelFlags.INDEX_SPO` will be set, so only ``(s p ?)`` and ``(s ? +?)`` queries will be fast. See the method documentation for how to control +things more precisely. + +Similarly, a model can be loaded from a string with the +:meth:`~serd.World.loads` method: + +>>> ttl = "<{}> <{}> <{}> .".format(eg.s, eg.p, eg.o) +>>> model4 = world.loads(ttl) +>>> for s in model4: print(s) + + +File Cursor +----------- + +When data is loaded from a file into a model with the flag +:data:`~serd.ModelFlags.STORE_CURSORS`, each statement will have a *cursor* +which describes the file name, line, and column where the statement originated: + +>>> model5 = world.load("out.ttl", model_flags=serd.ModelFlags.STORE_CURSORS) +>>> for s in model5: print(s.cursor()) +out.ttl:2:47 +out.ttl:3:25 +out.ttl:4:25 +out.ttl:7:47 + +Streaming Data +============== + +More advanced input and output can be performed by using the +:class:`~serd.Reader` and :class:`~serd.Writer` classes directly. The Reader +produces an :class:`~serd.Event` stream which describes the content of the +file, and the Writer consumes such a stream and writes syntax. + +Reading Files +------------- + +The reader reads from a source, which should be a :class:`~serd.FileSource` +to read from a file. Parsed input is sent to a sink, which is +called for each event: + +.. testcode:: + + def sink(event): + print(event) + + reader = serd.Reader(world, serd.Syntax.TURTLE, 0, sink, 4096) + with reader.open(serd.FileSource("out.ttl")) as context: + context.read_document() + +.. testoutput:: + :options: +ELLIPSIS + + serd.Event.statement(serd.Statement(serd.uri("http://example.org/s"), serd.uri("http://example.org/p"), serd.uri("http://example.org/o1"), serd.Cursor(serd.uri("out.ttl"), 2, 47))) + ... + +For more advanced use cases that keep track of state, the sink can be a custom +:class:`~serd.Sink` with a call operator: + +.. testcode:: + + class MySink(serd.Sink): + def __init__(self): + super().__init__() + self.events = [] + + def __call__(self, event: serd.Event) -> serd.Status: + self.events += [event] + return serd.Status.SUCCESS + + sink = MySink() + reader = serd.Reader(world, serd.Syntax.TURTLE, 0, sink, 4096) + with reader.open(serd.FileSource("out.ttl")) as context: + context.read_document() + + print(sink.events[0]) + +.. testoutput:: + + serd.Event.statement(serd.Statement(serd.uri("http://example.org/s"), serd.uri("http://example.org/p"), serd.uri("http://example.org/o1"), serd.Cursor(serd.uri("out.ttl"), 2, 47))) + +Reading Strings +--------------- + +To read from a string, use a :class:`~serd.StringSource` with the reader: + +.. testcode:: + + ttl = """ + @base . + @prefix eg: . + eg:name "Serd" . + """ + + def sink(event): + print(event) + + reader = serd.Reader(world, serd.Syntax.TURTLE, 0, sink, 4096) + with reader.open(serd.StringSource(ttl)) as context: + context.read_document() + +.. testoutput:: + + serd.Event.base("http://drobilla.net/") + serd.Event.prefix("eg", "http://example.org/") + serd.Event.statement(serd.Statement(serd.uri("sw/serd"), serd.curie("eg:name"), serd.string("Serd"), serd.Cursor(serd.string("string"), 4, 24))) + +Reading into a Model +-------------------- + +To read new data into an existing model, use an :class:`~serd.Inserter` as a sink: + +.. testcode:: + + ttl = """ + @prefix eg: . + eg:newSubject eg:p eg:o . + """ + + env = serd.Env() + sink = model.inserter(env) + reader = serd.Reader(world, serd.Syntax.TURTLE, 0, sink, 4096) + with reader.open(serd.StringSource(ttl)) as context: + context.read_document() + + for s in model: print(s) + +.. testoutput:: + + + + + + + +Writing Files +------------- + +.. testcode:: + + env = serd.Env() + byte_sink = serd.ByteSink(filename="written.ttl") + writer = serd.Writer(world, serd.Syntax.TURTLE, 0, env, byte_sink) + st = model.all().serialise(writer.sink(), 0) + writer.finish() + byte_sink.close() + print(open("written.ttl", "r").read()) + +.. testoutput:: + :options: +NORMALIZE_WHITESPACE + + + . + + + , + , + . + + + . diff --git a/bindings/python/reference.rst b/bindings/python/reference.rst new file mode 100644 index 00000000..0b64037e --- /dev/null +++ b/bindings/python/reference.rst @@ -0,0 +1,8 @@ +============= +API Reference +============= + +.. automodule:: serd + :members: + :undoc-members: + :inherited-members: diff --git a/wscript b/wscript index 3e1f5027..030b836e 100644 --- a/wscript +++ b/wscript @@ -63,6 +63,9 @@ def configure(conf): conf.load('autowaf', cache=True) + if conf.env.SERD_PYTHON and conf.env.DOCS: + conf.load('sphinx') + if not autowaf.set_c_lang(conf, 'c11', mandatory=False): autowaf.set_c_lang(conf, 'c99') @@ -425,6 +428,13 @@ def build(bld): name='index', SERD_VERSION=SERD_VERSION) + # Python Documentation + if bld.env.SERD_PYTHON and bld.env.DOCS: + bld.add_group('sphinx') + bld(features='sphinx', + sphinx_source='bindings/python', + sphinx_output_format='singlehtml') + # Man page bld.install_files('${MANDIR}/man1', 'doc/serdi.1') @@ -821,6 +831,11 @@ def test(tst): check([tst.env.PYTHON[0], '-m', 'unittest', 'discover', 'bindings/python']) + with tst.group('pythondoc') as check: + check([tst.env.SPHINX_BUILD[0], '-M', 'doctest', + '../bindings/python', + 'build/singlehtml']) + with tst.group('Unit') as check: check(['./base64_test']) check(['./bigint_test']) -- cgit v1.2.1