Better, tested, i18n system.

git-svn-id: http://svn.drobilla.net/lad/trunk/slv2@2916 a436a847-0d15-0410-975c-d299462d15a1

11 files changed, 202 insertions, 143 deletions

diff --git a/ChangeLog b/ChangeLog
index fc6a45d..4861b46 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -5,8 +5,10 @@ slv2 (UNRELEASED) unstable; urgency=low
   * Remove nonsensical slv2_plugin_get_properties and slv2_plugin_get_hints
   * Remove all use of SPARQL (including API that provided SPARQL support)
   * Remove use of redland (librdf) in favour of Serd and Sord
-  * *** API BREAK *** (remove slv2_world_new_using_rdf_world and
-    slv2_plugin_query_sparql)
+  * Remove slv2_world_new_using_rdf_world and slv2_plugin_query_sparql
+  * Remove separate i18n versions of functions and implement i18n everywhere
+  * Add slv2_world_filter_language (	to optionally disable i18n).
+  * *** API BREAK ***
 
  -- David Robillard <d@drobilla.net>  (UNRELEASED)
 
diff --git a/slv2/plugin.h b/slv2/plugin.h
index e94b4f3..e936bf4 100644
--- a/slv2/plugin.h
+++ b/slv2/plugin.h
@@ -178,18 +178,6 @@ SLV2Values
 slv2_plugin_get_value_by_qname(SLV2Plugin  p,
                                const char* predicate);
 
-/** Get a translated value associated with the plugin in a plugin's data files.
- *
- * This function is identical to slv2_plugin_get_value, but takes a QName
- * string parameter for a predicate instead of an SLV2Value, which may be
- * more convenient. It returns the value translated to the current language
- * if possible.
- */
-SLV2_API
-SLV2Values
-slv2_plugin_get_value_by_qname_i18n(SLV2Plugin  p,
-                                    const char* predicate);
-
 /** Get a value associated with some subject in a plugin's data files.
  *
  * Returns the ?object of all triples found of the form:
diff --git a/slv2/port.h b/slv2/port.h
index 0f3f2da..de34946 100644
--- a/slv2/port.h
+++ b/slv2/port.h
@@ -52,16 +52,6 @@ slv2_port_get_value_by_qname(SLV2Plugin  plugin,
                              SLV2Port    port,
                              const char* property_uri);
 
-/** Port analog of slv2_plugin_get_value_by_qname_i18n.
- *
- * Time = Query
- */
-SLV2_API
-SLV2Values
-slv2_port_get_value_by_qname_i18n(SLV2Plugin  plugin,
-				  SLV2Port    port,
-				  const char* property_uri);
-
 /** Return the LV2 port properties of a port.
  *
  * Time = Query
diff --git a/slv2/world.h b/slv2/world.h
index de3bf04..15dddf8 100644
--- a/slv2/world.h
+++ b/slv2/world.h
@@ -52,6 +52,15 @@ SLV2_API
 SLV2World
 slv2_world_new(void);
 
+/** Enable/disable language filtering for @a world.
+ * With filtering enabled, SLV2 will automatically return the best value(s)
+ * for the current LANG.  With filtering disabled, all matching values will
+ * be returned regardless of language tag.  Filtering is enabled by default.
+ */
+SLV2_API
+void
+slv2_world_filter_language(SLV2World world, bool filter);
+
 /** Destroy the world, mwahaha.
  *
  * NB: Destroying the world will leave dangling references in any plugin lists,
diff --git a/src/plugin.c b/src/plugin.c
index cb3d630..180c82a 100644
--- a/src/plugin.c
+++ b/src/plugin.c
@@ -406,7 +406,7 @@ SLV2_API
 SLV2Value
 slv2_plugin_get_name(SLV2Plugin plugin)
 {
-	SLV2Values results = slv2_plugin_get_value_by_qname_i18n(plugin, "doap:name");
+	SLV2Values results = slv2_plugin_get_value_by_qname(plugin, "doap:name");
 	SLV2Value  ret     = NULL;
 
 	if (results) {
@@ -456,30 +456,6 @@ slv2_plugin_get_value_by_qname(SLV2Plugin  p,
 
 SLV2_API
 SLV2Values
-slv2_plugin_get_value_by_qname_i18n(SLV2Plugin  p,
-                                    const char* predicate)
-{
-	uint8_t* pred_uri = slv2_qname_expand(p, predicate);
-	if (!pred_uri) {
-		return NULL;
-	}
-
-	SLV2Node pred_node = sord_get_uri(
-		p->world->model, true, (const uint8_t*)pred_uri);
-
-	SLV2Matches results = slv2_plugin_find_statements(
-		p,
-		p->plugin_uri->val.uri_val,
-		pred_node,
-		NULL);
-
-	slv2_node_free(pred_node);
-	free(pred_uri);
-	return slv2_values_from_stream_i18n(p, results);
-}
-
-SLV2_API
-SLV2Values
 slv2_plugin_get_value_for_subject(SLV2Plugin p,
                                   SLV2Value  subject,
                                   SLV2Value  predicate)
diff --git a/src/port.c b/src/port.c
index 80fa421..2ee3337 100644
--- a/src/port.c
+++ b/src/port.c
@@ -130,26 +130,6 @@ slv2_port_supports_event(SLV2Plugin p,
 	return ret;
 }
 
-static SLV2Values
-slv2_values_from_stream_objects(SLV2Plugin p, SLV2Matches stream)
-{
-	if (slv2_matches_end(stream)) {
-		slv2_match_end(stream);
-		return NULL;
-	}
-
-	SLV2Values values = slv2_values_new();
-	FOREACH_MATCH(stream) {
-		g_ptr_array_add(
-			values,
-			slv2_value_new_from_node(
-				p->world,
-				slv2_match_object(stream)));
-	}
-	slv2_match_end(stream);
-	return values;
-}
-
 SLV2_API
 SLV2Values
 slv2_port_get_value_by_qname(SLV2Plugin  p,
@@ -207,29 +187,6 @@ slv2_port_get_value(SLV2Plugin p,
 }
 
 SLV2_API
-SLV2Values
-slv2_port_get_value_by_qname_i18n(SLV2Plugin  p,
-                                  SLV2Port    port,
-                                  const char* predicate)
-{
-	assert(predicate);
-	uint8_t* pred_uri = slv2_qname_expand(p, predicate);
-	if (!pred_uri) {
-		return NULL;
-	}
-
-	SLV2Node    port_node = slv2_port_get_node(p, port);
-	SLV2Matches results   = slv2_plugin_find_statements(
-		p,
-		port_node,
-		sord_get_uri(p->world->model, true, pred_uri),
-		NULL);
-
-	free(pred_uri);
-	return slv2_values_from_stream_i18n(p, results);
-}
-
-SLV2_API
 SLV2Value
 slv2_port_get_symbol(SLV2Plugin p,
                      SLV2Port   port)
@@ -243,7 +200,7 @@ slv2_port_get_name(SLV2Plugin p,
                    SLV2Port   port)
 {
 	SLV2Value  ret     = NULL;
-	SLV2Values results = slv2_port_get_value_by_qname_i18n(p, port, "lv2:name");
+	SLV2Values results = slv2_port_get_value_by_qname(p, port, "lv2:name");
 
 	if (results && slv2_values_size(results) > 0) {
 		ret = slv2_value_duplicate(slv2_values_get_at(results, 0));
diff --git a/src/query.c b/src/query.c
index 0acd0d0..9382fdc 100644
--- a/src/query.c
+++ b/src/query.c
@@ -38,41 +38,118 @@ slv2_plugin_find_statements(SLV2Plugin plugin,
 	return sord_find(plugin->world->model, pat);
 }
 
+typedef enum {
+	SLV2_LANG_MATCH_NONE,    ///< Language does not match at all
+	SLV2_LANG_MATCH_PARTIAL, ///< Partial (language, but not country) match
+	SLV2_LANG_MATCH_EXACT    ///< Exact (language and country) match
+} SLV2LangMatch;
+
+static SLV2LangMatch
+slv2_lang_matches(const char* a, const char* b)
+{
+	if (!strcmp(a, b)) {
+		return SLV2_LANG_MATCH_EXACT;
+	}
+
+	const char*  a_dash     = strchr(a, '-');
+	const size_t a_lang_len = a_dash ? (a_dash - a) : 0;
+	const char*  b_dash     = strchr(b, '-');
+	const size_t b_lang_len = b_dash ? (b_dash - b) : 0;
+
+	if (a_lang_len && b_lang_len) {
+		if (a_lang_len == b_lang_len && !strncmp(a, b, a_lang_len)) {
+			return SLV2_LANG_MATCH_PARTIAL;  // e.g. a="en-gb", b="en-ca"
+		}
+	} else if (a_lang_len && !strncmp(a, b, a_lang_len)) {
+		return SLV2_LANG_MATCH_PARTIAL;  // e.g. a="en", b="en-ca"
+	} else if (b_lang_len && !strncmp(a, b, b_lang_len)) {
+		return SLV2_LANG_MATCH_PARTIAL;  // e.g. a="en-ca", b="en"
+	}
+	return SLV2_LANG_MATCH_NONE;
+}
+
 SLV2Values
-slv2_values_from_stream_i18n(SLV2Plugin  p,
-                             SLV2Matches stream)
+slv2_values_from_stream_objects_i18n(SLV2Plugin  p,
+                                     SLV2Matches stream)
 {
-	SLV2Values values = slv2_values_new();
-	SLV2Node   nolang = NULL;
+	SLV2Values values  = slv2_values_new();
+	SLV2Node   nolang  = NULL;  // Untranslated value
+	SLV2Node   partial = NULL;  // Partial language match
+	char*      syslang = slv2_get_lang();
 	FOREACH_MATCH(stream) {
 		SLV2Node value = slv2_match_object(stream);
 		if (sord_node_get_type(value) == SORD_LITERAL) {
-			const char* lang = sord_literal_get_lang(value);
+			const char*   lang = sord_literal_get_lang(value);
+			SLV2LangMatch lm   = SLV2_LANG_MATCH_NONE;
 			if (lang) {
-				if (!strcmp(lang, slv2_get_lang())) {
-					g_ptr_array_add(
-						values, (uint8_t*)slv2_value_new_string(
-							p->world, (const char*)sord_node_get_string(value)));
-				}
+				lm = (syslang)
+					? slv2_lang_matches(lang, syslang)
+					: SLV2_LANG_MATCH_PARTIAL;
 			} else {
 				nolang = value;
+				if (!syslang) {
+					lm = SLV2_LANG_MATCH_EXACT;
+				}
 			}
+			
+			if (lm == SLV2_LANG_MATCH_EXACT) {
+				// Exact language match, add to results
+				g_ptr_array_add(values, slv2_value_new_from_node(p->world, value));
+			} else if (lm == SLV2_LANG_MATCH_PARTIAL) {
+				// Partial language match, save in case we find no exact
+				partial = value;
+			}
+		} else {
+			g_ptr_array_add(values, slv2_value_new_from_node(p->world, value));
 		}
-		break;
 	}
 	slv2_match_end(stream);
+	free(syslang);
 
-	if (slv2_values_size(values) == 0) {
-		// No value with a matching language, use untranslated default
-		if (nolang) {
-			g_ptr_array_add(
-				values, (uint8_t*)slv2_value_new_string(
-					p->world, (const char*)sord_node_get_string(nolang)));
-		} else {
-			slv2_values_free(values);
-			values = NULL;
-		}
+	if (slv2_values_size(values) > 0) {
+		return values;
+	}
+
+	SLV2Node best = nolang;
+	if (syslang && partial) {
+		// Partial language match for system language
+		best = partial;
+	} else if (!best) {
+		// No languages matches at all, and no untranslated value
+		// Use any value, if possible
+		best = partial;
+	}
+
+	if (best) {
+		g_ptr_array_add(values, slv2_value_new_from_node(p->world, best));
+	} else {
+		// No matches whatsoever
+		slv2_values_free(values);
+		values = NULL;
 	}
 
 	return values;
 }
+
+SLV2Values
+slv2_values_from_stream_objects(SLV2Plugin  p,
+                                SLV2Matches stream)
+{
+	if (slv2_matches_end(stream)) {
+		slv2_match_end(stream);
+		return NULL;
+	} else if (p->world->filter_language) {
+		return slv2_values_from_stream_objects_i18n(p, stream);
+	} else {
+		SLV2Values values = slv2_values_new();
+		FOREACH_MATCH(stream) {
+			g_ptr_array_add(
+				values,
+				slv2_value_new_from_node(
+					p->world,
+					slv2_match_object(stream)));
+		}
+		slv2_match_end(stream);
+		return values;
+	}
+}
diff --git a/src/slv2_internal.h b/src/slv2_internal.h
index 7cf8d11..12a32a0 100644
--- a/src/slv2_internal.h
+++ b/src/slv2_internal.h
@@ -202,6 +202,7 @@ struct _SLV2World {
 	SLV2Node          slv2_dmanifest_node;
 	SLV2Node          xsd_integer_node;
 	SLV2Node          xsd_decimal_node;
+	bool              filter_language;
 };
 
 const uint8_t*
@@ -290,15 +291,15 @@ static inline bool slv2_matches_end(SLV2Matches matches) {
 	return sord_iter_end(matches);
 }
 
-SLV2Values slv2_values_from_stream_i18n(SLV2Plugin  p,
-                                        SLV2Matches stream);
+SLV2Values slv2_values_from_stream_objects(SLV2Plugin  p,
+                                           SLV2Matches stream);
 
 
 /* ********* Utilities ********* */
 
-char*       slv2_strjoin(const char* first, ...);
-const char* slv2_get_lang();
-uint8_t*    slv2_qname_expand(SLV2Plugin p, const char* qname);
+char*    slv2_strjoin(const char* first, ...);
+char*    slv2_get_lang();
+uint8_t* slv2_qname_expand(SLV2Plugin p, const char* qname);
 
 typedef void (*VoidFunc)();
 
diff --git a/src/util.c b/src/util.c
index 8024f2e..bcb5e6c 100644
--- a/src/util.c
+++ b/src/util.c
@@ -62,24 +62,37 @@ slv2_uri_to_path(const char* uri)
 		return NULL;
 }
 
-const char*
+/** Return the current LANG converted to Turtle (i.e. RFC3066) style.
+ * For example, if LANG is set to "en_CA.utf-8", this returns "en-ca".
+ */
+char*
 slv2_get_lang()
 {
-	static char lang[32];
-	lang[31] = '\0';
-	char* tmp = getenv("LANG");
-	if (!tmp) {
-		lang[0] = '\0';
-	} else {
-		strncpy(lang, tmp, 31);
-		for (int i = 0; i < 31 && lang[i]; ++i) {
-			if (lang[i] == '_') {
-				lang[i] = '-';
-			} else if (   !(lang[i] >= 'a' && lang[i] <= 'z')
-			           && !(lang[i] >= 'A' && lang[i] <= 'Z')) {
-				lang[i] = '\0';
-				break;
-			}
+	const char* const env_lang = getenv("LANG");
+	if (!env_lang || !strcmp(env_lang, "")
+	    || !strcmp(env_lang, "C") || !strcmp(env_lang, "POSIX")) {
+		return NULL;
+	}
+	
+	const size_t env_lang_len = strlen(env_lang);
+	char* const  lang         = malloc(env_lang_len + 1);
+	for (size_t i = 0; i < env_lang_len + 1; ++i) {
+		if (env_lang[i] == '_') {
+			lang[i] = '-';  // Convert _ to -
+		} else if (env_lang[i] >= 'A' && env_lang[i] <= 'Z') {
+			lang[i] = env_lang[i] + ('a' - 'A');  // Convert to lowercase
+		} else if (env_lang[i] >= 'a' && env_lang[i] <= 'z') {
+			lang[i] = env_lang[i];  // Lowercase letter, copy verbatim
+		} else if (env_lang[i] >= '0' && env_lang[i] <= '9') {
+			lang[i] = env_lang[i];  // Digit, copy verbatim
+		} else if (env_lang[i] == '\0' || env_lang[i] == '.') {
+			// End, or start of suffix (e.g. en_CA.utf-8), finished
+			lang[i] = '\0';
+			break;
+		} else {
+			SLV2_ERRORF("Illegal LANG `%s' ignored\n", env_lang);
+			free(lang);
+			return NULL;
 		}
 	}
 
diff --git a/src/world.c b/src/world.c
index 4c612ee..ae04b17 100644
--- a/src/world.c
+++ b/src/world.c
@@ -93,7 +93,8 @@ slv2_world_new()
 	slv2_world_set_prefix(world, "lv2",   "http://lv2plug.in/ns/lv2core#");
 	slv2_world_set_prefix(world, "lv2ev", "http://lv2plug.in/ns/ext/event#");
 
-	world->n_read_files = 0;
+	world->n_read_files    = 0;
+	world->filter_language = true;
 
 	return world;
 
@@ -148,6 +149,14 @@ slv2_world_free(SLV2World world)
 	free(world);
 }
 
+SLV2_API
+void
+slv2_world_filter_language(SLV2World world, bool filter)
+{
+	world->filter_language = filter;
+}
+
+
 static SLV2Matches
 slv2_world_find_statements(SLV2World world,
                            Sord      model,
diff --git a/test/slv2_test.c b/test/slv2_test.c
index 7daa5a5..a7f7faf 100644
--- a/test/slv2_test.c
+++ b/test/slv2_test.c
@@ -17,7 +17,7 @@
  * 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  */
 
-#define _XOPEN_SOURCE 500
+#define _XOPEN_SOURCE 600
 
 #include <unistd.h>
 #include <string.h>
@@ -369,7 +369,7 @@ discovery_verify_plugin(SLV2Plugin plugin)
 int
 test_discovery()
 {
-	SLV2Plugins plugins;
+	SLV2Plugins plugins = NULL;
 
 	if (!start_bundle(MANIFEST_PREFIXES
 			":plug a lv2:Plugin ; lv2:binary <foo.so> ; rdfs:seeAlso <plugin.ttl> .\n",
@@ -720,7 +720,9 @@ test_port()
 			"lv2:port [ "
 			"  a lv2:ControlPort ; a lv2:InputPort ; "
 			"  lv2:index 0 ; lv2:symbol \"foo\" ; "
-			"  lv2:name \"bar\" ; lv2:name \"le bar\"@fr ; "
+			"  lv2:name \"store\" ; "
+			"  lv2:name \"dépanneur\"@fr-ca ; lv2:name \"épicerie\"@fr-fr ; "
+			"  lv2:name \"tienda\"@es ; "
      		"  lv2:portProperty lv2:integer ; "
 			"  lv2:minimum -1.0 ; lv2:maximum 1.0 ; lv2:default 0.5 ; "
 			"  lv2:scalePoint [ rdfs:label \"Sin\"; rdf:value 3 ] ; "
@@ -769,11 +771,39 @@ test_port()
 	TEST_ASSERT(slv2_values_size(port_properties) == 1);
 	slv2_values_free(port_properties);
 
+	// Untranslated name (current locale is set to "C" in main)
 	TEST_ASSERT(!strcmp(slv2_value_as_string(slv2_port_get_symbol(plug, p)), "foo"));
 	SLV2Value name = slv2_port_get_name(plug, p);
-	TEST_ASSERT(!strcmp(slv2_value_as_string(name), "bar"));
+	TEST_ASSERT(!strcmp(slv2_value_as_string(name), "store"));
 	slv2_value_free(name);
 
+	// Exact language match
+	setenv("LANG", "fr_FR", 1);
+	name = slv2_port_get_name(plug, p);
+	TEST_ASSERT(!strcmp(slv2_value_as_string(name), "épicerie"));
+	slv2_value_free(name);
+
+	// Exact language match (with charset suffix)
+	setenv("LANG", "fr_CA.utf8", 1);
+	name = slv2_port_get_name(plug, p);
+	TEST_ASSERT(!strcmp(slv2_value_as_string(name), "dépanneur"));
+	slv2_value_free(name);
+
+	// Partial language match (choose value translated for different country)
+	setenv("LANG", "fr_BE", 1);
+	name = slv2_port_get_name(plug, p);
+	TEST_ASSERT((!strcmp(slv2_value_as_string(name), "dépanneur"))
+	            ||(!strcmp(slv2_value_as_string(name), "épicerie")));
+	slv2_value_free(name);
+
+	// Partial language match (choose country-less language tagged value)
+	setenv("LANG", "es_MX", 1);
+	name = slv2_port_get_name(plug, p);
+	TEST_ASSERT(!strcmp(slv2_value_as_string(name), "tienda"));
+	slv2_value_free(name);
+
+	setenv("LANG", "C", 1);  // Reset locale
+
 	SLV2ScalePoints points = slv2_port_get_scale_points(plug, p);
 	TEST_ASSERT(slv2_scale_points_size(points) == 2);
 
@@ -819,10 +849,17 @@ test_port()
 
 	SLV2Value name_p = slv2_value_new_uri(world, "http://lv2plug.in/ns/lv2core#name");
 	SLV2Values names = slv2_port_get_value(plug, p, name_p);
-	TEST_ASSERT(slv2_values_size(names) == 2);
+	TEST_ASSERT(slv2_values_size(names) == 1);
 	TEST_ASSERT(!strcmp(slv2_value_as_string(slv2_values_get_at(names, 0)),
-	                    "bar"));
+	                    "store"));
 	slv2_values_free(names);
+
+	slv2_world_filter_language(world, false);
+	names = slv2_port_get_value(plug, p, name_p);
+	TEST_ASSERT(slv2_values_size(names) == 4);
+	slv2_values_free(names);
+	slv2_world_filter_language(world, true);
+
 	names = slv2_port_get_value(plug, ep, name_p);
 	TEST_ASSERT(slv2_values_size(names) == 1);
 	TEST_ASSERT(!strcmp(slv2_value_as_string(slv2_values_get_at(names, 0)),
@@ -998,10 +1035,10 @@ main(int argc, char *argv[])
 		printf("Syntax: %s\n", argv[0]);
 		return 0;
 	}
+	setenv("LANG", "C", 1);
 	init_tests();
 	run_tests();
 	cleanup();
 	printf("\n*** Test Results: %d tests, %d errors\n\n", test_count, error_count);
 	return error_count ? 1 : 0;
 }
-