Mirrors/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/llama.cpp.git synced 2024-12-25 05:48:47 +01:00
Code Issues Packages Projects Releases Wiki Activity

json: better support for "type" unions (e.g. nullable arrays w/ typed items) (#7863)

Browse Source 
* json: better suport for "type" arrays (e.g. `{"type": ["array", "null"], "items": {"type": "string"}}`)

* json: add test for type: [array, null] fix

* update tests
This commit is contained in:
Olivier Chafik 2024-06-26 01:46:35 +01:00 committed by GitHub
parent 6777c544bd
commit 9b2f16f805
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 62 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
common/json-schema-to-grammar.cpp
View File

@ -893,7 +893,9 @@ public:
} else if (schema_type.is_array()) {
std::vector<json> schema_types;
for (const auto & t : schema_type) {
schema_types.push_back({{"type", t}});
json schema_copy(schema);
schema_copy["type"] = t;
schema_types.push_back(schema_copy);
}
return _add_rule(rule_name, _generate_union_rule(name, schema_types));
} else if (schema.contains("const")) {

2
examples/json_schema_to_grammar.py
View File

@ -565,7 +565,7 @@ class SchemaConverter:
return self._add_rule(rule_name, self._generate_union_rule(name, schema.get('oneOf') or schema['anyOf']))
elif isinstance(schema_type, list):
return self._add_rule(rule_name, self._generate_union_rule(name, [{'type': t} for t in schema_type]))
return self._add_rule(rule_name, self._generate_union_rule(name, [{**schema, 'type': t} for t in schema_type]))
elif 'const' in schema:
return self._add_rule(rule_name, self._generate_constant_rule(schema['const']) + ' space')

2
examples/server/public/json-schema-to-grammar.mjs
View File

@ -616,7 +616,7 @@ export class SchemaConverter {
} else if (schema.oneOf || schema.anyOf) {
return this._addRule(ruleName, this._generateUnionRule(name, schema.oneOf || schema.anyOf));
} else if (Array.isArray(schemaType)) {
return this._addRule(ruleName, this._generateUnionRule(name, schemaType.map(t => ({ type: t }))));
return this._addRule(ruleName, this._generateUnionRule(name, schemaType.map(t => ({...schema, type: t}))));
} else if ('const' in schema) {
return this._addRule(ruleName, this._generateConstantRule(schema.const) + ' space');
} else if ('enum' in schema) {

25
tests/test-grammar-integration.cpp
View File

@ -993,6 +993,31 @@ static void test_json_schema() {
}
);
test_schema(
"",
// Schema
R"""(
{
"type": ["array", "null"],
"items": { "type": "string" }
}
)""",
// Passing strings
{
"null",
"[]",
"[\"123\"]",
"[\"foo\", \"bar\"]",
},
// Failing strings
{
"",
"[123]",
"\"foo\"",
"[\"foo\", 42]",
}
);
test_schema(
"min+max items",
// Schema

32
tests/test-json-schema-to-grammar.cpp
View File

@ -502,6 +502,38 @@ static void test_all(const std::string & lang, std::function<void(const TestCase
)"""
});
test({
SUCCESS,
"string array",
R"""({
"type": "array",
"prefixItems": { "type": "string" }
})""",
R"""(
char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4})
root ::= "[" space (string ("," space string)*)? "]" space
space ::= | " " | "\n" [ \t]{0,20}
string ::= "\"" char* "\"" space
)"""
});
test({
SUCCESS,
"nullable string array",
R"""({
"type": ["array", "null"],
"prefixItems": { "type": "string" }
})""",
R"""(
alternative-0 ::= "[" space (string ("," space string)*)? "]" space
char ::= [^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4})
null ::= "null" space
root ::= alternative-0 | null
space ::= | " " | "\n" [ \t]{0,20}
string ::= "\"" char* "\"" space
)"""
});
test({
SUCCESS,
"tuple1",