Skip to content

Commit daa2414

Browse files
committed
grammar : prevent infinite expansion during left factoring
1 parent 7f10bc2 commit daa2414

2 files changed

Lines changed: 8 additions & 1 deletion

File tree

‎src/llama-grammar.cpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -981,6 +981,7 @@ static void llama_grammar_left_factor(llama_grammar_rules & rules) {
981981
size_t end;
982982
};
983983

984+
const size_t n_rules_original = rules.size();
984985
bool changed = true;
985986
while (changed) {
986987
changed = false;
@@ -1011,7 +1012,8 @@ static void llama_grammar_left_factor(llama_grammar_rules & rules) {
10111012

10121013
for (const auto & [astart, aend] : alt_ranges) {
10131014
size_t len = aend - astart;
1014-
if (len == 1 && rule[astart].type == LLAMA_GRETYPE_RULE_REF) {
1015+
// Resolving references in generated suffixes can expand recursive rules forever.
1016+
if (ri < n_rules_original && len == 1 && rule[astart].type == LLAMA_GRETYPE_RULE_REF) {
10151017
size_t ref_id = static_cast<size_t>(rule[astart].value);
10161018
if (ref_id < rules.size()) {
10171019
size_t ref_end = 0;

‎tests/test-grammar-integration.cpp‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,11 @@ static void test_schema(const std::string & test_desc, const std::string & schem
193193
}
194194

195195
static void test_simple_grammar() {
196+
test_grammar(
197+
"left factor recursive references",
198+
"root ::= \"ok\" | choice\nchoice ::= a | b | c | d\na ::= \"x\" a\nb ::= \"x\" b\nc ::= \"x\" c\nd ::= \"x\" d",
199+
{"ok"}, {"", "x", "xx", "okx"});
200+
196201
test_schema(
197202
"min 0",
198203
R"""({

0 commit comments

Comments
 (0)