mirror of
https://github.com/mudler/LocalAI.git
synced 2025-01-04 11:54:12 +00:00
5eda7f578d
* refactor: break down json grammar parser in different files Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * fix: patch to `refactor_grammars` - propagate errors (#3006) propagate errors around Signed-off-by: Dave Lee <dave@gray101.com> --------- Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Signed-off-by: Dave Lee <dave@gray101.com> Co-authored-by: Dave <dave@gray101.com>
48 lines
1.1 KiB
Go
48 lines
1.1 KiB
Go
package functions
|
|
|
|
import "regexp"
|
|
|
|
var (
|
|
PRIMITIVE_RULES = map[string]string{
|
|
"boolean": `("true" | "false") space`,
|
|
"number": `("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? space`,
|
|
"integer": `("-"? ([0-9] | [1-9] [0-9]*)) space`,
|
|
"string": `"\"" (
|
|
[^"\\] |
|
|
"\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F])
|
|
)* "\"" space`,
|
|
// TODO: we shouldn't forbid \" and \\ or all unicode and have this branch here,
|
|
// however, if we don't have it, the grammar will be ambiguous and
|
|
// empirically results are way worse.
|
|
"freestring": `(
|
|
[^\x00] |
|
|
"\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F])
|
|
)* space`,
|
|
"null": `"null" space`,
|
|
}
|
|
|
|
INVALID_RULE_CHARS_RE = regexp.MustCompile(`[^a-zA-Z0-9-]+`)
|
|
GRAMMAR_LITERAL_ESCAPE_RE = regexp.MustCompile(`[\r\n"]`)
|
|
GRAMMAR_LITERAL_ESCAPES = map[string]string{
|
|
"\r": `\r`,
|
|
"\n": `\n`,
|
|
`"`: `\"`,
|
|
}
|
|
)
|
|
|
|
const (
|
|
SPACE_RULE = `" "?`
|
|
|
|
arrayNewLines = `arr ::=
|
|
"[\n" (
|
|
realvalue
|
|
(",\n" realvalue)*
|
|
)? "]"`
|
|
|
|
array = `arr ::=
|
|
"[" (
|
|
realvalue
|
|
("," realvalue)*
|
|
)? "]"`
|
|
)
|