From adf810cae62c00b181e9277b725ffd49de01ccc1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 15 Jun 2021 13:12:24 +0530 Subject: [PATCH] Parse tokenizer options --- src/calibre/db/sqlite_extension.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/calibre/db/sqlite_extension.cpp b/src/calibre/db/sqlite_extension.cpp index 1c55e6c99f..8729fb38e7 100644 --- a/src/calibre/db/sqlite_extension.cpp +++ b/src/calibre/db/sqlite_extension.cpp @@ -18,6 +18,7 @@ typedef int (*token_callback_func)(void *, int, const char *, int, int, int); class Tokenizer { private: std::string ascii_folded_buf; + bool remove_diacritics; int ascii_tokenize(void *callback_ctx, int flags, const char *text, int text_sz, token_callback_func callback) { int pos = 0; @@ -40,8 +41,13 @@ private: return SQLITE_OK; } public: - Tokenizer(const char **args, int nargs) : ascii_folded_buf() { + Tokenizer(const char **args, int nargs) : ascii_folded_buf(), remove_diacritics(false) { ascii_folded_buf.reserve(128); + for (int i = 0; i < nargs; i++) { + if (strcmp(args[i], "remove_diacritics") == 0) { + remove_diacritics = true; + } + } } int tokenize(void *callback_ctx, int flags, const char *text, int text_sz, token_callback_func callback) {