Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Parse tokenizer options

Browse Source
This commit is contained in:
Kovid Goyal 2021-06-15 13:12:24 +05:30
parent 79ea88ddb8
commit adf810cae6
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
1 changed files with 7 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/calibre/db/sqlite_extension.cpp
View File

@ -18,6 +18,7 @@ typedef int (*token_callback_func)(void *, int, const char *, int, int, int);
class Tokenizer { class Tokenizer {
private: private:
std::string ascii_folded_buf; std::string ascii_folded_buf;
bool remove_diacritics;
int ascii_tokenize(void *callback_ctx, int flags, const char *text, int text_sz, token_callback_func callback) { int ascii_tokenize(void *callback_ctx, int flags, const char *text, int text_sz, token_callback_func callback) {
int pos = 0; int pos = 0;
@ -40,8 +41,13 @@ private:
return SQLITE_OK; return SQLITE_OK;
} }
public: public:
Tokenizer(const char **args, int nargs) : ascii_folded_buf() { Tokenizer(const char **args, int nargs) : ascii_folded_buf(), remove_diacritics(false) {
ascii_folded_buf.reserve(128); ascii_folded_buf.reserve(128);
for (int i = 0; i < nargs; i++) {
if (strcmp(args[i], "remove_diacritics") == 0) {
remove_diacritics = true;
}
}
} }
int tokenize(void *callback_ctx, int flags, const char *text, int text_sz, token_callback_func callback) { int tokenize(void *callback_ctx, int flags, const char *text, int text_sz, token_callback_func callback) {