diff options
Diffstat (limited to 'plugins/SpellChecker/hunspell/suggestmgr.cxx')
| -rw-r--r-- | plugins/SpellChecker/hunspell/suggestmgr.cxx | 2001 |
1 files changed, 2001 insertions, 0 deletions
diff --git a/plugins/SpellChecker/hunspell/suggestmgr.cxx b/plugins/SpellChecker/hunspell/suggestmgr.cxx new file mode 100644 index 0000000000..d08b506b9f --- /dev/null +++ b/plugins/SpellChecker/hunspell/suggestmgr.cxx @@ -0,0 +1,2001 @@ +#include "license.hunspell" +#include "license.myspell" + +#include <stdlib.h> +#include <string.h> +#include <stdio.h> +#include <ctype.h> + +#include "suggestmgr.hxx" +#include "htypes.hxx" +#include "csutil.hxx" + +const w_char W_VLINE = { '\0', '|' }; + +SuggestMgr::SuggestMgr(const char * tryme, int maxn, + AffixMgr * aptr) +{ + + // register affix manager and check in string of chars to + // try when building candidate suggestions + pAMgr = aptr; + + csconv = NULL; + + ckeyl = 0; + ckey = NULL; + ckey_utf = NULL; + + ctryl = 0; + ctry = NULL; + ctry_utf = NULL; + + utf8 = 0; + langnum = 0; + complexprefixes = 0; + + maxSug = maxn; + nosplitsugs = 0; + maxngramsugs = MAXNGRAMSUGS; + maxcpdsugs = MAXCOMPOUNDSUGS; + + if (pAMgr) { + langnum = pAMgr->get_langnum(); + ckey = pAMgr->get_key_string(); + nosplitsugs = pAMgr->get_nosplitsugs(); + if (pAMgr->get_maxngramsugs() >= 0) + maxngramsugs = pAMgr->get_maxngramsugs(); + utf8 = pAMgr->get_utf8(); + if (pAMgr->get_maxcpdsugs() >= 0) + maxcpdsugs = pAMgr->get_maxcpdsugs(); + if (!utf8) + { + char * enc = pAMgr->get_encoding(); + csconv = get_current_cs(enc); + free(enc); + } + complexprefixes = pAMgr->get_complexprefixes(); + } + + if (ckey) { + if (utf8) { + w_char t[MAXSWL]; + ckeyl = u8_u16(t, MAXSWL, ckey); + ckey_utf = (w_char *) malloc(ckeyl * sizeof(w_char)); + if (ckey_utf) memcpy(ckey_utf, t, ckeyl * sizeof(w_char)); + else ckeyl = 0; + } else { + ckeyl = strlen(ckey); + } + } + + if (tryme) { + ctry = mystrdup(tryme); + if (ctry) ctryl = strlen(ctry); + if (ctry && utf8) { + w_char t[MAXSWL]; + ctryl = u8_u16(t, MAXSWL, tryme); + ctry_utf = (w_char *) malloc(ctryl * sizeof(w_char)); + if (ctry_utf) memcpy(ctry_utf, t, ctryl * sizeof(w_char)); + else ctryl = 0; + } + } +} + + +SuggestMgr::~SuggestMgr() +{ + pAMgr = NULL; + if (ckey) free(ckey); + ckey = NULL; + if (ckey_utf) free(ckey_utf); + ckey_utf = NULL; + ckeyl = 0; + if (ctry) free(ctry); + ctry = NULL; + if (ctry_utf) free(ctry_utf); + ctry_utf = NULL; + ctryl = 0; + maxSug = 0; +#ifdef MOZILLA_CLIENT + delete [] csconv; +#endif +} + +int SuggestMgr::testsug(char** wlst, const char * candidate, int wl, int ns, int cpdsuggest, + int * timer, clock_t * timelimit) { + int cwrd = 1; + if (ns == maxSug) return maxSug; + for (int k=0; k < ns; k++) { + if (strcmp(candidate,wlst[k]) == 0) cwrd = 0; + } + if ((cwrd) && checkword(candidate, wl, cpdsuggest, timer, timelimit)) { + wlst[ns] = mystrdup(candidate); + if (wlst[ns] == NULL) { + for (int j=0; j<ns; j++) free(wlst[j]); + return -1; + } + ns++; + } + return ns; +} + +// generate suggestions for a misspelled word +// pass in address of array of char * pointers +// onlycompoundsug: probably bad suggestions (need for ngram sugs, too) + +int SuggestMgr::suggest(char*** slst, const char * w, int nsug, + int * onlycompoundsug) +{ + int nocompoundtwowords = 0; + char ** wlst; + w_char word_utf[MAXSWL]; + int wl = 0; + int nsugorig = nsug; + char w2[MAXWORDUTF8LEN]; + const char * word = w; + int oldSug = 0; + + // word reversing wrapper for complex prefixes + if (complexprefixes) { + strcpy(w2, w); + if (utf8) reverseword_utf(w2); else reverseword(w2); + word = w2; + } + + if (*slst) { + wlst = *slst; + } else { + wlst = (char **) malloc(maxSug * sizeof(char *)); + if (wlst == NULL) return -1; + for (int i = 0; i < maxSug; i++) { + wlst[i] = NULL; + } + } + + if (utf8) { + wl = u8_u16(word_utf, MAXSWL, word); + if (wl == -1) { + *slst = wlst; + return nsug; + } + } + + for (int cpdsuggest=0; (cpdsuggest<2) && (nocompoundtwowords==0); cpdsuggest++) { + + // limit compound suggestion + if (cpdsuggest > 0) oldSug = nsug; + + // suggestions for an uppercase word (html -> HTML) + if ((nsug < maxSug) && (nsug > -1)) { + nsug = (utf8) ? capchars_utf(wlst, word_utf, wl, nsug, cpdsuggest) : + capchars(wlst, word, nsug, cpdsuggest); + } + + // perhaps we made a typical fault of spelling + if ((nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) { + nsug = replchars(wlst, word, nsug, cpdsuggest); + } + + // perhaps we made chose the wrong char from a related set + if ((nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) { + nsug = mapchars(wlst, word, nsug, cpdsuggest); + } + + // only suggest compound words when no other suggestion + if ((cpdsuggest == 0) && (nsug > nsugorig)) nocompoundtwowords=1; + + // did we swap the order of chars by mistake + if ((nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) { + nsug = (utf8) ? swapchar_utf(wlst, word_utf, wl, nsug, cpdsuggest) : + swapchar(wlst, word, nsug, cpdsuggest); + } + + // did we swap the order of non adjacent chars by mistake + if ((nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) { + nsug = (utf8) ? longswapchar_utf(wlst, word_utf, wl, nsug, cpdsuggest) : + longswapchar(wlst, word, nsug, cpdsuggest); + } + + // did we just hit the wrong key in place of a good char (case and keyboard) + if ((nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) { + nsug = (utf8) ? badcharkey_utf(wlst, word_utf, wl, nsug, cpdsuggest) : + badcharkey(wlst, word, nsug, cpdsuggest); + } + + // did we add a char that should not be there + if ((nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) { + nsug = (utf8) ? extrachar_utf(wlst, word_utf, wl, nsug, cpdsuggest) : + extrachar(wlst, word, nsug, cpdsuggest); + } + + + // did we forgot a char + if ((nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) { + nsug = (utf8) ? forgotchar_utf(wlst, word_utf, wl, nsug, cpdsuggest) : + forgotchar(wlst, word, nsug, cpdsuggest); + } + + // did we move a char + if ((nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) { + nsug = (utf8) ? movechar_utf(wlst, word_utf, wl, nsug, cpdsuggest) : + movechar(wlst, word, nsug, cpdsuggest); + } + + // did we just hit the wrong key in place of a good char + if ((nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) { + nsug = (utf8) ? badchar_utf(wlst, word_utf, wl, nsug, cpdsuggest) : + badchar(wlst, word, nsug, cpdsuggest); + } + + // did we double two characters + if ((nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) { + nsug = (utf8) ? doubletwochars_utf(wlst, word_utf, wl, nsug, cpdsuggest) : + doubletwochars(wlst, word, nsug, cpdsuggest); + } + + // perhaps we forgot to hit space and two words ran together + if (!nosplitsugs && (nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) { + nsug = twowords(wlst, word, nsug, cpdsuggest); + } + + } // repeating ``for'' statement compounding support + + if (nsug < 0) { + // we ran out of memory - we should free up as much as possible + for (int i = 0; i < maxSug; i++) + if (wlst[i] != NULL) free(wlst[i]); + free(wlst); + wlst = NULL; + } + + if (!nocompoundtwowords && (nsug > 0) && onlycompoundsug) *onlycompoundsug = 1; + + *slst = wlst; + return nsug; +} + +// generate suggestions for a word with typical mistake +// pass in address of array of char * pointers +#ifdef HUNSPELL_EXPERIMENTAL +int SuggestMgr::suggest_auto(char*** slst, const char * w, int nsug) +{ + int nocompoundtwowords = 0; + char ** wlst; + int oldSug; + + char w2[MAXWORDUTF8LEN]; + const char * word = w; + + // word reversing wrapper for complex prefixes + if (complexprefixes) { + strcpy(w2, w); + if (utf8) reverseword_utf(w2); else reverseword(w2); + word = w2; + } + + if (*slst) { + wlst = *slst; + } else { + wlst = (char **) malloc(maxSug * sizeof(char *)); + if (wlst == NULL) return -1; + } + + for (int cpdsuggest=0; (cpdsuggest<2) && (nocompoundtwowords==0); cpdsuggest++) { + + // limit compound suggestion + if (cpdsuggest > 0) oldSug = nsug; + + // perhaps we made a typical fault of spelling + if ((nsug < maxSug) && (nsug > -1)) + nsug = replchars(wlst, word, nsug, cpdsuggest); + + // perhaps we made chose the wrong char from a related set + if ((nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs))) + nsug = mapchars(wlst, word, nsug, cpdsuggest); + + if ((cpdsuggest==0) && (nsug>0)) nocompoundtwowords=1; + + // perhaps we forgot to hit space and two words ran together + + if ((nsug < maxSug) && (nsug > -1) && (!cpdsuggest || (nsug < oldSug + maxcpdsugs)) && check_forbidden(word, strlen(word))) { + nsug = twowords(wlst, word, nsug, cpdsuggest); + } + + } // repeating ``for'' statement compounding support + + if (nsug < 0) { + for (int i=0;i<maxSug; i++) + if (wlst[i] != NULL) free(wlst[i]); + free(wlst); + return -1; + } + + *slst = wlst; + return nsug; +} +#endif // END OF HUNSPELL_EXPERIMENTAL CODE + +// suggestions for an uppercase word (html -> HTML) +int SuggestMgr::capchars_utf(char ** wlst, const w_char * word, int wl, int ns, int cpdsuggest) +{ + char candidate[MAXSWUTF8L]; + w_char candidate_utf[MAXSWL]; + memcpy(candidate_utf, word, wl * sizeof(w_char)); + mkallcap_utf(candidate_utf, wl, langnum); + u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl); + return testsug(wlst, candidate, strlen(candidate), ns, cpdsuggest, NULL, NULL); +} + +// suggestions for an uppercase word (html -> HTML) +int SuggestMgr::capchars(char** wlst, const char * word, int ns, int cpdsuggest) +{ + char candidate[MAXSWUTF8L]; + strcpy(candidate, word); + mkallcap(candidate, csconv); + return testsug(wlst, candidate, strlen(candidate), ns, cpdsuggest, NULL, NULL); +} + +// suggestions for when chose the wrong char out of a related set +int SuggestMgr::mapchars(char** wlst, const char * word, int ns, int cpdsuggest) +{ + char candidate[MAXSWUTF8L]; + clock_t timelimit; + int timer; + candidate[0] = '\0'; + + int wl = strlen(word); + if (wl < 2 || ! pAMgr) return ns; + + int nummap = pAMgr->get_nummap(); + struct mapentry* maptable = pAMgr->get_maptable(); + if (maptable==NULL) return ns; + + timelimit = clock(); + timer = MINTIMER; + return map_related(word, (char *) &candidate, 0, 0, wlst, cpdsuggest, ns, maptable, nummap, &timer, &timelimit); +} + +int SuggestMgr::map_related(const char * word, char * candidate, int wn, int cn, + char** wlst, int cpdsuggest, int ns, + const mapentry* maptable, int nummap, int * timer, clock_t * timelimit) +{ + if (*(word + wn) == '\0') { + int cwrd = 1; + *(candidate + cn) = '\0'; + int wl = strlen(candidate); + for (int m=0; m < ns; m++) + if (strcmp(candidate, wlst[m]) == 0) cwrd = 0; + if ((cwrd) && checkword(candidate, wl, cpdsuggest, timer, timelimit)) { + if (ns < maxSug) { + wlst[ns] = mystrdup(candidate); + if (wlst[ns] == NULL) return -1; + ns++; + } + } + return ns; + } + int in_map = 0; + for (int j = 0; j < nummap; j++) { + for (int k = 0; k < maptable[j].len; k++) { + int len = strlen(maptable[j].set[k]); + if (strncmp(maptable[j].set[k], word + wn, len) == 0) { + in_map = 1; + for (int l = 0; l < maptable[j].len; l++) { + strcpy(candidate + cn, maptable[j].set[l]); + ns = map_related(word, candidate, wn + len, strlen(candidate), wlst, + cpdsuggest, ns, maptable, nummap, timer, timelimit); + if (!(*timer)) return ns; + } + } + } + } + if (!in_map) { + *(candidate + cn) = *(word + wn); + ns = map_related(word, candidate, wn + 1, cn + 1, wlst, cpdsuggest, + ns, maptable, nummap, timer, timelimit); + } + return ns; +} + +// suggestions for a typical fault of spelling, that +// differs with more, than 1 letter from the right form. +int SuggestMgr::replchars(char** wlst, const char * word, int ns, int cpdsuggest) +{ + char candidate[MAXSWUTF8L]; + const char * r; + int lenr, lenp; + int wl = strlen(word); + if (wl < 2 || ! pAMgr) return ns; + int numrep = pAMgr->get_numrep(); + struct replentry* reptable = pAMgr->get_reptable(); + if (reptable==NULL) return ns; + for (int i=0; i < numrep; i++ ) { + r = word; + lenr = strlen(reptable[i].pattern2); + lenp = strlen(reptable[i].pattern); + // search every occurence of the pattern in the word + while ((r=strstr(r, reptable[i].pattern)) != NULL && (!reptable[i].end || strlen(r) == strlen(reptable[i].pattern)) && + (!reptable[i].start || r == word)) { + strcpy(candidate, word); + if (r-word + lenr + strlen(r+lenp) >= MAXSWUTF8L) break; + strcpy(candidate+(r-word),reptable[i].pattern2); + strcpy(candidate+(r-word)+lenr, r+lenp); + ns = testsug(wlst, candidate, wl-lenp+lenr, ns, cpdsuggest, NULL, NULL); + if (ns == -1) return -1; + // check REP suggestions with space + char * sp = strchr(candidate, ' '); + if (sp) { + char * prev = candidate; + while (sp) { + *sp = '\0'; + if (checkword(prev, strlen(prev), 0, NULL, NULL)) { + int oldns = ns; + *sp = ' '; + ns = testsug(wlst, sp + 1, strlen(sp + 1), ns, cpdsuggest, NULL, NULL); + if (ns == -1) return -1; + if (oldns < ns) { + free(wlst[ns - 1]); + wlst[ns - 1] = mystrdup(candidate); + if (!wlst[ns - 1]) return -1; + } + } + *sp = ' '; + prev = sp + 1; + sp = strchr(prev, ' '); + } + } + r++; // search for the next letter + } + } + return ns; +} + +// perhaps we doubled two characters (pattern aba -> ababa, for example vacation -> vacacation) +int SuggestMgr::doubletwochars(char** wlst, const char * word, int ns, int cpdsuggest) +{ + char candidate[MAXSWUTF8L]; + int state=0; + int wl = strlen(word); + if (wl < 5 || ! pAMgr) return ns; + for (int i=2; i < wl; i++ ) { + if (word[i]==word[i-2]) { + state++; + if (state==3) { + strcpy(candidate,word); + strcpy(candidate+i-1,word+i+1); + ns = testsug(wlst, candidate, wl-2, ns, cpdsuggest, NULL, NULL); + if (ns == -1) return -1; + state=0; + } + } else { + state=0; + } + } + return ns; +} + +// perhaps we doubled two characters (pattern aba -> ababa, for example vacation -> vacacation) +int SuggestMgr::doubletwochars_utf(char ** wlst, const w_char * word, int wl, int ns, int cpdsuggest) +{ + w_char candidate_utf[MAXSWL]; + char candidate[MAXSWUTF8L]; + int state=0; + if (wl < 5 || ! pAMgr) return ns; + for (int i=2; i < wl; i++) { + if (w_char_eq(word[i], word[i-2])) { + state++; + if (state==3) { + memcpy(candidate_utf, word, (i - 1) * sizeof(w_char)); + memcpy(candidate_utf+i-1, word+i+1, (wl-i-1) * sizeof(w_char)); + u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl-2); + ns = testsug(wlst, candidate, strlen(candidate), ns, cpdsuggest, NULL, NULL); + if (ns == -1) return -1; + state=0; + } + } else { + state=0; + } + } + return ns; +} + +// error is wrong char in place of correct one (case and keyboard related version) +int SuggestMgr::badcharkey(char ** wlst, const char * word, int ns, int cpdsuggest) +{ + char tmpc; + char candidate[MAXSWUTF8L]; + int wl = strlen(word); + strcpy(candidate, word); + // swap out each char one by one and try uppercase and neighbor + // keyboard chars in its place to see if that makes a good word + + for (int i=0; i < wl; i++) { + tmpc = candidate[i]; + // check with uppercase letters + candidate[i] = csconv[((unsigned char)tmpc)].cupper; + if (tmpc != candidate[i]) { + ns = testsug(wlst, candidate, wl, ns, cpdsuggest, NULL, NULL); + if (ns == -1) return -1; + candidate[i] = tmpc; + } + // check neighbor characters in keyboard string + if (!ckey) continue; + char * loc = strchr(ckey, tmpc); + while (loc) { + if ((loc > ckey) && (*(loc - 1) != '|')) { + candidate[i] = *(loc - 1); + ns = testsug(wlst, candidate, wl, ns, cpdsuggest, NULL, NULL); + if (ns == -1) return -1; + } + if ((*(loc + 1) != '|') && (*(loc + 1) != '\0')) { + candidate[i] = *(loc + 1); + ns = testsug(wlst, candidate, wl, ns, cpdsuggest, NULL, NULL); + if (ns == -1) return -1; + } + loc = strchr(loc + 1, tmpc); + } + candidate[i] = tmpc; + } + return ns; +} + +// error is wrong char in place of correct one (case and keyboard related version) +int SuggestMgr::badcharkey_utf(char ** wlst, const w_char * word, int wl, int ns, int cpdsuggest) +{ + w_char tmpc; + w_char candidate_utf[MAXSWL]; + char candidate[MAXSWUTF8L]; + memcpy(candidate_utf, word, wl * sizeof(w_char)); + // swap out each char one by one and try all the tryme + // chars in its place to see if that makes a good word + for (int i=0; i < wl; i++) { + tmpc = candidate_utf[i]; + // check with uppercase letters + mkallcap_utf(candidate_utf + i, 1, langnum); + if (!w_char_eq(tmpc, candidate_utf[i])) { + u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl); + ns = testsug(wlst, candidate, strlen(candidate), ns, cpdsuggest, NULL, NULL); + if (ns == -1) return -1; + candidate_utf[i] = tmpc; + } + // check neighbor characters in keyboard string + if (!ckey) continue; + w_char * loc = ckey_utf; + while ((loc < (ckey_utf + ckeyl)) && !w_char_eq(*loc, tmpc)) loc++; + while (loc < (ckey_utf + ckeyl)) { + if ((loc > ckey_utf) && !w_char_eq(*(loc - 1), W_VLINE)) { + candidate_utf[i] = *(loc - 1); + u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl); + ns = testsug(wlst, candidate, strlen(candidate), ns, cpdsuggest, NULL, NULL); + if (ns == -1) return -1; + } + if (((loc + 1) < (ckey_utf + ckeyl)) && !w_char_eq(*(loc + 1), W_VLINE)) { + candidate_utf[i] = *(loc + 1); + u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl); + ns = testsug(wlst, candidate, strlen(candidate), ns, cpdsuggest, NULL, NULL); + if (ns == -1) return -1; + } + do { loc++; } while ((loc < (ckey_utf + ckeyl)) && !w_char_eq(*loc, tmpc)); + } + candidate_utf[i] = tmpc; + } + return ns; +} + +// error is wrong char in place of correct one +int SuggestMgr::badchar(char ** wlst, const char * word, int ns, int cpdsuggest) +{ + char tmpc; + char candidate[MAXSWUTF8L]; + clock_t timelimit = clock(); + int timer = MINTIMER; + int wl = strlen(word); + strcpy(candidate, word); + // swap out each char one by one and try all the tryme + // chars in its place to see if that makes a good word + for (int j=0; j < ctryl; j++) { + for (int i=wl-1; i >= 0; i--) { + tmpc = candidate[i]; + if (ctry[j] == tmpc) continue; + candidate[i] = ctry[j]; + ns = testsug(wlst, candidate, wl, ns, cpdsuggest, &timer, &timelimit); + if (ns == -1) return -1; + if (!timer) return ns; + candidate[i] = tmpc; + } + } + return ns; +} + +// error is wrong char in place of correct one +int SuggestMgr::badchar_utf(char ** wlst, const w_char * word, int wl, int ns, int cpdsuggest) +{ + w_char tmpc; + w_char candidate_utf[MAXSWL]; + char candidate[MAXSWUTF8L]; + clock_t timelimit = clock(); + int timer = MINTIMER; + memcpy(candidate_utf, word, wl * sizeof(w_char)); + // swap out each char one by one and try all the tryme + // chars in its place to see if that makes a good word + for (int j=0; j < ctryl; j++) { + for (int i=wl-1; i >= 0; i--) { + tmpc = candidate_utf[i]; + if (w_char_eq(tmpc, ctry_utf[j])) continue; + candidate_utf[i] = ctry_utf[j]; + u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl); + ns = testsug(wlst, candidate, strlen(candidate), ns, cpdsuggest, &timer, &timelimit); + if (ns == -1) return -1; + if (!timer) return ns; + candidate_utf[i] = tmpc; + } + } + return ns; +} + +// error is word has an extra letter it does not need +int SuggestMgr::extrachar_utf(char** wlst, const w_char * word, int wl, int ns, int cpdsuggest) +{ + char candidate[MAXSWUTF8L]; + w_char candidate_utf[MAXSWL]; + w_char * p; + w_char tmpc = W_VLINE; // not used value, only for VCC warning message + if (wl < 2) return ns; + // try omitting one char of word at a time + memcpy(candidate_utf, word, wl * sizeof(w_char)); + for (p = candidate_utf + wl - 1; p >= candidate_utf; p--) { + w_char tmpc2 = *p; + if (p < candidate_utf + wl - 1) *p = tmpc; + u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl - 1); + ns = testsug(wlst, candidate, strlen(candidate), ns, cpdsuggest, NULL, NULL); + if (ns == -1) return -1; + tmpc = tmpc2; + } + return ns; +} + +// error is word has an extra letter it does not need +int SuggestMgr::extrachar(char** wlst, const char * word, int ns, int cpdsuggest) +{ + char tmpc = '\0'; + char candidate[MAXSWUTF8L]; + char * p; + int wl = strlen(word); + if (wl < 2) return ns; + // try omitting one char of word at a time + strcpy (candidate, word); + for (p = candidate + wl - 1; p >=candidate; p--) { + char tmpc2 = *p; + *p = tmpc; + ns = testsug(wlst, candidate, wl-1, ns, cpdsuggest, NULL, NULL); + if (ns == -1) return -1; + tmpc = tmpc2; + } + return ns; +} + +// error is missing a letter it needs +int SuggestMgr::forgotchar(char ** wlst, const char * word, int ns, int cpdsuggest) +{ + char candidate[MAXSWUTF8L]; + char * p; + clock_t timelimit = clock(); + int timer = MINTIMER; + int wl = strlen(word); + // try inserting a tryme character before every letter (and the null terminator) + for (int i = 0; i < ctryl; i++) { + strcpy(candidate, word); + for (p = candidate + wl; p >= candidate; p--) { + *(p+1) = *p; + *p = ctry[i]; + ns = testsug(wlst, candidate, wl+1, ns, cpdsuggest, &timer, &timelimit); + if (ns == -1) return -1; + if (!timer) return ns; + } + } + return ns; +} + +// error is missing a letter it needs +int SuggestMgr::forgotchar_utf(char ** wlst, const w_char * word, int wl, int ns, int cpdsuggest) +{ + w_char candidate_utf[MAXSWL]; + char candidate[MAXSWUTF8L]; + w_char * p; + clock_t timelimit = clock(); + int timer = MINTIMER; + // try inserting a tryme character at the end of the word and before every letter + for (int i = 0; i < ctryl; i++) { + memcpy (candidate_utf, word, wl * sizeof(w_char)); + for (p = candidate_utf + wl; p >= candidate_utf; p--) { + *(p + 1) = *p; + *p = ctry_utf[i]; + u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl + 1); + ns = testsug(wlst, candidate, strlen(candidate), ns, cpdsuggest, &timer, &timelimit); + if (ns == -1) return -1; + if (!timer) return ns; + } + } + return ns; +} + + +/* error is should have been two words */ +int SuggestMgr::twowords(char ** wlst, const char * word, int ns, int cpdsuggest) +{ + char candidate[MAXSWUTF8L]; + char * p; + int c1, c2; + int forbidden = 0; + int cwrd; + + int wl=strlen(word); + if (wl < 3) return ns; + + if (langnum == LANG_hu) forbidden = check_forbidden(word, wl); + + strcpy(candidate + 1, word); + // split the string into two pieces after every char + // if both pieces are good words make them a suggestion + for (p = candidate + 1; p[1] != '\0'; p++) { + p[-1] = *p; + // go to end of the UTF-8 character + while (utf8 && ((p[1] & 0xc0) == 0x80)) { + *p = p[1]; + p++; + } + if (utf8 && p[1] == '\0') break; // last UTF-8 character + *p = '\0'; + c1 = checkword(candidate,strlen(candidate), cpdsuggest, NULL, NULL); + if (c1) { + c2 = checkword((p+1),strlen(p+1), cpdsuggest, NULL, NULL); + if (c2) { + *p = ' '; + + // spec. Hungarian code (need a better compound word support) + if ((langnum == LANG_hu) && !forbidden && + // if 3 repeating letter, use - instead of space + (((p[-1] == p[1]) && (((p>candidate+1) && (p[-1] == p[-2])) || (p[-1] == p[2]))) || + // or multiple compounding, with more, than 6 syllables + ((c1 == 3) && (c2 >= 2)))) *p = '-'; + + cwrd = 1; + for (int k=0; k < ns; k++) + if (strcmp(candidate,wlst[k]) == 0) cwrd = 0; + if (ns < maxSug) { + if (cwrd) { + wlst[ns] = mystrdup(candidate); + if (wlst[ns] == NULL) return -1; + ns++; + } + } else return ns; + // add two word suggestion with dash, if TRY string contains + // "a" or "-" + // NOTE: cwrd doesn't modified for REP twoword sugg. + if (ctry && (strchr(ctry, 'a') || strchr(ctry, '-')) && + mystrlen(p + 1) > 1 && + mystrlen(candidate) - mystrlen(p) > 1) { + *p = '-'; + for (int k=0; k < ns; k++) + if (strcmp(candidate,wlst[k]) == 0) cwrd = 0; + if (ns < maxSug) { + if (cwrd) { + wlst[ns] = mystrdup(candidate); + if (wlst[ns] == NULL) return -1; + ns++; + } + } else return ns; + } + } + } + } + return ns; +} + + +// error is adjacent letter were swapped +int SuggestMgr::swapchar(char ** wlst, const char * word, int ns, int cpdsuggest) +{ + char candidate[MAXSWUTF8L]; + char * p; + char tmpc; + int wl=strlen(word); + // try swapping adjacent chars one by one + strcpy(candidate, word); + for (p = candidate; p[1] != 0; p++) { + tmpc = *p; + *p = p[1]; + p[1] = tmpc; + ns = testsug(wlst, candidate, wl, ns, cpdsuggest, NULL, NULL); + if (ns == -1) return -1; + p[1] = *p; + *p = tmpc; + } + // try double swaps for short words + // ahev -> have, owudl -> would + if (wl == 4 || wl == 5) { + candidate[0] = word[1]; + candidate[1] = word[0]; + candidate[2] = word[2]; + candidate[wl - 2] = word[wl - 1]; + candidate[wl - 1] = word[wl - 2]; + ns = testsug(wlst, candidate, wl, ns, cpdsuggest, NULL, NULL); + if (ns == -1) return -1; + if (wl == 5) { + candidate[0] = word[0]; + candidate[1] = word[2]; + candidate[2] = word[1]; + ns = testsug(wlst, candidate, wl, ns, cpdsuggest, NULL, NULL); + if (ns == -1) return -1; + } + } + return ns; +} + +// error is adjacent letter were swapped +int SuggestMgr::swapchar_utf(char ** wlst, const w_char * word, int wl, int ns, int cpdsuggest) +{ + w_char candidate_utf[MAXSWL]; + char candidate[MAXSWUTF8L]; + w_char * p; + w_char tmpc; + int len = 0; + // try swapping adjacent chars one by one + memcpy (candidate_utf, word, wl * sizeof(w_char)); + for (p = candidate_utf; p < (candidate_utf + wl - 1); p++) { + tmpc = *p; + *p = p[1]; + p[1] = tmpc; + u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl); + if (len == 0) len = strlen(candidate); + ns = testsug(wlst, candidate, len, ns, cpdsuggest, NULL, NULL); + if (ns == -1) return -1; + p[1] = *p; + *p = tmpc; + } + // try double swaps for short words + // ahev -> have, owudl -> would, suodn -> sound + if (wl == 4 || wl == 5) { + candidate_utf[0] = word[1]; + candidate_utf[1] = word[0]; + candidate_utf[2] = word[2]; + candidate_utf[wl - 2] = word[wl - 1]; + candidate_utf[wl - 1] = word[wl - 2]; + u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl); + ns = testsug(wlst, candidate, len, ns, cpdsuggest, NULL, NULL); + if (ns == -1) return -1; + if (wl == 5) { + candidate_utf[0] = word[0]; + candidate_utf[1] = word[2]; + candidate_utf[2] = word[1]; + u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl); + ns = testsug(wlst, candidate, len, ns, cpdsuggest, NULL, NULL); + if (ns == -1) return -1; + } + } + return ns; +} + +// error is not adjacent letter were swapped +int SuggestMgr::longswapchar(char ** wlst, const char * word, int ns, int cpdsuggest) +{ + char candidate[MAXSWUTF8L]; + char * p; + char * q; + char tmpc; + int wl=strlen(word); + // try swapping not adjacent chars one by one + strcpy(candidate, word); + for (p = candidate; *p != 0; p++) { + for (q = candidate; *q != 0; q++) { + if (abs((int)(p-q)) > 1) { + tmpc = *p; + *p = *q; + *q = tmpc; + ns = testsug(wlst, candidate, wl, ns, cpdsuggest, NULL, NULL); + if (ns == -1) return -1; + *q = *p; + *p = tmpc; + } + } + } + return ns; +} + + +// error is adjacent letter were swapped +int SuggestMgr::longswapchar_utf(char ** wlst, const w_char * word, int wl, int ns, int cpdsuggest) +{ + w_char candidate_utf[MAXSWL]; + char candidate[MAXSWUTF8L]; + w_char * p; + w_char * q; + w_char tmpc; + // try swapping not adjacent chars + memcpy (candidate_utf, word, wl * sizeof(w_char)); + for (p = candidate_utf; p < (candidate_utf + wl); p++) { + for (q = candidate_utf; q < (candidate_utf + wl); q++) { + if (abs((int)(p-q)) > 1) { + tmpc = *p; + *p = *q; + *q = tmpc; + u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl); + ns = testsug(wlst, candidate, strlen(candidate), ns, cpdsuggest, NULL, NULL); + if (ns == -1) return -1; + *q = *p; + *p = tmpc; + } + } + } + return ns; +} + +// error is a letter was moved +int SuggestMgr::movechar(char ** wlst, const char * word, int ns, int cpdsuggest) +{ + char candidate[MAXSWUTF8L]; + char * p; + char * q; + char tmpc; + + int wl=strlen(word); + // try moving a char + strcpy(candidate, word); + for (p = candidate; *p != 0; p++) { + for (q = p + 1; (*q != 0) && ((q - p) < 10); q++) { + tmpc = *(q-1); + *(q-1) = *q; + *q = tmpc; + if ((q-p) < 2) continue; // omit swap char + ns = testsug(wlst, candidate, wl, ns, cpdsuggest, NULL, NULL); + if (ns == -1) return -1; + } + strcpy(candidate, word); + } + for (p = candidate + wl - 1; p > candidate; p--) { + for (q = p - 1; (q >= candidate) && ((p - q) < 10); q--) { + tmpc = *(q+1); + *(q+1) = *q; + *q = tmpc; + if ((p-q) < 2) continue; // omit swap char + ns = testsug(wlst, candidate, wl, ns, cpdsuggest, NULL, NULL); + if (ns == -1) return -1; + } + strcpy(candidate, word); + } + return ns; +} + +// error is a letter was moved +int SuggestMgr::movechar_utf(char ** wlst, const w_char * word, int wl, int ns, int cpdsuggest) +{ + w_char candidate_utf[MAXSWL]; + char candidate[MAXSWUTF8L]; + w_char * p; + w_char * q; + w_char tmpc; + // try moving a char + memcpy (candidate_utf, word, wl * sizeof(w_char)); + for (p = candidate_utf; p < (candidate_utf + wl); p++) { + for (q = p + 1; (q < (candidate_utf + wl)) && ((q - p) < 10); q++) { + tmpc = *(q-1); + *(q-1) = *q; + *q = tmpc; + if ((q-p) < 2) continue; // omit swap char + u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl); + ns = testsug(wlst, candidate, strlen(candidate), ns, cpdsuggest, NULL, NULL); + if (ns == -1) return -1; + } + memcpy (candidate_utf, word, wl * sizeof(w_char)); + } + for (p = candidate_utf + wl - 1; p > candidate_utf; p--) { + for (q = p - 1; (q >= candidate_utf) && ((p - q) < 10); q--) { + tmpc = *(q+1); + *(q+1) = *q; + *q = tmpc; + if ((p-q) < 2) continue; // omit swap char + u16_u8(candidate, MAXSWUTF8L, candidate_utf, wl); + ns = testsug(wlst, candidate, strlen(candidate), ns, cpdsuggest, NULL, NULL); + if (ns == -1) return -1; + } + memcpy (candidate_utf, word, wl * sizeof(w_char)); + } + return ns; +} + +// generate a set of suggestions for very poorly spelled words +int SuggestMgr::ngsuggest(char** wlst, char * w, int ns, HashMgr** pHMgr, int md) +{ + + int i, j; + int lval; + int sc, scphon; + int lp, lpphon; + int nonbmp = 0; + + // exhaustively search through all root words + // keeping track of the MAX_ROOTS most similar root words + struct hentry * roots[MAX_ROOTS]; + char * rootsphon[MAX_ROOTS]; + int scores[MAX_ROOTS]; + int scoresphon[MAX_ROOTS]; + for (i = 0; i < MAX_ROOTS; i++) { + roots[i] = NULL; + scores[i] = -100 * i; + rootsphon[i] = NULL; + scoresphon[i] = -100 * i; + } + lp = MAX_ROOTS - 1; + lpphon = MAX_ROOTS - 1; + scphon = -20000; + int low = NGRAM_LOWERING; + + char w2[MAXWORDUTF8LEN]; + char f[MAXSWUTF8L]; + char * word = w; + + // word reversing wrapper for complex prefixes + if (complexprefixes) { + strcpy(w2, w); + if (utf8) reverseword_utf(w2); else reverseword(w2); + word = w2; + } + + char mw[MAXSWUTF8L]; + w_char u8[MAXSWL]; + int nc = strlen(word); + int n = (utf8) ? u8_u16(u8, MAXSWL, word) : nc; + + // set character based ngram suggestion for words with non-BMP Unicode characters + if (n == -1) { + utf8 = 0; // XXX not state-free + n = nc; + nonbmp = 1; + low = 0; + } + + struct hentry* hp = NULL; + int col = -1; + phonetable * ph = (pAMgr) ? pAMgr->get_phonetable() : NULL; + char target[MAXSWUTF8L]; + char candidate[MAXSWUTF8L]; + if (ph) { + if (utf8) { + w_char _w[MAXSWL]; + int _wl = u8_u16(_w, MAXSWL, word); + mkallcap_utf(_w, _wl, langnum); + u16_u8(candidate, MAXSWUTF8L, _w, _wl); + } else { + strcpy(candidate, word); + if (!nonbmp) mkallcap(candidate, csconv); + } + phonet(candidate, target, nc, *ph); // XXX phonet() is 8-bit (nc, not n) + } + + FLAG forbiddenword = pAMgr ? pAMgr->get_forbiddenword() : FLAG_NULL; + FLAG nosuggest = pAMgr ? pAMgr->get_nosuggest() : FLAG_NULL; + FLAG nongramsuggest = pAMgr ? pAMgr->get_nongramsuggest() : FLAG_NULL; + FLAG onlyincompound = pAMgr ? pAMgr->get_onlyincompound() : FLAG_NULL; + + for (i = 0; i < md; i++) { + while (0 != (hp = (pHMgr[i])->walk_hashtable(col, hp))) { + if ((hp->astr) && (pAMgr) && + (TESTAFF(hp->astr, forbiddenword, hp->alen) || + TESTAFF(hp->astr, ONLYUPCASEFLAG, hp->alen) || + TESTAFF(hp->astr, nosuggest, hp->alen) || + TESTAFF(hp->astr, nongramsuggest, hp->alen) || + TESTAFF(hp->astr, onlyincompound, hp->alen))) continue; + + sc = ngram(3, word, HENTRY_WORD(hp), NGRAM_LONGER_WORSE + low) + + leftcommonsubstring(word, HENTRY_WORD(hp)); + + // check special pronounciation + if ((hp->var & H_OPT_PHON) && copy_field(f, HENTRY_DATA(hp), MORPH_PHON)) { + int sc2 = ngram(3, word, f, NGRAM_LONGER_WORSE + low) + + + leftcommonsubstring(word, f); + if (sc2 > sc) sc = sc2; + } + + scphon = -20000; + if (ph && (sc > 2) && (abs(n - (int) hp->clen) <= 3)) { + char target2[MAXSWUTF8L]; + if (utf8) { + w_char _w[MAXSWL]; + int _wl = u8_u16(_w, MAXSWL, HENTRY_WORD(hp)); + mkallcap_utf(_w, _wl, langnum); + u16_u8(candidate, MAXSWUTF8L, _w, _wl); + } else { + strcpy(candidate, HENTRY_WORD(hp)); + mkallcap(candidate, csconv); + } + phonet(candidate, target2, -1, *ph); + scphon = 2 * ngram(3, target, target2, NGRAM_LONGER_WORSE); + } + + if (sc > scores[lp]) { + scores[lp] = sc; + roots[lp] = hp; + lval = sc; + for (j=0; j < MAX_ROOTS; j++) + if (scores[j] < lval) { + lp = j; + lval = scores[j]; + } + } + + + if (scphon > scoresphon[lpphon]) { + scoresphon[lpphon] = scphon; + rootsphon[lpphon] = HENTRY_WORD(hp); + lval = scphon; + for (j=0; j < MAX_ROOTS; j++) + if (scoresphon[j] < lval) { + lpphon = j; + lval = scoresphon[j]; + } + } + }} + + // find minimum threshold for a passable suggestion + // mangle original word three differnt ways + // and score them to generate a minimum acceptable score + int thresh = 0; + for (int sp = 1; sp < 4; sp++) { + if (utf8) { + for (int k=sp; k < n; k+=4) *((unsigned short *) u8 + k) = '*'; + u16_u8(mw, MAXSWUTF8L, u8, n); + thresh = thresh + ngram(n, word, mw, NGRAM_ANY_MISMATCH + low); + } else { + strcpy(mw, word); + for (int k=sp; k < n; k+=4) *(mw + k) = '*'; + thresh = thresh + ngram(n, word, mw, NGRAM_ANY_MISMATCH + low); + } + } + thresh = thresh / 3; + thresh--; + + // now expand affixes on each of these root words and + // and use length adjusted ngram scores to select + // possible suggestions + char * guess[MAX_GUESS]; + char * guessorig[MAX_GUESS]; + int gscore[MAX_GUESS]; + for(i=0;i<MAX_GUESS;i++) { + guess[i] = NULL; + guessorig[i] = NULL; + gscore[i] = -100 * i; + } + + lp = MAX_GUESS - 1; + + struct guessword * glst; + glst = (struct guessword *) calloc(MAX_WORDS,sizeof(struct guessword)); + if (! glst) { + if (nonbmp) utf8 = 1; + return ns; + } + + for (i = 0; i < MAX_ROOTS; i++) { + if (roots[i]) { + struct hentry * rp = roots[i]; + int nw = pAMgr->expand_rootword(glst, MAX_WORDS, HENTRY_WORD(rp), rp->blen, + rp->astr, rp->alen, word, nc, + ((rp->var & H_OPT_PHON) ? copy_field(f, HENTRY_DATA(rp), MORPH_PHON) : NULL)); + + for (int k = 0; k < nw ; k++) { + sc = ngram(n, word, glst[k].word, NGRAM_ANY_MISMATCH + low) + + leftcommonsubstring(word, glst[k].word); + + if (sc > thresh) { + if (sc > gscore[lp]) { + if (guess[lp]) { + free (guess[lp]); + if (guessorig[lp]) { + free(guessorig[lp]); + guessorig[lp] = NULL; + } + } + gscore[lp] = sc; + guess[lp] = glst[k].word; + guessorig[lp] = glst[k].orig; + lval = sc; + for (j=0; j < MAX_GUESS; j++) + if (gscore[j] < lval) { + lp = j; + lval = gscore[j]; + } + } else { + free(glst[k].word); + if (glst[k].orig) free(glst[k].orig); + } + } else { + free(glst[k].word); + if (glst[k].orig) free(glst[k].orig); + } + } + } + } + free(glst); + + // now we are done generating guesses + // sort in order of decreasing score + + + bubblesort(&guess[0], &guessorig[0], &gscore[0], MAX_GUESS); + if (ph) bubblesort(&rootsphon[0], NULL, &scoresphon[0], MAX_ROOTS); + + // weight suggestions with a similarity index, based on + // the longest common subsequent algorithm and resort + + int is_swap = 0; + int re = 0; + double fact = 1.0; + if (pAMgr) { + int maxd = pAMgr->get_maxdiff(); + if (maxd >= 0) fact = (10.0 - maxd)/5.0; + } + + for (i=0; i < MAX_GUESS; i++) { + if (guess[i]) { + // lowering guess[i] + char gl[MAXSWUTF8L]; + int len; + if (utf8) { + w_char _w[MAXSWL]; + len = u8_u16(_w, MAXSWL, guess[i]); + mkallsmall_utf(_w, len, langnum); + u16_u8(gl, MAXSWUTF8L, _w, len); + } else { + strcpy(gl, guess[i]); + if (!nonbmp) mkallsmall(gl, csconv); + len = strlen(guess[i]); + } + + int _lcs = lcslen(word, gl); + + // same characters with different casing + if ((n == len) && (n == _lcs)) { + gscore[i] += 2000; + break; + } + // using 2-gram instead of 3, and other weightening + gscore[i] = + // length of longest common subsequent minus length difference + 2 * _lcs - abs((int) (n - len)) + + // weight length of the left common substring + leftcommonsubstring(word, gl) + + // weight equal character positions + (!nonbmp && commoncharacterpositions(word, gl, &is_swap) ? 1: 0) + + // swap character (not neighboring) + ((is_swap) ? 10 : 0) + + // ngram + ngram(4, word, gl, NGRAM_ANY_MISMATCH + low) + + // weighted ngrams + (re = ngram(2, word, gl, NGRAM_ANY_MISMATCH + low + NGRAM_WEIGHTED)) + + (re += ngram(2, gl, word, NGRAM_ANY_MISMATCH + low + NGRAM_WEIGHTED)) + + // different limit for dictionaries with PHONE rules + (ph ? (re < len * fact ? -1000 : 0) : (re < (n + len)*fact? -1000 : 0)); + } + } + + bubblesort(&guess[0], &guessorig[0], &gscore[0], MAX_GUESS); + +// phonetic version + if (ph) for (i=0; i < MAX_ROOTS; i++) { + if (rootsphon[i]) { + // lowering rootphon[i] + char gl[MAXSWUTF8L]; + int len; + if (utf8) { + w_char _w[MAXSWL]; + len = u8_u16(_w, MAXSWL, rootsphon[i]); + mkallsmall_utf(_w, len, langnum); + u16_u8(gl, MAXSWUTF8L, _w, len); + } else { + strcpy(gl, rootsphon[i]); + if (!nonbmp) mkallsmall(gl, csconv); + len = strlen(rootsphon[i]); + } + + // heuristic weigthing of ngram scores + scoresphon[i] += 2 * lcslen(word, gl) - abs((int) (n - len)) + + // weight length of the left common substring + leftcommonsubstring(word, gl); + } + } + + if (ph) bubblesort(&rootsphon[0], NULL, &scoresphon[0], MAX_ROOTS); + + // copy over + int oldns = ns; + + int same = 0; + for (i=0; i < MAX_GUESS; i++) { + if (guess[i]) { + if ((ns < oldns + maxngramsugs) && (ns < maxSug) && (!same || (gscore[i] > 1000))) { + int unique = 1; + // leave only excellent suggestions, if exists + if (gscore[i] > 1000) same = 1; else if (gscore[i] < -100) { + same = 1; + // keep the best ngram suggestions, unless in ONLYMAXDIFF mode + if (ns > oldns || (pAMgr && pAMgr->get_onlymaxdiff())) { + free(guess[i]); + if (guessorig[i]) free(guessorig[i]); + continue; + } + } + for (j = 0; j < ns; j++) { + // don't suggest previous suggestions or a previous suggestion with prefixes or affixes + if ((!guessorig[i] && strstr(guess[i], wlst[j])) || + (guessorig[i] && strstr(guessorig[i], wlst[j])) || + // check forbidden words + !checkword(guess[i], strlen(guess[i]), 0, NULL, NULL)) unique = 0; + } + if (unique) { + wlst[ns++] = guess[i]; + if (guessorig[i]) { + free(guess[i]); + wlst[ns-1] = guessorig[i]; + } + } else { + free(guess[i]); + if (guessorig[i]) free(guessorig[i]); + } + } else { + free(guess[i]); + if (guessorig[i]) free(guessorig[i]); + } + } + } + + oldns = ns; + if (ph) for (i=0; i < MAX_ROOTS; i++) { + if (rootsphon[i]) { + if ((ns < oldns + MAXPHONSUGS) && (ns < maxSug)) { + int unique = 1; + for (j = 0; j < ns; j++) { + // don't suggest previous suggestions or a previous suggestion with prefixes or affixes + if (strstr(rootsphon[i], wlst[j]) || + // check forbidden words + !checkword(rootsphon[i], strlen(rootsphon[i]), 0, NULL, NULL)) unique = 0; + } + if (unique) { + wlst[ns++] = mystrdup(rootsphon[i]); + if (!wlst[ns - 1]) return ns - 1; + } + } + } + } + + if (nonbmp) utf8 = 1; + return ns; +} + + +// see if a candidate suggestion is spelled correctly +// needs to check both root words and words with affixes + +// obsolote MySpell-HU modifications: +// return value 2 and 3 marks compounding with hyphen (-) +// `3' marks roots without suffix +int SuggestMgr::checkword(const char * word, int len, int cpdsuggest, int * timer, clock_t * timelimit) +{ + struct hentry * rv=NULL; + struct hentry * rv2=NULL; + int nosuffix = 0; + + // check time limit + if (timer) { + (*timer)--; + if (!(*timer) && timelimit) { + if ((clock() - *timelimit) > TIMELIMIT) return 0; + *timer = MAXPLUSTIMER; + } + } + + if (pAMgr) { + if (cpdsuggest==1) { + if (pAMgr->get_compound()) { + rv = pAMgr->compound_check(word, len, 0, 0, 100, 0, NULL, 0, 1, 0); //EXT + if (rv && (!(rv2 = pAMgr->lookup(word)) || !rv2->astr || + !(TESTAFF(rv2->astr,pAMgr->get_forbiddenword(),rv2->alen) || + TESTAFF(rv2->astr,pAMgr->get_nosuggest(),rv2->alen)))) return 3; // XXX obsolote categorisation + only ICONV needs affix flag check? + } + return 0; + } + + rv = pAMgr->lookup(word); + + if (rv) { + if ((rv->astr) && (TESTAFF(rv->astr,pAMgr->get_forbiddenword(),rv->alen) + || TESTAFF(rv->astr,pAMgr->get_nosuggest(),rv->alen))) return 0; + while (rv) { + if (rv->astr && (TESTAFF(rv->astr,pAMgr->get_needaffix(),rv->alen) || + TESTAFF(rv->astr, ONLYUPCASEFLAG, rv->alen) || + TESTAFF(rv->astr,pAMgr->get_onlyincompound(),rv->alen))) { + rv = rv->next_homonym; + } else break; + } + } else rv = pAMgr->prefix_check(word, len, 0); // only prefix, and prefix + suffix XXX + + if (rv) { + nosuffix=1; + } else { + rv = pAMgr->suffix_check(word, len, 0, NULL, NULL, 0, NULL); // only suffix + } + + if (!rv && pAMgr->have_contclass()) { + rv = pAMgr->suffix_check_twosfx(word, len, 0, NULL, FLAG_NULL); + if (!rv) rv = pAMgr->prefix_check_twosfx(word, len, 1, FLAG_NULL); + } + + // check forbidden words + if ((rv) && (rv->astr) && (TESTAFF(rv->astr,pAMgr->get_forbiddenword(),rv->alen) || + TESTAFF(rv->astr, ONLYUPCASEFLAG, rv->alen) || + TESTAFF(rv->astr,pAMgr->get_nosuggest(),rv->alen) || + TESTAFF(rv->astr,pAMgr->get_onlyincompound(),rv->alen))) return 0; + + if (rv) { // XXX obsolote + if ((pAMgr->get_compoundflag()) && + TESTAFF(rv->astr, pAMgr->get_compoundflag(), rv->alen)) return 2 + nosuffix; + return 1; + } + } + return 0; +} + +int SuggestMgr::check_forbidden(const char * word, int len) +{ + struct hentry * rv = NULL; + + if (pAMgr) { + rv = pAMgr->lookup(word); + if (rv && rv->astr && (TESTAFF(rv->astr,pAMgr->get_needaffix(),rv->alen) || + TESTAFF(rv->astr,pAMgr->get_onlyincompound(),rv->alen))) rv = NULL; + if (!(pAMgr->prefix_check(word,len,1))) + rv = pAMgr->suffix_check(word,len, 0, NULL, NULL, 0, NULL); // prefix+suffix, suffix + // check forbidden words + if ((rv) && (rv->astr) && TESTAFF(rv->astr,pAMgr->get_forbiddenword(),rv->alen)) return 1; + } + return 0; +} + +#ifdef HUNSPELL_EXPERIMENTAL +// suggest possible stems +int SuggestMgr::suggest_pos_stems(char*** slst, const char * w, int nsug) +{ + char ** wlst; + + struct hentry * rv = NULL; + + char w2[MAXSWUTF8L]; + const char * word = w; + + // word reversing wrapper for complex prefixes + if (complexprefixes) { + strcpy(w2, w); + if (utf8) reverseword_utf(w2); else reverseword(w2); + word = w2; + } + + int wl = strlen(word); + + + if (*slst) { + wlst = *slst; + } else { + wlst = (char **) calloc(maxSug, sizeof(char *)); + if (wlst == NULL) return -1; + } + + rv = pAMgr->suffix_check(word, wl, 0, NULL, wlst, maxSug, &nsug); + + // delete dash from end of word + if (nsug > 0) { + for (int j=0; j < nsug; j++) { + if (wlst[j][strlen(wlst[j]) - 1] == '-') wlst[j][strlen(wlst[j]) - 1] = '\0'; + } + } + + *slst = wlst; + return nsug; +} +#endif // END OF HUNSPELL_EXPERIMENTAL CODE + + +char * SuggestMgr::suggest_morph(const char * w) +{ + char result[MAXLNLEN]; + char * r = (char *) result; + char * st; + + struct hentry * rv = NULL; + + *result = '\0'; + + if (! pAMgr) return NULL; + + char w2[MAXSWUTF8L]; + const char * word = w; + + // word reversing wrapper for complex prefixes + if (complexprefixes) { + strcpy(w2, w); + if (utf8) reverseword_utf(w2); else reverseword(w2); + word = w2; + } + + rv = pAMgr->lookup(word); + + while (rv) { + if ((!rv->astr) || !(TESTAFF(rv->astr, pAMgr->get_forbiddenword(), rv->alen) || + TESTAFF(rv->astr, pAMgr->get_needaffix(), rv->alen) || + TESTAFF(rv->astr,pAMgr->get_onlyincompound(),rv->alen))) { + if (!HENTRY_FIND(rv, MORPH_STEM)) { + mystrcat(result, " ", MAXLNLEN); + mystrcat(result, MORPH_STEM, MAXLNLEN); + mystrcat(result, word, MAXLNLEN); + } + if (HENTRY_DATA(rv)) { + mystrcat(result, " ", MAXLNLEN); + mystrcat(result, HENTRY_DATA2(rv), MAXLNLEN); + } + mystrcat(result, "\n", MAXLNLEN); + } + rv = rv->next_homonym; + } + + st = pAMgr->affix_check_morph(word,strlen(word)); + if (st) { + mystrcat(result, st, MAXLNLEN); + free(st); + } + + if (pAMgr->get_compound() && (*result == '\0')) + pAMgr->compound_check_morph(word, strlen(word), + 0, 0, 100, 0,NULL, 0, &r, NULL); + + return (*result) ? mystrdup(line_uniq(result, MSEP_REC)) : NULL; +} + +#ifdef HUNSPELL_EXPERIMENTAL +char * SuggestMgr::suggest_morph_for_spelling_error(const char * word) +{ + char * p = NULL; + char ** wlst = (char **) calloc(maxSug, sizeof(char *)); + if (!**wlst) return NULL; + // we will use only the first suggestion + for (int i = 0; i < maxSug - 1; i++) wlst[i] = ""; + int ns = suggest(&wlst, word, maxSug - 1, NULL); + if (ns == maxSug) { + p = suggest_morph(wlst[maxSug - 1]); + free(wlst[maxSug - 1]); + } + if (wlst) free(wlst); + return p; +} +#endif // END OF HUNSPELL_EXPERIMENTAL CODE + +/* affixation */ +char * SuggestMgr::suggest_hentry_gen(hentry * rv, char * pattern) +{ + char result[MAXLNLEN]; + *result = '\0'; + int sfxcount = get_sfxcount(pattern); + + if (get_sfxcount(HENTRY_DATA(rv)) > sfxcount) return NULL; + + if (HENTRY_DATA(rv)) { + char * aff = pAMgr->morphgen(HENTRY_WORD(rv), rv->blen, rv->astr, rv->alen, + HENTRY_DATA(rv), pattern, 0); + if (aff) { + mystrcat(result, aff, MAXLNLEN); + mystrcat(result, "\n", MAXLNLEN); + free(aff); + } + } + + // check all allomorphs + char allomorph[MAXLNLEN]; + char * p = NULL; + if (HENTRY_DATA(rv)) p = (char *) strstr(HENTRY_DATA2(rv), MORPH_ALLOMORPH); + while (p) { + struct hentry * rv2 = NULL; + p += MORPH_TAG_LEN; + int plen = fieldlen(p); + strncpy(allomorph, p, plen); + allomorph[plen] = '\0'; + rv2 = pAMgr->lookup(allomorph); + while (rv2) { +// if (HENTRY_DATA(rv2) && get_sfxcount(HENTRY_DATA(rv2)) <= sfxcount) { + if (HENTRY_DATA(rv2)) { + char * st = (char *) strstr(HENTRY_DATA2(rv2), MORPH_STEM); + if (st && (strncmp(st + MORPH_TAG_LEN, + HENTRY_WORD(rv), fieldlen(st + MORPH_TAG_LEN)) == 0)) { + char * aff = pAMgr->morphgen(HENTRY_WORD(rv2), rv2->blen, rv2->astr, rv2->alen, + HENTRY_DATA(rv2), pattern, 0); + if (aff) { + mystrcat(result, aff, MAXLNLEN); + mystrcat(result, "\n", MAXLNLEN); + free(aff); + } + } + } + rv2 = rv2->next_homonym; + } + p = strstr(p + plen, MORPH_ALLOMORPH); + } + + return (*result) ? mystrdup(result) : NULL; +} + +char * SuggestMgr::suggest_gen(char ** desc, int n, char * pattern) { + char result[MAXLNLEN]; + char result2[MAXLNLEN]; + char newpattern[MAXLNLEN]; + *newpattern = '\0'; + if (n == 0) return 0; + *result2 = '\0'; + struct hentry * rv = NULL; + if (!pAMgr) return NULL; + +// search affixed forms with and without derivational suffixes + while(1) { + + for (int k = 0; k < n; k++) { + *result = '\0'; + // add compound word parts (except the last one) + char * s = (char *) desc[k]; + char * part = strstr(s, MORPH_PART); + if (part) { + char * nextpart = strstr(part + 1, MORPH_PART); + while (nextpart) { + copy_field(result + strlen(result), part, MORPH_PART); + part = nextpart; + nextpart = strstr(part + 1, MORPH_PART); + } + s = part; + } + + char **pl; + char tok[MAXLNLEN]; + strcpy(tok, s); + char * alt = strstr(tok, " | "); + while (alt) { + alt[1] = MSEP_ALT; + alt = strstr(alt, " | "); + } + int pln = line_tok(tok, &pl, MSEP_ALT); + for (int i = 0; i < pln; i++) { + // remove inflectional and terminal suffixes + char * is = strstr(pl[i], MORPH_INFL_SFX); + if (is) *is = '\0'; + char * ts = strstr(pl[i], MORPH_TERM_SFX); + while (ts) { + *ts = '_'; + ts = strstr(pl[i], MORPH_TERM_SFX); + } + char * st = strstr(s, MORPH_STEM); + if (st) { + copy_field(tok, st, MORPH_STEM); + rv = pAMgr->lookup(tok); + while (rv) { + char newpat[MAXLNLEN]; + strcpy(newpat, pl[i]); + strcat(newpat, pattern); + char * sg = suggest_hentry_gen(rv, newpat); + if (!sg) sg = suggest_hentry_gen(rv, pattern); + if (sg) { + char ** gen; + int genl = line_tok(sg, &gen, MSEP_REC); + free(sg); + sg = NULL; + for (int j = 0; j < genl; j++) { + if (strstr(pl[i], MORPH_SURF_PFX)) { + int r2l = strlen(result2); + result2[r2l] = MSEP_REC; + strcpy(result2 + r2l + 1, result); + copy_field(result2 + strlen(result2), pl[i], MORPH_SURF_PFX); + mystrcat(result2, gen[j], MAXLNLEN); + } else { + sprintf(result2 + strlen(result2), "%c%s%s", + MSEP_REC, result, gen[j]); + } + } + freelist(&gen, genl); + } + rv = rv->next_homonym; + } + } + } + freelist(&pl, pln); + } + + if (*result2 || !strstr(pattern, MORPH_DERI_SFX)) break; + strcpy(newpattern, pattern); + pattern = newpattern; + char * ds = strstr(pattern, MORPH_DERI_SFX); + while (ds) { + strncpy(ds, MORPH_TERM_SFX, MORPH_TAG_LEN); + ds = strstr(pattern, MORPH_DERI_SFX); + } + } + return (*result2 ? mystrdup(result2) : NULL); +} + + +// generate an n-gram score comparing s1 and s2 +int SuggestMgr::ngram(int n, char * s1, const char * s2, int opt) +{ + int nscore = 0; + int ns; + int l1; + int l2; + int test = 0; + + if (utf8) { + w_char su1[MAXSWL]; + w_char su2[MAXSWL]; + l1 = u8_u16(su1, MAXSWL, s1); + l2 = u8_u16(su2, MAXSWL, s2); + if ((l2 <= 0) || (l1 == -1)) return 0; + // lowering dictionary word + if (opt & NGRAM_LOWERING) mkallsmall_utf(su2, l2, langnum); + for (int j = 1; j <= n; j++) { + ns = 0; + for (int i = 0; i <= (l1-j); i++) { + int k = 0; + for (int l = 0; l <= (l2-j); l++) { + for (k = 0; k < j; k++) { + w_char * c1 = su1 + i + k; + w_char * c2 = su2 + l + k; + if ((c1->l != c2->l) || (c1->h != c2->h)) break; + } + if (k == j) { + ns++; + break; + } + } + if (k != j && opt & NGRAM_WEIGHTED) { + ns--; + test++; + if (i == 0 || i == l1-j) ns--; // side weight + } + } + nscore = nscore + ns; + if (ns < 2 && !(opt & NGRAM_WEIGHTED)) break; + } + } else { + l2 = strlen(s2); + if (l2 == 0) return 0; + l1 = strlen(s1); + char *t = mystrdup(s2); + if (opt & NGRAM_LOWERING) mkallsmall(t, csconv); + for (int j = 1; j <= n; j++) { + ns = 0; + for (int i = 0; i <= (l1-j); i++) { + char c = *(s1 + i + j); + *(s1 + i + j) = '\0'; + if (strstr(t,(s1+i))) { + ns++; + } else if (opt & NGRAM_WEIGHTED) { + ns--; +test++; + if (i == 0 || i == l1-j) ns--; // side weight + } + *(s1 + i + j ) = c; + } + nscore = nscore + ns; + if (ns < 2 && !(opt & NGRAM_WEIGHTED)) break; + } + free(t); + } + + ns = 0; + if (opt & NGRAM_LONGER_WORSE) ns = (l2-l1)-2; + if (opt & NGRAM_ANY_MISMATCH) ns = abs(l2-l1)-2; + ns = (nscore - ((ns > 0) ? ns : 0)); + return ns; +} + +// length of the left common substring of s1 and (decapitalised) s2 +int SuggestMgr::leftcommonsubstring(char * s1, const char * s2) { + if (utf8) { + w_char su1[MAXSWL]; + w_char su2[MAXSWL]; + su1[0].l = su2[0].l = su1[0].h = su2[0].h = 0; + // decapitalize dictionary word + if (complexprefixes) { + int l1 = u8_u16(su1, MAXSWL, s1); + int l2 = u8_u16(su2, MAXSWL, s2); + if (*((short *)su1+l1-1) == *((short *)su2+l2-1)) return 1; + } else { + int i; + u8_u16(su1, 1, s1); + u8_u16(su2, 1, s2); + unsigned short idx = (su2->h << 8) + su2->l; + unsigned short otheridx = (su1->h << 8) + su1->l; + if (otheridx != idx && + (otheridx != unicodetolower(idx, langnum))) return 0; + int l1 = u8_u16(su1, MAXSWL, s1); + int l2 = u8_u16(su2, MAXSWL, s2); + for(i = 1; (i < l1) && (i < l2) && + (su1[i].l == su2[i].l) && (su1[i].h == su2[i].h); i++); + return i; + } + } else { + if (complexprefixes) { + int l1 = strlen(s1); + int l2 = strlen(s2); + if (*(s2+l1-1) == *(s2+l2-1)) return 1; + } else { + char * olds = s1; + // decapitalise dictionary word + if ((*s1 != *s2) && (*s1 != csconv[((unsigned char)*s2)].clower)) return 0; + do { + s1++; s2++; + } while ((*s1 == *s2) && (*s1 != '\0')); + return (int)(s1 - olds); + } + } + return 0; +} + +int SuggestMgr::commoncharacterpositions(char * s1, const char * s2, int * is_swap) { + int num = 0; + int diff = 0; + int diffpos[2]; + *is_swap = 0; + if (utf8) { + w_char su1[MAXSWL]; + w_char su2[MAXSWL]; + int l1 = u8_u16(su1, MAXSWL, s1); + int l2 = u8_u16(su2, MAXSWL, s2); + // decapitalize dictionary word + if (complexprefixes) { + mkallsmall_utf(su2+l2-1, 1, langnum); + } else { + mkallsmall_utf(su2, 1, langnum); + } + for (int i = 0; (i < l1) && (i < l2); i++) { + if (((short *) su1)[i] == ((short *) su2)[i]) { + num++; + } else { + if (diff < 2) diffpos[diff] = i; + diff++; + } + } + if ((diff == 2) && (l1 == l2) && + (((short *) su1)[diffpos[0]] == ((short *) su2)[diffpos[1]]) && + (((short *) su1)[diffpos[1]] == ((short *) su2)[diffpos[0]])) *is_swap = 1; + } else { + int i; + char t[MAXSWUTF8L]; + strcpy(t, s2); + // decapitalize dictionary word + if (complexprefixes) { + int l2 = strlen(t); + *(t+l2-1) = csconv[((unsigned char)*(t+l2-1))].clower; + } else { + mkallsmall(t, csconv); + } + for (i = 0; (*(s1+i) != 0) && (*(t+i) != 0); i++) { + if (*(s1+i) == *(t+i)) { + num++; + } else { + if (diff < 2) diffpos[diff] = i; + diff++; + } + } + if ((diff == 2) && (*(s1+i) == 0) && (*(t+i) == 0) && + (*(s1+diffpos[0]) == *(t+diffpos[1])) && + (*(s1+diffpos[1]) == *(t+diffpos[0]))) *is_swap = 1; + } + return num; +} + +int SuggestMgr::mystrlen(const char * word) { + if (utf8) { + w_char w[MAXSWL]; + return u8_u16(w, MAXSWL, word); + } else return strlen(word); +} + +// sort in decreasing order of score +void SuggestMgr::bubblesort(char** rword, char** rword2, int* rsc, int n ) +{ + int m = 1; + while (m < n) { + int j = m; + while (j > 0) { + if (rsc[j-1] < rsc[j]) { + int sctmp = rsc[j-1]; + char * wdtmp = rword[j-1]; + rsc[j-1] = rsc[j]; + rword[j-1] = rword[j]; + rsc[j] = sctmp; + rword[j] = wdtmp; + if (rword2) { + wdtmp = rword2[j-1]; + rword2[j-1] = rword2[j]; + rword2[j] = wdtmp; + } + j--; + } else break; + } + m++; + } + return; +} + +// longest common subsequence +void SuggestMgr::lcs(const char * s, const char * s2, int * l1, int * l2, char ** result) { + int n, m; + w_char su[MAXSWL]; + w_char su2[MAXSWL]; + char * b; + char * c; + int i; + int j; + if (utf8) { + m = u8_u16(su, MAXSWL, s); + n = u8_u16(su2, MAXSWL, s2); + } else { + m = strlen(s); + n = strlen(s2); + } + c = (char *) malloc((m + 1) * (n + 1)); + b = (char *) malloc((m + 1) * (n + 1)); + if (!c || !b) { + if (c) free(c); + if (b) free(b); + *result = NULL; + return; + } + for (i = 1; i <= m; i++) c[i*(n+1)] = 0; + for (j = 0; j <= n; j++) c[j] = 0; + for (i = 1; i <= m; i++) { + for (j = 1; j <= n; j++) { + if ( ((utf8) && (*((short *) su+i-1) == *((short *)su2+j-1))) + || ((!utf8) && ((*(s+i-1)) == (*(s2+j-1))))) { + c[i*(n+1) + j] = c[(i-1)*(n+1) + j-1]+1; + b[i*(n+1) + j] = LCS_UPLEFT; + } else if (c[(i-1)*(n+1) + j] >= c[i*(n+1) + j-1]) { + c[i*(n+1) + j] = c[(i-1)*(n+1) + j]; + b[i*(n+1) + j] = LCS_UP; + } else { + c[i*(n+1) + j] = c[i*(n+1) + j-1]; + b[i*(n+1) + j] = LCS_LEFT; + } + } + } + *result = b; + free(c); + *l1 = m; + *l2 = n; +} + +int SuggestMgr::lcslen(const char * s, const char* s2) { + int m; + int n; + int i; + int j; + char * result; + int len = 0; + lcs(s, s2, &m, &n, &result); + if (!result) return 0; + i = m; + j = n; + while ((i != 0) && (j != 0)) { + if (result[i*(n+1) + j] == LCS_UPLEFT) { + len++; + i--; + j--; + } else if (result[i*(n+1) + j] == LCS_UP) { + i--; + } else j--; + } + free(result); + return len; +} |