summaryrefslogtreecommitdiff
path: root/Plugins/eSpeak/eSpeak/compiledict.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'Plugins/eSpeak/eSpeak/compiledict.cpp')
-rw-r--r--Plugins/eSpeak/eSpeak/compiledict.cpp1649
1 files changed, 1649 insertions, 0 deletions
diff --git a/Plugins/eSpeak/eSpeak/compiledict.cpp b/Plugins/eSpeak/eSpeak/compiledict.cpp
new file mode 100644
index 0000000..9fcaa9a
--- /dev/null
+++ b/Plugins/eSpeak/eSpeak/compiledict.cpp
@@ -0,0 +1,1649 @@
+/***************************************************************************
+ * Copyright (C) 2005 to 2007 by Jonathan Duddington *
+ * email: jonsd@users.sourceforge.net *
+ * *
+ * This program is free software; you can redistribute it and/or modify *
+ * it under the terms of the GNU General Public License as published by *
+ * the Free Software Foundation; either version 3 of the License, or *
+ * (at your option) any later version. *
+ * *
+ * This program is distributed in the hope that it will be useful, *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
+ * GNU General Public License for more details. *
+ * *
+ * You should have received a copy of the GNU General Public License *
+ * along with this program; if not, write see: *
+ * <http://www.gnu.org/licenses/>. *
+ ***************************************************************************/
+
+#include "StdAfx.h"
+
+#include <stdio.h>
+#include <ctype.h>
+#include <stdlib.h>
+#include <string.h>
+#include <wctype.h>
+
+#include "speak_lib.h"
+#include "speech.h"
+#include "phoneme.h"
+#include "synthesize.h"
+#include "translate.h"
+
+//#define OPT_FORMAT // format the text and write formatted copy to Log file
+//#define OUTPUT_FORMAT
+
+extern void Write4Bytes(FILE *f, int value);
+int HashDictionary(const char *string);
+
+static FILE *f_log = NULL;
+extern char *dir_dictionary;
+
+static int linenum;
+static int error_count;
+static int transpose_offset; // transpose character range for LookupDictList()
+static int transpose_min;
+static int transpose_max;
+static int text_mode = 0;
+static int debug_flag = 0;
+
+static int hash_counts[N_HASH_DICT];
+static char *hash_chains[N_HASH_DICT];
+static char letterGroupsDefined[N_LETTER_GROUPS];
+
+MNEM_TAB mnem_flags[] = {
+ // these in the first group put a value in bits0-3 of dictionary_flags
+ {"$1", 0x41}, // stress on 1st syllable
+ {"$2", 0x42}, // stress on 2nd syllable
+ {"$3", 0x43},
+ {"$4", 0x44},
+ {"$5", 0x45},
+ {"$6", 0x46},
+ {"$7", 0x47},
+ {"$u", 0x48}, // reduce to unstressed
+ {"$u1", 0x49},
+ {"$u2", 0x4a},
+ {"$u3", 0x4b},
+ {"$u+", 0x4c}, // reduce to unstressed, but stress at end of clause
+ {"$u1+", 0x4d},
+ {"$u2+", 0x4e},
+ {"$u3+", 0x4f},
+
+
+ // these set the corresponding numbered bit if dictionary_flags
+ {"$pause", 8}, /* ensure pause before this word */
+ {"$only", 9}, /* only match on this word without suffix */
+ {"$onlys", 10}, /* only match with none, or with 's' suffix */
+ {"$strend", 11}, /* full stress if at end of clause */
+ {"$strend2", 12}, /* full stress if at end of clause, or only followed by unstressed */
+ {"$unstressend",13}, /* reduce stress at end of clause */
+ {"$atend", 14}, /* use this pronunciation if at end of clause */
+
+ {"$dot", 16}, /* ignore '.' after this word (abbreviation) */
+ {"$abbrev", 17}, /* use this pronuciation rather than split into letters */
+ {"$stem", 18}, // must have a suffix
+
+// language specific
+ {"$double", 19}, // IT double the initial consonant of next word
+ {"$alt", 20}, // use alternative pronunciation
+ {"$alt2", 21},
+
+
+ {"$max3", 27}, // limit to 3 repetitions
+ {"$brk", 28}, // a shorter $pause
+ {"$text", 29}, // word translates to replcement text, not phonemes
+
+// flags in dictionary word 2
+ {"$verbf", 0x20}, /* verb follows */
+ {"$verbsf", 0x21}, /* verb follows, allow -s suffix */
+ {"$nounf", 0x22}, /* noun follows */
+ {"$pastf", 0x23}, /* past tense follows */
+ {"$verb", 0x24}, /* use this pronunciation when its a verb */
+ {"$noun", 0x25}, /* use this pronunciation when its a noun */
+ {"$past", 0x26}, /* use this pronunciation when its past tense */
+ {"$verbextend",0x28}, /* extend influence of 'verb follows' */
+ {"$capital", 0x29}, /* use this pronunciation if initial letter is upper case */
+ {"$allcaps", 0x2a}, /* use this pronunciation if initial letter is upper case */
+ {"$accent", 0x2b}, // character name is base-character name + accent name
+
+ // doesn't set dictionary_flags
+ {"$?", 100}, // conditional rule, followed by byte giving the condition number
+
+ {"$textmode", 200},
+ {"$phonememode", 201},
+ {NULL, -1}
+};
+
+
+#define LEN_GROUP_NAME 12
+
+typedef struct {
+ char name[LEN_GROUP_NAME+1];
+ unsigned int start;
+ unsigned int length;
+} RGROUP;
+
+
+int isspace2(unsigned int c)
+{//=========================
+// can't use isspace() because on Windows, isspace(0xe1) gives TRUE !
+ int c2;
+
+ if(((c2 = (c & 0xff)) == 0) || (c > ' '))
+ return(0);
+ return(1);
+}
+
+
+static const char *LookupMnem2(MNEM_TAB *table, int value)
+{//=======================================================
+ while(table->mnem != NULL)
+ {
+ if(table->value == value)
+ return(table->mnem);
+ table++;
+ }
+ return("");
+}
+
+
+char *print_dictionary_flags(unsigned int *flags)
+{//==============================================
+ static char buf[20];
+
+ sprintf(buf,"%s 0x%x/%x",LookupMnem2(mnem_flags,(flags[0] & 0xf)+0x40), flags[0], flags[1]);
+ return(buf);
+}
+
+
+
+static FILE *fopen_log(const char *fname,const char *access)
+{//==================================================
+// performs fopen, but produces error message to f_log if it fails
+ FILE *f;
+
+ if((f = fopen(fname,access)) == NULL)
+ {
+ if(f_log != NULL)
+ fprintf(f_log,"Can't access (%s) file '%s'\n",access,fname);
+ }
+ return(f);
+}
+
+
+#ifdef OPT_FORMAT
+static const char *lookup_mnem(MNEM_TAB *table, int value)
+//========================================================
+/* Lookup a mnemonic string in a table, return its name */
+{
+ while(table->mnem != NULL)
+ {
+ if(table->value==value)
+ return(table->mnem);
+ table++;
+ }
+ return("??"); /* not found */
+} /* end of mnem */
+#endif
+
+
+
+
+static int compile_line(char *linebuf, char *dict_line, int *hash)
+{//===============================================================
+// Compile a line in the language_list file
+ unsigned char c;
+ char *p;
+ char *word;
+ char *phonetic;
+ unsigned int ix;
+ int step;
+ unsigned int n_flag_codes = 0;
+ int flag_offset;
+ int length;
+ int multiple_words = 0;
+ char *multiple_string = NULL;
+ char *multiple_string_end = NULL;
+
+ int len_word;
+ int len_phonetic;
+ int text_not_phonemes; // this word specifies replacement text, not phonemes
+ unsigned int wc;
+ int all_upper_case;
+
+ char *mnemptr;
+ char *comment;
+ unsigned char flag_codes[100];
+ char encoded_ph[200];
+ unsigned char bad_phoneme[4];
+static char nullstring[] = {0};
+
+ comment = NULL;
+ text_not_phonemes = 0;
+ phonetic = word = nullstring;
+
+if(memcmp(linebuf,"_-",2)==0)
+{
+step=1; // TEST
+}
+ p = linebuf;
+// while(isspace2(*p)) p++;
+
+#ifdef deleted
+ if(*p == '$')
+ {
+ if(memcmp(p,"$textmode",9) == 0)
+ {
+ text_mode = 1;
+ return(0);
+ }
+ if(memcmp(p,"$phonememode",12) == 0)
+ {
+ text_mode = 0;
+ return(0);
+ }
+ }
+#endif
+
+ step = 0;
+
+ c = 0;
+ while(c != '\n')
+ {
+ c = *p;
+
+ if((c == '?') && (step==0))
+ {
+ // conditional rule, allow only if the numbered condition is set for the voice
+ flag_offset = 100;
+
+ p++;
+ if(*p == '!')
+ {
+ // allow only if the numbered condition is NOT set
+ flag_offset = 132;
+ p++;
+ }
+
+ ix = 0;
+ if(isdigit(*p))
+ {
+ ix += (*p-'0');
+ p++;
+ }
+ if(isdigit(*p))
+ {
+ ix = ix*10 + (*p-'0');
+ p++;
+ }
+ flag_codes[n_flag_codes++] = ix + flag_offset;
+ c = *p;
+ }
+
+ if((c == '$') && isalnum(p[1]))
+ {
+ /* read keyword parameter */
+ mnemptr = p;
+ while(!isspace2(c = *p)) p++;
+ *p = 0;
+
+ ix = LookupMnem(mnem_flags,mnemptr);
+ if(ix > 0)
+ {
+ if(ix == 200)
+ {
+ text_mode = 1;
+ }
+ else
+ if(ix == 201)
+ {
+ text_mode = 0;
+ }
+ else
+ if(ix == BITNUM_FLAG_TEXTMODE)
+ {
+ text_not_phonemes = 1;
+ }
+ else
+ {
+ flag_codes[n_flag_codes++] = ix;
+ }
+ }
+ else
+ {
+ fprintf(f_log,"%5d: Unknown keyword: %s\n",linenum,mnemptr);
+ error_count++;
+ }
+ }
+
+ if((c == '/') && (p[1] == '/') && (multiple_words==0))
+ {
+ c = '\n'; /* "//" treat comment as end of line */
+ comment = p;
+ }
+
+ switch(step)
+ {
+ case 0:
+ if(c == '(')
+ {
+ multiple_words = 1;
+ word = p+1;
+ step = 1;
+ }
+ else
+ if(!isspace2(c))
+ {
+ word = p;
+ step = 1;
+ }
+ break;
+
+ case 1:
+ if((c == '-') && (word[0] != '_'))
+ {
+ flag_codes[n_flag_codes++] = BITNUM_FLAG_HYPHENATED;
+ c = ' ';
+ }
+ if(isspace2(c))
+ {
+ p[0] = 0; /* terminate english word */
+
+ if(multiple_words)
+ {
+ multiple_string = multiple_string_end = p+1;
+ step = 2;
+ }
+ else
+ {
+ step = 3;
+ }
+ }
+ else
+ if((c == ')') && multiple_words)
+ {
+ p[0] = 0;
+ step = 3;
+ multiple_words = 0;
+ }
+ break;
+
+ case 2:
+ if(isspace2(c))
+ {
+ multiple_words++;
+ }
+ else
+ if(c == ')')
+ {
+ p[0] = ' '; // terminate extra string
+ multiple_string_end = p+1;
+ step = 3;
+ }
+ break;
+
+ case 3:
+ if(!isspace2(c))
+ {
+ phonetic = p;
+ step = 4;
+ }
+ break;
+
+ case 4:
+ if(isspace2(c))
+ {
+ p[0] = 0; /* terminate phonetic */
+ step = 5;
+ }
+ break;
+
+ case 5:
+ break;
+ }
+ p++;
+ }
+
+ if(word[0] == 0)
+ {
+#ifdef OPT_FORMAT
+ if(comment != NULL)
+ fprintf(f_log,"%s",comment);
+ else
+ fputc('\n',f_log);
+#endif
+ return(0); /* blank line */
+ }
+
+ if(text_mode)
+ text_not_phonemes = 1;
+
+ if(text_not_phonemes != translator->langopts.textmode)
+ {
+ flag_codes[n_flag_codes++] = BITNUM_FLAG_TEXTMODE;
+ }
+
+ if(text_not_phonemes)
+ {
+ // this is replacement text, so don't encode as phonemes. Restrict the length of the replacement word
+ strncpy0(encoded_ph,phonetic,N_WORD_BYTES-4);
+ }
+ else
+ {
+ EncodePhonemes(phonetic,encoded_ph,bad_phoneme);
+ if(strchr(encoded_ph,phonSWITCH) != 0)
+ {
+ flag_codes[n_flag_codes++] = BITNUM_FLAG_ONLY_S; // don't match on suffixes (except 's') when switching languages
+ }
+
+ // check for errors in the phonemes codes
+ for(ix=0; ix<sizeof(encoded_ph); ix++)
+ {
+ c = encoded_ph[ix];
+ if(c == 0) break;
+
+ if(c == 255)
+ {
+ /* unrecognised phoneme, report error */
+ fprintf(f_log,"%5d: Bad phoneme [%c] (0x%x) in: %s %s\n",linenum,bad_phoneme[0],bad_phoneme[0],word,phonetic);
+ error_count++;
+ }
+ }
+ }
+
+ if(sscanf(word,"U+%x",&wc) == 1)
+ {
+ // Character code
+ ix = utf8_out(wc, word);
+ word[ix] = 0;
+ }
+ else
+ if(word[0] != '_')
+ {
+ // convert to lower case, and note if the word is all-capitals
+ int c2;
+
+ all_upper_case = 1;
+ p = word;
+ for(p=word;;)
+ {
+ // this assumes that the lower case char is the same length as the upper case char
+ // OK, except for Turkish "I", but use towlower() rather than towlower2()
+ ix = utf8_in(&c2,p);
+ if(c2 == 0)
+ break;
+ if(iswupper(c2))
+ {
+ utf8_out(towlower(c2),p);
+ }
+ else
+ {
+ all_upper_case = 0;
+ }
+ p += ix;
+ }
+ if(all_upper_case)
+ {
+ flag_codes[n_flag_codes++] = BITNUM_FLAG_ALLCAPS;
+ }
+ }
+
+ len_word = strlen(word);
+
+ if(transpose_offset > 0)
+ {
+ len_word = TransposeAlphabet(word, transpose_offset, transpose_min, transpose_max);
+ }
+
+ *hash = HashDictionary(word);
+ len_phonetic = strlen(encoded_ph);
+
+ dict_line[1] = len_word; // bit 6 indicates whether the word has been compressed
+ len_word &= 0x3f;
+
+ memcpy(&dict_line[2],word,len_word);
+
+ if(len_phonetic == 0)
+ {
+ // no phonemes specified. set bit 7
+ dict_line[1] |= 0x80;
+ length = len_word + 2;
+ }
+ else
+ {
+ length = len_word + len_phonetic + 3;
+ strcpy(&dict_line[(len_word)+2],encoded_ph);
+ }
+
+ for(ix=0; ix<n_flag_codes; ix++)
+ {
+ dict_line[ix+length] = flag_codes[ix];
+ }
+ length += n_flag_codes;
+
+ if((multiple_string != NULL) && (multiple_words > 0))
+ {
+ if(multiple_words > 10)
+ {
+ fprintf(f_log,"%5d: Two many parts in a multi-word entry: %d\n",linenum,multiple_words);
+ }
+ else
+ {
+ dict_line[length++] = 80 + multiple_words;
+ ix = multiple_string_end - multiple_string;
+ memcpy(&dict_line[length],multiple_string,ix);
+ length += ix;
+ }
+ }
+ dict_line[0] = length;
+
+#ifdef OPT_FORMAT
+ spaces = 16;
+ for(ix=0; ix<n_flag_codes; ix++)
+ {
+ if(flag_codes[ix] >= 100)
+ {
+ fprintf(f_log,"?%d ",flag_codes[ix]-100);
+ spaces -= 3;
+ }
+ }
+
+ fprintf(f_log,"%s",word);
+ spaces -= strlen(word);
+ DecodePhonemes(encoded_ph,decoded_ph);
+ while(spaces-- > 0) fputc(' ',f_log);
+ spaces += (14 - strlen(decoded_ph));
+
+ fprintf(f_log," %s",decoded_ph);
+ while(spaces-- > 0) fputc(' ',f_log);
+ for(ix=0; ix<n_flag_codes; ix++)
+ {
+ if(flag_codes[ix] < 100)
+ fprintf(f_log," %s",lookup_mnem(mnem_flags,flag_codes[ix]));
+ }
+ if(comment != NULL)
+ fprintf(f_log," %s",comment);
+ else
+ fputc('\n',f_log);
+#endif
+
+ return(length);
+} /* end of compile_line */
+
+
+
+static void compile_dictlist_start(void)
+{//=====================================
+// initialise dictionary list
+ int ix;
+ char *p;
+ char *p2;
+
+ for(ix=0; ix<N_HASH_DICT; ix++)
+ {
+ p = hash_chains[ix];
+ while(p != NULL)
+ {
+ memcpy(&p2,p,sizeof(char *));
+ free(p);
+ p = p2;
+ }
+ hash_chains[ix] = NULL;
+ hash_counts[ix]=0;
+ }
+}
+
+
+static void compile_dictlist_end(FILE *f_out)
+{//==========================================
+// Write out the compiled dictionary list
+ int hash;
+ int length;
+ char *p;
+
+ if(f_log != NULL)
+ {
+#ifdef OUTPUT_FORMAT
+ for(hash=0; hash<N_HASH_DICT; hash++)
+ {
+ fprintf(f_log,"%8d",hash_counts[hash]);
+ if((hash & 7) == 7)
+ fputc('\n',f_log);
+ }
+ fflush(f_log);
+#endif
+ }
+
+ for(hash=0; hash<N_HASH_DICT; hash++)
+ {
+ p = hash_chains[hash];
+ hash_counts[hash] = (int)ftell(f_out);
+
+ while(p != NULL)
+ {
+ length = *(p+sizeof(char *));
+ fwrite(p+sizeof(char *),length,1,f_out);
+ memcpy(&p,p,sizeof(char *));
+ }
+ fputc(0,f_out);
+ }
+}
+
+
+
+static int compile_dictlist_file(const char *path, const char* filename)
+{//=====================================================================
+ int length;
+ int hash;
+ char *p;
+ int count=0;
+ FILE *f_in;
+ char buf[200];
+ char fname[sizeof(path_home)+45];
+ char dict_line[128];
+
+ text_mode = 0;
+
+ sprintf(fname,"%s%s",path,filename);
+ if((f_in = fopen(fname,"r")) == NULL)
+ return(-1);
+
+ fprintf(f_log,"Compiling: '%s'\n",fname);
+
+ linenum=0;
+
+ while(fgets(buf,sizeof(buf),f_in) != NULL)
+ {
+ linenum++;
+
+ length = compile_line(buf,dict_line,&hash);
+ if(length == 0) continue; /* blank line */
+
+ hash_counts[hash]++;
+
+ p = (char *)malloc(length+sizeof(char *));
+ if(p == NULL)
+ {
+ if(f_log != NULL)
+ {
+ fprintf(f_log,"Can't allocate memory\n");
+ error_count++;
+ }
+ break;
+ }
+
+ memcpy(p,&hash_chains[hash],sizeof(char *));
+ hash_chains[hash] = p;
+ memcpy(p+sizeof(char *),dict_line,length);
+ count++;
+ }
+
+ fprintf(f_log,"\t%d entries\n",count);
+ fclose(f_in);
+ return(0);
+} /* end of compile_dictlist_file */
+
+
+
+static char rule_cond[80];
+static char rule_pre[80];
+static char rule_post[80];
+static char rule_match[80];
+static char rule_phonemes[80];
+static char group_name[LEN_GROUP_NAME+1];
+
+#define N_RULES 2000 // max rules for each group
+
+
+
+static void copy_rule_string(char *string, int &state)
+{//===================================================
+// state 0: conditional, 1=pre, 2=match, 3=post, 4=phonemes
+ static char *outbuf[5] = {rule_cond, rule_pre, rule_match, rule_post, rule_phonemes};
+ static int next_state[5] = {2,2,4,4,4};
+ char *output;
+ char *p;
+ int ix;
+ int len;
+ char c;
+ int sxflags;
+ int value;
+ int literal;
+
+ if(string[0] == 0) return;
+
+ output = outbuf[state];
+ if(state==4)
+ {
+ // append to any previous phoneme string, i.e. allow spaces in the phoneme string
+ len = strlen(rule_phonemes);
+ if(len > 0)
+ rule_phonemes[len++] = ' ';
+ output = &rule_phonemes[len];
+ }
+ sxflags = 0x808000; // to ensure non-zero bytes
+
+ for(p=string,ix=0;;)
+ {
+ literal = 0;
+ c = *p++;
+ if(c == '\\')
+ {
+ c = *p++; // treat next character literally
+ if((c >= '0') && (c <= '3') && (p[0] >= '0') && (p[0] <= '7') && (p[1] >= '0') && (p[1] <= '7'))
+ {
+ // character code given by 3 digit octal value;
+ c = (c-'0')*64 + (p[0]-'0')*8 + (p[1]-'0');
+ p += 2;
+ }
+ literal = 1;
+ }
+
+ if((state==1) || (state==3))
+ {
+ // replace special characters (note: 'E' is reserved for a replaced silent 'e')
+ if(literal == 0)
+ {
+ static const char lettergp_letters[9] = {LETTERGP_A,LETTERGP_B,LETTERGP_C,0,0,LETTERGP_F,LETTERGP_G,LETTERGP_H,LETTERGP_Y};
+ switch(c)
+ {
+ case '_':
+ c = RULE_SPACE;
+ break;
+
+ case 'Y':
+ c = 'I'; // drop through to next case
+ case 'A': // vowel
+ case 'B':
+ case 'C':
+ case 'H':
+ case 'F':
+ case 'G':
+ if(state == 1)
+ {
+ // pre-rule, put the number before the RULE_LETTERGP;
+ output[ix++] = lettergp_letters[c-'A'] + 'A';
+ c = RULE_LETTERGP;
+ }
+ else
+ {
+ output[ix++] = RULE_LETTERGP;
+ c = lettergp_letters[c-'A'] + 'A';
+ }
+ break;
+ case 'D':
+ c = RULE_DIGIT;
+ break;
+ case 'K':
+ c = RULE_NOTVOWEL;
+ break;
+ case 'N':
+ c = RULE_NO_SUFFIX;
+ break;
+ case 'V':
+ c = RULE_IFVERB;
+ break;
+ case 'Z':
+ c = RULE_NONALPHA;
+ break;
+ case '+':
+ c = RULE_INC_SCORE;
+ break;
+ case '@':
+ c = RULE_SYLLABLE;
+ break;
+ case '&':
+ c = RULE_STRESSED;
+ break;
+ case '%':
+ c = RULE_DOUBLE;
+ break;
+ case '#':
+ c = RULE_DEL_FWD;
+ break;
+ case '!':
+ c = RULE_CAPITAL;
+ break;
+ case 'T':
+ c = RULE_ALT1;
+ break;
+ case 'W':
+ c = RULE_SPELLING;
+ break;
+ case 'X':
+ c = RULE_NOVOWELS;
+ break;
+ case 'L':
+ // expect two digits
+ c = *p++ - '0';
+ value = *p++ - '0';
+ c = c * 10 + value;
+ if((value < 0) || (value > 9))
+ {
+ c = 0;
+ fprintf(f_log,"%5d: Expected 2 digits after 'L'\n",linenum);
+ error_count++;
+ }
+ else
+ if((c <= 0) || (c >= N_LETTER_GROUPS) || (letterGroupsDefined[(int)c] == 0))
+ {
+ fprintf(f_log,"%5d: Letter group L%.2d not defined\n",linenum,c);
+ error_count++;
+ }
+ c += 'A';
+ if(state == 1)
+ {
+ // pre-rule, put the group number before the RULE_LETTERGP command
+ output[ix++] = c;
+ c = RULE_LETTERGP2;
+ }
+ else
+ {
+ output[ix++] = RULE_LETTERGP2;
+ }
+ break;
+
+ case '$': // obsolete, replaced by S
+ fprintf(f_log,"%5d: $ now not allowed, use S for suffix",linenum);
+ error_count++;
+ break;
+ case 'P':
+ sxflags |= SUFX_P; // Prefix, now drop through to Suffix
+ case 'S':
+ output[ix++] = RULE_ENDING;
+ value = 0;
+ while(!isspace2(c = *p++) && (c != 0))
+ {
+ switch(c)
+ {
+ case 'e':
+ sxflags |= SUFX_E;
+ break;
+ case 'i':
+ sxflags |= SUFX_I;
+ break;
+ case 'p': // obsolete, replaced by 'P' above
+ sxflags |= SUFX_P;
+ break;
+ case 'v':
+ sxflags |= SUFX_V;
+ break;
+ case 'd':
+ sxflags |= SUFX_D;
+ break;
+ case 'f':
+ sxflags |= SUFX_F;
+ break;
+ case 'q':
+ sxflags |= SUFX_Q;
+ break;
+ case 't':
+ sxflags |= SUFX_T;
+ break;
+ case 'b':
+ sxflags |= SUFX_B;
+ break;
+ default:
+ if(isdigit(c))
+ value = (value*10) + (c - '0');
+ break;
+ }
+ }
+ p--;
+ output[ix++] = sxflags >> 16;
+ output[ix++] = sxflags >> 8;
+ c = value | 0x80;
+ break;
+ }
+ }
+ }
+ output[ix++] = c;
+ if(c == 0) break;
+ }
+
+ state = next_state[state];
+} // end of copy_rule_string
+
+
+
+static char *compile_rule(char *input)
+{//===================================
+ int ix;
+ unsigned char c;
+ int wc;
+ char *p;
+ char *prule;
+ int len;
+ int len_name;
+ int state=2;
+ int finish=0;
+ int pre_bracket=0;
+ char buf[80];
+ char output[150];
+ unsigned char bad_phoneme[4];
+
+ buf[0]=0;
+ rule_cond[0]=0;
+ rule_pre[0]=0;
+ rule_post[0]=0;
+ rule_match[0]=0;
+ rule_phonemes[0]=0;
+
+ p = buf;
+
+ for(ix=0; finish==0; ix++)
+ {
+ c = input[ix];
+
+ switch(c = input[ix])
+ {
+ case ')': // end of prefix section
+ *p = 0;
+ state = 1;
+ pre_bracket = 1;
+ copy_rule_string(buf,state);
+ p = buf;
+ break;
+
+ case '(': // start of suffix section
+ *p = 0;
+ state = 2;
+ copy_rule_string(buf,state);
+ state = 3;
+ p = buf;
+ break;
+
+ case '\n': // end of line
+ case '\r':
+ case 0: // end of line
+ *p = 0;
+ copy_rule_string(buf,state);
+ finish=1;
+ break;
+
+ case '\t': // end of section section
+ case ' ':
+ *p = 0;
+ copy_rule_string(buf,state);
+ p = buf;
+ break;
+
+ case '?':
+ if(state==2)
+ state=0;
+ else
+ *p++ = c;
+ break;
+
+ default:
+ *p++ = c;
+ break;
+ }
+ }
+
+ if(strcmp(rule_match,"$group")==0)
+ strcpy(rule_match,group_name);
+
+ if(rule_match[0]==0)
+ return(NULL);
+
+ EncodePhonemes(rule_phonemes,buf,bad_phoneme);
+ for(ix=0;; ix++)
+ {
+ if((c = buf[ix])==0) break;
+ if(c==255)
+ {
+ fprintf(f_log,"%5d: Bad phoneme [%c] in %s",linenum,bad_phoneme[0],input);
+ error_count++;
+ break;
+ }
+ }
+ strcpy(output,buf);
+ len = strlen(buf)+1;
+
+ len_name = strlen(group_name);
+ if((len_name > 0) && (memcmp(rule_match,group_name,len_name) != 0))
+ {
+ utf8_in(&wc,rule_match);
+ if((group_name[0] == '9') && IsDigit(wc))
+ {
+ // numeric group, rule_match starts with a digit, so OK
+ }
+ else
+ {
+ fprintf(f_log,"%5d: Wrong initial letters '%s' for group '%s'\n",linenum,rule_match,group_name);
+ error_count++;
+ }
+ }
+ strcpy(&output[len],rule_match);
+ len += strlen(rule_match);
+
+ if(debug_flag)
+ {
+ output[len] = RULE_LINENUM;
+ output[len+1] = (linenum % 255) + 1;
+ output[len+2] = (linenum / 255) + 1;
+ len+=3;
+ }
+
+ if(rule_cond[0] != 0)
+ {
+ ix = -1;
+ if(rule_cond[0] == '!')
+ {
+ // allow the rule only if the condition number is NOT set for the voice
+ ix = atoi(&rule_cond[1]) + 32;
+ }
+ else
+ {
+ // allow the rule only if the condition number is set for the voice
+ ix = atoi(rule_cond);
+ }
+
+ if((ix > 0) && (ix < 255))
+ {
+ output[len++] = RULE_CONDITION;
+ output[len++] = ix;
+ }
+ else
+ {
+ fprintf(f_log,"%5d: bad condition number ?%d\n",linenum,ix);
+ error_count++;
+ }
+ }
+ if(rule_pre[0] != 0)
+ {
+ output[len++] = RULE_PRE;
+ // output PRE string in reverse order
+ for(ix = strlen(rule_pre)-1; ix>=0; ix--)
+ output[len++] = rule_pre[ix];
+ }
+
+ if(rule_post[0] != 0)
+ {
+ sprintf(&output[len],"%c%s",RULE_POST,rule_post);
+ len += (strlen(rule_post)+1);
+ }
+ output[len++]=0;
+ prule = (char *)malloc(len);
+ memcpy(prule,output,len);
+ return(prule);
+} // end of compile_rule
+
+
+static int __cdecl string_sorter(char **a, char **b)
+{//=================================================
+ char *pa, *pb;
+ int ix;
+
+ if((ix = strcmp(pa = *a,pb = *b)) != 0)
+ return(ix);
+ pa += (strlen(pa)+1);
+ pb += (strlen(pb)+1);
+ return(strcmp(pa,pb));
+} /* end of string_sorter */
+
+
+static int __cdecl rgroup_sorter(RGROUP *a, RGROUP *b)
+{//===================================================
+ int ix;
+ ix = strcmp(a->name,b->name);
+ if(ix != 0) return(ix);
+ return(a->start-b->start);
+}
+
+
+#ifdef OUTPUT_FORMAT
+static void print_rule_group(FILE *f_out, int n_rules, char **rules, char *name)
+{//=============================================================================
+ int rule;
+ int ix;
+ unsigned char c;
+ int len1;
+ int len2;
+ int spaces;
+ char *p;
+ char *pout;
+ int condition;
+ char buf[80];
+ char suffix[12];
+
+ static unsigned char symbols[] = {'@','&','%','+','#','$','D','Z','A','B','C','F'};
+
+ fprintf(f_out,"\n$group %s\n",name);
+
+ for(rule=0; rule<n_rules; rule++)
+ {
+ p = rules[rule];
+ len1 = strlen(p) + 1;
+ p = &p[len1];
+ len2 = strlen(p);
+
+ rule_match[0]=0;
+ rule_pre[0]=0;
+ rule_post[0]=0;
+ condition = 0;
+
+ pout = rule_match;
+ for(ix=0; ix<len2; ix++)
+ {
+ switch(c = p[ix])
+ {
+ case RULE_PRE:
+ *pout = 0;
+ pout = rule_pre;
+ break;
+ case RULE_POST:
+ *pout = 0;
+ pout = rule_post;
+ break;
+ case RULE_CONDITION:
+ condition = p[++ix];
+ break;
+ case RULE_ENDING:
+ sprintf(suffix,"$%d[%x]",(p[ix+2]),p[ix+1] & 0x7f);
+ ix += 2;
+ strcpy(pout,suffix);
+ pout += strlen(suffix);
+ break;
+ default:
+ if(c <= RULE_LETTER7)
+ c = symbols[c-RULE_SYLLABLE];
+ if(c == ' ')
+ c = '_';
+ *pout++ = c;
+ break;
+ }
+ }
+ *pout = 0;
+
+ spaces = 12;
+ if(condition > 0)
+ {
+ sprintf(buf,"?%d ",condition);
+ spaces -= strlen(buf);
+ fprintf(f_out,"%s",buf);
+ }
+
+ if(rule_pre[0] != 0)
+ {
+ p = buf;
+ for(ix=strlen(rule_pre)-1;ix>=0;ix--)
+ *p++ = rule_pre[ix];
+ sprintf(p,") ");
+ spaces -= strlen(buf);
+ for(ix=0; ix<spaces; ix++)
+ fputc(' ',f_out);
+ fprintf(f_out,"%s",buf);
+ spaces = 0;
+ }
+
+ for(ix=0; ix<spaces; ix++)
+ fputc(' ',f_out);
+
+ spaces = 14;
+ sprintf(buf," %s ",rule_match);
+ if(rule_post[0] != 0)
+ {
+ p = &buf[strlen(buf)];
+ sprintf(p,"(%s ",rule_post);
+ }
+ fprintf(f_out,"%s",buf);
+ spaces -= strlen(buf);
+
+ for(ix=0; ix<spaces; ix++)
+ fputc(' ',f_out);
+ DecodePhonemes(rules[rule],buf);
+ fprintf(f_out,"%s\n",buf); // phonemes
+ }
+}
+#endif
+
+
+//#define LIST_GROUP_INFO
+static void output_rule_group(FILE *f_out, int n_rules, char **rules, char *name)
+{//==============================================================================
+ int ix;
+ int len1;
+ int len2;
+ int len_name;
+ char *p;
+ char *p2, *p3;
+ const char *common;
+
+ short nextchar_count[256];
+ memset(nextchar_count,0,sizeof(nextchar_count));
+
+ len_name = strlen(name);
+
+#ifdef OUTPUT_FORMAT
+ print_rule_group(f_log,n_rules,rules,name);
+#endif
+
+ // sort the rules in this group by their phoneme string
+ common = "";
+ qsort((void *)rules,n_rules,sizeof(char *),(int (__cdecl *)(const void *,const void *))string_sorter);
+
+ if(strcmp(name,"9")==0)
+ len_name = 0; // don't remove characters from numeric match strings
+
+ for(ix=0; ix<n_rules; ix++)
+ {
+ p = rules[ix];
+ len1 = strlen(p) + 1; // phoneme string
+ p3 = &p[len1];
+ p2 = p3 + len_name; // remove group name from start of match string
+ len2 = strlen(p2);
+
+ nextchar_count[(unsigned char)(p2[0])]++; // the next byte after the group name
+
+ if((common[0] != 0) && (strcmp(p,common)==0))
+ {
+ fwrite(p2,len2,1,f_out);
+ fputc(0,f_out); // no phoneme string, it's the same as previous rule
+ }
+ else
+ {
+ if((ix < n_rules-1) && (strcmp(p,rules[ix+1])==0))
+ {
+ common = rules[ix]; // phoneme string is same as next, set as common
+ fputc(RULE_PH_COMMON,f_out);
+ }
+
+ fwrite(p2,len2,1,f_out);
+ fputc(RULE_PHONEMES,f_out);
+ fwrite(p,len1,1,f_out);
+ }
+ }
+
+#ifdef LIST_GROUP_INFO
+ for(ix=32; ix<256; ix++)
+ {
+ if(nextchar_count[ix] > 30)
+ printf("Group %s %c %d\n",name,ix,nextchar_count[ix]);
+ }
+#endif
+} // end of output_rule_group
+
+
+
+static int compile_lettergroup(char *input, FILE *f_out)
+{//=====================================================
+ char *p;
+ char *p_start;
+ int group;
+ int ix;
+ int n_items;
+ int length;
+ int max_length = 0;
+
+ #define N_LETTERGP_ITEMS 200
+ char *items[N_LETTERGP_ITEMS];
+ char item_length[N_LETTERGP_ITEMS];
+
+ p = input;
+ if(!isdigit(p[0]) || !isdigit(p[1]))
+ {
+ fprintf(f_log,"%5d: Expected 2 digits after '.L'\n",linenum);
+ error_count++;
+ return(1);
+ }
+
+ group = atoi(&p[0]);
+ if(group >= N_LETTER_GROUPS)
+ {
+ fprintf(f_log,"%5d: lettergroup out of range (01-%.2d)\n",linenum,N_LETTER_GROUPS-1);
+ error_count++;
+ return(1);
+ }
+
+ while(!isspace2(*p)) p++;
+
+ fputc(RULE_GROUP_START,f_out);
+ fputc(RULE_LETTERGP2,f_out);
+ fputc(group + 'A', f_out);
+ letterGroupsDefined[group] = 1;
+
+ n_items = 0;
+ while(n_items < N_LETTERGP_ITEMS)
+ {
+ while(isspace2(*p)) p++;
+ if(*p == 0)
+ break;
+
+ items[n_items] = p_start = p;
+ while((*p & 0xff) > ' ')
+ {
+ p++;
+ }
+ *p++ = 0;
+ length = p - p_start;
+ if(length > max_length)
+ max_length = length;
+ item_length[n_items++] = length;
+ }
+
+ // write out the items, longest first
+ while(max_length > 1)
+ {
+ for(ix=0; ix < n_items; ix++)
+ {
+ if(item_length[ix] == max_length)
+ {
+ fwrite(items[ix],1,max_length,f_out);
+ }
+ }
+ max_length--;
+ }
+
+ fputc(RULE_GROUP_END,f_out);
+
+ return(0);
+}
+
+
+static int compile_dictrules(FILE *f_in, FILE *f_out, char *fname_temp)
+{//====================================================================
+ char *prule;
+ unsigned char *p;
+ int ix;
+ int c;
+ int gp;
+ FILE *f_temp;
+ int n_rules=0;
+ int count=0;
+ int different;
+ const char *prev_rgroup_name;
+ unsigned int char_code;
+ int compile_mode=0;
+ char *buf;
+ char buf1[200];
+ char *rules[N_RULES];
+
+ int n_rgroups = 0;
+ RGROUP rgroup[N_RULE_GROUP2];
+
+ linenum = 0;
+ group_name[0] = 0;
+
+ if((f_temp = fopen_log(fname_temp,"wb")) == NULL)
+ return(1);
+
+ for(;;)
+ {
+ linenum++;
+ buf = fgets(buf1,sizeof(buf1),f_in);
+ if(buf != NULL)
+ {
+ if((p = (unsigned char *)strstr(buf,"//")) != NULL)
+ *p = 0;
+
+ if(buf[0] == '\r') buf++; // ignore extra \r in \r\n
+ }
+
+ if((buf == NULL) || (buf[0] == '.'))
+ {
+ // next .group or end of file, write out the previous group
+
+ if(n_rules > 0)
+ {
+ strcpy(rgroup[n_rgroups].name,group_name);
+ rgroup[n_rgroups].start = ftell(f_temp);
+ output_rule_group(f_temp,n_rules,rules,group_name);
+ rgroup[n_rgroups].length = ftell(f_temp) - rgroup[n_rgroups].start;
+ n_rgroups++;
+
+ count += n_rules;
+ }
+ n_rules = 0;
+
+ if(compile_mode == 2)
+ {
+ // end of the character replacements section
+ fwrite(&n_rules,1,4,f_out); // write a zero word to terminate the replacemenmt list
+ compile_mode = 0;
+ }
+
+ if(buf == NULL) break; // end of file
+
+ if(memcmp(buf,".L",2)==0)
+ {
+ compile_lettergroup(&buf[2], f_out);
+ continue;
+ }
+
+ if(memcmp(buf,".replace",8)==0)
+ {
+ compile_mode = 2;
+ fputc(RULE_GROUP_START,f_out);
+ fputc(RULE_REPLACEMENTS,f_out);
+
+ // advance to next word boundary
+ while((ftell(f_out) & 3) != 0)
+ fputc(0,f_out);
+ }
+
+ if(memcmp(buf,".group",6)==0)
+ {
+ compile_mode = 1;
+
+ p = (unsigned char *)&buf[6];
+ while((p[0]==' ') || (p[0]=='\t')) p++; // Note: Windows isspace(0xe1) gives TRUE !
+ ix = 0;
+ while((*p > ' ') && (ix < LEN_GROUP_NAME))
+ group_name[ix++] = *p++;
+ group_name[ix]=0;
+
+ if(sscanf(group_name,"0x%x",&char_code)==1)
+ {
+ // group character is given as a character code (max 16 bits)
+ p = (unsigned char *)group_name;
+
+ if(char_code > 0x100)
+ {
+ *p++ = (char_code >> 8);
+ }
+ *p++ = char_code;
+ *p = 0;
+ }
+
+ if(strlen(group_name) > 2)
+ {
+ if(utf8_in(&c,group_name) < 2)
+ {
+ fprintf(f_log,"%5d: Group name longer than 2 bytes (UTF8)",linenum);
+ error_count++;
+ }
+
+ group_name[2] = 0;
+ }
+ }
+
+ continue;
+ }
+
+ switch(compile_mode)
+ {
+ case 1: // .group
+ prule = compile_rule(buf);
+ if((prule != NULL) && (n_rules < N_RULES))
+ {
+ rules[n_rules++] = prule;
+ }
+ break;
+
+ case 2: // .replace
+ {
+ int replace1;
+ int replace2;
+ char *p;
+
+ p = buf;
+ replace1 = 0;
+ replace2 = 0;
+ while(isspace2(*p)) p++;
+ ix = 0;
+ while((unsigned char)(*p) > 0x20) // not space or zero-byte
+ {
+ p += utf8_in(&c,p);
+ replace1 += (c << ix);
+ ix += 16;
+ }
+ while(isspace2(*p)) p++;
+ ix = 0;
+ while((unsigned char)(*p) > 0x20)
+ {
+ p += utf8_in(&c,p);
+ replace2 += (c << ix);
+ ix += 16;
+ }
+ if(replace1 != 0)
+ {
+ Write4Bytes(f_out,replace1); // write as little-endian
+ Write4Bytes(f_out,replace2); // if big-endian, reverse the bytes in LoadDictionary()
+ }
+ }
+ break;
+ }
+ }
+ fclose(f_temp);
+
+ qsort((void *)rgroup,n_rgroups,sizeof(rgroup[0]),(int (__cdecl *)(const void *,const void *))rgroup_sorter);
+
+ if((f_temp = fopen(fname_temp,"rb"))==NULL)
+ return(2);
+
+ prev_rgroup_name = "\n";
+
+ for(gp = 0; gp < n_rgroups; gp++)
+ {
+ fseek(f_temp,rgroup[gp].start,SEEK_SET);
+
+ if((different = strcmp(rgroup[gp].name, prev_rgroup_name)) != 0)
+ {
+ // not the same as the previous group
+ if(gp > 0)
+ fputc(RULE_GROUP_END,f_out);
+ fputc(RULE_GROUP_START,f_out);
+ fprintf(f_out, prev_rgroup_name = rgroup[gp].name);
+ fputc(0,f_out);
+ }
+
+ for(ix=rgroup[gp].length; ix>0; ix--)
+ {
+ c = fgetc(f_temp);
+ fputc(c,f_out);
+ }
+
+ if(different)
+ {
+ }
+ }
+ fputc(RULE_GROUP_END,f_out);
+ fputc(0,f_out);
+
+ fclose(f_temp);
+ remove(fname_temp);
+
+ fprintf(f_log,"\t%d rules, %d groups\n\n",count,n_rgroups);
+ return(0);
+} // end of compile_dictrules
+
+
+
+
+int CompileDictionary(const char *dsource, const char *dict_name, FILE *log, char *fname_err, int flags)
+{//=====================================================================================================
+// fname: space to write the filename in case of error
+// flags: bit 0: include source line number information, for debug purposes.
+
+ FILE *f_in;
+ FILE *f_out;
+ int offset_rules=0;
+ int value;
+ char fname_in[sizeof(path_home)+45];
+ char fname_out[sizeof(path_home)+15];
+ char fname_temp[sizeof(path_home)+15];
+ char path[sizeof(path_home)+40]; // path_dsource+20
+
+ error_count = 0;
+ memset(letterGroupsDefined,0,sizeof(letterGroupsDefined));
+
+ debug_flag = flags & 1;
+
+ if(dsource == NULL)
+ dsource = "";
+
+ f_log = log;
+//f_log = fopen("log2.txt","w");
+ if(f_log == NULL)
+ f_log = stderr;
+
+ sprintf(path,"%s%s_",dsource,dict_name);
+ sprintf(fname_in,"%srules",path);
+ f_in = fopen_log(fname_in,"r");
+ if(f_in == NULL)
+ {
+ if(fname_err)
+ strcpy(fname_err,fname_in);
+ return(-1);
+ }
+
+ sprintf(fname_out,"%s%c%s_dict",path_home,PATHSEP,dict_name);
+ if((f_out = fopen_log(fname_out,"wb+")) == NULL)
+ {
+ if(fname_err)
+ strcpy(fname_err,fname_in);
+ return(-1);
+ }
+ sprintf(fname_temp,"%s%ctemp",path_home,PATHSEP);
+
+ transpose_offset = 0;
+
+ if(strcmp(dict_name,"ru") == 0)
+ {
+ // transpose cyrillic alphabet from unicode to iso8859-5
+// transpose_offset = 0x430-0xd0;
+ transpose_offset = 0x42f; // range 0x01 to 0x22
+ transpose_min = 0x430;
+ transpose_max = 0x451;
+ }
+
+ value = N_HASH_DICT;
+ Write4Bytes(f_out,value);
+ Write4Bytes(f_out,offset_rules);
+
+ compile_dictlist_start();
+
+ fprintf(f_log,"Using phonemetable: '%s'\n",phoneme_tab_list[phoneme_tab_number].name);
+ compile_dictlist_file(path,"roots");
+ if(translator->langopts.listx)
+ {
+ compile_dictlist_file(path,"list");
+ compile_dictlist_file(path,"listx");
+ }
+ else
+ {
+ compile_dictlist_file(path,"listx");
+ compile_dictlist_file(path,"list");
+ }
+ compile_dictlist_file(path,"extra");
+
+ compile_dictlist_end(f_out);
+ offset_rules = ftell(f_out);
+
+ fprintf(f_log,"Compiling: '%s'\n",fname_in);
+
+ compile_dictrules(f_in,f_out,fname_temp);
+ fclose(f_in);
+
+ fseek(f_out,4,SEEK_SET);
+ Write4Bytes(f_out,offset_rules);
+ fclose(f_out);
+
+ LoadDictionary(translator, dict_name, 0);
+
+ return(error_count);
+} // end of compile_dictionary
+