tesseract
3.03
|
00001 00002 // File: wordlist2dawg.cpp 00003 // Description: Program to generate a DAWG from a word list file 00004 // Author: Thomas Kielbus 00005 // Created: Thu May 10 18:11:42 PDT 2007 00006 // 00007 // (C) Copyright 2006, Google Inc. 00008 // Licensed under the Apache License, Version 2.0 (the "License"); 00009 // you may not use this file except in compliance with the License. 00010 // You may obtain a copy of the License at 00011 // http://www.apache.org/licenses/LICENSE-2.0 00012 // Unless required by applicable law or agreed to in writing, software 00013 // distributed under the License is distributed on an "AS IS" BASIS, 00014 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 00015 // See the License for the specific language governing permissions and 00016 // limitations under the License. 00017 // 00019 00020 // Given a file that contains a list of words (one word per line) this program 00021 // generates the corresponding squished DAWG file. 00022 00023 #include <stdio.h> 00024 00025 #include "classify.h" 00026 #include "dawg.h" 00027 #include "dict.h" 00028 #include "emalloc.h" 00029 #include "freelist.h" 00030 #include "helpers.h" 00031 #include "serialis.h" 00032 #include "trie.h" 00033 #include "unicharset.h" 00034 00035 static const int kMaxNumEdges = 30000000; 00036 00037 int main(int argc, char** argv) { 00038 if (!(argc == 4 || (argc == 5 && strcmp(argv[1], "-t") == 0) || 00039 (argc == 6 && strcmp(argv[1], "-r") == 0))) { 00040 printf("Usage: %s [-t | -r [reverse policy] ] word_list_file" 00041 " dawg_file unicharset_file\n", argv[0]); 00042 return 1; 00043 } 00044 tesseract::Classify *classify = new tesseract::Classify(); 00045 int argv_index = 0; 00046 if (argc == 5) ++argv_index; 00047 tesseract::Trie::RTLReversePolicy reverse_policy = 00048 tesseract::Trie::RRP_DO_NO_REVERSE; 00049 if (argc == 6) { 00050 ++argv_index; 00051 int tmp_int; 00052 sscanf(argv[++argv_index], "%d", &tmp_int); 00053 reverse_policy = static_cast<tesseract::Trie::RTLReversePolicy>(tmp_int); 00054 tprintf("Set reverse_policy to %s\n", 00055 tesseract::Trie::get_reverse_policy_name(reverse_policy)); 00056 } 00057 if (argc == 7) argv_index += 3; 00058 const char* wordlist_filename = argv[++argv_index]; 00059 const char* dawg_filename = argv[++argv_index]; 00060 const char* unicharset_file = argv[++argv_index]; 00061 tprintf("Loading unicharset from '%s'\n", unicharset_file); 00062 if (!classify->getDict().getUnicharset().load_from_file(unicharset_file)) { 00063 tprintf("Failed to load unicharset from '%s'\n", unicharset_file); 00064 delete classify; 00065 return 1; 00066 } 00067 const UNICHARSET &unicharset = classify->getDict().getUnicharset(); 00068 if (argc == 4 || argc == 6) { 00069 tesseract::Trie trie( 00070 // the first 3 arguments are not used in this case 00071 tesseract::DAWG_TYPE_WORD, "", SYSTEM_DAWG_PERM, 00072 kMaxNumEdges, unicharset.size(), 00073 classify->getDict().dawg_debug_level); 00074 tprintf("Reading word list from '%s'\n", wordlist_filename); 00075 if (!trie.read_and_add_word_list(wordlist_filename, unicharset, 00076 reverse_policy)) { 00077 tprintf("Failed to add word list from '%s'\n", wordlist_filename); 00078 exit(1); 00079 } 00080 tprintf("Reducing Trie to SquishedDawg\n"); 00081 tesseract::SquishedDawg *dawg = trie.trie_to_dawg(); 00082 if (dawg != NULL && dawg->NumEdges() > 0) { 00083 tprintf("Writing squished DAWG to '%s'\n", dawg_filename); 00084 dawg->write_squished_dawg(dawg_filename); 00085 } else { 00086 tprintf("Dawg is empty, skip producing the output file\n"); 00087 } 00088 delete dawg; 00089 } else if (argc == 5) { 00090 tprintf("Loading dawg DAWG from '%s'\n", dawg_filename); 00091 tesseract::SquishedDawg words( 00092 dawg_filename, 00093 // these 3 arguments are not used in this case 00094 tesseract::DAWG_TYPE_WORD, "", SYSTEM_DAWG_PERM, 00095 classify->getDict().dawg_debug_level); 00096 tprintf("Checking word list from '%s'\n", wordlist_filename); 00097 words.check_for_words(wordlist_filename, unicharset, true); 00098 } else { // should never get here 00099 tprintf("Invalid command-line options\n"); 00100 exit(1); 00101 } 00102 delete classify; 00103 return 0; 00104 }