tesseract
3.03
|
00001 00002 // File: params_model.cpp 00003 // Description: Trained language model parameters. 00004 // Author: David Eger 00005 // Created: Mon Jun 11 11:26:42 PDT 2012 00006 // 00007 // (C) Copyright 2012, Google Inc. 00008 // Licensed under the Apache License, Version 2.0 (the "License"); 00009 // you may not use this file except in compliance with the License. 00010 // You may obtain a copy of the License at 00011 // http://www.apache.org/licenses/LICENSE-2.0 00012 // Unless required by applicable law or agreed to in writing, software 00013 // distributed under the License is distributed on an "AS IS" BASIS, 00014 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 00015 // See the License for the specific language governing permissions and 00016 // limitations under the License. 00017 // 00019 00020 #include "params_model.h" 00021 00022 #include <ctype.h> 00023 #include <math.h> 00024 #include <stdio.h> 00025 00026 #include "bitvector.h" 00027 #include "tprintf.h" 00028 00029 namespace tesseract { 00030 00031 // Scale factor to apply to params model scores. 00032 static const float kScoreScaleFactor = 100.0f; 00033 // Minimum cost result to return. 00034 static const float kMinFinalCost = 0.001f; 00035 // Maximum cost result to return. 00036 static const float kMaxFinalCost = 100.0f; 00037 00038 void ParamsModel::Print() { 00039 for (int p = 0; p < PTRAIN_NUM_PASSES; ++p) { 00040 tprintf("ParamsModel for pass %d lang %s\n", p, lang_.string()); 00041 for (int i = 0; i < weights_vec_[p].size(); ++i) { 00042 tprintf("%s = %g\n", kParamsTrainingFeatureTypeName[i], 00043 weights_vec_[p][i]); 00044 } 00045 } 00046 } 00047 00048 void ParamsModel::Copy(const ParamsModel &other_model) { 00049 for (int p = 0; p < PTRAIN_NUM_PASSES; ++p) { 00050 weights_vec_[p] = other_model.weights_for_pass( 00051 static_cast<PassEnum>(p)); 00052 } 00053 } 00054 00055 // Given a (modifiable) line, parse out a key / value pair. 00056 // Return true on success. 00057 bool ParamsModel::ParseLine(char *line, char** key, float *val) { 00058 if (line[0] == '#') 00059 return false; 00060 int end_of_key = 0; 00061 while (line[end_of_key] && !isspace(line[end_of_key])) end_of_key++; 00062 if (!line[end_of_key]) { 00063 tprintf("ParamsModel::Incomplete line %s\n", line); 00064 return false; 00065 } 00066 line[end_of_key++] = 0; 00067 *key = line; 00068 if (sscanf(line + end_of_key, " %f", val) != 1) 00069 return false; 00070 return true; 00071 } 00072 00073 // Applies params model weights to the given features. 00074 // Assumes that features is an array of size PTRAIN_NUM_FEATURE_TYPES. 00075 // The cost is set to a number that can be multiplied by the outline length, 00076 // as with the old ratings scheme. This enables words of different length 00077 // and combinations of words to be compared meaningfully. 00078 float ParamsModel::ComputeCost(const float features[]) const { 00079 float unnorm_score = 0.0; 00080 for (int f = 0; f < PTRAIN_NUM_FEATURE_TYPES; ++f) { 00081 unnorm_score += weights_vec_[pass_][f] * features[f]; 00082 } 00083 return ClipToRange(-unnorm_score / kScoreScaleFactor, 00084 kMinFinalCost, kMaxFinalCost); 00085 } 00086 00087 bool ParamsModel::Equivalent(const ParamsModel &that) const { 00088 float epsilon = 0.0001; 00089 for (int p = 0; p < PTRAIN_NUM_PASSES; ++p) { 00090 if (weights_vec_[p].size() != that.weights_vec_[p].size()) return false; 00091 for (int i = 0; i < weights_vec_[p].size(); i++) { 00092 if (weights_vec_[p][i] != that.weights_vec_[p][i] && 00093 fabs(weights_vec_[p][i] - that.weights_vec_[p][i]) > epsilon) 00094 return false; 00095 } 00096 } 00097 return true; 00098 } 00099 00100 bool ParamsModel::LoadFromFile( 00101 const char *lang, 00102 const char *full_path) { 00103 FILE *fp = fopen(full_path, "rb"); 00104 if (!fp) { 00105 tprintf("Error opening file %s\n", full_path); 00106 return false; 00107 } 00108 bool result = LoadFromFp(lang, fp, -1); 00109 fclose(fp); 00110 return result; 00111 } 00112 00113 bool ParamsModel::LoadFromFp(const char *lang, FILE *fp, inT64 end_offset) { 00114 const int kMaxLineSize = 100; 00115 char line[kMaxLineSize]; 00116 BitVector present; 00117 present.Init(PTRAIN_NUM_FEATURE_TYPES); 00118 lang_ = lang; 00119 // Load weights for passes with adaption on. 00120 GenericVector<float> &weights = weights_vec_[pass_]; 00121 weights.init_to_size(PTRAIN_NUM_FEATURE_TYPES, 0.0); 00122 00123 while ((end_offset < 0 || ftell(fp) < end_offset) && 00124 fgets(line, kMaxLineSize, fp)) { 00125 char *key = NULL; 00126 float value; 00127 if (!ParseLine(line, &key, &value)) 00128 continue; 00129 int idx = ParamsTrainingFeatureByName(key); 00130 if (idx < 0) { 00131 tprintf("ParamsModel::Unknown parameter %s\n", key); 00132 continue; 00133 } 00134 if (!present[idx]) { 00135 present.SetValue(idx, true); 00136 } 00137 weights[idx] = value; 00138 } 00139 bool complete = (present.NumSetBits() == PTRAIN_NUM_FEATURE_TYPES); 00140 if (!complete) { 00141 for (int i = 0; i < PTRAIN_NUM_FEATURE_TYPES; i++) { 00142 if (!present[i]) { 00143 tprintf("Missing field %s.\n", kParamsTrainingFeatureTypeName[i]); 00144 } 00145 } 00146 lang_ = ""; 00147 weights.truncate(0); 00148 } 00149 return complete; 00150 } 00151 00152 bool ParamsModel::SaveToFile(const char *full_path) const { 00153 const GenericVector<float> &weights = weights_vec_[pass_]; 00154 if (weights.size() != PTRAIN_NUM_FEATURE_TYPES) { 00155 tprintf("Refusing to save ParamsModel that has not been initialized.\n"); 00156 return false; 00157 } 00158 FILE *fp = fopen(full_path, "wb"); 00159 if (!fp) { 00160 tprintf("Could not open %s for writing.\n", full_path); 00161 return false; 00162 } 00163 bool all_good = true; 00164 for (int i = 0; i < weights.size(); i++) { 00165 if (fprintf(fp, "%s %f\n", kParamsTrainingFeatureTypeName[i], weights[i]) 00166 < 0) { 00167 all_good = false; 00168 } 00169 } 00170 fclose(fp); 00171 return all_good; 00172 } 00173 00174 } // namespace tesseract