tesseract  3.03
/usr/local/google/home/jbreiden/tesseract-ocr-read-only/classify/mastertrainer.cpp
Go to the documentation of this file.
00001 // Copyright 2010 Google Inc. All Rights Reserved.
00002 // Author: rays@google.com (Ray Smith)
00004 // File:        mastertrainer.cpp
00005 // Description: Trainer to build the MasterClassifier.
00006 // Author:      Ray Smith
00007 // Created:     Wed Nov 03 18:10:01 PDT 2010
00008 //
00009 // (C) Copyright 2010, Google Inc.
00010 // Licensed under the Apache License, Version 2.0 (the "License");
00011 // you may not use this file except in compliance with the License.
00012 // You may obtain a copy of the License at
00013 // http://www.apache.org/licenses/LICENSE-2.0
00014 // Unless required by applicable law or agreed to in writing, software
00015 // distributed under the License is distributed on an "AS IS" BASIS,
00016 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00017 // See the License for the specific language governing permissions and
00018 // limitations under the License.
00019 //
00021 
00022 // Include automatically generated configuration file if running autoconf.
00023 #ifdef HAVE_CONFIG_H
00024 #include "config_auto.h"
00025 #endif
00026 
00027 #include "mastertrainer.h"
00028 #include <math.h>
00029 #include <time.h>
00030 #include "allheaders.h"
00031 #include "boxread.h"
00032 #include "classify.h"
00033 #include "efio.h"
00034 #include "errorcounter.h"
00035 #include "featdefs.h"
00036 #include "sampleiterator.h"
00037 #include "shapeclassifier.h"
00038 #include "shapetable.h"
00039 #include "svmnode.h"
00040 
00041 namespace tesseract {
00042 
00043 // Constants controlling clustering. With a low kMinClusteredShapes and a high
00044 // kMaxUnicharsPerCluster, then kFontMergeDistance is the only limiting factor.
00045 // Min number of shapes in the output.
00046 const int kMinClusteredShapes = 1;
00047 // Max number of unichars in any individual cluster.
00048 const int kMaxUnicharsPerCluster = 2000;
00049 // Mean font distance below which to merge fonts and unichars.
00050 const float kFontMergeDistance = 0.025;
00051 
00052 MasterTrainer::MasterTrainer(NormalizationMode norm_mode,
00053                              bool shape_analysis,
00054                              bool replicate_samples,
00055                              int debug_level)
00056   : norm_mode_(norm_mode), samples_(fontinfo_table_),
00057     junk_samples_(fontinfo_table_), verify_samples_(fontinfo_table_),
00058     charsetsize_(0),
00059     enable_shape_anaylsis_(shape_analysis),
00060     enable_replication_(replicate_samples),
00061     fragments_(NULL), prev_unichar_id_(-1), debug_level_(debug_level) {
00062 }
00063 
00064 MasterTrainer::~MasterTrainer() {
00065   delete [] fragments_;
00066   for (int p = 0; p < page_images_.size(); ++p)
00067     pixDestroy(&page_images_[p]);
00068 }
00069 
00070 // WARNING! Serialize/DeSerialize are only partial, providing
00071 // enough data to get the samples back and display them.
00072 // Writes to the given file. Returns false in case of error.
00073 bool MasterTrainer::Serialize(FILE* fp) const {
00074   if (fwrite(&norm_mode_, sizeof(norm_mode_), 1, fp) != 1) return false;
00075   if (!unicharset_.save_to_file(fp)) return false;
00076   if (!feature_space_.Serialize(fp)) return false;
00077   if (!samples_.Serialize(fp)) return false;
00078   if (!junk_samples_.Serialize(fp)) return false;
00079   if (!verify_samples_.Serialize(fp)) return false;
00080   if (!master_shapes_.Serialize(fp)) return false;
00081   if (!flat_shapes_.Serialize(fp)) return false;
00082   if (!fontinfo_table_.Serialize(fp)) return false;
00083   if (!xheights_.Serialize(fp)) return false;
00084   return true;
00085 }
00086 
00087 // Reads from the given file. Returns false in case of error.
00088 // If swap is true, assumes a big/little-endian swap is needed.
00089 bool MasterTrainer::DeSerialize(bool swap, FILE* fp) {
00090   if (fread(&norm_mode_, sizeof(norm_mode_), 1, fp) != 1) return false;
00091   if (swap) {
00092     ReverseN(&norm_mode_, sizeof(norm_mode_));
00093   }
00094   if (!unicharset_.load_from_file(fp)) return false;
00095   charsetsize_ = unicharset_.size();
00096   if (!feature_space_.DeSerialize(swap, fp)) return false;
00097   feature_map_.Init(feature_space_);
00098   if (!samples_.DeSerialize(swap, fp)) return false;
00099   if (!junk_samples_.DeSerialize(swap, fp)) return false;
00100   if (!verify_samples_.DeSerialize(swap, fp)) return false;
00101   if (!master_shapes_.DeSerialize(swap, fp)) return false;
00102   if (!flat_shapes_.DeSerialize(swap, fp)) return false;
00103   if (!fontinfo_table_.DeSerialize(swap, fp)) return false;
00104   if (!xheights_.DeSerialize(swap, fp)) return false;
00105   return true;
00106 }
00107 
00108 // Load an initial unicharset, or set one up if the file cannot be read.
00109 void MasterTrainer::LoadUnicharset(const char* filename) {
00110   if (!unicharset_.load_from_file(filename)) {
00111     tprintf("Failed to load unicharset from file %s\n"
00112             "Building unicharset for training from scratch...\n",
00113             filename);
00114     unicharset_.clear();
00115     UNICHARSET initialized;
00116     // Add special characters, as they were removed by the clear, but the
00117     // default constructor puts them in.
00118     unicharset_.AppendOtherUnicharset(initialized);
00119   }
00120   charsetsize_ = unicharset_.size();
00121   delete [] fragments_;
00122   fragments_ = new int[charsetsize_];
00123   memset(fragments_, 0, sizeof(*fragments_) * charsetsize_);
00124   samples_.LoadUnicharset(filename);
00125   junk_samples_.LoadUnicharset(filename);
00126   verify_samples_.LoadUnicharset(filename);
00127 }
00128 
00129 // Reads the samples and their features from the given .tr format file,
00130 // adding them to the trainer with the font_id from the content of the file.
00131 // See mftraining.cpp for a description of the file format.
00132 // If verification, then these are verification samples, not training.
00133 void MasterTrainer::ReadTrainingSamples(const char* page_name,
00134                                         const FEATURE_DEFS_STRUCT& feature_defs,
00135                                         bool verification) {
00136   char buffer[2048];
00137   int int_feature_type = ShortNameToFeatureType(feature_defs, kIntFeatureType);
00138   int micro_feature_type = ShortNameToFeatureType(feature_defs,
00139                                                   kMicroFeatureType);
00140   int cn_feature_type = ShortNameToFeatureType(feature_defs, kCNFeatureType);
00141   int geo_feature_type = ShortNameToFeatureType(feature_defs, kGeoFeatureType);
00142 
00143   FILE* fp = Efopen(page_name, "rb");
00144   if (fp == NULL) {
00145     tprintf("Failed to open tr file: %s\n", page_name);
00146     return;
00147   }
00148   tr_filenames_.push_back(STRING(page_name));
00149   while (fgets(buffer, sizeof(buffer), fp) != NULL) {
00150     if (buffer[0] == '\n')
00151       continue;
00152 
00153     char* space = strchr(buffer, ' ');
00154     if (space == NULL) {
00155       tprintf("Bad format in tr file, reading fontname, unichar\n");
00156       continue;
00157     }
00158     *space++ = '\0';
00159     int font_id = GetFontInfoId(buffer);
00160     if (font_id < 0) font_id = 0;
00161     int page_number;
00162     STRING unichar;
00163     TBOX bounding_box;
00164     if (!ParseBoxFileStr(space, &page_number, &unichar, &bounding_box)) {
00165       tprintf("Bad format in tr file, reading box coords\n");
00166       continue;
00167     }
00168     CHAR_DESC char_desc = ReadCharDescription(feature_defs, fp);
00169     TrainingSample* sample = new TrainingSample;
00170     sample->set_font_id(font_id);
00171     sample->set_page_num(page_number + page_images_.size());
00172     sample->set_bounding_box(bounding_box);
00173     sample->ExtractCharDesc(int_feature_type, micro_feature_type,
00174                             cn_feature_type, geo_feature_type, char_desc);
00175     AddSample(verification, unichar.string(), sample);
00176     FreeCharDescription(char_desc);
00177   }
00178   charsetsize_ = unicharset_.size();
00179   fclose(fp);
00180 }
00181 
00182 // Adds the given single sample to the trainer, setting the classid
00183 // appropriately from the given unichar_str.
00184 void MasterTrainer::AddSample(bool verification, const char* unichar,
00185                               TrainingSample* sample) {
00186   if (verification) {
00187     verify_samples_.AddSample(unichar, sample);
00188     prev_unichar_id_ = -1;
00189   } else if (unicharset_.contains_unichar(unichar)) {
00190     if (prev_unichar_id_ >= 0)
00191       fragments_[prev_unichar_id_] = -1;
00192     prev_unichar_id_ = samples_.AddSample(unichar, sample);
00193     if (flat_shapes_.FindShape(prev_unichar_id_, sample->font_id()) < 0)
00194       flat_shapes_.AddShape(prev_unichar_id_, sample->font_id());
00195   } else {
00196     int junk_id = junk_samples_.AddSample(unichar, sample);
00197     if (prev_unichar_id_ >= 0) {
00198       CHAR_FRAGMENT* frag = CHAR_FRAGMENT::parse_from_string(unichar);
00199       if (frag != NULL && frag->is_natural()) {
00200         if (fragments_[prev_unichar_id_] == 0)
00201           fragments_[prev_unichar_id_] = junk_id;
00202         else if (fragments_[prev_unichar_id_] != junk_id)
00203           fragments_[prev_unichar_id_] = -1;
00204       }
00205       delete frag;
00206     }
00207     prev_unichar_id_ = -1;
00208   }
00209 }
00210 
00211 // Loads all pages from the given tif filename and append to page_images_.
00212 // Must be called after ReadTrainingSamples, as the current number of images
00213 // is used as an offset for page numbers in the samples.
00214 void MasterTrainer::LoadPageImages(const char* filename) {
00215   int page;
00216   Pix* pix;
00217   for (page = 0; (pix = pixReadTiff(filename, page)) != NULL; ++page) {
00218     page_images_.push_back(pix);
00219   }
00220   tprintf("Loaded %d page images from %s\n", page, filename);
00221 }
00222 
00223 // Cleans up the samples after initial load from the tr files, and prior to
00224 // saving the MasterTrainer:
00225 // Remaps fragmented chars if running shape anaylsis.
00226 // Sets up the samples appropriately for class/fontwise access.
00227 // Deletes outlier samples.
00228 void MasterTrainer::PostLoadCleanup() {
00229   if (debug_level_ > 0)
00230     tprintf("PostLoadCleanup...\n");
00231   if (enable_shape_anaylsis_)
00232     ReplaceFragmentedSamples();
00233   SampleIterator sample_it;
00234   sample_it.Init(NULL, NULL, true, &verify_samples_);
00235   sample_it.NormalizeSamples();
00236   verify_samples_.OrganizeByFontAndClass();
00237 
00238   samples_.IndexFeatures(feature_space_);
00239   // TODO(rays) DeleteOutliers is currently turned off to prove NOP-ness
00240   // against current training.
00241   //  samples_.DeleteOutliers(feature_space_, debug_level_ > 0);
00242   samples_.OrganizeByFontAndClass();
00243   if (debug_level_ > 0)
00244     tprintf("ComputeCanonicalSamples...\n");
00245   samples_.ComputeCanonicalSamples(feature_map_, debug_level_ > 0);
00246 }
00247 
00248 // Gets the samples ready for training. Use after both
00249 // ReadTrainingSamples+PostLoadCleanup or DeSerialize.
00250 // Re-indexes the features and computes canonical and cloud features.
00251 void MasterTrainer::PreTrainingSetup() {
00252   if (debug_level_ > 0)
00253     tprintf("PreTrainingSetup...\n");
00254   samples_.IndexFeatures(feature_space_);
00255   samples_.ComputeCanonicalFeatures();
00256   if (debug_level_ > 0)
00257     tprintf("ComputeCloudFeatures...\n");
00258   samples_.ComputeCloudFeatures(feature_space_.Size());
00259 }
00260 
00261 // Sets up the master_shapes_ table, which tells which fonts should stay
00262 // together until they get to a leaf node classifier.
00263 void MasterTrainer::SetupMasterShapes() {
00264   tprintf("Building master shape table\n");
00265   int num_fonts = samples_.NumFonts();
00266 
00267   ShapeTable char_shapes_begin_fragment(samples_.unicharset());
00268   ShapeTable char_shapes_end_fragment(samples_.unicharset());
00269   ShapeTable char_shapes(samples_.unicharset());
00270   for (int c = 0; c < samples_.charsetsize(); ++c) {
00271     ShapeTable shapes(samples_.unicharset());
00272     for (int f = 0; f < num_fonts; ++f) {
00273       if (samples_.NumClassSamples(f, c, true) > 0)
00274         shapes.AddShape(c, f);
00275     }
00276     ClusterShapes(kMinClusteredShapes, 1, kFontMergeDistance, &shapes);
00277 
00278     const CHAR_FRAGMENT *fragment = samples_.unicharset().get_fragment(c);
00279 
00280     if (fragment == NULL)
00281       char_shapes.AppendMasterShapes(shapes, NULL);
00282     else if (fragment->is_beginning())
00283       char_shapes_begin_fragment.AppendMasterShapes(shapes, NULL);
00284     else if (fragment->is_ending())
00285       char_shapes_end_fragment.AppendMasterShapes(shapes, NULL);
00286     else
00287       char_shapes.AppendMasterShapes(shapes, NULL);
00288   }
00289   ClusterShapes(kMinClusteredShapes, kMaxUnicharsPerCluster,
00290                 kFontMergeDistance, &char_shapes_begin_fragment);
00291   char_shapes.AppendMasterShapes(char_shapes_begin_fragment, NULL);
00292   ClusterShapes(kMinClusteredShapes, kMaxUnicharsPerCluster,
00293                 kFontMergeDistance, &char_shapes_end_fragment);
00294   char_shapes.AppendMasterShapes(char_shapes_end_fragment, NULL);
00295   ClusterShapes(kMinClusteredShapes, kMaxUnicharsPerCluster,
00296                 kFontMergeDistance, &char_shapes);
00297   master_shapes_.AppendMasterShapes(char_shapes, NULL);
00298   tprintf("Master shape_table:%s\n", master_shapes_.SummaryStr().string());
00299 }
00300 
00301 // Adds the junk_samples_ to the main samples_ set. Junk samples are initially
00302 // fragments and n-grams (all incorrectly segmented characters).
00303 // Various training functions may result in incorrectly segmented characters
00304 // being added to the unicharset of the main samples, perhaps because they
00305 // form a "radical" decomposition of some (Indic) grapheme, or because they
00306 // just look the same as a real character (like rn/m)
00307 // This function moves all the junk samples, to the main samples_ set, but
00308 // desirable junk, being any sample for which the unichar already exists in
00309 // the samples_ unicharset gets the unichar-ids re-indexed to match, but
00310 // anything else gets re-marked as unichar_id 0 (space character) to identify
00311 // it as junk to the error counter.
00312 void MasterTrainer::IncludeJunk() {
00313   // Get ids of fragments in junk_samples_ that replace the dead chars.
00314   const UNICHARSET& junk_set = junk_samples_.unicharset();
00315   const UNICHARSET& sample_set = samples_.unicharset();
00316   int num_junks = junk_samples_.num_samples();
00317   tprintf("Moving %d junk samples to master sample set.\n", num_junks);
00318   for (int s = 0; s < num_junks; ++s) {
00319     TrainingSample* sample = junk_samples_.mutable_sample(s);
00320     int junk_id = sample->class_id();
00321     const char* junk_utf8 = junk_set.id_to_unichar(junk_id);
00322     int sample_id = sample_set.unichar_to_id(junk_utf8);
00323     if (sample_id == INVALID_UNICHAR_ID)
00324       sample_id = 0;
00325     sample->set_class_id(sample_id);
00326     junk_samples_.extract_sample(s);
00327     samples_.AddSample(sample_id, sample);
00328   }
00329   junk_samples_.DeleteDeadSamples();
00330   samples_.OrganizeByFontAndClass();
00331 }
00332 
00333 // Replicates the samples and perturbs them if the enable_replication_ flag
00334 // is set. MUST be used after the last call to OrganizeByFontAndClass on
00335 // the training samples, ie after IncludeJunk if it is going to be used, as
00336 // OrganizeByFontAndClass will eat the replicated samples into the regular
00337 // samples.
00338 void MasterTrainer::ReplicateAndRandomizeSamplesIfRequired() {
00339   if (enable_replication_) {
00340     if (debug_level_ > 0)
00341       tprintf("ReplicateAndRandomize...\n");
00342     verify_samples_.ReplicateAndRandomizeSamples();
00343     samples_.ReplicateAndRandomizeSamples();
00344     samples_.IndexFeatures(feature_space_);
00345   }
00346 }
00347 
00348 // Loads the basic font properties file into fontinfo_table_.
00349 // Returns false on failure.
00350 bool MasterTrainer::LoadFontInfo(const char* filename) {
00351   FILE* fp = fopen(filename, "rb");
00352   if (fp == NULL) {
00353     fprintf(stderr, "Failed to load font_properties from %s\n", filename);
00354     return false;
00355   }
00356   int italic, bold, fixed, serif, fraktur;
00357   while (!feof(fp)) {
00358     FontInfo fontinfo;
00359     char* font_name = new char[1024];
00360     fontinfo.name = font_name;
00361     fontinfo.properties = 0;
00362     fontinfo.universal_id = 0;
00363     if (fscanf(fp, "%1024s %i %i %i %i %i\n", font_name,
00364                &italic, &bold, &fixed, &serif, &fraktur) != 6)
00365       continue;
00366     fontinfo.properties =
00367         (italic << 0) +
00368         (bold << 1) +
00369         (fixed << 2) +
00370         (serif << 3) +
00371         (fraktur << 4);
00372     if (!fontinfo_table_.contains(fontinfo)) {
00373       fontinfo_table_.push_back(fontinfo);
00374     }
00375   }
00376   fclose(fp);
00377   return true;
00378 }
00379 
00380 // Loads the xheight font properties file into xheights_.
00381 // Returns false on failure.
00382 bool MasterTrainer::LoadXHeights(const char* filename) {
00383   tprintf("fontinfo table is of size %d\n", fontinfo_table_.size());
00384   xheights_.init_to_size(fontinfo_table_.size(), -1);
00385   if (filename == NULL) return true;
00386   FILE *f = fopen(filename, "rb");
00387   if (f == NULL) {
00388     fprintf(stderr, "Failed to load font xheights from %s\n", filename);
00389     return false;
00390   }
00391   tprintf("Reading x-heights from %s ...\n", filename);
00392   FontInfo fontinfo;
00393   fontinfo.properties = 0;  // Not used to lookup in the table.
00394   fontinfo.universal_id = 0;
00395   char buffer[1024];
00396   int xht;
00397   int total_xheight = 0;
00398   int xheight_count = 0;
00399   while (!feof(f)) {
00400     if (fscanf(f, "%1023s %d\n", buffer, &xht) != 2)
00401       continue;
00402     buffer[1023] = '\0';
00403     fontinfo.name = buffer;
00404     if (!fontinfo_table_.contains(fontinfo)) continue;
00405     int fontinfo_id = fontinfo_table_.get_index(fontinfo);
00406     xheights_[fontinfo_id] = xht;
00407     total_xheight += xht;
00408     ++xheight_count;
00409   }
00410   if (xheight_count == 0) {
00411     fprintf(stderr, "No valid xheights in %s!\n", filename);
00412     fclose(f);
00413     return false;
00414   }
00415   int mean_xheight = DivRounded(total_xheight, xheight_count);
00416   for (int i = 0; i < fontinfo_table_.size(); ++i) {
00417     if (xheights_[i] < 0)
00418       xheights_[i] = mean_xheight;
00419   }
00420   fclose(f);
00421   return true;
00422 }  // LoadXHeights
00423 
00424 // Reads spacing stats from filename and adds them to fontinfo_table.
00425 bool MasterTrainer::AddSpacingInfo(const char *filename) {
00426   FILE* fontinfo_file = fopen(filename, "rb");
00427   if (fontinfo_file == NULL)
00428     return true;  // We silently ignore missing files!
00429   // Find the fontinfo_id.
00430   int fontinfo_id = GetBestMatchingFontInfoId(filename);
00431   if (fontinfo_id < 0) {
00432     tprintf("No font found matching fontinfo filename %s\n", filename);
00433     fclose(fontinfo_file);
00434     return false;
00435   }
00436   tprintf("Reading spacing from %s for font %d...\n", filename, fontinfo_id);
00437   // TODO(rays) scale should probably be a double, but keep as an int for now
00438   // to duplicate current behavior.
00439   int scale = kBlnXHeight / xheights_[fontinfo_id];
00440   int num_unichars;
00441   char uch[UNICHAR_LEN];
00442   char kerned_uch[UNICHAR_LEN];
00443   int x_gap, x_gap_before, x_gap_after, num_kerned;
00444   ASSERT_HOST(fscanf(fontinfo_file, "%d\n", &num_unichars) == 1);
00445   FontInfo *fi = &fontinfo_table_.get(fontinfo_id);
00446   fi->init_spacing(unicharset_.size());
00447   FontSpacingInfo *spacing = NULL;
00448   for (int l = 0; l < num_unichars; ++l) {
00449     if (fscanf(fontinfo_file, "%s %d %d %d",
00450                uch, &x_gap_before, &x_gap_after, &num_kerned) != 4) {
00451       tprintf("Bad format of font spacing file %s\n", filename);
00452       fclose(fontinfo_file);
00453       return false;
00454     }
00455     bool valid = unicharset_.contains_unichar(uch);
00456     if (valid) {
00457       spacing = new FontSpacingInfo();
00458       spacing->x_gap_before = static_cast<inT16>(x_gap_before * scale);
00459       spacing->x_gap_after = static_cast<inT16>(x_gap_after * scale);
00460     }
00461     for (int k = 0; k < num_kerned; ++k) {
00462       if (fscanf(fontinfo_file, "%s %d", kerned_uch, &x_gap) != 2) {
00463         tprintf("Bad format of font spacing file %s\n", filename);
00464         fclose(fontinfo_file);
00465         return false;
00466       }
00467       if (!valid || !unicharset_.contains_unichar(kerned_uch)) continue;
00468       spacing->kerned_unichar_ids.push_back(
00469           unicharset_.unichar_to_id(kerned_uch));
00470       spacing->kerned_x_gaps.push_back(static_cast<inT16>(x_gap * scale));
00471     }
00472     if (valid) fi->add_spacing(unicharset_.unichar_to_id(uch), spacing);
00473   }
00474   fclose(fontinfo_file);
00475   return true;
00476 }
00477 
00478 // Returns the font id corresponding to the given font name.
00479 // Returns -1 if the font cannot be found.
00480 int MasterTrainer::GetFontInfoId(const char* font_name) {
00481   FontInfo fontinfo;
00482   // We are only borrowing the string, so it is OK to const cast it.
00483   fontinfo.name = const_cast<char*>(font_name);
00484   fontinfo.properties = 0;  // Not used to lookup in the table
00485   fontinfo.universal_id = 0;
00486   return fontinfo_table_.get_index(fontinfo);
00487 }
00488 // Returns the font_id of the closest matching font name to the given
00489 // filename. It is assumed that a substring of the filename will match
00490 // one of the fonts. If more than one is matched, the longest is returned.
00491 int MasterTrainer::GetBestMatchingFontInfoId(const char* filename) {
00492   int fontinfo_id = -1;
00493   int best_len = 0;
00494   for (int f = 0; f < fontinfo_table_.size(); ++f) {
00495     if (strstr(filename, fontinfo_table_.get(f).name) != NULL) {
00496       int len = strlen(fontinfo_table_.get(f).name);
00497       // Use the longest matching length in case a substring of a font matched.
00498       if (len > best_len) {
00499         best_len = len;
00500         fontinfo_id = f;
00501       }
00502     }
00503   }
00504   return fontinfo_id;
00505 }
00506 
00507 // Sets up a flat shapetable with one shape per class/font combination.
00508 void MasterTrainer::SetupFlatShapeTable(ShapeTable* shape_table) {
00509   // To exactly mimic the results of the previous implementation, the shapes
00510   // must be clustered in order the fonts arrived, and reverse order of the
00511   // characters within each font.
00512   // Get a list of the fonts in the order they appeared.
00513   GenericVector<int> active_fonts;
00514   int num_shapes = flat_shapes_.NumShapes();
00515   for (int s = 0; s < num_shapes; ++s) {
00516     int font = flat_shapes_.GetShape(s)[0].font_ids[0];
00517     int f = 0;
00518     for (f = 0; f < active_fonts.size(); ++f) {
00519       if (active_fonts[f] == font)
00520         break;
00521     }
00522     if (f == active_fonts.size())
00523       active_fonts.push_back(font);
00524   }
00525   // For each font in order, add all the shapes with that font in reverse order.
00526   int num_fonts = active_fonts.size();
00527   for (int f = 0; f < num_fonts; ++f) {
00528     for (int s = num_shapes - 1; s >= 0; --s) {
00529       int font = flat_shapes_.GetShape(s)[0].font_ids[0];
00530       if (font == active_fonts[f]) {
00531         shape_table->AddShape(flat_shapes_.GetShape(s));
00532       }
00533     }
00534   }
00535 }
00536 
00537 // Sets up a Clusterer for mftraining on a single shape_id.
00538 // Call FreeClusterer on the return value after use.
00539 CLUSTERER* MasterTrainer::SetupForClustering(
00540     const ShapeTable& shape_table,
00541     const FEATURE_DEFS_STRUCT& feature_defs,
00542     int shape_id,
00543     int* num_samples) {
00544 
00545   int desc_index = ShortNameToFeatureType(feature_defs, kMicroFeatureType);
00546   int num_params = feature_defs.FeatureDesc[desc_index]->NumParams;
00547   ASSERT_HOST(num_params == MFCount);
00548   CLUSTERER* clusterer = MakeClusterer(
00549       num_params, feature_defs.FeatureDesc[desc_index]->ParamDesc);
00550 
00551   // We want to iterate over the samples of just the one shape.
00552   IndexMapBiDi shape_map;
00553   shape_map.Init(shape_table.NumShapes(), false);
00554   shape_map.SetMap(shape_id, true);
00555   shape_map.Setup();
00556   // Reverse the order of the samples to match the previous behavior.
00557   GenericVector<const TrainingSample*> sample_ptrs;
00558   SampleIterator it;
00559   it.Init(&shape_map, &shape_table, false, &samples_);
00560   for (it.Begin(); !it.AtEnd(); it.Next()) {
00561     sample_ptrs.push_back(&it.GetSample());
00562   }
00563   int sample_id = 0;
00564   for (int i = sample_ptrs.size() - 1; i >= 0; --i) {
00565     const TrainingSample* sample = sample_ptrs[i];
00566     int num_features = sample->num_micro_features();
00567     for (int f = 0; f < num_features; ++f)
00568       MakeSample(clusterer, sample->micro_features()[f], sample_id);
00569     ++sample_id;
00570   }
00571   *num_samples = sample_id;
00572   return clusterer;
00573 }
00574 
00575 // Writes the given float_classes (produced by SetupForFloat2Int) as inttemp
00576 // to the given inttemp_file, and the corresponding pffmtable.
00577 // The unicharset is the original encoding of graphemes, and shape_set should
00578 // match the size of the shape_table, and may possibly be totally fake.
00579 void MasterTrainer::WriteInttempAndPFFMTable(const UNICHARSET& unicharset,
00580                                              const UNICHARSET& shape_set,
00581                                              const ShapeTable& shape_table,
00582                                              CLASS_STRUCT* float_classes,
00583                                              const char* inttemp_file,
00584                                              const char* pffmtable_file) {
00585   tesseract::Classify *classify = new tesseract::Classify();
00586   // Move the fontinfo table to classify.
00587   fontinfo_table_.MoveTo(&classify->get_fontinfo_table());
00588   INT_TEMPLATES int_templates = classify->CreateIntTemplates(float_classes,
00589                                                              shape_set);
00590   FILE* fp = fopen(inttemp_file, "wb");
00591   classify->WriteIntTemplates(fp, int_templates, shape_set);
00592   fclose(fp);
00593   // Now write pffmtable. This is complicated by the fact that the adaptive
00594   // classifier still wants one indexed by unichar-id, but the static
00595   // classifier needs one indexed by its shape class id.
00596   // We put the shapetable_cutoffs in a GenericVector, and compute the
00597   // unicharset cutoffs along the way.
00598   GenericVector<uinT16> shapetable_cutoffs;
00599   GenericVector<uinT16> unichar_cutoffs;
00600   for (int c = 0; c < unicharset.size(); ++c)
00601     unichar_cutoffs.push_back(0);
00602   /* then write out each class */
00603   for (int i = 0; i < int_templates->NumClasses; ++i) {
00604     INT_CLASS Class = ClassForClassId(int_templates, i);
00605     // Todo: Test with min instead of max
00606     // int MaxLength = LengthForConfigId(Class, 0);
00607     uinT16 max_length = 0;
00608     for (int config_id = 0; config_id < Class->NumConfigs; config_id++) {
00609       // Todo: Test with min instead of max
00610       // if (LengthForConfigId (Class, config_id) < MaxLength)
00611       uinT16 length = Class->ConfigLengths[config_id];
00612       if (length > max_length)
00613         max_length = Class->ConfigLengths[config_id];
00614       int shape_id = float_classes[i].font_set.get(config_id);
00615       const Shape& shape = shape_table.GetShape(shape_id);
00616       for (int c = 0; c < shape.size(); ++c) {
00617         int unichar_id = shape[c].unichar_id;
00618         if (length > unichar_cutoffs[unichar_id])
00619           unichar_cutoffs[unichar_id] = length;
00620       }
00621     }
00622     shapetable_cutoffs.push_back(max_length);
00623   }
00624   fp = fopen(pffmtable_file, "wb");
00625   shapetable_cutoffs.Serialize(fp);
00626   for (int c = 0; c < unicharset.size(); ++c) {
00627     const char *unichar = unicharset.id_to_unichar(c);
00628     if (strcmp(unichar, " ") == 0) {
00629       unichar = "NULL";
00630     }
00631     fprintf(fp, "%s %d\n", unichar, unichar_cutoffs[c]);
00632   }
00633   fclose(fp);
00634   free_int_templates(int_templates);
00635 }
00636 
00637 // Generate debug output relating to the canonical distance between the
00638 // two given UTF8 grapheme strings.
00639 void MasterTrainer::DebugCanonical(const char* unichar_str1,
00640                                    const char* unichar_str2) {
00641   int class_id1 = unicharset_.unichar_to_id(unichar_str1);
00642   int class_id2 = unicharset_.unichar_to_id(unichar_str2);
00643   if (class_id2 == INVALID_UNICHAR_ID)
00644     class_id2 = class_id1;
00645   if (class_id1 == INVALID_UNICHAR_ID) {
00646     tprintf("No unicharset entry found for %s\n", unichar_str1);
00647     return;
00648   } else {
00649     tprintf("Font ambiguities for unichar %d = %s and %d = %s\n",
00650             class_id1, unichar_str1, class_id2, unichar_str2);
00651   }
00652   int num_fonts = samples_.NumFonts();
00653   const IntFeatureMap& feature_map = feature_map_;
00654   // Iterate the fonts to get the similarity with other fonst of the same
00655   // class.
00656   tprintf("      ");
00657   for (int f = 0; f < num_fonts; ++f) {
00658     if (samples_.NumClassSamples(f, class_id2, false) == 0)
00659       continue;
00660     tprintf("%6d", f);
00661   }
00662   tprintf("\n");
00663   for (int f1 = 0; f1 < num_fonts; ++f1) {
00664     // Map the features of the canonical_sample.
00665     if (samples_.NumClassSamples(f1, class_id1, false) == 0)
00666       continue;
00667     tprintf("%4d  ", f1);
00668     for (int f2 = 0; f2 < num_fonts; ++f2) {
00669       if (samples_.NumClassSamples(f2, class_id2, false) == 0)
00670         continue;
00671       float dist = samples_.ClusterDistance(f1, class_id1, f2, class_id2,
00672                                             feature_map);
00673       tprintf(" %5.3f", dist);
00674     }
00675     tprintf("\n");
00676   }
00677   // Build a fake ShapeTable containing all the sample types.
00678   ShapeTable shapes(unicharset_);
00679   for (int f = 0; f < num_fonts; ++f) {
00680     if (samples_.NumClassSamples(f, class_id1, true) > 0)
00681       shapes.AddShape(class_id1, f);
00682     if (class_id1 != class_id2 &&
00683         samples_.NumClassSamples(f, class_id2, true) > 0)
00684       shapes.AddShape(class_id2, f);
00685   }
00686 }
00687 
00688 #ifndef GRAPHICS_DISABLED
00689 // Debugging for cloud/canonical features.
00690 // Displays a Features window containing:
00691 // If unichar_str2 is in the unicharset, and canonical_font is non-negative,
00692 // displays the canonical features of the char/font combination in red.
00693 // If unichar_str1 is in the unicharset, and cloud_font is non-negative,
00694 // displays the cloud feature of the char/font combination in green.
00695 // The canonical features are drawn first to show which ones have no
00696 // matches in the cloud features.
00697 // Until the features window is destroyed, each click in the features window
00698 // will display the samples that have that feature in a separate window.
00699 void MasterTrainer::DisplaySamples(const char* unichar_str1, int cloud_font,
00700                                    const char* unichar_str2,
00701                                    int canonical_font) {
00702   const IntFeatureMap& feature_map = feature_map_;
00703   const IntFeatureSpace& feature_space = feature_map.feature_space();
00704   ScrollView* f_window = CreateFeatureSpaceWindow("Features", 100, 500);
00705   ClearFeatureSpaceWindow(norm_mode_ == NM_BASELINE ? baseline : character,
00706                           f_window);
00707   int class_id2 = samples_.unicharset().unichar_to_id(unichar_str2);
00708   if (class_id2 != INVALID_UNICHAR_ID && canonical_font >= 0) {
00709     const TrainingSample* sample = samples_.GetCanonicalSample(canonical_font,
00710                                                                class_id2);
00711     for (int f = 0; f < sample->num_features(); ++f) {
00712       RenderIntFeature(f_window, &sample->features()[f], ScrollView::RED);
00713     }
00714   }
00715   int class_id1 = samples_.unicharset().unichar_to_id(unichar_str1);
00716   if (class_id1 != INVALID_UNICHAR_ID && cloud_font >= 0) {
00717     const BitVector& cloud = samples_.GetCloudFeatures(cloud_font, class_id1);
00718     for (int f = 0; f < cloud.size(); ++f) {
00719       if (cloud[f]) {
00720         INT_FEATURE_STRUCT feature =
00721             feature_map.InverseIndexFeature(f);
00722         RenderIntFeature(f_window, &feature, ScrollView::GREEN);
00723       }
00724     }
00725   }
00726   f_window->Update();
00727   ScrollView* s_window = CreateFeatureSpaceWindow("Samples", 100, 500);
00728   SVEventType ev_type;
00729   do {
00730     SVEvent* ev;
00731     // Wait until a click or popup event.
00732     ev = f_window->AwaitEvent(SVET_ANY);
00733     ev_type = ev->type;
00734     if (ev_type == SVET_CLICK) {
00735       int feature_index = feature_space.XYToFeatureIndex(ev->x, ev->y);
00736       if (feature_index >= 0) {
00737         // Iterate samples and display those with the feature.
00738         Shape shape;
00739         shape.AddToShape(class_id1, cloud_font);
00740         s_window->Clear();
00741         samples_.DisplaySamplesWithFeature(feature_index, shape,
00742                                            feature_space, ScrollView::GREEN,
00743                                            s_window);
00744         s_window->Update();
00745       }
00746     }
00747     delete ev;
00748   } while (ev_type != SVET_DESTROY);
00749 }
00750 #endif  // GRAPHICS_DISABLED
00751 
00752 void MasterTrainer::TestClassifierVOld(bool replicate_samples,
00753                                        ShapeClassifier* test_classifier,
00754                                        ShapeClassifier* old_classifier) {
00755   SampleIterator sample_it;
00756   sample_it.Init(NULL, NULL, replicate_samples, &samples_);
00757   ErrorCounter::DebugNewErrors(test_classifier, old_classifier,
00758                                CT_UNICHAR_TOPN_ERR, fontinfo_table_,
00759                                page_images_, &sample_it);
00760 }
00761 
00762 // Tests the given test_classifier on the internal samples.
00763 // See TestClassifier for details.
00764 void MasterTrainer::TestClassifierOnSamples(CountTypes error_mode,
00765                                             int report_level,
00766                                             bool replicate_samples,
00767                                             ShapeClassifier* test_classifier,
00768                                             STRING* report_string) {
00769   TestClassifier(error_mode, report_level, replicate_samples, &samples_,
00770                  test_classifier, report_string);
00771 }
00772 
00773 // Tests the given test_classifier on the given samples.
00774 // error_mode indicates what counts as an error.
00775 // report_levels:
00776 // 0 = no output.
00777 // 1 = bottom-line error rate.
00778 // 2 = bottom-line error rate + time.
00779 // 3 = font-level error rate + time.
00780 // 4 = list of all errors + short classifier debug output on 16 errors.
00781 // 5 = list of all errors + short classifier debug output on 25 errors.
00782 // If replicate_samples is true, then the test is run on an extended test
00783 // sample including replicated and systematically perturbed samples.
00784 // If report_string is non-NULL, a summary of the results for each font
00785 // is appended to the report_string.
00786 double MasterTrainer::TestClassifier(CountTypes error_mode,
00787                                      int report_level,
00788                                      bool replicate_samples,
00789                                      TrainingSampleSet* samples,
00790                                      ShapeClassifier* test_classifier,
00791                                      STRING* report_string) {
00792   SampleIterator sample_it;
00793   sample_it.Init(NULL, NULL, replicate_samples, samples);
00794   if (report_level > 0) {
00795     int num_samples = 0;
00796     for (sample_it.Begin(); !sample_it.AtEnd(); sample_it.Next())
00797       ++num_samples;
00798     tprintf("Iterator has charset size of %d/%d, %d shapes, %d samples\n",
00799             sample_it.SparseCharsetSize(), sample_it.CompactCharsetSize(),
00800             test_classifier->GetShapeTable()->NumShapes(), num_samples);
00801     tprintf("Testing %sREPLICATED:\n", replicate_samples ? "" : "NON-");
00802   }
00803   double unichar_error = 0.0;
00804   ErrorCounter::ComputeErrorRate(test_classifier, report_level,
00805                                  error_mode, fontinfo_table_,
00806                                  page_images_, &sample_it, &unichar_error,
00807                                  NULL, report_string);
00808   return unichar_error;
00809 }
00810 
00811 // Returns the average (in some sense) distance between the two given
00812 // shapes, which may contain multiple fonts and/or unichars.
00813 float MasterTrainer::ShapeDistance(const ShapeTable& shapes, int s1, int s2) {
00814   const IntFeatureMap& feature_map = feature_map_;
00815   const Shape& shape1 = shapes.GetShape(s1);
00816   const Shape& shape2 = shapes.GetShape(s2);
00817   int num_chars1 = shape1.size();
00818   int num_chars2 = shape2.size();
00819   float dist_sum = 0.0f;
00820   int dist_count = 0;
00821   if (num_chars1 > 1 || num_chars2 > 1) {
00822     // In the multi-char case try to optimize the calculation by computing
00823     // distances between characters of matching font where possible.
00824     for (int c1 = 0; c1 < num_chars1; ++c1) {
00825       for (int c2 = 0; c2 < num_chars2; ++c2) {
00826         dist_sum += samples_.UnicharDistance(shape1[c1], shape2[c2],
00827                                              true, feature_map);
00828         ++dist_count;
00829       }
00830     }
00831   } else {
00832     // In the single unichar case, there is little alternative, but to compute
00833     // the squared-order distance between pairs of fonts.
00834     dist_sum = samples_.UnicharDistance(shape1[0], shape2[0],
00835                                         false, feature_map);
00836     ++dist_count;
00837   }
00838   return dist_sum / dist_count;
00839 }
00840 
00841 // Replaces samples that are always fragmented with the corresponding
00842 // fragment samples.
00843 void MasterTrainer::ReplaceFragmentedSamples() {
00844   if (fragments_ == NULL) return;
00845   // Remove samples that are replaced by fragments. Each class that was
00846   // always naturally fragmented should be replaced by its fragments.
00847   int num_samples = samples_.num_samples();
00848   for (int s = 0; s < num_samples; ++s) {
00849     TrainingSample* sample = samples_.mutable_sample(s);
00850     if (fragments_[sample->class_id()] > 0)
00851       samples_.KillSample(sample);
00852   }
00853   samples_.DeleteDeadSamples();
00854 
00855   // Get ids of fragments in junk_samples_ that replace the dead chars.
00856   const UNICHARSET& frag_set = junk_samples_.unicharset();
00857 #if 0
00858   // TODO(rays) The original idea was to replace only graphemes that were
00859   // always naturally fragmented, but that left a lot of the Indic graphemes
00860   // out. Determine whether we can go back to that idea now that spacing
00861   // is fixed in the training images, or whether this code is obsolete.
00862   bool* good_junk = new bool[frag_set.size()];
00863   memset(good_junk, 0, sizeof(*good_junk) * frag_set.size());
00864   for (int dead_ch = 1; dead_ch < unicharset_.size(); ++dead_ch) {
00865     int frag_ch = fragments_[dead_ch];
00866     if (frag_ch <= 0) continue;
00867     const char* frag_utf8 = frag_set.id_to_unichar(frag_ch);
00868     CHAR_FRAGMENT* frag = CHAR_FRAGMENT::parse_from_string(frag_utf8);
00869     // Mark the chars for all parts of the fragment as good in good_junk.
00870     for (int part = 0; part < frag->get_total(); ++part) {
00871       frag->set_pos(part);
00872       int good_ch = frag_set.unichar_to_id(frag->to_string().string());
00873       if (good_ch != INVALID_UNICHAR_ID)
00874         good_junk[good_ch] = true;  // We want this one.
00875     }
00876   }
00877 #endif
00878   // For now just use all the junk that was from natural fragments.
00879   // Get samples of fragments in junk_samples_ that replace the dead chars.
00880   int num_junks = junk_samples_.num_samples();
00881   for (int s = 0; s < num_junks; ++s) {
00882     TrainingSample* sample = junk_samples_.mutable_sample(s);
00883     int junk_id = sample->class_id();
00884     const char* frag_utf8 = frag_set.id_to_unichar(junk_id);
00885     CHAR_FRAGMENT* frag = CHAR_FRAGMENT::parse_from_string(frag_utf8);
00886     if (frag != NULL && frag->is_natural()) {
00887       junk_samples_.extract_sample(s);
00888       samples_.AddSample(frag_set.id_to_unichar(junk_id), sample);
00889     }
00890   }
00891   junk_samples_.DeleteDeadSamples();
00892   junk_samples_.OrganizeByFontAndClass();
00893   samples_.OrganizeByFontAndClass();
00894   unicharset_.clear();
00895   unicharset_.AppendOtherUnicharset(samples_.unicharset());
00896   // delete [] good_junk;
00897   // Fragments_ no longer needed?
00898   delete [] fragments_;
00899   fragments_ = NULL;
00900 }
00901 
00902 // Runs a hierarchical agglomerative clustering to merge shapes in the given
00903 // shape_table, while satisfying the given constraints:
00904 // * End with at least min_shapes left in shape_table,
00905 // * No shape shall have more than max_shape_unichars in it,
00906 // * Don't merge shapes where the distance between them exceeds max_dist.
00907 const float kInfiniteDist = 999.0f;
00908 void MasterTrainer::ClusterShapes(int min_shapes,  int max_shape_unichars,
00909                                   float max_dist, ShapeTable* shapes) {
00910   int num_shapes = shapes->NumShapes();
00911   int max_merges = num_shapes - min_shapes;
00912   GenericVector<ShapeDist>* shape_dists =
00913       new GenericVector<ShapeDist>[num_shapes];
00914   float min_dist = kInfiniteDist;
00915   int min_s1 = 0;
00916   int min_s2 = 0;
00917   tprintf("Computing shape distances...");
00918   for (int s1 = 0; s1 < num_shapes; ++s1) {
00919     for (int s2 = s1 + 1; s2 < num_shapes; ++s2) {
00920       ShapeDist dist(s1, s2, ShapeDistance(*shapes, s1, s2));
00921       shape_dists[s1].push_back(dist);
00922       if (dist.distance < min_dist) {
00923         min_dist = dist.distance;
00924         min_s1 = s1;
00925         min_s2 = s2;
00926       }
00927     }
00928     tprintf(" %d", s1);
00929   }
00930   tprintf("\n");
00931   int num_merged = 0;
00932   while (num_merged < max_merges && min_dist < max_dist) {
00933     tprintf("Distance = %f: ", min_dist);
00934     int num_unichars = shapes->MergedUnicharCount(min_s1, min_s2);
00935     shape_dists[min_s1][min_s2 - min_s1 - 1].distance = kInfiniteDist;
00936     if (num_unichars > max_shape_unichars) {
00937       tprintf("Merge of %d and %d with %d would exceed max of %d unichars\n",
00938               min_s1, min_s2, num_unichars, max_shape_unichars);
00939     } else {
00940       shapes->MergeShapes(min_s1, min_s2);
00941       shape_dists[min_s2].clear();
00942       ++num_merged;
00943 
00944       for (int s = 0; s < min_s1; ++s) {
00945         if (!shape_dists[s].empty()) {
00946           shape_dists[s][min_s1 - s - 1].distance =
00947               ShapeDistance(*shapes, s, min_s1);
00948           shape_dists[s][min_s2 - s -1].distance = kInfiniteDist;
00949         }
00950       }
00951       for (int s2 = min_s1 + 1; s2 < num_shapes; ++s2) {
00952         if (shape_dists[min_s1][s2 - min_s1 - 1].distance < kInfiniteDist)
00953           shape_dists[min_s1][s2 - min_s1 - 1].distance =
00954               ShapeDistance(*shapes, min_s1, s2);
00955       }
00956       for (int s = min_s1 + 1; s < min_s2; ++s) {
00957         if (!shape_dists[s].empty()) {
00958           shape_dists[s][min_s2 - s - 1].distance = kInfiniteDist;
00959         }
00960       }
00961     }
00962     min_dist = kInfiniteDist;
00963     for (int s1 = 0; s1 < num_shapes; ++s1) {
00964       for (int i = 0; i < shape_dists[s1].size(); ++i) {
00965         if (shape_dists[s1][i].distance < min_dist) {
00966           min_dist = shape_dists[s1][i].distance;
00967           min_s1 = s1;
00968           min_s2 = s1 + 1 + i;
00969         }
00970       }
00971     }
00972   }
00973   tprintf("Stopped with %d merged, min dist %f\n", num_merged, min_dist);
00974   delete [] shape_dists;
00975   if (debug_level_ > 1) {
00976     for (int s1 = 0; s1 < num_shapes; ++s1) {
00977       if (shapes->MasterDestinationIndex(s1) == s1) {
00978         tprintf("Master shape:%s\n", shapes->DebugStr(s1).string());
00979       }
00980     }
00981   }
00982 }
00983 
00984 
00985 }  // namespace tesseract.
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines