tesseract 5.2.0
Loading...
Searching...
No Matches
unicharset_extractor.cpp
Go to the documentation of this file.
1
2// File: unicharset_extractor.cpp
3// Description: Unicode character/ligature set extractor.
4// Author: Thomas Kielbus
5//
6// (C) Copyright 2006, Google Inc.
7// Licensed under the Apache License, Version 2.0 (the "License");
8// you may not use this file except in compliance with the License.
9// You may obtain a copy of the License at
10// http://www.apache.org/licenses/LICENSE-2.0
11// Unless required by applicable law or agreed to in writing, software
12// distributed under the License is distributed on an "AS IS" BASIS,
13// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14// See the License for the specific language governing permissions and
15// limitations under the License.
16//
18
19// Given a list of box files or text files on the command line, this program
20// normalizes the text according to command-line options and generates
21// a unicharset.
22
23#include <cstdlib>
24#include "boxread.h"
25#include "commandlineflags.h"
26#include "commontraining.h" // CheckSharedLibraryVersion
27#include "lang_model_helpers.h"
28#include "normstrngs.h"
29#include "unicharset.h"
31
32using namespace tesseract;
33
34static STRING_PARAM_FLAG(output_unicharset, "unicharset", "Output file path");
35static INT_PARAM_FLAG(norm_mode, 1,
36 "Normalization mode: 1=Combine graphemes, "
37 "2=Split graphemes, 3=Pure unicode");
38
39namespace tesseract {
40
41// Helper normalizes and segments the given strings according to norm_mode, and
42// adds the segmented parts to unicharset.
43static void AddStringsToUnicharset(const std::vector<std::string> &strings, int norm_mode,
44 UNICHARSET *unicharset) {
45 for (const auto &string : strings) {
46 std::vector<std::string> normalized;
48 static_cast<GraphemeNormMode>(norm_mode),
49 /*report_errors*/ true, string.c_str(), &normalized)) {
50 for (const std::string &normed : normalized) {
51 // normed is a UTF-8 encoded string
52 if (normed.empty() || IsUTF8Whitespace(normed.c_str())) {
53 continue;
54 }
55 unicharset->unichar_insert(normed.c_str());
56 }
57 } else {
58 tprintf("Normalization failed for string '%s'\n", string.c_str());
59 }
60 }
61}
62
63static int Main(int argc, char **argv) {
64 UNICHARSET unicharset;
65 // Load input files
66 for (int arg = 1; arg < argc; ++arg) {
67 std::string file_data = tesseract::ReadFile(argv[arg]);
68 if (file_data.empty()) {
69 continue;
70 }
71 std::vector<std::string> texts;
72 if (ReadMemBoxes(-1, /*skip_blanks*/ true, &file_data[0],
73 /*continue_on_failure*/ false, /*boxes*/ nullptr, &texts,
74 /*box_texts*/ nullptr, /*pages*/ nullptr)) {
75 tprintf("Extracting unicharset from box file %s\n", argv[arg]);
76 } else {
77 tprintf("Extracting unicharset from plain text file %s\n", argv[arg]);
78 texts.clear();
79 texts = split(file_data, '\n');
80 }
81 AddStringsToUnicharset(texts, FLAGS_norm_mode, &unicharset);
82 }
83 SetupBasicProperties(/*report_errors*/ true, /*decompose*/ false, &unicharset);
84 // Write unicharset file.
85 if (unicharset.save_to_file(FLAGS_output_unicharset.c_str())) {
86 tprintf("Wrote unicharset file %s\n", FLAGS_output_unicharset.c_str());
87 } else {
88 tprintf("Cannot save unicharset file %s\n", FLAGS_output_unicharset.c_str());
89 return EXIT_FAILURE;
90 }
91 return EXIT_SUCCESS;
92}
93
94} // namespace tesseract
95
96int main(int argc, char **argv) {
97 tesseract::CheckSharedLibraryVersion();
98 if (argc > 1) {
99 tesseract::ParseCommandLineFlags(argv[0], &argc, &argv, true);
100 }
101 if (argc < 2) {
102 tprintf(
103 "Usage: %s [--output_unicharset filename] [--norm_mode mode]"
104 " box_or_text_file [...]\n",
105 argv[0]);
106 tprintf("Where mode means:\n");
107 tprintf(" 1=combine graphemes (use for Latin and other simple scripts)\n");
108 tprintf(" 2=split graphemes (use for Indic/Khmer/Myanmar)\n");
109 tprintf(" 3=pure unicode (use for Arabic/Hebrew/Thai/Tibetan)\n");
110 tprintf("Reads box or plain text files to extract the unicharset.\n");
111 return EXIT_FAILURE;
112 }
113 return tesseract::Main(argc, argv);
114}
#define INT_PARAM_FLAG(name, val, comment)
#define STRING_PARAM_FLAG(name, val, comment)
int main()
GraphemeNormMode
Definition: validator.h:36
bool ReadMemBoxes(int target_page, bool skip_blanks, const char *box_data, bool continue_on_failure, std::vector< TBOX > *boxes, std::vector< std::string > *texts, std::vector< std::string > *box_texts, std::vector< int > *pages)
Definition: boxread.cpp:90
void ParseCommandLineFlags(const char *usage, int *argc, char ***argv, const bool remove_flags)
void tprintf(const char *format,...)
Definition: tprintf.cpp:41
void SetupBasicProperties(bool report_errors, bool decompose, UNICHARSET *unicharset)
bool NormalizeCleanAndSegmentUTF8(UnicodeNormMode u_mode, OCRNorm ocr_normalize, GraphemeNormMode g_mode, bool report_errors, const char *str8, std::vector< std::string > *graphemes)
Definition: normstrngs.cpp:179
std::string ReadFile(const std::string &filename, FileReader reader)
const std::vector< std::string > split(const std::string &s, char c)
Definition: helpers.h:41
bool IsUTF8Whitespace(const char *text)
Definition: normstrngs.cpp:233
void unichar_insert(const char *const unichar_repr, OldUncleanUnichars old_style)
Definition: unicharset.cpp:654
bool save_to_file(const char *const filename) const
Definition: unicharset.h:361