FST  openfst-1.7.2
OpenFst Library
farcompilestrings.cc
Go to the documentation of this file.
1 // See www.openfst.org for extensive documentation on this weighted
2 // finite-state transducer library.
3 //
4 // Compiles a set of stings as FSTs and stores them in a finite-state archive.
5 
6 #include <string>
7 #include <vector>
8 
9 #include <fst/flags.h>
12 #include <fstream>
13 
14 DEFINE_string(key_prefix, "", "Prefix to append to keys");
15 DEFINE_string(key_suffix, "", "Suffix to append to keys");
16 DEFINE_int32(generate_keys, 0,
17  "Generate N digit numeric keys (def: use file basenames)");
18 DEFINE_string(far_type, "default",
19  "FAR file format type: one of: \"default\", \"fst\", "
20  "\"stlist\", \"sttable\"");
21 DEFINE_bool(allow_negative_labels, false,
22  "Allow negative labels (not recommended; may cause conflicts)");
23 DEFINE_string(arc_type, "standard", "Output arc type");
24 DEFINE_string(entry_type, "line",
25  "Entry type: one of : "
26  "\"file\" (one FST per file), \"line\" (one FST per line)");
27 DEFINE_string(fst_type, "vector", "Output FST type");
28 DEFINE_string(token_type, "symbol",
29  "Token type: one of : "
30  "\"symbol\", \"byte\", \"utf8\"");
31 DEFINE_string(symbols, "", "Label symbol table");
32 DEFINE_string(unknown_symbol, "", "");
33 DEFINE_bool(file_list_input, false,
34  "Each input file contains a list of files to be processed");
35 DEFINE_bool(keep_symbols, false, "Store symbol table in the FAR file");
36 DEFINE_bool(initial_symbols, true,
37  "When keep_symbols is true, stores symbol table only for the first"
38  " FST in archive.");
39 
40 int main(int argc, char **argv) {
41  namespace s = fst::script;
42 
43  string usage = "Compiles a set of strings as FSTs and stores them in";
44  usage += " a finite-state archive.\n\n Usage:";
45  usage += argv[0];
46  usage += " [in1.txt [[in2.txt ...] out.far]]\n";
47 
48  std::set_new_handler(FailedNewHandler);
49  SET_FLAGS(usage.c_str(), &argc, &argv, true);
50  s::ExpandArgs(argc, argv, &argc, &argv);
51 
52  std::vector<string> in_fnames;
53  if (FLAGS_file_list_input) {
54  for (int i = 1; i < argc - 1; ++i) {
55  std::ifstream istrm(argv[i]);
56  string str;
57  while (getline(istrm, str)) in_fnames.push_back(str);
58  }
59  } else {
60  for (int i = 1; i < argc - 1; ++i)
61  in_fnames.push_back(argv[i]);
62  }
63  if (in_fnames.empty()) {
64  in_fnames.push_back(argc == 2 && strcmp(argv[1], "-") != 0 ? argv[1] : "");
65  }
66 
67  string out_fname =
68  argc > 2 && strcmp(argv[argc - 1], "-") != 0 ? argv[argc - 1] : "";
69 
70  fst::FarEntryType entry_type;
71  if (!s::GetFarEntryType(FLAGS_entry_type, &entry_type)) {
72  LOG(ERROR) << "Unknown or unsupported FAR entry type: " << FLAGS_entry_type;
73  return 1;
74  }
75 
76  fst::FarTokenType token_type;
77  if (!s::GetFarTokenType(FLAGS_token_type, &token_type)) {
78  LOG(ERROR) << "Unkonwn or unsupported FAR token type: " << FLAGS_token_type;
79  return 1;
80  }
81 
82  const auto far_type = s::GetFarType(FLAGS_far_type);
83 
84  s::FarCompileStrings(in_fnames, out_fname, FLAGS_arc_type, FLAGS_fst_type,
85  far_type, FLAGS_generate_keys, entry_type, token_type,
86  FLAGS_symbols, FLAGS_unknown_symbol, FLAGS_keep_symbols,
87  FLAGS_initial_symbols, FLAGS_allow_negative_labels,
88  FLAGS_key_prefix, FLAGS_key_suffix);
89 
90  return 0;
91 }
DEFINE_int32(generate_keys, 0,"Generate N digit numeric keys (def: use file basenames)")
FarType GetFarType(const string &str)
Definition: getters.cc:19
void ExpandArgs(int argc, char **argv, int *argcp, char ***argvp)
Definition: getters.cc:55
DEFINE_bool(allow_negative_labels, false,"Allow negative labels (not recommended; may cause conflicts)")
#define LOG(type)
Definition: log.h:48
bool GetFarTokenType(const string &str, FarTokenType *token_type)
Definition: getters.cc:42
void FailedNewHandler()
Definition: compat.cc:25
#define SET_FLAGS(usage, argc, argv, rmflags)
Definition: flags.h:214
FarTokenType
Definition: far.h:23
DEFINE_string(key_prefix,"","Prefix to append to keys")
int main(int argc, char **argv)
bool GetFarEntryType(const string &str, FarEntryType *entry_type)
Definition: getters.cc:31
void FarCompileStrings(const std::vector< string > &in_fnames, const string &out_fname, const string &arc_type, const string &fst_type, const FarType &far_type, int32 generate_keys, FarEntryType fet, FarTokenType tt, const string &symbols_fname, const string &unknown_symbol, bool keep_symbols, bool initial_symbols, bool allow_negative_labels, const string &key_prefix, const string &key_suffix)
Definition: farscript.cc:14
FarEntryType
Definition: far.h:21