FST  openfst-1.7.1
OpenFst Library
print-strings.h
Go to the documentation of this file.
1 // See www.openfst.org for extensive documentation on this weighted
2 // finite-state transducer library.
3 //
4 // Outputs as strings the string FSTs in a finite-state archive.
5 
6 #ifndef FST_EXTENSIONS_FAR_PRINT_STRINGS_H_
7 #define FST_EXTENSIONS_FAR_PRINT_STRINGS_H_
8 
9 #include <iomanip>
10 #include <string>
11 #include <vector>
12 
13 #include <fst/flags.h>
14 #include <fst/extensions/far/far.h>
15 #include <fstream>
16 #include <fst/shortest-distance.h>
17 #include <fst/string.h>
18 
19 DECLARE_string(far_field_separator);
20 
21 namespace fst {
22 
23 template <class Arc>
24 void FarPrintStrings(const std::vector<string> &ifilenames,
25  FarEntryType entry_type, FarTokenType far_token_type,
26  const string &begin_key, const string &end_key,
27  bool print_key, bool print_weight,
28  const string &symbols_fname, bool initial_symbols,
29  int32 generate_filenames, const string &filename_prefix,
30  const string &filename_suffix) {
31  StringTokenType token_type;
32  if (far_token_type == FTT_SYMBOL) {
33  token_type = StringTokenType::SYMBOL;
34  } else if (far_token_type == FTT_BYTE) {
35  token_type = StringTokenType::BYTE;
36  } else if (far_token_type == FTT_UTF8) {
37  token_type = StringTokenType::UTF8;
38  } else {
39  FSTERROR() << "FarPrintStrings: Unknown token type";
40  return;
41  }
42  std::unique_ptr<const SymbolTable> syms;
43  if (!symbols_fname.empty()) {
44  // TODO(kbg): Allow negative flag?
45  const SymbolTableTextOptions opts(true);
46  syms.reset(SymbolTable::ReadText(symbols_fname, opts));
47  if (!syms) {
48  LOG(ERROR) << "FarPrintStrings: Error reading symbol table "
49  << symbols_fname;
50  return;
51  }
52  }
53  std::unique_ptr<FarReader<Arc>> far_reader(FarReader<Arc>::Open(ifilenames));
54  if (!far_reader) return;
55  if (!begin_key.empty()) far_reader->Find(begin_key);
56  string okey;
57  int nrep = 0;
58  for (int i = 1; !far_reader->Done(); far_reader->Next(), ++i) {
59  const auto &key = far_reader->GetKey();
60  if (!end_key.empty() && end_key < key) break;
61  if (okey == key) {
62  ++nrep;
63  } else {
64  nrep = 0;
65  }
66  okey = key;
67  const auto *fst = far_reader->GetFst();
68  if (i == 1 && initial_symbols && !syms && fst->InputSymbols())
69  syms.reset(fst->InputSymbols()->Copy());
70  string str;
71  VLOG(2) << "Handling key: " << key;
72  StringPrinter<Arc> string_printer(token_type,
73  syms ? syms.get() : fst->InputSymbols());
74  string_printer(*fst, &str);
75  if (entry_type == FET_LINE) {
76  if (print_key) std::cout << key << FLAGS_far_field_separator[0];
77  std::cout << str;
78  if (print_weight)
79  std::cout << FLAGS_far_field_separator[0] << ShortestDistance(*fst);
80  std::cout << std::endl;
81  } else if (entry_type == FET_FILE) {
82  std::stringstream sstrm;
83  if (generate_filenames) {
84  sstrm.fill('0');
85  sstrm << std::right << std::setw(generate_filenames) << i;
86  } else {
87  sstrm << key;
88  if (nrep > 0) sstrm << "." << nrep;
89  }
90  string filename;
91  filename = filename_prefix + sstrm.str() + filename_suffix;
92  std::ofstream ostrm(filename);
93  if (!ostrm) {
94  LOG(ERROR) << "FarPrintStrings: Can't open file: " << filename;
95  return;
96  }
97  ostrm << str;
98  if (token_type == StringTokenType::SYMBOL) ostrm << "\n";
99  }
100  }
101 }
102 
103 } // namespace fst
104 
105 #endif // FST_EXTENSIONS_FAR_PRINT_STRINGS_H_
void FarPrintStrings(const std::vector< string > &ifilenames, FarEntryType entry_type, FarTokenType far_token_type, const string &begin_key, const string &end_key, bool print_key, bool print_weight, const string &symbols_fname, bool initial_symbols, int32 generate_filenames, const string &filename_prefix, const string &filename_suffix)
Definition: print-strings.h:24
StringTokenType
Definition: string.h:27
#define LOG(type)
Definition: log.h:48
FarTokenType
Definition: far.h:23
#define FSTERROR()
Definition: util.h:35
#define VLOG(level)
Definition: log.h:49
int32_t int32
Definition: types.h:26
void ShortestDistance(const Fst< Arc > &fst, std::vector< typename Arc::Weight > *distance, const ShortestDistanceOptions< Arc, Queue, ArcFilter > &opts)
FarEntryType
Definition: far.h:21
static SymbolTable * ReadText(std::istream &strm, const string &name, const SymbolTableTextOptions &opts=SymbolTableTextOptions())
Definition: symbol-table.h:216