FST  openfst-1.7.2
OpenFst Library
symbol-table-ops.h
Go to the documentation of this file.
1 // See www.openfst.org for extensive documentation on this weighted
2 // finite-state transducer library.
3 
4 #ifndef FST_SYMBOL_TABLE_OPS_H_
5 #define FST_SYMBOL_TABLE_OPS_H_
6 
7 #include <string>
8 #include <unordered_set>
9 #include <vector>
10 
11 
12 #include <fst/fst.h>
13 #include <fst/symbol-table.h>
14 
15 
16 namespace fst {
17 
18 // Returns a minimal symbol table containing only symbols referenced by the
19 // passed fst. Symbols preserve their original numbering, so fst does not
20 // require relabeling.
21 template <class Arc>
23  bool input) {
24  std::unordered_set<typename Arc::Label> seen;
25  seen.insert(0); // Always keep epsilon.
26  for (StateIterator<Fst<Arc>> siter(fst); !siter.Done(); siter.Next()) {
27  for (ArcIterator<Fst<Arc>> aiter(fst, siter.Value()); !aiter.Done();
28  aiter.Next()) {
29  const auto sym = (input) ? aiter.Value().ilabel : aiter.Value().olabel;
30  seen.insert(sym);
31  }
32  }
33  auto *pruned = new SymbolTable(syms.Name() + "_pruned");
34  for (SymbolTableIterator stiter(syms); !stiter.Done(); stiter.Next()) {
35  const auto label = stiter.Value();
36  if (seen.count(label)) pruned->AddSymbol(stiter.Symbol(), label);
37  }
38  return pruned;
39 }
40 
41 // Relabels a symbol table to make it a contiguous mapping.
43 
44 // Merges two SymbolTables, all symbols from left will be merged into right
45 // with the same ids. Symbols in right that have conflicting ids with those
46 // in left will be assigned to value assigned from the left SymbolTable.
47 // The returned symbol table will never modify symbol assignments from the left
48 // side, but may do so on the right. If right_relabel_output is non-NULL, it
49 // will be assigned true if the symbols from the right table needed to be
50 // reassigned.
51 // A potential use case is to Compose two Fst's that have different symbol
52 // tables. You can reconcile them in the following way:
53 // Fst<Arc> a, b;
54 // bool relabel;
55 // std::unique_ptr<SymbolTable> bnew(MergeSymbolTable(a.OutputSymbols(),
56 // b.InputSymbols(), &relabel);
57 // if (relabel) {
58 // Relabel(b, bnew.get(), nullptr);
59 // }
60 // b.SetInputSymbols(bnew);
61 SymbolTable *MergeSymbolTable(const SymbolTable &left, const SymbolTable &right,
62  bool *right_relabel_output = nullptr);
63 
64 // Read the symbol table from any Fst::Read()able file, without loading the
65 // corresponding Fst. Returns nullptr if the Fst does not contain a symbol
66 // table or the symbol table cannot be read.
67 SymbolTable *FstReadSymbols(const string &filename, bool input);
68 
69 // Adds a contiguous range of symbols to a symbol table using a simple prefix
70 // for the string, returning false if the inserted symbol string clashes with
71 // any currently present.
72 bool AddAuxiliarySymbols(const string &prefix, int64 start_label,
73  int64 nlabels, SymbolTable *syms);
74 
75 } // namespace fst
76 #endif // FST_SYMBOL_TABLE_OPS_H_
int64_t int64
Definition: types.h:27
SymbolTable * CompactSymbolTable(const SymbolTable &syms)
SymbolTable * FstReadSymbols(const string &filename, bool input)
bool AddAuxiliarySymbols(const string &prefix, int64 start_label, int64 nlabels, SymbolTable *syms)
virtual int64 AddSymbol(const string &symbol, int64 key)
Definition: symbol-table.h:270
SymbolTable * PruneSymbolTable(const Fst< Arc > &fst, const SymbolTable &syms, bool input)
virtual const string & Name() const
Definition: symbol-table.h:319
SymbolTable * MergeSymbolTable(const SymbolTable &left, const SymbolTable &right, bool *right_relabel_output=nullptr)