FST  openfst-1.7.3
OpenFst Library
symbol-table-ops.h
Go to the documentation of this file.
1 // See www.openfst.org for extensive documentation on this weighted
2 // finite-state transducer library.
3 
4 #ifndef FST_SYMBOL_TABLE_OPS_H_
5 #define FST_SYMBOL_TABLE_OPS_H_
6 
7 #include <string>
8 #include <unordered_set>
9 #include <vector>
10 
11 
12 #include <fst/fst.h>
13 #include <fst/symbol-table.h>
14 
15 
16 namespace fst {
17 
18 // Returns a minimal symbol table containing only symbols referenced by the
19 // passed fst. Symbols preserve their original numbering, so fst does not
20 // require relabeling.
21 template <class Arc>
23  bool input) {
24  std::unordered_set<typename Arc::Label> seen;
25  seen.insert(0); // Always keep epsilon.
26  for (StateIterator<Fst<Arc>> siter(fst); !siter.Done(); siter.Next()) {
27  for (ArcIterator<Fst<Arc>> aiter(fst, siter.Value()); !aiter.Done();
28  aiter.Next()) {
29  const auto sym = (input) ? aiter.Value().ilabel : aiter.Value().olabel;
30  seen.insert(sym);
31  }
32  }
33  auto *pruned = new SymbolTable(syms.Name() + "_pruned");
34  for (SymbolTableIterator stiter(syms); !stiter.Done(); stiter.Next()) {
35  const auto label = stiter.Value();
36  if (seen.count(label)) pruned->AddSymbol(stiter.Symbol(), label);
37  }
38  return pruned;
39 }
40 
41 // Relabels a symbol table to make it a contiguous mapping.
43 
44 // Merges two SymbolTables, all symbols from left will be merged into right
45 // with the same IDs. Symbols in right that have conflicting IDs with those
46 // in left will be assigned to value assigned from the left SymbolTable.
47 // The returned symbol table will never modify symbol assignments from the left
48 // side, but may do so on the right. If right_relabel_output is non-null, it
49 // will be assigned true if the symbols from the right table needed to be
50 // reassigned.
51 //
52 // A potential use case is to compose two FSTs that have different symbol
53 // tables. You can reconcile them in the following way:
54 //
55 // Fst<Arc> a, b;
56 // bool relabel;
57 // std::unique_ptr<SymbolTable> bnew(MergeSymbolTable(a.OutputSymbols(),
58 // b.InputSymbols(), &relabel);
59 // if (relabel) Relabel(b, bnew.get(), nullptr);
60 // b.SetInputSymbols(bnew);
61 SymbolTable *MergeSymbolTable(const SymbolTable &left, const SymbolTable &right,
62  bool *right_relabel_output = nullptr);
63 
64 // Read the symbol table from any Fst::Read()able file, without loading the
65 // corresponding FST. Returns nullptr if the FST does not contain a symbol
66 // table or the symbol table cannot be read.
67 SymbolTable *FstReadSymbols(const std::string &filename, bool input);
68 
69 // Adds a contiguous range of symbols to a symbol table using a simple prefix
70 // for the string, returning false if the inserted symbol string clashes with
71 // any currently present.
72 bool AddAuxiliarySymbols(const std::string &prefix, int64 start_label,
73  int64 nlabels, SymbolTable *syms);
74 
75 } // namespace fst
76 
77 #endif // FST_SYMBOL_TABLE_OPS_H_
virtual int64 AddSymbol(SymbolType symbol, int64 key)
Definition: symbol-table.h:300
SymbolTable * FstReadSymbols(const std::string &filename, bool input)
int64_t int64
Definition: types.h:27
virtual const std::string & Name() const
Definition: symbol-table.h:347
SymbolTable * CompactSymbolTable(const SymbolTable &syms)
bool AddAuxiliarySymbols(const std::string &prefix, int64 start_label, int64 nlabels, SymbolTable *syms)
SymbolTable * PruneSymbolTable(const Fst< Arc > &fst, const SymbolTable &syms, bool input)
SymbolTable * MergeSymbolTable(const SymbolTable &left, const SymbolTable &right, bool *right_relabel_output=nullptr)