FST  openfst-1.8.2.post1
OpenFst Library
encode.h
Go to the documentation of this file.
1 // Copyright 2005-2020 Google LLC
2 //
3 // Licensed under the Apache License, Version 2.0 (the 'License');
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an 'AS IS' BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 //
15 // See www.openfst.org for extensive documentation on this weighted
16 // finite-state transducer library.
17 //
18 // Class to encode and decode an FST.
19 
20 #ifndef FST_ENCODE_H_
21 #define FST_ENCODE_H_
22 
23 #include <cstdint>
24 #include <iostream>
25 #include <memory>
26 #include <ostream>
27 #include <string>
28 #include <utility>
29 #include <vector>
30 
31 #include <fst/log.h>
32 #include <fst/arc-map.h>
33 #include <fstream>
34 #include <fst/properties.h>
35 #include <fst/rmfinalepsilon.h>
36 #include <fst/util.h>
37 #include <unordered_map>
38 
39 namespace fst {
40 
41 enum EncodeType { ENCODE = 1, DECODE = 2 };
42 
43 inline constexpr uint8_t kEncodeLabels = 0x01;
44 inline constexpr uint8_t kEncodeWeights = 0x02;
45 inline constexpr uint8_t kEncodeFlags = 0x03;
46 
47 namespace internal {
48 
49 // Bits storing whether or not an encode table has input and/or output symbol
50 // tables, for internal use only.
51 inline constexpr uint8_t kEncodeHasISymbols = 0x04;
52 inline constexpr uint8_t kEncodeHasOSymbols = 0x08;
53 
54 // Identifies stream data as an encode table (and its endianity).
55 inline constexpr int32_t kEncodeMagicNumber = 2128178506;
56 
57 } // namespace internal
58 
59 // Header for the encoder table.
61  public:
62  EncodeTableHeader() = default;
63 
64  // Getters.
65 
66  const std::string &ArcType() const { return arctype_; }
67 
68  uint8_t Flags() const { return flags_; }
69 
70  size_t Size() const { return size_; }
71 
72  // Setters.
73 
74  void SetArcType(const std::string &arctype) { arctype_ = arctype; }
75 
76  void SetFlags(uint8_t flags) { flags_ = flags; }
77 
78  void SetSize(size_t size) { size_ = size; }
79 
80  // IO.
81 
82  bool Read(std::istream &strm, const std::string &source);
83 
84  bool Write(std::ostream &strm, const std::string &source) const;
85 
86  private:
87  std::string arctype_;
88  uint8_t flags_;
89  size_t size_;
90 };
91 
92 namespace internal {
93 
94 // The following class encapsulates implementation details for the encoding and
95 // decoding of label/weight triples used for encoding and decoding of FSTs. The
96 // EncodeTable is bidirectional, i.e, it stores both the Triple of encode labels
97 // and weights to a unique label, and the reverse.
98 template <class Arc>
99 class EncodeTable {
100  public:
101  using Label = typename Arc::Label;
102  using Weight = typename Arc::Weight;
103 
104  // Encoded data consists of arc input/output labels and arc weight.
105  struct Triple {
106  Triple() = default;
107 
108  Triple(Label ilabel, Label olabel, Weight weight)
109  : ilabel(ilabel), olabel(olabel), weight(std::move(weight)) {}
110 
111  // Constructs from arc and flags.
112  Triple(const Arc &arc, uint8_t flags)
113  : ilabel(arc.ilabel),
114  olabel(flags & kEncodeLabels ? arc.olabel : 0),
115  weight(flags & kEncodeWeights ? arc.weight : Weight::One()) {}
116 
117  static std::unique_ptr<Triple> Read(std::istream &strm) {
118  auto triple = std::make_unique<Triple>();
119  ReadType(strm, &triple->ilabel);
120  ReadType(strm, &triple->olabel);
121  ReadType(strm, &triple->weight);
122  return triple;
123  }
124 
125  void Write(std::ostream &strm) const {
126  WriteType(strm, ilabel);
127  WriteType(strm, olabel);
128  WriteType(strm, weight);
129  }
130 
131  // Exploited below for TripleEqual functor.
132  bool operator==(const Triple &other) const {
133  return (ilabel == other.ilabel && olabel == other.olabel &&
134  weight == other.weight);
135  }
136 
140  };
141 
142  // Equality functor for two Triple pointers.
143  struct TripleEqual {
144  bool operator()(const Triple *x, const Triple *y) const { return *x == *y; }
145  };
146 
147  // Hash functor for one Triple pointer.
148  class TripleHash {
149  public:
150  explicit TripleHash(uint8_t flags) : flags_(flags) {}
151 
152  size_t operator()(const Triple *triple) const {
153  size_t hash = triple->ilabel;
154  static constexpr int lshift = 5;
155  static constexpr int rshift = CHAR_BIT * sizeof(size_t) - 5;
156  if (flags_ & kEncodeLabels) {
157  hash = hash << lshift ^ hash >> rshift ^ triple->olabel;
158  }
159  if (flags_ & kEncodeWeights) {
160  hash = hash << lshift ^ hash >> rshift ^ triple->weight.Hash();
161  }
162  return hash;
163  }
164 
165  private:
166  uint8_t flags_;
167  };
168 
169  explicit EncodeTable(uint8_t flags)
170  : flags_(flags), triple2label_(1024, TripleHash(flags)) {}
171 
172  // Given an arc, encodes either input/output labels or input/costs or both.
173  Label Encode(const Arc &arc) {
174  // Encoding weights of a weighted superfinal transition could result in
175  // a clash with a true epsilon arc; to avoid this we hallucinate kNoLabel
176  // labels instead.
177  if (arc.nextstate == kNoStateId && (flags_ & kEncodeWeights)) {
178  return Encode(std::make_unique<Triple>(kNoLabel, kNoLabel, arc.weight));
179  } else {
180  return Encode(std::make_unique<Triple>(arc, flags_));
181  }
182  }
183 
184  // Given an encoded arc label, decodes back to input/output labels and costs.
185  const Triple *Decode(Label label) const {
186  if (label < 1 || label > triples_.size()) {
187  LOG(ERROR) << "EncodeTable::Decode: Unknown decode label: " << label;
188  return nullptr;
189  }
190  return triples_[label - 1].get();
191  }
192 
193  size_t Size() const { return triples_.size(); }
194 
195  static EncodeTable *Read(std::istream &strm, const std::string &source);
196 
197  bool Write(std::ostream &strm, const std::string &source) const;
198 
199  // This is masked to hide internal-only isymbol and osymbol bits.
200 
201  uint8_t Flags() const { return flags_ & kEncodeFlags; }
202 
203  const SymbolTable *InputSymbols() const { return isymbols_.get(); }
204 
205  const SymbolTable *OutputSymbols() const { return osymbols_.get(); }
206 
207  void SetInputSymbols(const SymbolTable *syms) {
208  if (syms) {
209  isymbols_.reset(syms->Copy());
210  flags_ |= kEncodeHasISymbols;
211  } else {
212  isymbols_.reset();
213  flags_ &= ~kEncodeHasISymbols;
214  }
215  }
216 
217  void SetOutputSymbols(const SymbolTable *syms) {
218  if (syms) {
219  osymbols_.reset(syms->Copy());
220  flags_ |= kEncodeHasOSymbols;
221  } else {
222  osymbols_.reset();
223  flags_ &= ~kEncodeHasOSymbols;
224  }
225  }
226 
227  private:
228  Label Encode(std::unique_ptr<Triple> triple) {
229  auto insert_result =
230  triple2label_.emplace(triple.get(), triples_.size() + 1);
231  if (insert_result.second) triples_.push_back(std::move(triple));
232  return insert_result.first->second;
233  }
234 
235  uint8_t flags_;
236  std::vector<std::unique_ptr<Triple>> triples_;
237  std::unordered_map<const Triple *, Label, TripleHash, TripleEqual>
238  triple2label_;
239  std::unique_ptr<SymbolTable> isymbols_;
240  std::unique_ptr<SymbolTable> osymbols_;
241 
242  EncodeTable(const EncodeTable &) = delete;
243  EncodeTable &operator=(const EncodeTable &) = delete;
244 };
245 
246 template <class Arc>
248  const std::string &source) {
249  EncodeTableHeader hdr;
250  if (!hdr.Read(strm, source)) return nullptr;
251  const auto flags = hdr.Flags();
252  const auto size = hdr.Size();
253  auto table = std::make_unique<EncodeTable>(flags);
254  for (int64_t i = 0; i < size; ++i) {
255  table->triples_.emplace_back(std::move(Triple::Read(strm)));
256  table->triple2label_[table->triples_.back().get()] = table->triples_.size();
257  }
258  if (flags & kEncodeHasISymbols) {
259  table->isymbols_.reset(SymbolTable::Read(strm, source));
260  }
261  if (flags & kEncodeHasOSymbols) {
262  table->osymbols_.reset(SymbolTable::Read(strm, source));
263  }
264  if (!strm) {
265  LOG(ERROR) << "EncodeTable::Read: Read failed: " << source;
266  return nullptr;
267  }
268  return table.release();
269 }
270 
271 template <class Arc>
272 bool EncodeTable<Arc>::Write(std::ostream &strm,
273  const std::string &source) const {
274  EncodeTableHeader hdr;
275  hdr.SetArcType(Arc::Type());
276  hdr.SetFlags(flags_); // Real flags, not masked ones.
277  hdr.SetSize(Size());
278  if (!hdr.Write(strm, source)) return false;
279  for (const auto &triple : triples_) triple->Write(strm);
280  if (flags_ & kEncodeHasISymbols) isymbols_->Write(strm);
281  if (flags_ & kEncodeHasOSymbols) osymbols_->Write(strm);
282  strm.flush();
283  if (!strm) {
284  LOG(ERROR) << "EncodeTable::Write: Write failed: " << source;
285  return false;
286  }
287  return true;
288 }
289 
290 } // namespace internal
291 
292 // A mapper to encode/decode weighted transducers. Encoding of an FST is used
293 // for performing classical determinization or minimization on a weighted
294 // transducer viewing it as an unweighted acceptor over encoded labels.
295 //
296 // The mapper stores the encoding in a local hash table (EncodeTable). This
297 // table is shared (and reference-counted) between the encoder and decoder.
298 // A decoder has read-only access to the EncodeTable.
299 //
300 // The EncodeMapper allows on the fly encoding of the machine. As the
301 // EncodeTable is generated the same table may by used to decode the machine
302 // on the fly. For example in the following sequence of operations
303 //
304 // Encode -> Determinize -> Decode
305 //
306 // we will use the encoding table generated during the encode step in the
307 // decode, even though the encoding is not complete.
308 template <class Arc>
310  using Label = typename Arc::Label;
311  using Weight = typename Arc::Weight;
312 
313  public:
314  explicit EncodeMapper(uint8_t flags, EncodeType type = ENCODE)
315  : flags_(flags),
316  type_(type),
317  table_(std::make_shared<internal::EncodeTable<Arc>>(flags)),
318  error_(false) {}
319 
320  EncodeMapper(const EncodeMapper &mapper)
321  : flags_(mapper.flags_),
322  type_(mapper.type_),
323  table_(mapper.table_),
324  error_(false) {}
325 
326  // Copy constructor but setting the type, typically to DECODE.
327  EncodeMapper(const EncodeMapper &mapper, EncodeType type)
328  : flags_(mapper.flags_),
329  type_(type),
330  table_(mapper.table_),
331  error_(mapper.error_) {}
332 
333  Arc operator()(const Arc &arc);
334 
336  return (type_ == ENCODE && (flags_ & kEncodeWeights))
339  }
340 
342  return MAP_CLEAR_SYMBOLS;
343  }
344 
346  return MAP_CLEAR_SYMBOLS;
347  }
348 
349  uint8_t Flags() const { return flags_; }
350 
351  uint64_t Properties(uint64_t inprops) {
352  uint64_t outprops = inprops;
353  if (error_) outprops |= kError;
354  uint64_t mask = kFstProperties;
355  if (flags_ & kEncodeLabels) {
357  }
358  if (flags_ & kEncodeWeights) {
360  (type_ == ENCODE ? kAddSuperFinalProperties
362  }
363  if (type_ == ENCODE) mask |= kIDeterministic;
364  return outprops & mask;
365  }
366 
367  EncodeType Type() const { return type_; }
368 
369  static EncodeMapper *Read(std::istream &strm, const std::string &source,
370  EncodeType type = ENCODE) {
371  auto *table = internal::EncodeTable<Arc>::Read(strm, source);
372  return table ? new EncodeMapper(table->Flags(), type, table) : nullptr;
373  }
374 
375  static EncodeMapper *Read(const std::string &source,
376  EncodeType type = ENCODE) {
377  std::ifstream strm(source, std::ios_base::in | std::ios_base::binary);
378  if (!strm) {
379  LOG(ERROR) << "EncodeMapper: Can't open file: " << source;
380  return nullptr;
381  }
382  return Read(strm, source, type);
383  }
384 
385  bool Write(std::ostream &strm, const std::string &source) const {
386  return table_->Write(strm, source);
387  }
388 
389  bool Write(const std::string &source) const {
390  std::ofstream strm(source,
391  std::ios_base::out | std::ios_base::binary);
392  if (!strm) {
393  LOG(ERROR) << "EncodeMapper: Can't open file: " << source;
394  return false;
395  }
396  return Write(strm, source);
397  }
398 
399  const SymbolTable *InputSymbols() const { return table_->InputSymbols(); }
400 
401  const SymbolTable *OutputSymbols() const { return table_->OutputSymbols(); }
402 
403  void SetInputSymbols(const SymbolTable *syms) {
404  table_->SetInputSymbols(syms);
405  }
406 
407  void SetOutputSymbols(const SymbolTable *syms) {
408  table_->SetOutputSymbols(syms);
409  }
410 
411  private:
412  uint8_t flags_;
413  EncodeType type_;
414  std::shared_ptr<internal::EncodeTable<Arc>> table_;
415  bool error_;
416 
417  explicit EncodeMapper(uint8_t flags, EncodeType type,
419  : flags_(flags), type_(type), table_(table), error_(false) {}
420 
421  EncodeMapper &operator=(const EncodeMapper &) = delete;
422 };
423 
424 template <class Arc>
425 Arc EncodeMapper<Arc>::operator()(const Arc &arc) {
426  if (type_ == ENCODE) {
427  // If this arc is a hallucinated final state, and we're either not encoding
428  // weights, or we're encoding weights but this is non-final, we use an
429  // identity-encoding.
430  if (arc.nextstate == kNoStateId &&
431  ((!(flags_ & kEncodeWeights) ||
432  ((flags_ & kEncodeWeights) && arc.weight == Weight::Zero())))) {
433  return arc;
434  } else {
435  const auto label = table_->Encode(arc);
436  return Arc(label, flags_ & kEncodeLabels ? label : arc.olabel,
437  flags_ & kEncodeWeights ? Weight::One() : arc.weight,
438  arc.nextstate);
439  }
440  } else { // type_ == DECODE
441  if (arc.nextstate == kNoStateId) {
442  return arc;
443  } else {
444  if (arc.ilabel == 0) return arc;
445  if (flags_ & kEncodeLabels && arc.ilabel != arc.olabel) {
446  FSTERROR() << "EncodeMapper: Label-encoded arc has different "
447  "input and output labels";
448  error_ = true;
449  }
450  if (flags_ & kEncodeWeights && arc.weight != Weight::One()) {
451  FSTERROR() << "EncodeMapper: Weight-encoded arc has non-trivial weight";
452  error_ = true;
453  }
454  const auto triple = table_->Decode(arc.ilabel);
455  if (!triple) {
456  FSTERROR() << "EncodeMapper: Decode failed";
457  error_ = true;
458  return Arc(kNoLabel, kNoLabel, Weight::NoWeight(), arc.nextstate);
459  } else if (triple->ilabel == kNoLabel) {
460  // Hallucinated kNoLabel from a weighted superfinal transition.
461  return Arc(0, 0, triple->weight, arc.nextstate);
462  } else {
463  return Arc(triple->ilabel,
464  flags_ & kEncodeLabels ? triple->olabel : arc.olabel,
465  flags_ & kEncodeWeights ? triple->weight : arc.weight,
466  arc.nextstate);
467  }
468  }
469  }
470 }
471 
472 // Complexity: O(E + V).
473 template <class Arc>
474 inline void Encode(MutableFst<Arc> *fst, EncodeMapper<Arc> *mapper) {
475  mapper->SetInputSymbols(fst->InputSymbols());
476  mapper->SetOutputSymbols(fst->OutputSymbols());
477  ArcMap(fst, mapper);
478 }
479 
480 template <class Arc>
481 inline void Decode(MutableFst<Arc> *fst, const EncodeMapper<Arc> &mapper) {
482  ArcMap(fst, EncodeMapper<Arc>(mapper, DECODE));
483  RmFinalEpsilon(fst);
484  fst->SetInputSymbols(mapper.InputSymbols());
485  fst->SetOutputSymbols(mapper.OutputSymbols());
486 }
487 
488 // On-the-fly encoding of an input FST.
489 //
490 // Complexity:
491 //
492 // Construction: O(1)
493 // Traversal: O(e + v)
494 //
495 // where e is the number of arcs visited and v is the number of states visited.
496 // Constant time and space to visit an input state or arc is assumed and
497 // exclusive of caching.
498 template <class Arc>
499 class EncodeFst : public ArcMapFst<Arc, Arc, EncodeMapper<Arc>> {
500  public:
503 
504  EncodeFst(const Fst<Arc> &fst, Mapper *encoder)
505  : ArcMapFst<Arc, Arc, Mapper>(fst, encoder, ArcMapFstOptions()) {
506  encoder->SetInputSymbols(fst.InputSymbols());
507  encoder->SetOutputSymbols(fst.OutputSymbols());
508  }
509 
510  EncodeFst(const Fst<Arc> &fst, const Mapper &encoder)
511  : ArcMapFst<Arc, Arc, Mapper>(fst, encoder, ArcMapFstOptions()) {}
512 
513  // See Fst<>::Copy() for doc.
514  EncodeFst(const EncodeFst &fst, bool copy = false)
515  : ArcMapFst<Arc, Arc, Mapper>(fst, copy) {}
516 
517  // Makes a copy of this EncodeFst. See Fst<>::Copy() for further doc.
518  EncodeFst *Copy(bool safe = false) const override {
519  if (safe) {
520  FSTERROR() << "EncodeFst::Copy(true): Not allowed";
521  GetImpl()->SetProperties(kError, kError);
522  }
523  return new EncodeFst(*this);
524  }
525 
526  private:
529 };
530 
531 // On-the-fly decoding of an input FST.
532 //
533 // Complexity:
534 //
535 // Construction: O(1).
536 // Traversal: O(e + v)
537 //
538 // Constant time and space to visit an input state or arc is assumed and
539 // exclusive of caching.
540 template <class Arc>
541 class DecodeFst : public ArcMapFst<Arc, Arc, EncodeMapper<Arc>> {
542  public:
545 
546  DecodeFst(const Fst<Arc> &fst, const Mapper &encoder)
547  : ArcMapFst<Arc, Arc, Mapper>(fst, Mapper(encoder, DECODE),
548  ArcMapFstOptions()) {
549  GetMutableImpl()->SetInputSymbols(encoder.InputSymbols());
550  GetMutableImpl()->SetOutputSymbols(encoder.OutputSymbols());
551  }
552 
553  // See Fst<>::Copy() for doc.
554  DecodeFst(const DecodeFst &fst, bool safe = false)
555  : ArcMapFst<Arc, Arc, Mapper>(fst, safe) {}
556 
557  // Makes a copy of this DecodeFst. See Fst<>::Copy() for further doc.
558  DecodeFst *Copy(bool safe = false) const override {
559  return new DecodeFst(*this, safe);
560  }
561 
562  private:
565 };
566 
567 // Specialization for EncodeFst.
568 template <class Arc>
570  : public StateIterator<ArcMapFst<Arc, Arc, EncodeMapper<Arc>>> {
571  public:
574 };
575 
576 // Specialization for EncodeFst.
577 template <class Arc>
579  : public ArcIterator<ArcMapFst<Arc, Arc, EncodeMapper<Arc>>> {
580  public:
581  ArcIterator(const EncodeFst<Arc> &fst, typename Arc::StateId s)
582  : ArcIterator<ArcMapFst<Arc, Arc, EncodeMapper<Arc>>>(fst, s) {}
583 };
584 
585 // Specialization for DecodeFst.
586 template <class Arc>
588  : public StateIterator<ArcMapFst<Arc, Arc, EncodeMapper<Arc>>> {
589  public:
592 };
593 
594 // Specialization for DecodeFst.
595 template <class Arc>
597  : public ArcIterator<ArcMapFst<Arc, Arc, EncodeMapper<Arc>>> {
598  public:
599  ArcIterator(const DecodeFst<Arc> &fst, typename Arc::StateId s)
600  : ArcIterator<ArcMapFst<Arc, Arc, EncodeMapper<Arc>>>(fst, s) {}
601 };
602 
603 // Useful aliases when using StdArc.
604 
606 
608 
609 } // namespace fst
610 
611 #endif // FST_ENCODE_H_
typename Impl::Arc Arc
Definition: fst.h:946
bool Write(const std::string &source) const
Definition: encode.h:389
MapSymbolsAction
Definition: arc-map.h:55
void ArcMap(MutableFst< A > *fst, C *mapper)
Definition: arc-map.h:110
EncodeType Type() const
Definition: encode.h:367
typename ArcMapFst< Arc, Arc, EncodeMapper< Arc > >::Arc Arc
Definition: fst.h:518
constexpr uint64_t kWeightInvariantProperties
Definition: properties.h:279
constexpr int kNoLabel
Definition: fst.h:201
const SymbolTable * OutputSymbols() const
Definition: encode.h:205
constexpr uint8_t kEncodeFlags
Definition: encode.h:45
const SymbolTable * OutputSymbols() const
Definition: encode.h:401
bool Write(std::ostream &strm, const std::string &source) const
Definition: encode.h:272
void Encode(MutableFst< Arc > *fst, EncodeMapper< Arc > *mapper)
Definition: encode.h:474
static EncodeMapper * Read(const std::string &source, EncodeType type=ENCODE)
Definition: encode.h:375
ArcIterator(const DecodeFst< Arc > &fst, typename Arc::StateId s)
Definition: encode.h:599
constexpr uint8_t kEncodeHasISymbols
Definition: encode.h:51
EncodeFst(const Fst< Arc > &fst, Mapper *encoder)
Definition: encode.h:504
virtual SymbolTable * Copy() const
Definition: symbol-table.h:407
size_t Size() const
Definition: encode.h:193
DecodeFst(const Fst< Arc > &fst, const Mapper &encoder)
Definition: encode.h:546
bool Write(std::ostream &strm, const std::string &source) const
Definition: encode.h:385
const SymbolTable * InputSymbols() const override=0
void SetInputSymbols(const SymbolTable *syms)
Definition: encode.h:207
constexpr uint64_t kError
Definition: properties.h:51
constexpr uint64_t kRmSuperFinalProperties
Definition: properties.h:300
virtual void SetInputSymbols(const SymbolTable *isyms)=0
Triple(Label ilabel, Label olabel, Weight weight)
Definition: encode.h:108
#define LOG(type)
Definition: log.h:49
void RmFinalEpsilon(MutableFst< Arc > *fst)
MapFinalAction FinalAction() const
Definition: encode.h:335
DecodeFst(const DecodeFst &fst, bool safe=false)
Definition: encode.h:554
constexpr MapSymbolsAction InputSymbolsAction() const
Definition: encode.h:341
typename ArcMapFst< Arc, Arc, EncodeMapper< Arc > >::Arc Arc
Definition: fst.h:408
void SetOutputSymbols(const SymbolTable *syms)
Definition: encode.h:217
DecodeFst * Copy(bool safe=false) const override
Definition: encode.h:558
constexpr int kNoStateId
Definition: fst.h:202
static EncodeMapper * Read(std::istream &strm, const std::string &source, EncodeType type=ENCODE)
Definition: encode.h:369
constexpr uint8_t kEncodeHasOSymbols
Definition: encode.h:52
EncodeType
Definition: encode.h:41
std::ostream & WriteType(std::ostream &strm, const T t)
Definition: util.h:214
#define FSTERROR()
Definition: util.h:53
EncodeFst(const Fst< Arc > &fst, const Mapper &encoder)
Definition: encode.h:510
EncodeMapper(const EncodeMapper &mapper)
Definition: encode.h:320
const SymbolTable * OutputSymbols() const override=0
EncodeMapper(const EncodeMapper &mapper, EncodeType type)
Definition: encode.h:327
MapFinalAction
Definition: arc-map.h:40
EncodeTable(uint8_t flags)
Definition: encode.h:169
EncodeMapper(uint8_t flags, EncodeType type=ENCODE)
Definition: encode.h:314
ArcIterator(const EncodeFst< Arc > &fst, typename Arc::StateId s)
Definition: encode.h:581
void Write(std::ostream &strm) const
Definition: encode.h:125
constexpr int32_t kEncodeMagicNumber
Definition: encode.h:55
constexpr MapSymbolsAction OutputSymbolsAction() const
Definition: encode.h:345
void SetSize(size_t size)
Definition: encode.h:78
Arc operator()(const Arc &arc)
Definition: encode.h:425
StateIterator(const DecodeFst< Arc > &fst)
Definition: encode.h:590
constexpr uint8_t kEncodeLabels
Definition: encode.h:43
static EncodeTable * Read(std::istream &strm, const std::string &source)
Definition: encode.h:247
void SetInputSymbols(const SymbolTable *syms)
Definition: encode.h:403
void SetArcType(const std::string &arctype)
Definition: encode.h:74
const Triple * Decode(Label label) const
Definition: encode.h:185
void SetFlags(uint8_t flags)
Definition: encode.h:76
size_t Size() const
Definition: encode.h:70
static SymbolTable * Read(std::istream &strm, const std::string &source)
Definition: symbol-table.h:391
constexpr uint64_t kIDeterministic
Definition: properties.h:68
constexpr uint64_t kOLabelInvariantProperties
Definition: properties.h:269
uint8_t Flags() const
Definition: encode.h:349
StateIterator(const EncodeFst< Arc > &fst)
Definition: encode.h:572
const SymbolTable * InputSymbols() const
Definition: encode.h:399
Label Encode(const Arc &arc)
Definition: encode.h:173
constexpr uint64_t kILabelInvariantProperties
Definition: properties.h:260
constexpr uint8_t kEncodeWeights
Definition: encode.h:44
constexpr uint64_t kFstProperties
Definition: properties.h:325
virtual const SymbolTable * InputSymbols() const =0
bool Write(std::ostream &strm, const std::string &source) const
Definition: encode.cc:44
Triple(const Arc &arc, uint8_t flags)
Definition: encode.h:112
static std::unique_ptr< Triple > Read(std::istream &strm)
Definition: encode.h:117
bool operator()(const Triple *x, const Triple *y) const
Definition: encode.h:144
const SymbolTable * InputSymbols() const
Definition: encode.h:203
uint8_t Flags() const
Definition: encode.h:68
std::istream & ReadType(std::istream &strm, T *t)
Definition: util.h:68
virtual void SetOutputSymbols(const SymbolTable *osyms)=0
size_t operator()(const Triple *triple) const
Definition: encode.h:152
uint64_t Properties(uint64_t inprops)
Definition: encode.h:351
typename Arc::Label Label
Definition: encode.h:101
EncodeFst(const EncodeFst &fst, bool copy=false)
Definition: encode.h:514
typename Arc::Weight Weight
Definition: encode.h:102
void Decode(MutableFst< Arc > *fst, const EncodeMapper< Arc > &mapper)
Definition: encode.h:481
void SetOutputSymbols(const SymbolTable *syms)
Definition: encode.h:407
bool Read(std::istream &strm, const std::string &source)
Definition: encode.cc:26
constexpr uint64_t kAddSuperFinalProperties
Definition: properties.h:290
uint8_t Flags() const
Definition: encode.h:201
bool operator==(const Triple &other) const
Definition: encode.h:132
EncodeFst * Copy(bool safe=false) const override
Definition: encode.h:518
const std::string & ArcType() const
Definition: encode.h:66
virtual const SymbolTable * OutputSymbols() const =0