FST  openfst-1.8.3
OpenFst Library
vector-fst.h
Go to the documentation of this file.
1 // Copyright 2005-2024 Google LLC
2 //
3 // Licensed under the Apache License, Version 2.0 (the 'License');
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an 'AS IS' BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 //
15 // See www.openfst.org for extensive documentation on this weighted
16 // finite-state transducer library.
17 //
18 // Simple concrete, mutable FST whose states and arcs are stored in STL vectors.
19 
20 #ifndef FST_VECTOR_FST_H_
21 #define FST_VECTOR_FST_H_
22 
23 #include <algorithm>
24 #include <atomic>
25 #include <cstddef>
26 #include <cstdint>
27 #include <ios>
28 #include <iosfwd>
29 #include <istream>
30 #include <memory>
31 #include <new>
32 #include <ostream>
33 #include <string>
34 #include <utility>
35 #include <vector>
36 
37 #include <fst/log.h>
38 #include <fst/arc.h>
39 #include <fst/expanded-fst.h>
40 #include <fst/float-weight.h>
41 #include <fst/fst-decl.h> // For optional argument declarations
42 #include <fst/fst.h>
43 #include <fst/mutable-fst.h>
44 #include <fst/properties.h>
45 #include <fst/util.h>
46 #include <string_view>
47 
48 namespace fst {
49 
50 template <class A, class S>
51 class VectorFst;
52 
53 template <class F, class G>
54 void Cast(const F &, G *);
55 
56 // Arcs (of type A) implemented by an STL vector per state. M specifies Arc
57 // allocator (default declared in fst-decl.h).
58 template <class A, class M /* = std::allocator<A> */>
59 class VectorState {
60  public:
61  using Arc = A;
62  using StateId = typename Arc::StateId;
63  using Weight = typename Arc::Weight;
64  using ArcAllocator = M;
65  using StateAllocator = typename std::allocator_traits<
66  ArcAllocator>::template rebind_alloc<VectorState<Arc, M>>;
67 
68  // Provide STL allocator for arcs.
69  explicit VectorState(const ArcAllocator &alloc) : arcs_(alloc) {}
70 
71  VectorState(const VectorState<A, M> &state, const ArcAllocator &alloc)
72  : final_weight_(state.Final()),
73  niepsilons_(state.NumInputEpsilons()),
74  noepsilons_(state.NumOutputEpsilons()),
75  arcs_(state.arcs_.begin(), state.arcs_.end(), alloc) {}
76 
77  void Reset() {
78  final_weight_ = Weight::Zero();
79  niepsilons_ = 0;
80  noepsilons_ = 0;
81  arcs_.clear();
82  }
83 
84  Weight Final() const { return final_weight_; }
85 
86  size_t NumInputEpsilons() const { return niepsilons_; }
87 
88  size_t NumOutputEpsilons() const { return noepsilons_; }
89 
90  size_t NumArcs() const { return arcs_.size(); }
91 
92  const Arc &GetArc(size_t n) const { return arcs_[n]; }
93 
94  const Arc *Arcs() const { return !arcs_.empty() ? &arcs_[0] : nullptr; }
95 
96  Arc *MutableArcs() { return !arcs_.empty() ? &arcs_[0] : nullptr; }
97 
98  void ReserveArcs(size_t n) { arcs_.reserve(n); }
99 
100  void SetFinal(Weight weight) { final_weight_ = std::move(weight); }
101 
102  void SetNumInputEpsilons(size_t n) { niepsilons_ = n; }
103 
104  void SetNumOutputEpsilons(size_t n) { noepsilons_ = n; }
105 
106  void AddArc(const Arc &arc) {
107  IncrementNumEpsilons(arc);
108  arcs_.push_back(arc);
109  }
110 
111  void AddArc(Arc &&arc) {
112  IncrementNumEpsilons(arc);
113  arcs_.push_back(std::move(arc));
114  }
115 
116  template <class... T>
117  void EmplaceArc(T &&...ctor_args) {
118  arcs_.emplace_back(std::forward<T>(ctor_args)...);
119  IncrementNumEpsilons(arcs_.back());
120  }
121 
122  void SetArc(const Arc &arc, size_t n) {
123  if (arcs_[n].ilabel == 0) --niepsilons_;
124  if (arcs_[n].olabel == 0) --noepsilons_;
125  IncrementNumEpsilons(arc);
126  arcs_[n] = arc;
127  }
128 
129  void DeleteArcs() {
130  niepsilons_ = 0;
131  noepsilons_ = 0;
132  arcs_.clear();
133  }
134 
135  void DeleteArcs(size_t n) {
136  for (size_t i = 0; i < n; ++i) {
137  if (arcs_.back().ilabel == 0) --niepsilons_;
138  if (arcs_.back().olabel == 0) --noepsilons_;
139  arcs_.pop_back();
140  }
141  }
142 
143  // For state class allocation.
144  void *operator new(size_t size, StateAllocator *alloc) {
145  return alloc->allocate(1);
146  }
147 
148  // For state destruction and memory freeing.
149  static void Destroy(VectorState<A, M> *state, StateAllocator *alloc) {
150  if (state) {
151  state->~VectorState<A, M>();
152  alloc->deallocate(state, 1);
153  }
154  }
155 
156  private:
157  // Update the number of epsilons as a result of having added an arc.
158  void IncrementNumEpsilons(const Arc &arc) {
159  if (arc.ilabel == 0) ++niepsilons_;
160  if (arc.olabel == 0) ++noepsilons_;
161  }
162 
163  Weight final_weight_ = Weight::Zero(); // Final weight.
164  size_t niepsilons_ = 0; // # of input epsilons
165  size_t noepsilons_ = 0; // # of output epsilons
166  std::vector<A, ArcAllocator> arcs_; // Arc container.
167 };
168 
169 namespace internal {
170 
171 // States are implemented by STL vectors, templated on the
172 // State definition. This does not manage the Fst properties.
173 template <class S>
174 class VectorFstBaseImpl : public FstImpl<typename S::Arc> {
175  public:
176  using State = S;
177  using Arc = typename State::Arc;
178  using StateId = typename Arc::StateId;
179  using Weight = typename Arc::Weight;
180 
181  VectorFstBaseImpl() = default;
182 
183  ~VectorFstBaseImpl() override {
184  for (auto *state : states_) State::Destroy(state, &state_alloc_);
185  }
186 
187  // Copying is not permitted.
188  VectorFstBaseImpl(const VectorFstBaseImpl<S> &) = delete;
189  VectorFstBaseImpl &operator=(const VectorFstBaseImpl &) = delete;
190 
191  // Moving is permitted.
193  : FstImpl<typename S::Arc>(),
194  states_(std::move(impl.states_)),
195  start_(impl.start_) {
196  impl.states_.clear();
197  impl.start_ = kNoStateId;
198  }
199 
201  for (auto *state : states_) {
202  State::Destroy(state, &state_alloc_);
203  }
204  states_.clear();
205  std::swap(states_, impl.states_);
206  start_ = impl.start_;
207  impl.start_ = kNoStateId;
208  return *this;
209  }
210 
211  StateId Start() const { return start_; }
212 
213  Weight Final(StateId state) const { return states_[state]->Final(); }
214 
215  StateId NumStates() const { return states_.size(); }
216 
217  size_t NumArcs(StateId state) const { return states_[state]->NumArcs(); }
218 
219  size_t NumInputEpsilons(StateId state) const {
220  return GetState(state)->NumInputEpsilons();
221  }
222 
223  size_t NumOutputEpsilons(StateId state) const {
224  return GetState(state)->NumOutputEpsilons();
225  }
226 
227  void SetStart(StateId state) { start_ = state; }
228 
229  void SetFinal(StateId state, Weight weight) {
230  states_[state]->SetFinal(std::move(weight));
231  }
232 
234  states_.push_back(state);
235  return states_.size() - 1;
236  }
237 
238  StateId AddState() { return AddState(CreateState()); }
239 
240  void AddStates(size_t n) {
241  const auto curr_num_states = NumStates();
242  states_.resize(n + curr_num_states);
243  std::generate(states_.begin() + curr_num_states, states_.end(),
244  [this] { return CreateState(); });
245  }
246 
247  void AddArc(StateId state, const Arc &arc) { states_[state]->AddArc(arc); }
248 
249  void AddArc(StateId state, Arc &&arc) {
250  states_[state]->AddArc(std::move(arc));
251  }
252 
253  template <class... T>
254  void EmplaceArc(StateId state, T &&...ctor_args) {
255  states_[state]->EmplaceArc(std::forward<T>(ctor_args)...);
256  }
257 
258  void DeleteStates(const std::vector<StateId> &dstates) {
259  std::vector<StateId> newid(states_.size(), 0);
260  for (size_t i = 0; i < dstates.size(); ++i) newid[dstates[i]] = kNoStateId;
261  StateId nstates = 0;
262  for (StateId state = 0; state < states_.size(); ++state) {
263  if (newid[state] != kNoStateId) {
264  newid[state] = nstates;
265  if (state != nstates) states_[nstates] = states_[state];
266  ++nstates;
267  } else {
268  State::Destroy(states_[state], &state_alloc_);
269  }
270  }
271  states_.resize(nstates);
272  for (StateId state = 0; state < states_.size(); ++state) {
273  auto *arcs = states_[state]->MutableArcs();
274  size_t narcs = 0;
275  auto nieps = states_[state]->NumInputEpsilons();
276  auto noeps = states_[state]->NumOutputEpsilons();
277  for (size_t i = 0; i < states_[state]->NumArcs(); ++i) {
278  const auto t = newid[arcs[i].nextstate];
279  if (t != kNoStateId) {
280  arcs[i].nextstate = t;
281  if (i != narcs) arcs[narcs] = arcs[i];
282  ++narcs;
283  } else {
284  if (arcs[i].ilabel == 0) --nieps;
285  if (arcs[i].olabel == 0) --noeps;
286  }
287  }
288  states_[state]->DeleteArcs(states_[state]->NumArcs() - narcs);
289  states_[state]->SetNumInputEpsilons(nieps);
290  states_[state]->SetNumOutputEpsilons(noeps);
291  }
292  if (Start() != kNoStateId) SetStart(newid[Start()]);
293  }
294 
295  void DeleteStates() {
296  for (size_t state = 0; state < states_.size(); ++state) {
297  State::Destroy(states_[state], &state_alloc_);
298  }
299  states_.clear();
300  SetStart(kNoStateId);
301  }
302 
303  void DeleteArcs(StateId state, size_t n) { states_[state]->DeleteArcs(n); }
304 
305  void DeleteArcs(StateId state) { states_[state]->DeleteArcs(); }
306 
307  State *GetState(StateId state) { return states_[state]; }
308 
309  const State *GetState(StateId state) const { return states_[state]; }
310 
311  void SetState(StateId state, State *vstate) { states_[state] = vstate; }
312 
313  void ReserveStates(size_t n) { states_.reserve(n); }
314 
315  void ReserveArcs(StateId state, size_t n) { states_[state]->ReserveArcs(n); }
316 
317  // Provide information needed for generic state iterator.
319  data->base = nullptr;
320  data->nstates = states_.size();
321  }
322 
323  // Provide information needed for generic arc iterator.
324  void InitArcIterator(StateId state, ArcIteratorData<Arc> *data) const {
325  data->base = nullptr;
326  data->narcs = states_[state]->NumArcs();
327  data->arcs = states_[state]->Arcs();
328  data->ref_count = nullptr;
329  }
330 
331  private:
332  State *CreateState() { return new (&state_alloc_) State(arc_alloc_); }
333 
334  std::vector<State *> states_;
335  StateId start_ = kNoStateId;
336  typename State::StateAllocator state_alloc_;
337  typename State::ArcAllocator arc_alloc_;
338 };
339 
340 // This is a VectorFstBaseImpl container that holds VectorStates and manages FST
341 // properties.
342 template <class S>
343 class VectorFstImpl : public VectorFstBaseImpl<S> {
344  public:
345  using State = S;
346  using Arc = typename State::Arc;
347  using Label = typename Arc::Label;
348  using StateId = typename Arc::StateId;
349  using Weight = typename Arc::Weight;
350 
353  using FstImpl<Arc>::SetType;
356 
361 
362  friend class MutableArcIterator<VectorFst<Arc, S>>;
363 
365 
367  SetType("vector");
368  SetProperties(kNullProperties | kStaticProperties);
369  }
370 
371  explicit VectorFstImpl(const Fst<Arc> &fst);
372 
373  static VectorFstImpl *Read(std::istream &strm, const FstReadOptions &opts);
374 
375  void SetStart(StateId state) {
376  BaseImpl::SetStart(state);
377  SetProperties(SetStartProperties(Properties()));
378  }
379 
380  void SetFinal(StateId state, Weight weight) {
381  const auto old_weight = BaseImpl::Final(state);
382  const auto properties =
383  SetFinalProperties(Properties(), old_weight, weight);
384  BaseImpl::SetFinal(state, std::move(weight));
385  SetProperties(properties);
386  }
387 
389  const auto state = BaseImpl::AddState();
390  SetProperties(AddStateProperties(Properties()));
391  return state;
392  }
393 
394  void AddStates(size_t n) {
395  BaseImpl::AddStates(n);
396  SetProperties(AddStateProperties(Properties()));
397  }
398 
399  void AddArc(StateId state, const Arc &arc) {
400  BaseImpl::AddArc(state, arc);
401  UpdatePropertiesAfterAddArc(state);
402  }
403 
404  void AddArc(StateId state, Arc &&arc) {
405  BaseImpl::AddArc(state, std::move(arc));
406  UpdatePropertiesAfterAddArc(state);
407  }
408 
409  template <class... T>
410  void EmplaceArc(StateId state, T &&...ctor_args) {
411  BaseImpl::EmplaceArc(state, std::forward<T>(ctor_args)...);
412  UpdatePropertiesAfterAddArc(state);
413  }
414 
415  void DeleteStates(const std::vector<StateId> &dstates) {
416  BaseImpl::DeleteStates(dstates);
417  SetProperties(DeleteStatesProperties(Properties()));
418  }
419 
420  void DeleteStates() {
421  BaseImpl::DeleteStates();
422  SetProperties(DeleteAllStatesProperties(Properties(), kStaticProperties));
423  }
424 
425  void DeleteArcs(StateId state, size_t n) {
426  BaseImpl::DeleteArcs(state, n);
427  SetProperties(DeleteArcsProperties(Properties()));
428  }
429 
430  void DeleteArcs(StateId state) {
431  BaseImpl::DeleteArcs(state);
432  SetProperties(DeleteArcsProperties(Properties()));
433  }
434 
435  // Properties always true of this FST class
436  static constexpr uint64_t kStaticProperties = kExpanded | kMutable;
437 
438  private:
439  void UpdatePropertiesAfterAddArc(StateId state) {
440  auto *vstate = GetState(state);
441  const size_t num_arcs{vstate->NumArcs()};
442  if (num_arcs) {
443  const auto &arc = vstate->GetArc(num_arcs - 1);
444  const auto *parc =
445  (num_arcs < 2) ? nullptr : &(vstate->GetArc(num_arcs - 2));
446  SetProperties(AddArcProperties(Properties(), state, arc, parc));
447  }
448  }
449 
450  // Minimum file format version supported.
451  static constexpr int kMinFileVersion = 2;
452 };
453 
454 template <class S>
456  SetType("vector");
457  SetInputSymbols(fst.InputSymbols());
458  SetOutputSymbols(fst.OutputSymbols());
459  BaseImpl::SetStart(fst.Start());
460  if (std::optional<StateId> num_states = fst.NumStatesIfKnown()) {
461  BaseImpl::ReserveStates(*num_states);
462  }
463  for (StateIterator<Fst<Arc>> siter(fst); !siter.Done(); siter.Next()) {
464  const auto state = siter.Value();
465  BaseImpl::AddState();
466  BaseImpl::SetFinal(state, fst.Final(state));
467  ReserveArcs(state, fst.NumArcs(state));
468  for (ArcIterator<Fst<Arc>> aiter(fst, state); !aiter.Done(); aiter.Next()) {
469  const auto &arc = aiter.Value();
470  BaseImpl::AddArc(state, arc);
471  }
472  }
473  SetProperties(fst.Properties(kCopyProperties, false) | kStaticProperties);
474 }
475 
476 template <class S>
478  const FstReadOptions &opts) {
479  auto impl = std::make_unique<VectorFstImpl>();
480  FstHeader hdr;
481  if (!impl->ReadHeader(strm, opts, kMinFileVersion, &hdr)) return nullptr;
482  impl->BaseImpl::SetStart(hdr.Start());
483  if (hdr.NumStates() != kNoStateId) impl->ReserveStates(hdr.NumStates());
484  StateId state = 0;
485  for (; hdr.NumStates() == kNoStateId || state < hdr.NumStates(); ++state) {
486  Weight weight;
487  if (!weight.Read(strm)) break;
488  impl->BaseImpl::AddState();
489  auto *vstate = impl->GetState(state);
490  vstate->SetFinal(weight);
491  int64_t narcs;
492  ReadType(strm, &narcs);
493  if (!strm) {
494  LOG(ERROR) << "VectorFst::Read: Read failed: " << opts.source;
495  return nullptr;
496  }
497  impl->ReserveArcs(state, narcs);
498  for (int64_t i = 0; i < narcs; ++i) {
499  Arc arc;
500  ReadType(strm, &arc.ilabel);
501  ReadType(strm, &arc.olabel);
502  arc.weight.Read(strm);
503  ReadType(strm, &arc.nextstate);
504  if (!strm) {
505  LOG(ERROR) << "VectorFst::Read: Read failed: " << opts.source;
506  return nullptr;
507  }
508  impl->BaseImpl::AddArc(state, std::move(arc));
509  }
510  }
511  if (hdr.NumStates() != kNoStateId && state != hdr.NumStates()) {
512  LOG(ERROR) << "VectorFst::Read: Unexpected end of file: " << opts.source;
513  return nullptr;
514  }
515  return impl.release();
516 }
517 
518 } // namespace internal
519 
520 // Simple concrete, mutable FST. This class attaches interface to implementation
521 // and handles reference counting, delegating most methods to ImplToMutableFst.
522 // Also supports ReserveStates and ReserveArcs methods (cf. STL vector methods).
523 // The second optional template argument gives the State definition.
524 //
525 // VectorFst is thread-compatible.
526 template <class A, class S /* = VectorState<A> */>
527 class VectorFst : public ImplToMutableFst<internal::VectorFstImpl<S>> {
528  public:
529  using Arc = A;
530  using StateId = typename Arc::StateId;
531 
532  using State = S;
534 
535  friend class StateIterator<VectorFst<Arc, State>>;
536  friend class ArcIterator<VectorFst<Arc, State>>;
537  friend class MutableArcIterator<VectorFst<A, S>>;
538 
539  template <class F, class G>
540  friend void Cast(const F &, G *);
541 
542  VectorFst() : ImplToMutableFst<Impl>(std::make_shared<Impl>()) {}
543 
544  explicit VectorFst(const Fst<Arc> &fst)
545  : ImplToMutableFst<Impl>(std::make_shared<Impl>(fst)) {}
546 
547  VectorFst(const VectorFst &fst, bool unused_safe = false)
548  : ImplToMutableFst<Impl>(fst.GetSharedImpl()) {}
549 
550  VectorFst(VectorFst &&) noexcept;
551 
552  // Get a copy of this VectorFst. See Fst<>::Copy() for further doc.
553  VectorFst *Copy(bool safe = false) const override {
554  return new VectorFst(*this, safe);
555  }
556 
557  VectorFst &operator=(const VectorFst &) = default;
558 
559  VectorFst &operator=(VectorFst &&) noexcept;
560 
561  VectorFst &operator=(const Fst<Arc> &fst) override {
562  if (this != &fst) SetImpl(std::make_shared<Impl>(fst));
563  return *this;
564  }
565 
566  template <class... T>
567  void EmplaceArc(StateId state, T &&...ctor_args) {
568  MutateCheck();
569  GetMutableImpl()->EmplaceArc(state, std::forward<T>(ctor_args)...);
570  }
571 
572  // Reads a VectorFst from an input stream, returning nullptr on error.
573  static VectorFst *Read(std::istream &strm, const FstReadOptions &opts) {
574  auto *impl = Impl::Read(strm, opts);
575  return impl ? new VectorFst(std::shared_ptr<Impl>(impl)) : nullptr;
576  }
577 
578  // Read a VectorFst from a file, returning nullptr on error; empty source
579  // reads from standard input.
580  static VectorFst *Read(std::string_view source) {
581  auto *impl = ImplToExpandedFst<Impl, MutableFst<Arc>>::Read(source);
582  return impl ? new VectorFst(std::shared_ptr<Impl>(impl)) : nullptr;
583  }
584 
585  bool Write(std::ostream &strm, const FstWriteOptions &opts) const override {
586  return WriteFst(*this, strm, opts);
587  }
588 
589  bool Write(const std::string &source) const override {
590  return Fst<Arc>::WriteFile(source);
591  }
592 
593  template <class FST>
594  static bool WriteFst(const FST &fst, std::ostream &strm,
595  const FstWriteOptions &opts);
596 
597  void InitStateIterator(StateIteratorData<Arc> *data) const override {
598  GetImpl()->InitStateIterator(data);
599  }
600 
601  void InitArcIterator(StateId s, ArcIteratorData<Arc> *data) const override {
602  GetImpl()->InitArcIterator(s, data);
603  }
604 
605  inline void InitMutableArcIterator(StateId s,
606  MutableArcIteratorData<Arc> *) override;
607 
609  using ImplToMutableFst<Impl, MutableFst<Arc>>::ReserveStates;
610 
611  private:
613  using ImplToMutableFst<Impl, MutableFst<Arc>>::GetMutableImpl;
614  using ImplToMutableFst<Impl, MutableFst<Arc>>::MutateCheck;
616 
617  explicit VectorFst(std::shared_ptr<Impl> impl)
618  : ImplToMutableFst<Impl>(impl) {}
619 };
620 
621 template <class Arc, class State>
622 inline VectorFst<Arc, State>::VectorFst(VectorFst &&fst) noexcept = default;
623 
624 template <class Arc, class State>
626  VectorFst &&fst) noexcept = default;
627 
628 // Writes FST to file in Vector format, potentially with a pass over the machine
629 // before writing to compute number of states.
630 template <class Arc, class State>
631 template <class FST>
632 bool VectorFst<Arc, State>::WriteFst(const FST &fst, std::ostream &strm,
633  const FstWriteOptions &opts) {
634  static constexpr int file_version = 2;
635  bool update_header = true;
636  FstHeader hdr;
637  hdr.SetStart(fst.Start());
639  std::streampos start_offset = 0;
640  if (fst.Properties(kExpanded, false) || opts.stream_write ||
641  (start_offset = strm.tellp()) != -1) {
642  hdr.SetNumStates(CountStates(fst));
643  update_header = false;
644  }
645  const auto properties =
646  fst.Properties(kCopyProperties, false) | Impl::kStaticProperties;
647  internal::FstImpl<Arc>::WriteFstHeader(fst, strm, opts, file_version,
648  "vector", properties, &hdr);
649  StateId num_states = 0;
650  for (StateIterator<FST> siter(fst); !siter.Done(); siter.Next()) {
651  const auto s = siter.Value();
652  fst.Final(s).Write(strm);
653  const int64_t narcs = fst.NumArcs(s);
654  WriteType(strm, narcs);
655  for (ArcIterator<FST> aiter(fst, s); !aiter.Done(); aiter.Next()) {
656  const auto &arc = aiter.Value();
657  WriteType(strm, arc.ilabel);
658  WriteType(strm, arc.olabel);
659  arc.weight.Write(strm);
660  WriteType(strm, arc.nextstate);
661  }
662  ++num_states;
663  }
664  strm.flush();
665  if (!strm) {
666  LOG(ERROR) << "VectorFst::Write: Write failed: " << opts.source;
667  return false;
668  }
669  if (update_header) {
670  hdr.SetNumStates(num_states);
672  fst, strm, opts, file_version, "vector", properties, &hdr,
673  start_offset);
674  } else {
675  if (num_states != hdr.NumStates()) {
676  LOG(ERROR) << "Inconsistent number of states observed during write";
677  return false;
678  }
679  }
680  return true;
681 }
682 
683 // Specialization for VectorFst; see generic version in fst.h for sample usage
684 // (but use the VectorFst type instead). This version should inline.
685 template <class Arc, class State>
686 class StateIterator<VectorFst<Arc, State>> {
687  public:
688  using StateId = typename Arc::StateId;
689 
691  : nstates_(fst.GetImpl()->NumStates()) {}
692 
693  bool Done() const { return s_ >= nstates_; }
694 
695  StateId Value() const { return s_; }
696 
697  void Next() { ++s_; }
698 
699  void Reset() { s_ = 0; }
700 
701  private:
702  const StateId nstates_;
703  StateId s_ = 0;
704 };
705 
706 // Specialization for VectorFst; see generic version in fst.h for sample usage
707 // (but use the VectorFst type instead). This version should inline.
708 template <class Arc, class State>
709 class ArcIterator<VectorFst<Arc, State>> {
710  public:
711  using StateId = typename Arc::StateId;
712 
714  : arcs_(fst.GetImpl()->GetState(s)->Arcs()),
715  narcs_(fst.GetImpl()->GetState(s)->NumArcs()) {}
716 
717  bool Done() const { return i_ >= narcs_; }
718 
719  const Arc &Value() const { return arcs_[i_]; }
720 
721  void Next() { ++i_; }
722 
723  void Reset() { i_ = 0; }
724 
725  void Seek(size_t a) { i_ = a; }
726 
727  size_t Position() const { return i_; }
728 
729  constexpr uint8_t Flags() const { return kArcValueFlags; }
730 
731  void SetFlags(uint8_t, uint8_t) {}
732 
733  private:
734  const Arc *arcs_;
735  size_t narcs_;
736  size_t i_ = 0;
737 };
738 
739 // Specialization for VectorFst; see generic version in mutable-fst.h for sample
740 // usage (but use the VectorFst type instead). This version should inline.
741 template <class Arc, class State>
743  : public MutableArcIteratorBase<Arc> {
744  public:
745  using StateId = typename Arc::StateId;
746  using Weight = typename Arc::Weight;
747 
749  fst->MutateCheck();
750  state_ = fst->GetMutableImpl()->GetState(s);
751  properties_ = &fst->GetImpl()->properties_;
752  }
753 
754  bool Done() const final { return i_ >= state_->NumArcs(); }
755 
756  const Arc &Value() const final { return state_->GetArc(i_); }
757 
758  void Next() final { ++i_; }
759 
760  size_t Position() const final { return i_; }
761 
762  void Reset() final { i_ = 0; }
763 
764  void Seek(size_t a) final { i_ = a; }
765 
766  void SetValue(const Arc &arc) final {
767  const auto &oarc = state_->GetArc(i_);
768  uint64_t properties = properties_->load(std::memory_order_relaxed);
769  if (oarc.ilabel != oarc.olabel) properties &= ~kNotAcceptor;
770  if (oarc.ilabel == 0) {
771  properties &= ~kIEpsilons;
772  if (oarc.olabel == 0) properties &= ~kEpsilons;
773  }
774  if (oarc.olabel == 0) properties &= ~kOEpsilons;
775  if (oarc.weight != Weight::Zero() && oarc.weight != Weight::One()) {
776  properties &= ~kWeighted;
777  }
778  state_->SetArc(arc, i_);
779  if (arc.ilabel != arc.olabel) {
780  properties |= kNotAcceptor;
781  properties &= ~kAcceptor;
782  }
783  if (arc.ilabel == 0) {
784  properties |= kIEpsilons;
785  properties &= ~kNoIEpsilons;
786  if (arc.olabel == 0) {
787  properties |= kEpsilons;
788  properties &= ~kNoEpsilons;
789  }
790  }
791  if (arc.olabel == 0) {
792  properties |= kOEpsilons;
793  properties &= ~kNoOEpsilons;
794  }
795  if (arc.weight != Weight::Zero() && arc.weight != Weight::One()) {
796  properties |= kWeighted;
797  properties &= ~kUnweighted;
798  }
799  properties &= kSetArcProperties | kAcceptor | kNotAcceptor | kEpsilons |
802  properties_->store(properties, std::memory_order_relaxed);
803  }
804 
805  uint8_t Flags() const final { return kArcValueFlags; }
806 
807  void SetFlags(uint8_t, uint8_t) final {}
808 
809  private:
810  State *state_;
811  std::atomic<uint64_t> *properties_;
812  size_t i_ = 0;
813 };
814 
815 // Provides information needed for the generic mutable arc iterator.
816 template <class Arc, class State>
819  data->base =
820  std::make_unique<MutableArcIterator<VectorFst<Arc, State>>>(this, s);
821 }
822 
823 // A useful alias when using StdArc.
825 
826 } // namespace fst
827 
828 #endif // FST_VECTOR_FST_H_
typename std::allocator_traits< ArcAllocator >::template rebind_alloc< VectorState< Arc, M >> StateAllocator
Definition: vector-fst.h:66
void AddArc(StateId state, const Arc &arc)
Definition: vector-fst.h:247
typename Arc::Weight Weight
Definition: vector-fst.h:179
void SetState(StateId state, State *vstate)
Definition: vector-fst.h:311
uint64_t AddArcProperties(uint64_t inprops, typename A::StateId s, const A &arc, const A *prev_arc)
void DeleteArcs(StateId state)
Definition: vector-fst.h:305
constexpr uint64_t kMutable
Definition: properties.h:49
constexpr uint8_t kArcValueFlags
Definition: fst.h:453
void AddArc(StateId state, Arc &&arc)
Definition: vector-fst.h:404
uint64_t SetStartProperties(uint64_t inprops)
Definition: properties.h:396
void Cast(const F &, G *)
virtual uint64_t Properties(uint64_t mask, bool test) const =0
constexpr uint64_t kOEpsilons
Definition: properties.h:89
void DeleteStates(const std::vector< StateId > &dstates)
Definition: vector-fst.h:415
std::unique_ptr< MutableArcIteratorBase< Arc > > base
Definition: mutable-fst.h:206
virtual size_t NumArcs(StateId) const =0
VectorFstBaseImpl & operator=(VectorFstBaseImpl &&impl) noexcept
Definition: vector-fst.h:200
VectorState(const ArcAllocator &alloc)
Definition: vector-fst.h:69
void DeleteArcs(StateId state, size_t n)
Definition: vector-fst.h:425
std::string source
Definition: fst.h:73
Weight Final() const
Definition: vector-fst.h:84
Arc::Weight Final(const ExpandedFst< Arc > &fst, typename Arc::StateId s)
Definition: expanded-fst.h:107
void ReserveArcs(size_t n)
Definition: vector-fst.h:98
void InitArcIterator(StateId s, ArcIteratorData< Arc > *data) const override
Definition: vector-fst.h:601
uint64_t AddStateProperties(uint64_t inprops)
Definition: properties.h:404
StateIterator(const VectorFst< Arc, State > &fst)
Definition: vector-fst.h:690
uint64_t DeleteAllStatesProperties(uint64_t inprops, uint64_t staticProps)
Definition: properties.h:412
Arc * MutableArcs()
Definition: vector-fst.h:96
uint64_t DeleteStatesProperties(uint64_t inprops)
Definition: properties.h:408
bool Write(std::ostream &strm, const FstWriteOptions &opts) const override
Definition: vector-fst.h:585
void SetNumOutputEpsilons(size_t n)
Definition: vector-fst.h:104
size_t NumArcs(StateId state) const
Definition: vector-fst.h:217
#define LOG(type)
Definition: log.h:53
void SetFinal(StateId state, Weight weight)
Definition: vector-fst.h:229
virtual Weight Final(StateId) const =0
SetType
Definition: set-weight.h:59
static VectorFst * Read(std::istream &strm, const FstReadOptions &opts)
Definition: vector-fst.h:573
typename Arc::StateId StateId
Definition: fst.h:462
constexpr uint64_t kEpsilons
Definition: properties.h:79
uint64_t SetFinalProperties(uint64_t inprops, const Weight &old_weight, const Weight &new_weight)
Definition: properties.h:424
void InitMutableArcIterator(StateId s, MutableArcIteratorData< Arc > *) override
Definition: vector-fst.h:817
typename Arc::Label Label
Definition: vector-fst.h:347
bool stream_write
Definition: fst.h:107
constexpr int kNoStateId
Definition: fst.h:196
void DeleteStates(const std::vector< StateId > &dstates)
Definition: vector-fst.h:258
void AddArc(StateId state, Arc &&arc)
Definition: vector-fst.h:249
typename Arc::StateId StateId
Definition: vector-fst.h:62
VectorFst(const Fst< Arc > &fst)
Definition: vector-fst.h:544
std::ostream & WriteType(std::ostream &strm, const T t)
Definition: util.h:228
std::string source
Definition: fst.h:102
void InitStateIterator(StateIteratorData< Arc > *data) const
Definition: vector-fst.h:318
static void WriteFstHeader(const Fst< Arc > &fst, std::ostream &strm, const FstWriteOptions &opts, int version, std::string_view type, uint64_t properties, FstHeader *hdr)
Definition: fst.h:811
Weight Final(StateId state) const
Definition: vector-fst.h:213
const State * GetState(StateId state) const
Definition: vector-fst.h:309
MutableArcIterator(VectorFst< Arc, State > *fst, StateId s)
Definition: vector-fst.h:748
State * GetState(StateId state)
Definition: vector-fst.h:307
size_t NumArcs() const
Definition: vector-fst.h:90
void SetFinal(Weight weight)
Definition: vector-fst.h:100
void EmplaceArc(StateId state, T &&...ctor_args)
Definition: vector-fst.h:410
static void Destroy(VectorState< A, M > *state, StateAllocator *alloc)
Definition: vector-fst.h:149
constexpr uint64_t kNoOEpsilons
Definition: properties.h:91
void InitStateIterator(StateIteratorData< Arc > *data) const override
Definition: vector-fst.h:597
constexpr uint64_t kNotAcceptor
Definition: properties.h:66
constexpr uint64_t kCopyProperties
Definition: properties.h:163
VectorFst & operator=(const Fst< Arc > &fst) override
Definition: vector-fst.h:561
void AddArc(const Arc &arc)
Definition: vector-fst.h:106
constexpr uint64_t kNoEpsilons
Definition: properties.h:81
ArcIterator(const VectorFst< Arc, State > &fst, StateId s)
Definition: vector-fst.h:713
static bool UpdateFstHeader(const Fst< Arc > &fst, std::ostream &strm, const FstWriteOptions &opts, int version, std::string_view type, uint64_t properties, FstHeader *hdr, size_t header_offset)
Definition: fst.h:844
void SetStart(StateId state)
Definition: vector-fst.h:375
StateId nstates
Definition: fst.h:384
VectorFst * Copy(bool safe=false) const override
Definition: vector-fst.h:553
virtual StateId Start() const =0
bool Done() const
Definition: fst.h:532
std::unique_ptr< StateIteratorBase< Arc > > base
Definition: fst.h:382
void DeleteArcs(size_t n)
Definition: vector-fst.h:135
void ReserveArcs(StateId state, size_t n)
Definition: vector-fst.h:315
void AddArc(StateId state, const Arc &arc)
Definition: vector-fst.h:399
size_t NumOutputEpsilons() const
Definition: vector-fst.h:88
VectorFst & operator=(const VectorFst &)=default
void EmplaceArc(StateId state, T &&...ctor_args)
Definition: vector-fst.h:567
typename Arc::StateId StateId
Definition: vector-fst.h:178
constexpr uint64_t kNullProperties
Definition: properties.h:150
std::unique_ptr< ArcIteratorBase< Arc > > base
Definition: fst.h:494
constexpr uint64_t kIEpsilons
Definition: properties.h:84
size_t NumInputEpsilons(StateId state) const
Definition: vector-fst.h:219
virtual std::optional< StateId > NumStatesIfKnown() const
Definition: fst.h:228
VectorState(const VectorState< A, M > &state, const ArcAllocator &alloc)
Definition: vector-fst.h:71
size_t NumOutputEpsilons(StateId state) const
Definition: vector-fst.h:223
bool WriteFile(const std::string &source) const
Definition: fst.h:332
constexpr uint8_t Flags() const
Definition: vector-fst.h:729
void SetNumInputEpsilons(size_t n)
Definition: vector-fst.h:102
bool Write(const std::string &source) const override
Definition: vector-fst.h:589
constexpr uint64_t kUnweighted
Definition: properties.h:106
void EmplaceArc(StateId state, T &&...ctor_args)
Definition: vector-fst.h:254
void SetNumStates(int64_t numstates)
Definition: fst.h:164
A Arc
Definition: fst.h:205
const Arc & GetArc(size_t n) const
Definition: vector-fst.h:92
virtual const SymbolTable * InputSymbols() const =0
Arc::StateId CountStates(const Fst< Arc > &fst)
Definition: expanded-fst.h:179
int64_t NumStates() const
Definition: fst.h:148
typename Arc::Weight Weight
Definition: vector-fst.h:63
VectorFst(const VectorFst &fst, bool unused_safe=false)
Definition: vector-fst.h:547
bool Done() const
Definition: fst.h:415
const Arc * Arcs() const
Definition: vector-fst.h:94
void SetStart(int64_t start)
Definition: fst.h:162
const Arc * arcs
Definition: fst.h:495
void DeleteArcs(StateId state, size_t n)
Definition: vector-fst.h:303
void SetStart(StateId state)
Definition: vector-fst.h:227
VectorFstBaseImpl(VectorFstBaseImpl &&impl) noexcept
Definition: vector-fst.h:192
static VectorFst * Read(std::string_view source)
Definition: vector-fst.h:580
std::istream & ReadType(std::istream &strm, T *t)
Definition: util.h:80
uint64_t DeleteArcsProperties(uint64_t inprops)
Definition: properties.h:417
typename Arc::StateId StateId
Definition: fst.h:206
void DeleteArcs(StateId state)
Definition: vector-fst.h:430
size_t NumInputEpsilons() const
Definition: vector-fst.h:86
void AddArc(Arc &&arc)
Definition: vector-fst.h:111
void InitArcIterator(StateId state, ArcIteratorData< Arc > *data) const
Definition: vector-fst.h:324
constexpr uint64_t kWeighted
Definition: properties.h:104
void EmplaceArc(T &&...ctor_args)
Definition: vector-fst.h:117
static bool WriteFst(const FST &fst, std::ostream &strm, const FstWriteOptions &opts)
Definition: vector-fst.h:632
constexpr uint64_t kExpanded
Definition: properties.h:46
void SetFinal(StateId state, Weight weight)
Definition: vector-fst.h:380
void SetArc(const Arc &arc, size_t n)
Definition: vector-fst.h:122
int * ref_count
Definition: fst.h:497
constexpr uint64_t kNoIEpsilons
Definition: properties.h:86
constexpr uint64_t kSetArcProperties
Definition: properties.h:224
void Destroy(ArcIterator< FST > *aiter, MemoryPool< ArcIterator< FST >> *pool)
Definition: fst.h:593
size_t narcs
Definition: fst.h:496
StateId AddState(State *state)
Definition: vector-fst.h:233
constexpr uint64_t kAcceptor
Definition: properties.h:64
virtual const SymbolTable * OutputSymbols() const =0