FST  openfst-1.8.3
OpenFst Library
add-on.h
Go to the documentation of this file.
1 // Copyright 2005-2024 Google LLC
2 //
3 // Licensed under the Apache License, Version 2.0 (the 'License');
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an 'AS IS' BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 //
15 // See www.openfst.org for extensive documentation on this weighted
16 // finite-state transducer library.
17 //
18 // FST implementation class to attach an arbitrary object with a read/write
19 // method to an FST and its file representation. The FST is given a new type
20 // name.
21 
22 #ifndef FST_ADD_ON_H_
23 #define FST_ADD_ON_H_
24 
25 #include <cstddef>
26 #include <cstdint>
27 #include <istream>
28 #include <memory>
29 #include <ostream>
30 #include <string>
31 #include <utility>
32 
33 #include <fst/log.h>
34 #include <fst/fst.h>
35 #include <fst/properties.h>
36 #include <fst/util.h>
37 #include <string_view>
38 
39 namespace fst {
40 
41 // Identifies stream data as an add-on FST.
42 inline constexpr int32_t kAddOnMagicNumber = 446681434;
43 
44 // Nothing to save.
45 class NullAddOn {
46  public:
47  NullAddOn() = default;
48 
49  static NullAddOn *Read(std::istream &strm, const FstReadOptions &opts) {
50  return new NullAddOn();
51  }
52 
53  bool Write(std::ostream &ostrm, const FstWriteOptions &opts) const {
54  return true;
55  }
56 };
57 
58 // Create a new add-on from a pair of add-ons.
59 template <class A1, class A2>
60 class AddOnPair {
61  public:
62  // Argument reference count incremented.
63  AddOnPair(std::shared_ptr<A1> a1, std::shared_ptr<A2> a2)
64  : a1_(std::move(a1)), a2_(std::move(a2)) {}
65 
66  const A1 *First() const { return a1_.get(); }
67 
68  const A2 *Second() const { return a2_.get(); }
69 
70  std::shared_ptr<A1> SharedFirst() const { return a1_; }
71 
72  std::shared_ptr<A2> SharedSecond() const { return a2_; }
73 
74  static AddOnPair *Read(std::istream &istrm, const FstReadOptions &opts) {
75  bool have_addon1 = false;
76  ReadType(istrm, &have_addon1);
77  std::unique_ptr<A1> a1;
78  if (have_addon1) a1 = fst::WrapUnique(A1::Read(istrm, opts));
79 
80  bool have_addon2 = false;
81  ReadType(istrm, &have_addon2);
82  std::unique_ptr<A1> a2;
83  if (have_addon2) a2 = fst::WrapUnique(A2::Read(istrm, opts));
84 
85  return new AddOnPair(std::move(a1), std::move(a2));
86  }
87 
88  bool Write(std::ostream &ostrm, const FstWriteOptions &opts) const {
89  bool have_addon1 = a1_ != nullptr;
90  WriteType(ostrm, have_addon1);
91  if (have_addon1) a1_->Write(ostrm, opts);
92  bool have_addon2 = a2_ != nullptr;
93  WriteType(ostrm, have_addon2);
94  if (have_addon2) a2_->Write(ostrm, opts);
95  return true;
96  }
97 
98  private:
99  std::shared_ptr<A1> a1_;
100  std::shared_ptr<A2> a2_;
101 };
102 
103 namespace internal {
104 
105 // Adds an object of type T to an FST. T must support:
106 //
107 // T* Read(std::istream &);
108 // bool Write(std::ostream &);
109 //
110 // The resulting type is a new FST implementation.
111 template <class FST, class T>
112 class AddOnImpl : public FstImpl<typename FST::Arc> {
113  public:
114  using FstType = FST;
115  using Arc = typename FST::Arc;
116  using Label = typename Arc::Label;
117  using StateId = typename Arc::StateId;
118  using Weight = typename Arc::Weight;
119 
120  using FstImpl<Arc>::SetType;
125 
126  // We make a thread-safe copy of the FST by default since an FST
127  // implementation is expected to not share mutable data between objects.
128  AddOnImpl(const FST &fst, std::string_view type,
129  std::shared_ptr<T> t = nullptr)
130  : fst_(fst, true), t_(std::move(t)) {
131  SetType(type);
132  SetProperties(fst_.Properties(kFstProperties, false));
133  SetInputSymbols(fst_.InputSymbols());
134  SetOutputSymbols(fst_.OutputSymbols());
135  }
136 
137  // Conversion from const Fst<Arc> & to F always copies the underlying
138  // implementation.
139  AddOnImpl(const Fst<Arc> &fst, std::string_view type,
140  std::shared_ptr<T> t = nullptr)
141  : fst_(fst), t_(std::move(t)) {
142  SetType(type);
143  SetProperties(fst_.Properties(kFstProperties, false));
144  SetInputSymbols(fst_.InputSymbols());
145  SetOutputSymbols(fst_.OutputSymbols());
146  }
147 
148  // We make a thread-safe copy of the FST by default since an FST
149  // implementation is expected to not share mutable data between objects.
150  AddOnImpl(const AddOnImpl &impl) : fst_(impl.fst_, true), t_(impl.t_) {
151  SetType(impl.Type());
152  SetProperties(fst_.Properties(kCopyProperties, false));
153  SetInputSymbols(fst_.InputSymbols());
154  SetOutputSymbols(fst_.OutputSymbols());
155  }
156 
157  StateId Start() const { return fst_.Start(); }
158 
159  Weight Final(StateId s) const { return fst_.Final(s); }
160 
161  size_t NumArcs(StateId s) const { return fst_.NumArcs(s); }
162 
163  size_t NumInputEpsilons(StateId s) const { return fst_.NumInputEpsilons(s); }
164 
165  size_t NumOutputEpsilons(StateId s) const {
166  return fst_.NumOutputEpsilons(s);
167  }
168 
169  size_t NumStates() const { return fst_.NumStates(); }
170 
171  static AddOnImpl *Read(std::istream &strm, const FstReadOptions &opts) {
172  FstReadOptions nopts(opts);
173  FstHeader hdr;
174  if (!nopts.header) {
175  hdr.Read(strm, nopts.source);
176  nopts.header = &hdr;
177  }
178  // Using `new` to access private constructor for `AddOnImpl`.
179  auto impl = fst::WrapUnique(new AddOnImpl(nopts.header->FstType()));
180  if (!impl->ReadHeader(strm, nopts, kMinFileVersion, &hdr)) return nullptr;
181  impl.reset();
182  int32_t magic_number = 0;
183  ReadType(strm, &magic_number); // Ensures this is an add-on FST.
184  if (magic_number != kAddOnMagicNumber) {
185  LOG(ERROR) << "AddOnImpl::Read: Bad add-on header: " << nopts.source;
186  return nullptr;
187  }
188  FstReadOptions fopts(opts);
189  fopts.header = nullptr; // Contained header was written out.
190  std::unique_ptr<FST> fst(FST::Read(strm, fopts));
191  if (!fst) return nullptr;
192  std::shared_ptr<T> t;
193  bool have_addon = false;
194  ReadType(strm, &have_addon);
195  if (have_addon) { // Reads add-on object if present.
196  t = std::shared_ptr<T>(T::Read(strm, fopts));
197  if (!t) return nullptr;
198  }
199  return new AddOnImpl(*fst, nopts.header->FstType(), t);
200  }
201 
202  bool Write(std::ostream &strm, const FstWriteOptions &opts) const {
203  FstHeader hdr;
204  FstWriteOptions nopts(opts);
205  nopts.write_isymbols = false; // Allows contained FST to hold any symbols.
206  nopts.write_osymbols = false;
207  WriteHeader(strm, nopts, kFileVersion, &hdr);
208  WriteType(strm, kAddOnMagicNumber); // Ensures this is an add-on FST.
209  FstWriteOptions fopts(opts);
210  fopts.write_header = true; // Forces writing contained header.
211  if (!fst_.Write(strm, fopts)) return false;
212  bool have_addon = !!t_;
213  WriteType(strm, have_addon);
214  // Writes add-on object if present.
215  if (have_addon) t_->Write(strm, opts);
216  return true;
217  }
218 
220  fst_.InitStateIterator(data);
221  }
222 
224  fst_.InitArcIterator(s, data);
225  }
226 
227  FST &GetFst() { return fst_; }
228 
229  const FST &GetFst() const { return fst_; }
230 
231  const T *GetAddOn() const { return t_.get(); }
232 
233  std::shared_ptr<T> GetSharedAddOn() const { return t_; }
234 
235  void SetAddOn(std::shared_ptr<T> t) { t_ = t; }
236 
237  private:
238  explicit AddOnImpl(std::string_view type) : t_() {
239  SetType(type);
240  SetProperties(kExpanded);
241  }
242 
243  // Current file format version.
244  static constexpr int kFileVersion = 1;
245  // Minimum file format version supported.
246  static constexpr int kMinFileVersion = 1;
247 
248  FST fst_;
249  std::shared_ptr<T> t_;
250 
251  AddOnImpl &operator=(const AddOnImpl &) = delete;
252 };
253 
254 } // namespace internal
255 } // namespace fst
256 
257 #endif // FST_ADD_ON_H_
static AddOnPair * Read(std::istream &istrm, const FstReadOptions &opts)
Definition: add-on.h:74
bool Read(std::istream &strm, const std::string &source, bool rewind=false)
Definition: fst.cc:56
const FstHeader * header
Definition: fst.h:74
void InitStateIterator(StateIteratorData< Arc > *data) const
Definition: add-on.h:219
bool Write(std::ostream &strm, const FstWriteOptions &opts) const
Definition: add-on.h:202
std::shared_ptr< A2 > SharedSecond() const
Definition: add-on.h:72
std::string source
Definition: fst.h:73
bool Write(std::ostream &ostrm, const FstWriteOptions &opts) const
Definition: add-on.h:53
#define LOG(type)
Definition: log.h:53
SetType
Definition: set-weight.h:59
std::unique_ptr< T > WrapUnique(T *ptr)
Definition: compat.h:132
std::shared_ptr< A1 > SharedFirst() const
Definition: add-on.h:70
void InitArcIterator(StateId s, ArcIteratorData< Arc > *data) const
Definition: add-on.h:223
std::ostream & WriteType(std::ostream &strm, const T t)
Definition: util.h:228
NullAddOn()=default
size_t NumOutputEpsilons(StateId s) const
Definition: add-on.h:165
const std::string & FstType() const
Definition: fst.h:136
AddOnPair(std::shared_ptr< A1 > a1, std::shared_ptr< A2 > a2)
Definition: add-on.h:63
bool write_osymbols
Definition: fst.h:105
AddOnImpl(const FST &fst, std::string_view type, std::shared_ptr< T > t=nullptr)
Definition: add-on.h:128
constexpr uint64_t kCopyProperties
Definition: properties.h:163
size_t NumArcs(StateId s) const
Definition: add-on.h:161
bool write_isymbols
Definition: fst.h:104
StateId Start() const
Definition: add-on.h:157
size_t NumStates() const
Definition: add-on.h:169
std::shared_ptr< T > GetSharedAddOn() const
Definition: add-on.h:233
typename FST::Arc Arc
Definition: add-on.h:115
AddOnImpl(const AddOnImpl &impl)
Definition: add-on.h:150
constexpr int32_t kAddOnMagicNumber
Definition: add-on.h:42
bool write_header
Definition: fst.h:103
const A2 * Second() const
Definition: add-on.h:68
const std::string & Type() const
Definition: fst.h:697
const A1 * First() const
Definition: add-on.h:66
constexpr uint64_t kFstProperties
Definition: properties.h:326
typename Arc::Weight Weight
Definition: add-on.h:118
const T * GetAddOn() const
Definition: add-on.h:231
size_t NumInputEpsilons(StateId s) const
Definition: add-on.h:163
typename Arc::Label Label
Definition: add-on.h:116
std::istream & ReadType(std::istream &strm, T *t)
Definition: util.h:80
bool Write(std::ostream &ostrm, const FstWriteOptions &opts) const
Definition: add-on.h:88
const FST & GetFst() const
Definition: add-on.h:229
static NullAddOn * Read(std::istream &strm, const FstReadOptions &opts)
Definition: add-on.h:49
static AddOnImpl * Read(std::istream &strm, const FstReadOptions &opts)
Definition: add-on.h:171
constexpr uint64_t kExpanded
Definition: properties.h:46
Weight Final(StateId s) const
Definition: add-on.h:159
AddOnImpl(const Fst< Arc > &fst, std::string_view type, std::shared_ptr< T > t=nullptr)
Definition: add-on.h:139
void SetAddOn(std::shared_ptr< T > t)
Definition: add-on.h:235
typename Arc::StateId StateId
Definition: add-on.h:117