FST  openfst-1.7.8
OpenFst Library
fst_test.h
Go to the documentation of this file.
1 // See www.openfst.org for extensive documentation on this weighted
2 // finite-state transducer library.
3 //
4 // Regression test for FST classes.
5 
6 #ifndef FST_TEST_FST_TEST_H_
7 #define FST_TEST_FST_TEST_H_
8 
9 #include <fst/equal.h>
10 #include <fstream>
11 #include <fst/matcher.h>
12 #include <fst/vector-fst.h>
13 #include <fst/verify.h>
14 
15 DECLARE_string(tmpdir);
16 
17 namespace fst {
18 
19 // This tests an Fst F that is assumed to have a copy method from an
20 // arbitrary Fst. Some test functions make further assumptions mostly
21 // obvious from their name. These tests are written as member temple
22 // functions that take a test fst as its argument so that different
23 // Fsts in the interface hierarchy can be tested separately and so
24 // that we can instantiate only those tests that make sense for a
25 // particular Fst.
26 template <class F>
27 class FstTester {
28  public:
29  using Arc = typename F::Arc;
30  using StateId = typename Arc::StateId;
31  using Weight = typename Arc::Weight;
32  using Label = typename Arc::Label;
33 
34  explicit FstTester(size_t num_states = 128, bool weighted = true)
35  : num_states_(num_states), weighted_(weighted) {
36  VectorFst<Arc> vfst;
37  InitFst(&vfst, num_states);
38  testfst_ = new F(vfst);
39  }
40 
41  ~FstTester() { delete testfst_; }
42 
43  // This verifies the contents described in InitFst() using
44  // methods defined in a generic Fst.
45  template <class G>
46  void TestBase(const G &fst) const {
47  StateId ns = 0;
48  StateIterator<G> siter(fst);
49  Matcher<G> matcher(fst, MATCH_INPUT);
50  MatchType match_type = matcher.Type(true);
51  bool has_states = false;
52  for (; !siter.Done(); siter.Next()) {
53  has_states = true;
54  }
55  CHECK_EQ(fst.Start(), has_states ? 0 : kNoStateId);
56  for (siter.Reset(); !siter.Done(); siter.Next()) {
57  StateId s = siter.Value();
58  matcher.SetState(s);
59  CHECK_EQ(fst.Final(s), NthWeight(s));
60  size_t na = 0;
61  ArcIterator<G> aiter(fst, s);
62  for (; !aiter.Done(); aiter.Next()) {
63  }
64  for (aiter.Reset(); !aiter.Done(); aiter.Next()) {
65  ++na;
66  const Arc &arc = aiter.Value();
67  CHECK_EQ(arc.ilabel, na);
68  CHECK_EQ(arc.olabel, 0);
69  CHECK_EQ(arc.weight, NthWeight(na));
70  if (na == ns + 1) {
71  CHECK_EQ(arc.nextstate, s == num_states_ - 1 ? 0 : s + 1);
72  } else {
73  CHECK_EQ(arc.nextstate, s);
74  }
75  if (match_type == MATCH_INPUT) {
76  CHECK(matcher.Find(arc.ilabel));
77  CHECK_EQ(matcher.Value().ilabel, arc.ilabel);
78  }
79  }
80  CHECK_EQ(na, s + 1);
81  CHECK_EQ(na, aiter.Position());
82  CHECK_EQ(fst.NumArcs(s), s + 1);
83  CHECK_EQ(fst.NumInputEpsilons(s), 0);
84  CHECK_EQ(fst.NumOutputEpsilons(s), s + 1);
85  CHECK(!matcher.Find(s + 2)); // out-of-range
86  CHECK(!matcher.Find(kNoLabel)); // no explicit input epsilons
87  CHECK(matcher.Find(0));
88  CHECK_EQ(matcher.Value().ilabel, kNoLabel); // implicit epsilon loop
89  ++ns;
90  }
91  CHECK_EQ(num_states_, ns);
92  CHECK(Verify(fst));
93  CHECK(fst.Properties(ns > 0 ? kNotAcceptor : kAcceptor, true));
94  CHECK(fst.Properties(ns > 0 ? kOEpsilons : kNoOEpsilons, true));
95  }
96 
97  void TestBase() const { TestBase(*testfst_); }
98 
99  // This verifies methods specfic to an ExpandedFst.
100  template <class G>
101  void TestExpanded(const G &fst) const {
102  CHECK_EQ(fst.NumStates(), num_states_);
103  StateId ns = 0;
104  for (StateIterator<G> siter(fst); !siter.Done(); siter.Next()) {
105  ++ns;
106  }
107  CHECK_EQ(fst.NumStates(), ns);
108  CHECK(fst.Properties(kExpanded, false));
109  }
110 
111  void TestExpanded() const { TestExpanded(*testfst_); }
112 
113  // This verifies methods specific to a MutableFst.
114  template <class G>
115  void TestMutable(G *fst) const {
116  for (StateIterator<G> siter(*fst); !siter.Done(); siter.Next()) {
117  StateId s = siter.Value();
118  size_t na = 0;
119  size_t ni = fst->NumInputEpsilons(s);
120  MutableArcIterator<G> aiter(fst, s);
121  for (; !aiter.Done(); aiter.Next()) {
122  }
123  for (aiter.Reset(); !aiter.Done(); aiter.Next()) {
124  ++na;
125  Arc arc = aiter.Value();
126  arc.ilabel = 0;
127  aiter.SetValue(arc);
128  arc = aiter.Value();
129  CHECK_EQ(arc.ilabel, 0);
130  CHECK_EQ(fst->NumInputEpsilons(s), ni + 1);
131  arc.ilabel = na;
132  aiter.SetValue(arc);
133  CHECK_EQ(fst->NumInputEpsilons(s), ni);
134  }
135  }
136 
137  G *cfst1 = fst->Copy();
138  cfst1->DeleteStates();
139  CHECK_EQ(cfst1->NumStates(), 0);
140  delete cfst1;
141 
142  G *cfst2 = fst->Copy();
143  for (StateIterator<G> siter(*cfst2); !siter.Done(); siter.Next()) {
144  StateId s = siter.Value();
145  cfst2->DeleteArcs(s);
146  CHECK_EQ(cfst2->NumArcs(s), 0);
147  CHECK_EQ(cfst2->NumInputEpsilons(s), 0);
148  CHECK_EQ(cfst2->NumOutputEpsilons(s), 0);
149  }
150  delete cfst2;
151  }
152 
153  void TestMutable() { TestMutable(testfst_); }
154 
155  // This verifies operator=
156  template <class G>
157  void TestAssign(const G &fst) const {
158  // Assignment from G
159  G afst1;
160  afst1 = fst;
161  CHECK(Equal(fst, afst1));
162 
163  // Assignment from Fst
164  G afst2;
165  afst2 = static_cast<const Fst<Arc> &>(fst);
166  CHECK(Equal(fst, afst2));
167 
168  // Assignment from self
169  afst2.operator=(afst2);
170  CHECK(Equal(fst, afst2));
171  }
172 
173  void TestAssign() { TestAssign(*testfst_); }
174 
175  // This verifies the copy constructor and Copy method.
176  template <class G>
177  void TestCopy(const G &fst) const {
178  // Copy from G
179  G c1fst(fst);
180  TestBase(c1fst);
181 
182  // Copy from Fst
183  const G c2fst(static_cast<const Fst<Arc> &>(fst));
184  TestBase(c2fst);
185 
186  // Copy from self
187  const G *c3fst = fst.Copy();
188  TestBase(*c3fst);
189  delete c3fst;
190  }
191 
192  void TestCopy() const { TestCopy(*testfst_); }
193 
194  // This verifies the read/write methods.
195  template <class G>
196  void TestIO(const G &fst) const {
197  const std::string filename = FLAGS_tmpdir + "/test.fst";
198  const std::string aligned = FLAGS_tmpdir + "/aligned.fst";
199  {
200  // write/read
201  CHECK(fst.Write(filename));
202  G *ffst = G::Read(filename);
203  CHECK(ffst);
204  TestBase(*ffst);
205  delete ffst;
206  }
207 
208  {
209  // generic read/cast/test
210  Fst<Arc> *gfst = Fst<Arc>::Read(filename);
211  CHECK(gfst);
212  G *dfst = static_cast<G *>(gfst);
213  TestBase(*dfst);
214 
215  // generic write/read/test
216  CHECK(gfst->Write(filename));
217  Fst<Arc> *hfst = Fst<Arc>::Read(filename);
218  CHECK(hfst);
219  TestBase(*hfst);
220  delete gfst;
221  delete hfst;
222  }
223 
224  {
225  // check mmaping by first writing the file with the aligned attribute set
226  {
227  std::ofstream ostr(aligned);
228  FstWriteOptions opts;
229  opts.source = aligned;
230  opts.align = true;
231  CHECK(fst.Write(ostr, opts));
232  }
233  std::ifstream istr(aligned);
234  FstReadOptions opts;
235  opts.mode = FstReadOptions::ReadMode("map");
236  opts.source = aligned;
237  G *gfst = G::Read(istr, opts);
238  CHECK(gfst);
239  TestBase(*gfst);
240  delete gfst;
241  }
242 
243  // check mmaping of unaligned files to make sure it does not fail.
244  {
245  {
246  std::ofstream ostr(aligned);
247  FstWriteOptions opts;
248  opts.source = aligned;
249  opts.align = false;
250  CHECK(fst.Write(ostr, opts));
251  }
252  std::ifstream istr(aligned);
253  FstReadOptions opts;
254  opts.mode = FstReadOptions::ReadMode("map");
255  opts.source = aligned;
256  G *gfst = G::Read(istr, opts);
257  CHECK(gfst);
258  TestBase(*gfst);
259  delete gfst;
260  }
261 
262  // expanded write/read/test
263  if (fst.Properties(kExpanded, false)) {
264  ExpandedFst<Arc> *efst = ExpandedFst<Arc>::Read(filename);
265  CHECK(efst);
266  TestBase(*efst);
267  TestExpanded(*efst);
268  delete efst;
269  }
270 
271  // mutable write/read/test
272  if (fst.Properties(kMutable, false)) {
273  MutableFst<Arc> *mfst = MutableFst<Arc>::Read(filename);
274  CHECK(mfst);
275  TestBase(*mfst);
276  TestExpanded(*mfst);
277  TestMutable(mfst);
278  delete mfst;
279  }
280  }
281 
282  void TestIO() const { TestIO(*testfst_); }
283 
284  private:
285  // This constructs test FSTs. Given a mutable FST, will leave
286  // the FST as follows:
287  // (I) NumStates() = nstates
288  // (II) Start() = 0
289  // (III) Final(s) = NthWeight(s)
290  // (IV) For state s:
291  // (a) NumArcs(s) == s + 1
292  // (b) For ith arc (i: 1 to s) of s:
293  // (1) ilabel = i
294  // (2) olabel = 0
295  // (3) weight = NthWeight(i)
296  // (4) nextstate = s
297  // (c) s+1st arc of s:
298  // (1) ilabel = s + 1
299  // (2) olabel = 0
300  // (3) weight = NthWeight(s + 1)
301  // (4) nextstate = s + 1 if s < nstates - 1
302  // 0 if s == nstates - 1
303  void InitFst(MutableFst<Arc> *fst, size_t nstates) const {
304  fst->DeleteStates();
305 
306  for (StateId s = 0; s < nstates; ++s) {
307  fst->AddState();
308  fst->SetFinal(s, NthWeight(s));
309  for (size_t i = 1; i <= s; ++i) {
310  Arc arc(i, 0, NthWeight(i), s);
311  fst->AddArc(s, arc);
312  }
313  fst->AddArc(
314  s, Arc(s + 1, 0, NthWeight(s + 1), s == nstates - 1 ? 0 : s + 1));
315  }
316 
317  if (nstates > 0) fst->SetStart(0);
318  }
319 
320  // Generates One() + ... + One() (n times) if weighted_,
321  // otherwise One().
322  Weight NthWeight(int n) const {
323  if (!weighted_) return Weight::One();
324  Weight w = Weight::Zero();
325  for (int i = 0; i < n; ++i) w = Plus(w, Weight::One());
326  return w;
327  }
328 
329  size_t num_states_ = 0;
330  bool weighted_ = true;
331  F *testfst_; // what we're testing
332 };
333 
334 } // namespace fst
335 
336 #endif // FST_TEST_FST_TEST_H_
void TestExpanded(const G &fst) const
Definition: fst_test.h:101
DECLARE_string(tmpdir)
size_t Position() const
Definition: fst.h:553
MatchType Type(bool test) const
Definition: matcher.h:1532
bool Equal(const Fst< Arc > &fst1, const Fst< Arc > &fst2, WeightEqual weight_equal, uint8 etype=kEqualFsts)
Definition: equal.h:45
constexpr int kNoLabel
Definition: fst.h:185
static MutableFst * Read(std::istream &strm, const FstReadOptions &opts)
Definition: mutable-fst.h:107
void TestBase() const
Definition: fst_test.h:97
static FileReadMode ReadMode(const std::string &mode)
Definition: fst.cc:137
void Reset()
Definition: fst.h:537
static Fst * Read(std::istream &strm, const FstReadOptions &opts)
Definition: fst.h:241
void TestIO(const G &fst) const
Definition: fst_test.h:196
FstTester(size_t num_states=128, bool weighted=true)
Definition: fst_test.h:34
std::string source
Definition: fst.h:58
MatchType
Definition: fst.h:177
virtual bool Write(std::ostream &strm, const FstWriteOptions &opts) const
Definition: fst.h:277
virtual void SetStart(StateId)=0
void TestAssign(const G &fst) const
Definition: fst_test.h:157
void SetValue(const Arc &arc)
Definition: mutable-fst.h:227
constexpr int kNoStateId
Definition: fst.h:186
constexpr uint64 kExpanded
Definition: properties.h:30
const Arc & Value() const
Definition: fst.h:525
const Arc & Value() const
Definition: mutable-fst.h:217
std::string source
Definition: fst.h:86
void TestCopy(const G &fst) const
Definition: fst_test.h:177
void TestIO() const
Definition: fst_test.h:282
void TestCopy() const
Definition: fst_test.h:192
typename Arc::Weight Weight
Definition: fst_test.h:31
void TestExpanded() const
Definition: fst_test.h:111
StateId Value() const
Definition: fst.h:405
bool Done() const
Definition: fst.h:521
bool Find(Label label)
Definition: matcher.h:1536
constexpr uint64 kAcceptor
Definition: properties.h:48
constexpr uint64 kNoOEpsilons
Definition: properties.h:75
void TestAssign()
Definition: fst_test.h:173
bool Verify(const Fst< Arc > &fst, bool allow_negative_labels=false)
Definition: verify.h:20
typename F::Arc Arc
Definition: fst_test.h:29
void SetState(StateId s)
Definition: matcher.h:1534
virtual void AddArc(StateId, const Arc &)=0
void Next()
Definition: fst.h:407
constexpr uint64 kNotAcceptor
Definition: properties.h:50
#define CHECK_EQ(x, y)
Definition: log.h:60
void TestMutable(G *fst) const
Definition: fst_test.h:115
bool Done() const
Definition: fst.h:401
static ExpandedFst * Read(std::istream &strm, const FstReadOptions &opts)
Definition: expanded-fst.h:37
virtual StateId AddState()=0
virtual void SetFinal(StateId s, Weight weight=Weight::One())=0
FileReadMode mode
Definition: fst.h:65
virtual void DeleteStates(const std::vector< StateId > &)=0
void TestMutable()
Definition: fst_test.h:153
typename Arc::StateId StateId
Definition: fst_test.h:30
ExpectationWeight< W1, W2 > Plus(const ExpectationWeight< W1, W2 > &w1, const ExpectationWeight< W1, W2 > &w2)
#define CHECK(x)
Definition: log.h:59
const Arc & Value() const
Definition: matcher.h:1540
void Reset()
Definition: fst.h:415
void TestBase(const G &fst) const
Definition: fst_test.h:46
typename Arc::Label Label
Definition: fst_test.h:32
constexpr uint64 kMutable
Definition: properties.h:33
constexpr uint64 kOEpsilons
Definition: properties.h:73
void Next()
Definition: fst.h:529