FST  openfst-1.6.1
OpenFst Library
fst_test.h
Go to the documentation of this file.
1 // See www.openfst.org for extensive documentation on this weighted
2 // finite-state transducer library.
3 //
4 // Regression test for FST classes.
5 
6 #ifndef FST_TEST_FST_TEST_H_
7 #define FST_TEST_FST_TEST_H_
8 
9 #include <fst/equal.h>
10 #include <fstream>
11 #include <fst/matcher.h>
12 #include <fst/vector-fst.h>
13 #include <fst/verify.h>
14 
15 DECLARE_string(tmpdir);
16 
17 namespace fst {
18 
19 // This tests an Fst F that is assumed to have a copy method from an
20 // arbitrary Fst. Some test functions make further assumptions mostly
21 // obvious from their name. These tests are written as member temple
22 // functions that take a test fst as its argument so that different
23 // Fsts in the interface hierarchy can be tested separately and so
24 // that we can instantiate only those tests that make sense for a
25 // particular Fst.
26 template <class F>
27 class FstTester {
28  public:
29  typedef typename F::Arc Arc;
30  typedef typename Arc::StateId StateId;
31  typedef typename Arc::Weight Weight;
32  typedef typename Arc::Label Label;
33 
35  VectorFst<Arc> vfst;
36  InitFst(&vfst, 128);
37  testfst_ = new F(vfst);
38  }
39 
40  explicit FstTester(F *testfst) : testfst_(testfst) {}
41 
42  ~FstTester() { delete testfst_; }
43 
44  // This verifies the contents described in InitFst() using
45  // methods defined in a generic Fst.
46  template <class G>
47  void TestBase(const G &fst) const {
48  CHECK(Verify(fst));
49  CHECK_EQ(fst.Start(), 0);
50  StateId ns = 0;
51  StateIterator<G> siter(fst);
52  Matcher<G> matcher(fst, MATCH_INPUT);
53  MatchType match_type = matcher.Type(true);
54  for (; !siter.Done(); siter.Next()) {
55  }
56  for (siter.Reset(); !siter.Done(); siter.Next()) {
57  StateId s = siter.Value();
58  matcher.SetState(s);
59  CHECK_EQ(fst.Final(s), NthWeight(s));
60  size_t na = 0;
61  ArcIterator<G> aiter(fst, s);
62  for (; !aiter.Done(); aiter.Next()) {
63  }
64  for (aiter.Reset(); !aiter.Done(); aiter.Next()) {
65  ++na;
66  const Arc &arc = aiter.Value();
67  CHECK_EQ(arc.ilabel, na);
68  CHECK_EQ(arc.olabel, 0);
69  CHECK_EQ(arc.weight, NthWeight(na));
70  CHECK_EQ(arc.nextstate, s);
71  if (match_type == MATCH_INPUT) {
72  CHECK(matcher.Find(arc.ilabel));
73  CHECK_EQ(matcher.Value().ilabel, arc.ilabel);
74  }
75  }
76  CHECK_EQ(na, s);
77  CHECK_EQ(na, aiter.Position());
78  CHECK_EQ(fst.NumArcs(s), s);
79  CHECK_EQ(fst.NumInputEpsilons(s), 0);
80  CHECK_EQ(fst.NumOutputEpsilons(s), s);
81  CHECK(!matcher.Find(s + 1)); // out-of-range
82  CHECK(!matcher.Find(kNoLabel)); // no explicit epsilons
83  CHECK(matcher.Find(0));
84  CHECK_EQ(matcher.Value().ilabel, kNoLabel); // implicit epsilon loop
85  ++ns;
86  }
87  CHECK(fst.Properties(kNotAcceptor, true));
88  CHECK(fst.Properties(kOEpsilons, true));
89  }
90 
91  void TestBase() const { TestBase(*testfst_); }
92 
93  // This verifies methods specfic to an ExpandedFst.
94  template <class G>
95  void TestExpanded(const G &fst) const {
96  StateId ns = 0;
97  for (StateIterator<G> siter(fst); !siter.Done(); siter.Next()) {
98  ++ns;
99  }
100  CHECK_EQ(fst.NumStates(), ns);
101  CHECK(fst.Properties(kExpanded, false));
102  }
103 
104  void TestExpanded() const { TestExpanded(*testfst_); }
105 
106  // This verifies methods specific to a MutableFst.
107  template <class G>
108  void TestMutable(G *fst) const {
109  for (StateIterator<G> siter(*fst); !siter.Done(); siter.Next()) {
110  StateId s = siter.Value();
111  size_t na = 0;
112  size_t ni = fst->NumInputEpsilons(s);
113  MutableArcIterator<G> aiter(fst, s);
114  for (; !aiter.Done(); aiter.Next()) {
115  }
116  for (aiter.Reset(); !aiter.Done(); aiter.Next()) {
117  ++na;
118  Arc arc = aiter.Value();
119  arc.ilabel = 0;
120  aiter.SetValue(arc);
121  arc = aiter.Value();
122  CHECK_EQ(arc.ilabel, 0);
123  CHECK_EQ(fst->NumInputEpsilons(s), ni + 1);
124  arc.ilabel = na;
125  aiter.SetValue(arc);
126  CHECK_EQ(fst->NumInputEpsilons(s), ni);
127  }
128  }
129 
130  G *cfst1 = fst->Copy();
131  cfst1->DeleteStates();
132  CHECK_EQ(cfst1->NumStates(), 0);
133  delete cfst1;
134 
135  G *cfst2 = fst->Copy();
136  for (StateIterator<G> siter(*cfst2); !siter.Done(); siter.Next()) {
137  StateId s = siter.Value();
138  cfst2->DeleteArcs(s);
139  CHECK_EQ(cfst2->NumArcs(s), 0);
140  CHECK_EQ(cfst2->NumInputEpsilons(s), 0);
141  CHECK_EQ(cfst2->NumOutputEpsilons(s), 0);
142  }
143  delete cfst2;
144  }
145 
146  void TestMutable() { TestMutable(testfst_); }
147 
148  // This verifies the copy methods.
149  template <class G>
150  void TestAssign(G *fst) const {
151  // Assignment from G
152  G afst1;
153  afst1 = *fst;
154  CHECK(Equal(*fst, afst1));
155 
156  // Assignment from Fst
157  G afst2;
158  afst2 = *static_cast<const Fst<Arc> *>(fst);
159  CHECK(Equal(*fst, afst2));
160 
161  // Assignment from self
162  afst2.operator=(afst2);
163  CHECK(Equal(*fst, afst2));
164  }
165 
166  void TestAssign() { TestAssign(testfst_); }
167 
168  // This verifies the copy methods.
169  template <class G>
170  void TestCopy(const G &fst) const {
171  // Copy from G
172  G c1fst(fst);
173  TestBase(c1fst);
174 
175  // Copy from Fst
176  const G c2fst(static_cast<const Fst<Arc> &>(fst));
177  TestBase(c2fst);
178 
179  // Copy from self
180  const G *c3fst = fst.Copy();
181  TestBase(*c3fst);
182  delete c3fst;
183  }
184 
185  void TestCopy() const { TestCopy(*testfst_); }
186 
187  // This verifies the read/write methods.
188  template <class G>
189  void TestIO(const G &fst) const {
190  const string filename = FLAGS_tmpdir + "/test.fst";
191  const string aligned = FLAGS_tmpdir + "/aligned.fst";
192  {
193  // write/read
194  CHECK(fst.Write(filename));
195  G *ffst = G::Read(filename);
196  CHECK(ffst);
197  TestBase(*ffst);
198  delete ffst;
199  }
200 
201  {
202  // generic read/cast/test
203  Fst<Arc> *gfst = Fst<Arc>::Read(filename);
204  CHECK(gfst);
205  G *dfst = static_cast<G *>(gfst);
206  TestBase(*dfst);
207 
208  // generic write/read/test
209  CHECK(gfst->Write(filename));
210  Fst<Arc> *hfst = Fst<Arc>::Read(filename);
211  CHECK(hfst);
212  TestBase(*hfst);
213  delete gfst;
214  delete hfst;
215  }
216 
217  {
218  // check mmaping by first writing the file with the aligned attribute set
219  {
220  std::ofstream ostr(aligned.c_str());
221  FstWriteOptions opts;
222  opts.source = aligned;
223  opts.align = true;
224  CHECK(fst.Write(ostr, opts));
225  }
226  std::ifstream istr(aligned.c_str());
227  FstReadOptions opts;
228  opts.mode = FstReadOptions::ReadMode("map");
229  opts.source = aligned;
230  G *gfst = G::Read(istr, opts);
231  CHECK(gfst);
232  TestBase(*gfst);
233  delete gfst;
234  }
235 
236  // check mmaping of unaligned files to make sure it does not fail.
237  {
238  {
239  std::ofstream ostr(aligned.c_str());
240  FstWriteOptions opts;
241  opts.source = aligned;
242  opts.align = false;
243  CHECK(fst.Write(ostr, opts));
244  }
245  std::ifstream istr(aligned.c_str());
246  FstReadOptions opts;
247  opts.mode = FstReadOptions::ReadMode("map");
248  opts.source = aligned;
249  G *gfst = G::Read(istr, opts);
250  CHECK(gfst);
251  TestBase(*gfst);
252  delete gfst;
253  }
254 
255  // expanded write/read/test
256  if (fst.Properties(kExpanded, false)) {
257  ExpandedFst<Arc> *efst = ExpandedFst<Arc>::Read(filename);
258  CHECK(efst);
259  TestBase(*efst);
260  TestExpanded(*efst);
261  delete efst;
262  }
263 
264  // mutable write/read/test
265  if (fst.Properties(kMutable, false)) {
266  MutableFst<Arc> *mfst = MutableFst<Arc>::Read(filename);
267  CHECK(mfst);
268  TestBase(*mfst);
269  TestExpanded(*mfst);
270  TestMutable(mfst);
271  delete mfst;
272  }
273  }
274 
275  void TestIO() const { TestIO(*testfst_); }
276 
277  private:
278  // This constructs test FSTs. Given a mutable FST, will leave
279  // the FST as follows:
280  // (I) NumStates() = nstates
281  // (II) Start() = 0
282  // (III) Final(s) = NthWeight(s)
283  // (IV) For state s:
284  // (a) NumArcs(s) == s
285  // (b) For ith arc of s:
286  // (1) ilabel = i
287  // (2) olabel = 0
288  // (3) weight = NthWeight(i)
289  // (4) nextstate = s
290  void InitFst(MutableFst<Arc> *fst, size_t nstates) const {
291  fst->DeleteStates();
292  CHECK_GT(nstates, 0);
293 
294  for (StateId s = 0; s < nstates; ++s) {
295  fst->AddState();
296  fst->SetFinal(s, NthWeight(s));
297  for (size_t i = 1; i <= s; ++i) {
298  Arc arc(i, 0, NthWeight(i), s);
299  fst->AddArc(s, arc);
300  }
301  }
302 
303  fst->SetStart(0);
304  }
305 
306  // Generates One() + ... + One() (n times)
307  Weight NthWeight(int n) const {
308  Weight w = Weight::Zero();
309  for (int i = 0; i < n; ++i) w = Plus(w, Weight::One());
310  return w;
311  }
312 
313  F *testfst_; // what we're testing
314 };
315 
316 } // namespace fst
317 
318 #endif // FST_TEST_FST_TEST_H_
void TestExpanded(const G &fst) const
Definition: fst_test.h:95
DECLARE_string(tmpdir)
size_t Position() const
Definition: fst.h:528
MatchType Type(bool test) const
Definition: matcher.h:1422
string source
Definition: fst.h:84
void TestAssign(G *fst) const
Definition: fst_test.h:150
constexpr int kNoLabel
Definition: fst.h:180
void TestBase() const
Definition: fst_test.h:91
void Reset()
Definition: fst.h:512
void TestIO(const G &fst) const
Definition: fst_test.h:189
F::Arc Arc
Definition: fst_test.h:29
virtual void AddArc(StateId, const Arc &arc)=0
MatchType
Definition: fst.h:171
virtual bool Write(std::ostream &strm, const FstWriteOptions &opts) const
Definition: fst.h:270
virtual void SetStart(StateId)=0
void SetValue(const Arc &arc)
Definition: mutable-fst.h:219
FstTester(F *testfst)
Definition: fst_test.h:40
constexpr uint64 kExpanded
Definition: properties.h:27
const Arc & Value() const
Definition: fst.h:500
static Fst< Arc > * Read(std::istream &strm, const FstReadOptions &opts)
Definition: fst.h:234
const Arc & Value() const
Definition: mutable-fst.h:209
void TestCopy(const G &fst) const
Definition: fst_test.h:170
Arc::Label Label
Definition: fst_test.h:32
virtual void SetFinal(StateId, Weight)=0
void TestIO() const
Definition: fst_test.h:275
static FileReadMode ReadMode(const string &mode)
Definition: fst.cc:160
void TestCopy() const
Definition: fst_test.h:185
Arc::Weight Weight
Definition: fst_test.h:31
#define CHECK_GT(x, y)
Definition: log.h:64
void TestExpanded() const
Definition: fst_test.h:104
ExpectationWeight< X1, X2 > Plus(const ExpectationWeight< X1, X2 > &w1, const ExpectationWeight< X1, X2 > &w2)
StateId Value() const
Definition: fst.h:385
bool Done() const
Definition: fst.h:496
bool Find(Label label)
Definition: matcher.h:1426
static ExpandedFst< Arc > * Read(std::istream &strm, const FstReadOptions &opts)
Definition: expanded-fst.h:34
void TestAssign()
Definition: fst_test.h:166
bool Verify(const Fst< Arc > &fst, bool allow_negative_labels=false)
Definition: verify.h:19
void SetState(StateId s)
Definition: matcher.h:1424
bool Equal(const Fst< Arc > &fst1, const Fst< Arc > &fst2, float delta=kDelta, uint32 etype=kEqualFsts)
Definition: equal.h:30
void Next()
Definition: fst.h:387
constexpr uint64 kNotAcceptor
Definition: properties.h:47
#define CHECK_EQ(x, y)
Definition: log.h:62
void TestMutable(G *fst) const
Definition: fst_test.h:108
bool Done() const
Definition: fst.h:381
virtual StateId AddState()=0
FileReadMode mode
Definition: fst.h:64
virtual void DeleteStates(const std::vector< StateId > &)=0
void TestMutable()
Definition: fst_test.h:146
#define CHECK(x)
Definition: log.h:61
const Arc & Value() const
Definition: matcher.h:1430
static MutableFst< Arc > * Read(std::istream &strm, const FstReadOptions &opts)
Definition: mutable-fst.h:99
void Reset()
Definition: fst.h:395
void TestBase(const G &fst) const
Definition: fst_test.h:47
constexpr uint64 kMutable
Definition: properties.h:30
constexpr uint64 kOEpsilons
Definition: properties.h:70
Arc::StateId StateId
Definition: fst_test.h:30
void Next()
Definition: fst.h:504