FST  openfst-1.8.2.post1
OpenFst Library
map-reduce.h
Go to the documentation of this file.
1 // Copyright 2005-2020 Google LLC
2 //
3 // Licensed under the Apache License, Version 2.0 (the 'License');
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an 'AS IS' BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 //
15 // See www.openfst.org for extensive documentation on this weighted
16 // finite-state transducer library.
17 //
18 // Map helper for FAR processing.
19 
20 #ifndef FST_EXTENSIONS_FAR_MAP_REDUCE_H_
21 #define FST_EXTENSIONS_FAR_MAP_REDUCE_H_
22 
23 #include <fst/extensions/far/far.h>
24 #include <fst/arc.h>
25 #include <fst/fst.h>
26 #include <string_view>
27 
28 namespace fst {
29 namespace internal {
30 
31 // This function applies a functor to each FST in a FAR reader, writing the
32 // resulting FST to a FAR writer. The functor must support the following
33 // interface:
34 //
35 // std::unique_ptr<Fst<Arc>> operator()(std::string_view key,
36 // const Fst<Arc> *fst);
37 //
38 // The functor signals an error by returning a null pointer.
39 //
40 // One can imagine more expressive variants of this: e.g., one which also
41 // transforms the key, but this can be added as needed.
42 //
43 // The caller is responsible for rewinding the reader afterwards, if desired,
44 // and for checking the error bit of the reader and writer.
45 template <class Arc, class Functor>
46 void Map(FarReader<Arc> &reader, FarWriter<Arc> &writer, Functor functor) {
47  for (; !reader.Done(); reader.Next()) {
48  const auto &key = reader.GetKey();
49  const auto fst = functor(key, reader.GetFst());
50  if (!fst) return;
51  writer.Add(key, *fst);
52  }
53 }
54 
55 // This function applies a boolean functor to pairs of FSTs passed via FAR
56 // readers, returning false if any of the keys do not match or if any of the
57 // functor calls are false. The functor must support the following interface:
58 //
59 // bool operator()(std::string_view key, const Fst<Arc> *fst1,
60 // const Fst<Arc> *fst2);
61 //
62 // One can imagine more expressive variants of this, but these can be added as
63 // needed.
64 //
65 // The caller is responsible for rewinding the readers afterwards, if desired,
66 // and for checking the error bits of the readers.
67 template <class Arc, class Functor>
68 bool MapAllReduce(FarReader<Arc> &reader1, FarReader<Arc> &reader2,
69  Functor functor, std::string_view begin_key = "",
70  std::string_view end_key = "") {
71  if (!begin_key.empty()) {
72  const bool find_begin1 = reader1.Find(begin_key);
73  const bool find_begin2 = reader2.Find(begin_key);
74  if (!find_begin1 || !find_begin2) {
75  const bool ret = !find_begin1 && !find_begin2;
76  if (!ret) {
77  LOG(ERROR) << "MapAllReduce: Key " << begin_key << " missing from "
78  << (find_begin1 ? "second" : "first") << " FAR";
79  }
80  return ret;
81  }
82  }
83  for (; !reader1.Done() && !reader2.Done(); reader1.Next(), reader2.Next()) {
84  const auto &key1 = reader1.GetKey();
85  const auto &key2 = reader2.GetKey();
86  if (!end_key.empty() && end_key < key1 && end_key < key2) {
87  return true;
88  }
89  if (key1 != key2) {
90  LOG(ERROR) << "MapAllReduce: Mismatched keys " << key1 << " and " << key2;
91  return false;
92  }
93  if (!functor(key1, reader1.GetFst(), reader2.GetFst())) return false;
94  }
95  if (reader1.Done() && !reader2.Done()) {
96  LOG(ERROR) << "MapAllReduce: Key " << reader2.GetKey()
97  << " missing from first FAR";
98  return false;
99  } else if (reader2.Done() && !reader1.Done()) {
100  LOG(ERROR) << "MapAllReduce: Key " << reader1.GetKey()
101  << " missing from second FAR";
102  return false;
103  }
104  return true;
105 }
106 
107 } // namespace internal
108 } // namespace fst
109 
110 #endif // FST_EXTENSIONS_FAR_MAP_REDUCE_H_
virtual const std::string & GetKey() const =0
bool MapAllReduce(FarReader< Arc > &reader1, FarReader< Arc > &reader2, Functor functor, std::string_view begin_key="", std::string_view end_key="")
Definition: map-reduce.h:68
virtual bool Done() const =0
#define LOG(type)
Definition: log.h:49
virtual const Fst< Arc > * GetFst() const =0
void Map(FarReader< Arc > &reader, FarWriter< Arc > &writer, Functor functor)
Definition: map-reduce.h:46
virtual void Add(std::string_view key, const Fst< Arc > &fst)=0
virtual void Next()=0
virtual bool Find(std::string_view key)=0