20 #ifndef FST_EXTENSIONS_LINEAR_LINEAR_FST_DATA_H_ 21 #define FST_EXTENSIONS_LINEAR_LINEAR_FST_DATA_H_ 60 typedef typename A::Label
Label;
71 : max_future_size_(0), max_input_label_(1), input_attribs_(1) {}
86 template <
class Iterator>
87 void TakeTransition(Iterator buffer_end, Iterator trie_state_begin,
88 Iterator trie_state_end, Label ilabel, Label olabel,
89 std::vector<Label> *next, Weight *weight)
const;
92 template <
class Iterator>
93 Weight
FinalWeight(Iterator trie_state_begin, Iterator trie_state_end)
const;
97 return groups_[group_id]->Start();
103 Label olabel, Weight *weight)
const;
107 return groups_[group_id]->FinalWeight(trie_state);
125 std::pair<typename std::vector<Label>::const_iterator,
126 typename std::vector<Label>::const_iterator>
130 std::ostream &
Write(std::ostream &strm)
const;
134 struct InputAttribute {
135 size_t output_begin, output_length;
137 std::istream &
Read(std::istream &strm);
138 std::ostream &
Write(std::ostream &strm)
const;
147 Label FindFeature(
size_t group, Label word)
const;
149 size_t max_future_size_;
150 Label max_input_label_;
151 std::vector<std::unique_ptr<const FeatureGroup<A>>> groups_;
152 std::vector<InputAttribute> input_attribs_;
153 std::vector<Label> output_pool_, output_set_;
161 template <
class Iterator>
163 Iterator trie_state_begin,
164 Iterator trie_state_end,
Label ilabel,
165 Label olabel, std::vector<Label> *next,
167 DCHECK_EQ(trie_state_end - trie_state_begin, groups_.size());
171 for (Iterator it = trie_state_begin; it != trie_state_end; ++it, ++group_id) {
172 size_t delay = groups_[group_id]->Delay();
175 Label real_ilabel = delay == 0 ? ilabel : *(buffer_end - delay);
186 Label group_ilabel = FindFeature(group_id, ilabel);
187 return groups_[group_id]->Walk(trie_state, group_ilabel, olabel, weight);
191 template <
class Iterator>
193 Iterator trie_state_begin, Iterator trie_state_end)
const {
194 DCHECK_EQ(trie_state_end - trie_state_begin, groups_.size());
196 Weight accum = Weight::One();
197 for (Iterator it = trie_state_begin; it != trie_state_end; ++it, ++group_id)
203 inline std::pair<typename std::vector<typename A::Label>::const_iterator,
204 typename std::vector<typename A::Label>::const_iterator>
206 const InputAttribute &attrib = input_attribs_[word];
207 if (attrib.output_length == 0)
208 return std::make_pair(output_set_.begin(), output_set_.end());
210 return std::make_pair(
211 output_pool_.begin() + attrib.output_begin,
212 output_pool_.begin() + attrib.output_begin + attrib.output_length);
218 ReadType(strm, &(data->max_future_size_));
219 ReadType(strm, &(data->max_input_label_));
221 size_t num_groups = 0;
223 data->groups_.resize(num_groups);
224 for (
size_t i = 0; i < num_groups; ++i)
227 ReadType(strm, &(data->input_attribs_));
228 ReadType(strm, &(data->output_pool_));
229 ReadType(strm, &(data->output_set_));
230 ReadType(strm, &(data->group_feat_map_));
232 return data.release();
244 for (
size_t i = 0; i < groups_.size(); ++i) {
245 groups_[i]->Write(strm);
262 return group_feat_map_.Find(group, word);
267 std::istream &strm) {
275 std::ostream &strm)
const {
305 int Start()
const {
return start_; }
309 int Walk(
int cur, Label ilabel, Label olabel, Weight *weight)
const;
315 return trie_[trie_state].final_weight;
325 std::unique_ptr<FeatureGroup<A>> ret(
new FeatureGroup<A>(delay, start));
326 ret->trie_.swap(trie);
329 return ret.release();
335 std::ostream &
Write(std::ostream &strm)
const {
343 size_t Delay()
const {
return delay_; }
345 std::string Stats()
const;
356 struct WeightBackLink {
358 Weight weight, final_weight;
362 weight(Weight::One()),
363 final_weight(Weight::One()) {}
365 std::istream &
Read(std::istream &strm) {
372 std::ostream &
Write(std::ostream &strm)
const {
384 : delay_(delay), start_(start) {}
398 std::vector<int> next_state_;
409 : input(i), output(o) {}
412 return input == that.
input && output == that.
output;
415 std::istream &
Read(std::istream &strm) {
421 std::ostream &
Write(std::ostream &strm)
const {
431 return static_cast<size_t>(label.
input * 7853 + label.
output);
460 *weight =
Times(*weight, trie_[next].weight);
461 next = next_state_[next];
471 for (; parent !=
kNoTrieNodeId; parent = trie_[parent].back_link) {
472 int next = trie_.Find(parent, label);
480 std::ostringstream strm;
482 for (
int i = 2; i < next_state_.size(); ++i)
483 num_states += i == next_state_[i];
484 strm << trie_.NumNodes() <<
" node(s); " << num_states <<
" state(s)";
493 void Init(
size_t num_groups,
size_t num_words) {
494 num_groups_ = num_groups;
496 pool_.resize(num_groups * num_words,
kNoLabel);
500 return pool_[IndexOf(group_id, ilabel)];
504 size_t i = IndexOf(group_id, ilabel);
505 if (pool_[i] !=
kNoLabel && pool_[i] != feat) {
506 FSTERROR() <<
"Feature group " << group_id
507 <<
" already has feature for word " << ilabel;
514 std::istream &
Read(std::istream &strm) {
520 std::ostream &
Write(std::ostream &strm)
const {
527 size_t IndexOf(
size_t group_id,
Label ilabel)
const {
528 return ilabel * num_groups_ + group_id;
534 std::vector<Label> pool_;
539 #endif // FST_EXTENSIONS_LINEAR_LINEAR_FST_DATA_H_ std::string Stats() const
bool Set(size_t group_id, Label ilabel, Label feat)
std::ostream & Write(std::ostream &strm) const
Label GroupStartState(int group_id) const
std::pair< typename std::vector< Label >::const_iterator, typename std::vector< Label >::const_iterator > PossibleOutputLabels(Label word) const
void TakeTransition(Iterator buffer_end, Iterator trie_state_begin, Iterator trie_state_end, Label ilabel, Label olabel, std::vector< Label > *next, Weight *weight) const
int Walk(int cur, Label ilabel, Label olabel, Weight *weight) const
ErrorWeight Times(const ErrorWeight &, const ErrorWeight &)
std::istream & Read(std::istream &strm)
Weight GroupFinalWeight(int group_id, int trie_state) const
void Init(size_t num_groups, size_t num_words)
void EncodeStartState(std::vector< Label > *output) const
static constexpr Label kEndOfSentence
Label GroupTransition(int group_id, int trie_state, Label ilabel, Label olabel, Weight *weight) const
Label Find(size_t group_id, Label ilabel) const
std::ostream & WriteType(std::ostream &strm, const T t)
Weight FinalWeight(Iterator trie_state_begin, Iterator trie_state_end) const
constexpr int kNoTrieNodeId
std::ostream & Write(std::ostream &strm) const
static constexpr Label kStartOfSentence
static LinearFstData< A > * Read(std::istream &strm)
Weight FinalWeight(int trie_state) const
Label MinInputLabel() const
size_t MaxFutureSize() const
std::istream & ReadType(std::istream &strm, T *t)
Label MaxInputLabel() const
std::ostream & Write(std::ostream &strm) const
static FeatureGroup< A > * Read(std::istream &strm)