39 #ifndef FST_EDIT_FST_H_ 40 #define FST_EDIT_FST_H_ 59 #include <unordered_map> 60 #include <string_view> 80 template <
typename Arc,
typename WrappedFstT = ExpandedFst<Arc>,
81 typename MutableFstT = VectorFst<Arc>>
90 : edits_(other.edits_),
91 external_to_internal_ids_(other.external_to_internal_ids_),
92 edited_final_weights_(other.edited_final_weights_),
93 num_new_states_(other.num_new_states_) {}
103 edits_.Write(strm, edits_opts);
104 WriteType(strm, external_to_internal_ids_);
108 LOG(ERROR) <<
"EditFstData::Write: Write failed: " << opts.
source;
120 auto final_weight_it = GetFinalWeightIterator(s);
121 if (final_weight_it == NotInFinalWeightMap()) {
122 const auto it = GetEditedIdMapIterator(s);
123 return it == NotInEditedMap() ? wrapped->Final(s)
124 : edits_.Final(it->second);
126 return final_weight_it->second;
131 const auto it = GetEditedIdMapIterator(s);
132 return it == NotInEditedMap() ? wrapped->NumArcs(s)
133 : edits_.NumArcs(it->second);
137 const auto it = GetEditedIdMapIterator(s);
138 return it == NotInEditedMap() ? wrapped->NumInputEpsilons(s)
139 : edits_.NumInputEpsilons(it->second);
143 const auto it = GetEditedIdMapIterator(s);
144 return it == NotInEditedMap() ? wrapped->NumOutputEpsilons(s)
145 : edits_.NumOutputEpsilons(it->second);
149 edits_.SetProperties(props, mask);
159 const auto old_weight =
Final(s, wrapped);
160 const auto it = GetEditedIdMapIterator(s);
164 if (it == NotInEditedMap()) {
165 edited_final_weights_[s] = weight;
167 edits_.SetFinal(GetEditableInternalId(s, wrapped), weight);
174 external_to_internal_ids_[curr_num_states] = edits_.AddState();
176 return curr_num_states;
181 for (
size_t i = 0; i < n; ++i) {
182 curr_num_states =
AddState(curr_num_states);
188 const auto internal_id = GetEditableInternalId(s, wrapped);
189 const auto num_arcs = edits_.NumArcs(internal_id);
191 const Arc *prev_arc =
nullptr;
194 arc_it.
Seek(num_arcs - 1);
195 prev_arc = &(arc_it.
Value());
197 edits_.AddArc(internal_id, arc);
202 edits_.DeleteStates();
204 external_to_internal_ids_.clear();
205 edited_final_weights_.clear();
210 edits_.DeleteArcs(GetEditableInternalId(s, wrapped), n);
215 edits_.DeleteArcs(GetEditableInternalId(s, wrapped));
222 const WrappedFstT *wrapped)
const {
223 const auto it = GetEditedIdMapIterator(s);
224 if (it == NotInEditedMap()) {
225 VLOG(3) <<
"EditFstData::InitArcIterator: iterating on state " << s
226 <<
" of original FST";
227 wrapped->InitArcIterator(s, data);
229 VLOG(2) <<
"EditFstData::InitArcIterator: iterating on edited state " << s
230 <<
" (internal state ID: " << it->second <<
")";
231 edits_.InitArcIterator(it->second, data);
237 const WrappedFstT *wrapped) {
238 data->
base = std::make_unique<MutableArcIterator<MutableFstT>>(
239 &edits_, GetEditableInternalId(s, wrapped));
245 for (
auto it = external_to_internal_ids_.begin(); it != NotInEditedMap();
247 LOG(INFO) <<
"(external,internal)=(" << it->first <<
"," << it->second
255 typename std::unordered_map<StateId, StateId>::const_iterator
256 GetEditedIdMapIterator(
StateId s)
const {
257 return external_to_internal_ids_.find(s);
260 typename std::unordered_map<StateId, StateId>::const_iterator
261 NotInEditedMap()
const {
262 return external_to_internal_ids_.end();
265 typename std::unordered_map<StateId, Weight>::const_iterator
266 GetFinalWeightIterator(
StateId s)
const {
267 return edited_final_weights_.find(s);
270 typename std::unordered_map<StateId, Weight>::const_iterator
271 NotInFinalWeightMap()
const {
272 return edited_final_weights_.end();
278 StateId GetEditableInternalId(
StateId s,
const WrappedFstT *wrapped) {
279 auto id_map_it = GetEditedIdMapIterator(s);
280 if (id_map_it == NotInEditedMap()) {
281 StateId new_internal_id = edits_.AddState();
282 VLOG(2) <<
"EditFstData::GetEditableInternalId: editing state " << s
283 <<
" of original FST; new internal state id:" << new_internal_id;
284 external_to_internal_ids_[s] = new_internal_id;
286 !arc_iterator.Done(); arc_iterator.Next()) {
287 edits_.AddArc(new_internal_id, arc_iterator.Value());
290 auto final_weight_it = GetFinalWeightIterator(s);
291 if (final_weight_it == NotInFinalWeightMap()) {
292 edits_.SetFinal(new_internal_id, wrapped->Final(s));
294 edits_.SetFinal(new_internal_id, final_weight_it->second);
295 edited_final_weights_.erase(s);
297 return new_internal_id;
299 return id_map_it->second;
309 std::unordered_map<StateId, StateId> external_to_internal_ids_;
314 std::unordered_map<StateId, Weight> edited_final_weights_;
322 template <
typename A,
typename WrappedFstT,
typename MutableFstT>
326 auto data = fst::make_unique_for_overwrite<EditFstData>();
330 edits_opts.header =
nullptr;
338 std::unique_ptr<MutableFstT> edits(MutableFstT::Read(strm, edits_opts));
339 if (!edits)
return nullptr;
340 data->edits_ = *edits;
343 ReadType(strm, &data->external_to_internal_ids_);
344 ReadType(strm, &data->edited_final_weights_);
345 ReadType(strm, &data->num_new_states_);
347 LOG(ERROR) <<
"EditFst::Read: read failed: " << opts.
source;
350 return data.release();
370 template <
typename A,
typename WrappedFstT = ExpandedFst<A>,
371 typename MutableFstT = VectorFst<A>>
391 InheritPropertiesFromWrapped();
392 data_ = std::make_shared<EditFstData<Arc, WrappedFstT, MutableFstT>>();
409 : wrapped_(
down_cast<WrappedFstT *>(wrapped.Copy())) {
411 data_ = std::make_shared<EditFstData<Arc, WrappedFstT, MutableFstT>>();
413 data_->SetEditedProperties(wrapped_->Properties(
kFstProperties,
false),
415 InheritPropertiesFromWrapped();
422 wrapped_(
down_cast<WrappedFstT *>(impl.wrapped_->Copy(true))),
430 const auto edited_start = data_->EditedStart();
431 return edited_start ==
kNoStateId ? wrapped_->Start() : edited_start;
439 return data_->NumInputEpsilons(s, wrapped_.get());
443 return data_->NumOutputEpsilons(s, wrapped_.get());
447 return wrapped_->NumStates() + data_->NumNewStates();
460 WriteHeader(strm, header_opts, kFileVersion, &hdr);
465 wrapped_->Write(strm, wrapped_opts);
466 data_->Write(strm, opts);
469 LOG(ERROR) <<
"EditFst::Write: Write failed: " << opts.
source;
485 Weight old_weight = data_->SetFinal(s, weight, wrapped_.get());
494 return data_->AddState(NumStates());
501 return data_->AddStates(NumStates(), n);
507 const auto *prev_arc = data_->AddArc(s, arc, wrapped_.get());
513 FSTERROR() <<
": EditFstImpl::DeleteStates(const std::vector<StateId>&): " 514 <<
" not implemented";
524 data_->DeleteArcs(s, n, wrapped_.get());
531 data_->DeleteArcs(s, wrapped_.get());
543 data->
base =
nullptr;
549 data_->InitArcIterator(s, data, wrapped_.get());
555 data_->InitMutableArcIterator(s, data, wrapped_.get());
562 static constexpr
int kFileVersion = 2;
564 static constexpr
int kMinFileVersion = 2;
569 void InheritPropertiesFromWrapped() {
572 SetInputSymbols(wrapped_->InputSymbols());
573 SetOutputSymbols(wrapped_->OutputSymbols());
587 if (!data_.unique()) {
589 std::make_shared<EditFstData<Arc, WrappedFstT, MutableFstT>>(*data_);
595 std::unique_ptr<const WrappedFstT> wrapped_;
598 std::shared_ptr<EditFstData<Arc, WrappedFstT, MutableFstT>> data_;
601 template <
typename Arc,
typename WrappedFstT,
typename MutableFstT>
603 data_->DeleteStates();
606 wrapped_ = std::make_unique<MutableFstT>();
607 const auto new_props =
612 template <
typename Arc,
typename WrappedFstT,
typename MutableFstT>
616 auto impl = std::make_unique<EditFstImpl>();
618 if (!impl->ReadHeader(strm, opts, kMinFileVersion, &hdr))
return nullptr;
623 wrapped_opts.
header =
nullptr;
624 std::unique_ptr<Fst<Arc>> wrapped_fst(
Fst<Arc>::Read(strm, wrapped_opts));
625 if (!wrapped_fst)
return nullptr;
626 impl->wrapped_.reset(down_cast<WrappedFstT *>(wrapped_fst.release()));
627 impl->data_ = std::shared_ptr<EditFstData<Arc, WrappedFstT, MutableFstT>>(
629 if (!impl->data_)
return nullptr;
630 return impl.release();
638 template <
typename A,
typename WrappedFstT = ExpandedFst<A>,
639 typename MutableFstT = VectorFst<A>>
641 internal::EditFstImpl<A, WrappedFstT, MutableFstT>> {
666 return new EditFst(*
this, safe);
675 SetImpl(std::make_shared<Impl>(fst));
681 auto *impl = Impl::Read(strm, opts);
682 return impl ?
new EditFst(std::shared_ptr<Impl>(impl)) :
nullptr;
689 return impl ?
new EditFst(std::shared_ptr<Impl>(impl)) :
nullptr;
693 return GetImpl()->Write(strm, opts);
696 bool Write(
const std::string &source)
const override {
701 GetImpl()->InitStateIterator(data);
705 GetImpl()->InitArcIterator(s, data);
710 GetMutableImpl()->InitMutableArcIterator(s, data);
723 #endif // FST_EDIT_FST_H_
EditFst * Copy(bool safe=false) const override
static EditFst * Read(std::istream &strm, const FstReadOptions &opts)
void ReserveArcs(StateId s, size_t n)
uint64_t AddArcProperties(uint64_t inprops, typename A::StateId s, const A &arc, const A *prev_arc)
void SetProperties(uint64_t props)
constexpr uint64_t kMutable
static EditFst * Read(std::string_view source)
void InitMutableArcIterator(StateId s, MutableArcIteratorData< Arc > *data, const WrappedFstT *wrapped)
typename Arc::Weight Weight
Weight SetFinal(StateId s, Weight weight, const WrappedFstT *wrapped)
uint64_t SetStartProperties(uint64_t inprops)
bool Write(const std::string &source) const override
void AddArc(StateId s, const Arc &arc)
std::unique_ptr< MutableArcIteratorBase< Arc > > base
std::shared_ptr< Impl > GetSharedImpl() const
void InitArcIterator(StateId s, ArcIteratorData< Arc > *data) const
StateId NumStates() const
size_t NumInputEpsilons(StateId s, const WrappedFstT *wrapped) const
StateId AddState(StateId curr_num_states)
uint64_t AddStateProperties(uint64_t inprops)
uint64_t DeleteAllStatesProperties(uint64_t inprops, uint64_t staticProps)
constexpr uint64_t kError
size_t NumArcs(StateId s, const WrappedFstT *wrapped) const
EditFstImpl(const Fst< Arc > &wrapped)
void DeleteArcs(StateId s)
StateId NumNewStates() const
static EditFstData * Read(std::istream &strm, const FstReadOptions &opts)
void SetEditedProperties(uint64_t props, uint64_t mask)
uint64_t SetFinalProperties(uint64_t inprops, const Weight &old_weight, const Weight &new_weight)
typename Arc::StateId StateId
void DeleteStates(const std::vector< StateId > &dstates)
EditFst(const WrappedFstT &fst)
const Arc & Value() const
typename Arc::StateId StateId
std::ostream & WriteType(std::ostream &strm, const T t)
void AddStates(StateId curr_num_states, size_t n)
void DeleteArcs(StateId s, size_t n, const WrappedFstT *wrapped)
static EditFstImpl * Read(std::istream &strm, const FstReadOptions &opts)
typename Arc::Weight Weight
virtual uint64_t Properties() const
constexpr uint64_t kCopyProperties
Weight Final(StateId s, const WrappedFstT *wrapped) const
void DeleteArcs(StateId s, size_t n)
StateId EditedStart() const
void InitArcIterator(StateId s, ArcIteratorData< Arc > *data, const WrappedFstT *wrapped) const
void InitArcIterator(StateId s, ArcIteratorData< Arc > *data) const override
EditFst(const Fst< Arc > &fst)
size_t NumInputEpsilons(StateId s) const
std::unique_ptr< StateIteratorBase< Arc > > base
size_t NumOutputEpsilons(StateId s, const WrappedFstT *wrapped) const
bool Write(std::ostream &strm, const FstWriteOptions &opts) const override
EditFstImpl(const EditFstImpl &impl)
EditFstData(const EditFstData &other)
bool WriteFile(const std::string &source) const
void ReserveStates(StateId s)
constexpr uint64_t kFstProperties
Weight Final(StateId s) const
void InitStateIterator(StateIteratorData< Arc > *data) const override
void SetType(std::string_view type)
const Arc * AddArc(StateId s, const Arc &arc, const WrappedFstT *wrapped)
void SetFinal(StateId s, Weight weight)
EditFst & operator=(const Fst< Arc > &fst) override
std::istream & ReadType(std::istream &strm, T *t)
uint64_t DeleteArcsProperties(uint64_t inprops)
void InitMutableArcIterator(StateId s, MutableArcIteratorData< Arc > *data)
typename Arc::StateId StateId
size_t NumArcs(StateId s) const
EditFst(const EditFst &fst, bool safe=false)
bool Write(std::ostream &strm, const FstWriteOptions &opts) const
size_t NumOutputEpsilons(StateId s) const
void InitMutableArcIterator(StateId s, MutableArcIteratorData< A > *data) override
void InitStateIterator(StateIteratorData< Arc > *data) const
constexpr uint64_t kExpanded
void DeleteArcs(StateId s, const WrappedFstT *wrapped)
bool Write(std::ostream &strm, const FstWriteOptions &opts) const
EditFst & operator=(const EditFst &fst)