24 #include <sys/types.h> 50 #include <string_view> 85 const std::string_view source =
"<unspecified>",
110 bool write_header =
true,
bool write_isymbols =
true,
111 bool write_osymbols =
true,
112 bool align = FST_FLAGS_fst_align,
113 bool stream_write =
false)
115 write_header(write_header),
116 write_isymbols(write_isymbols),
117 write_osymbols(write_osymbols),
119 stream_write(stream_write) {}
136 const std::string &
FstType()
const {
return fsttype_; }
138 const std::string &
ArcType()
const {
return arctype_; }
146 int64_t
Start()
const {
return start_; }
152 void SetFstType(std::string_view type) { fsttype_ = std::string(type); }
154 void SetArcType(std::string_view type) { arctype_ = std::string(type); }
168 bool Read(std::istream &strm,
const std::string &
source,
bool rewind =
false);
170 bool Write(std::ostream &strm, std::string_view source)
const;
176 std::string fsttype_;
177 std::string arctype_;
178 int32_t version_ = 0;
180 uint64_t properties_ = 0;
182 int64_t numstates_ = 0;
183 int64_t numarcs_ = 0;
209 virtual ~
Fst() =
default;
212 virtual StateId Start()
const = 0;
235 virtual uint64_t Properties(uint64_t mask,
bool test)
const = 0;
238 virtual const std::string &Type()
const = 0;
254 virtual Fst *Copy(
bool safe =
false)
const = 0;
263 if (!hdr.
Read(strm, opts.
source))
return nullptr;
266 const auto &fst_type = hdr.
FstType();
269 LOG(ERROR) <<
"Fst::Read: Unknown FST type " << fst_type
270 <<
" (arc type = " << Arc::Type() <<
"): " << ropts.
source;
273 return reader(strm, ropts);
279 if (!source.empty()) {
280 std::ifstream strm(source,
281 std::ios_base::in | std::ios_base::binary);
283 LOG(ERROR) <<
"Fst::Read: Can't open file: " <<
source;
294 LOG(ERROR) <<
"Fst::Write: No write stream method for " << Type()
302 LOG(ERROR) <<
"Fst::Write: No write source method for " << Type()
314 virtual const SymbolTable *InputSymbols()
const = 0;
317 virtual const SymbolTable *OutputSymbols()
const = 0;
333 if (!source.empty()) {
334 std::ofstream strm(source,
335 std::ios_base::out | std::ios_base::binary);
337 LOG(ERROR) <<
"Fst::WriteFile: Can't open file: " <<
source;
341 LOG(ERROR) <<
"Fst::WriteFile: Write failed: " <<
source;
366 virtual bool Done()
const = 0;
368 virtual StateId Value()
const = 0;
370 virtual void Next() = 0;
372 virtual void Reset() = 0;
382 std::unique_ptr<StateIteratorBase<Arc>>
base;
408 using Arc =
typename FST::Arc;
412 fst.InitStateIterator(&data_);
416 return data_.base ? data_.base->Done() : s_ >= data_.nstates;
419 StateId Value()
const {
return data_.base ? data_.base->Value() : s_; }
467 virtual bool Done()
const = 0;
469 virtual const Arc &Value()
const = 0;
471 virtual void Next() = 0;
473 virtual size_t Position()
const = 0;
475 virtual void Reset() = 0;
477 virtual void Seek(
size_t) = 0;
479 virtual uint8_t Flags()
const = 0;
481 virtual void SetFlags(uint8_t, uint8_t) = 0;
493 std::unique_ptr<ArcIteratorBase<Arc>>
495 const Arc *arcs =
nullptr;
497 int *ref_count =
nullptr;
517 using Arc =
typename FST::Arc;
521 fst.InitArcIterator(s, &data_);
527 if (data_.ref_count) {
528 --(*data_.ref_count);
533 return data_.base ? data_.base->Done() : i_ >= data_.narcs;
537 return data_.base ? data_.base->Value() : data_.arcs[i_];
564 size_t Position()
const {
return data_.base ? data_.base->Position() : i_; }
571 if (data_.base) data_.base->SetFlags(flags, mask);
585 void *
operator new(
size_t size,
587 return pool->Allocate();
614 inline typename F::Arc::Weight
Final(
const F &
fst,
typename F::Arc::StateId s) {
615 return fst.F::Final(s);
619 inline ssize_t
NumArcs(
const F &
fst,
typename F::Arc::StateId s) {
620 return fst.F::NumArcs(s);
625 return fst.F::NumInputEpsilons(s);
630 return fst.F::NumOutputEpsilons(s);
637 typename Arc::StateId s) {
677 : properties_(impl.properties_.load(std::memory_order_relaxed)),
679 isymbols_(impl.isymbols_ ? impl.isymbols_->Copy() : nullptr),
680 osymbols_(impl.osymbols_ ? impl.osymbols_->Copy() : nullptr) {}
687 properties_.store(impl.
properties_.load(std::memory_order_relaxed),
688 std::memory_order_relaxed);
690 isymbols_ = impl.isymbols_ ? impl.isymbols_->Copy() :
nullptr;
691 osymbols_ = impl.osymbols_ ? impl.osymbols_->Copy() :
nullptr;
697 const std::string &
Type()
const {
return type_; }
699 void SetType(std::string_view type) { type_ = std::string(type); }
702 return properties_.load(std::memory_order_relaxed);
706 return properties_.load(std::memory_order_relaxed) & mask;
710 uint64_t properties = properties_.load(std::memory_order_relaxed);
713 properties_.store(properties, std::memory_order_relaxed);
720 uint64_t properties = properties_.load(std::memory_order_relaxed);
721 properties &= ~mask |
kError;
722 properties |= props & mask;
723 properties_.store(properties, std::memory_order_relaxed);
729 FSTERROR() <<
"FstImpl::SetProperties() const: Can only set kError";
731 properties_.fetch_or(
kError, std::memory_order_relaxed);
746 const uint64_t properties = properties_.load(std::memory_order_relaxed);
748 const uint64_t old_props = properties & mask;
750 const uint64_t discovered_mask = mask & ~old_mask;
751 const uint64_t discovered_props = props & discovered_mask;
754 if (discovered_props != 0) {
755 properties_.fetch_or(discovered_props, std::memory_order_relaxed);
768 isymbols_.reset(isyms ? isyms->
Copy() :
nullptr);
772 osymbols_.reset(osyms ? osyms->
Copy() :
nullptr);
791 hdr->
SetProperties(properties_.load(std::memory_order_relaxed));
792 int32_t file_flags = 0;
813 std::string_view type, uint64_t properties,
820 int32_t file_flags = 0;
846 std::string_view type, uint64_t properties,
848 strm.seekp(header_offset);
850 LOG(ERROR) <<
"Fst::UpdateFstHeader: Write failed: " << opts.
source;
853 WriteFstHeader(fst, strm, opts, version, type, properties, hdr);
855 LOG(ERROR) <<
"Fst::UpdateFstHeader: Write failed: " << opts.
source;
858 strm.seekp(0, std::ios_base::end);
860 LOG(ERROR) <<
"Fst::UpdateFstHeader: Write failed: " << opts.
source;
870 mutable std::atomic<uint64_t> properties_ = 0;
873 std::string type_ =
"null";
874 std::unique_ptr<SymbolTable> isymbols_;
875 std::unique_ptr<SymbolTable> osymbols_;
890 }
else if (!hdr->Read(strm, opts.source)) {
893 VLOG(2) <<
"FstImpl::ReadHeader: source: " << opts.source
894 <<
", fst_type: " << hdr->FstType() <<
", arc_type: " << Arc::Type()
895 <<
", version: " << hdr->Version() <<
", flags: " << hdr->GetFlags();
896 if (hdr->FstType() != type_) {
897 LOG(ERROR) <<
"FstImpl::ReadHeader: FST not of type " << type_ <<
", found " 898 << hdr->FstType() <<
": " << opts.source;
901 if (hdr->ArcType() != Arc::Type()) {
902 LOG(ERROR) <<
"FstImpl::ReadHeader: Arc not of type " << Arc::Type()
903 <<
", found " << hdr->ArcType() <<
": " << opts.source;
906 if (hdr->Version() < min_version) {
907 LOG(ERROR) <<
"FstImpl::ReadHeader: Obsolete " << type_ <<
" FST version " 908 << hdr->Version() <<
", min_version=" << min_version <<
": " 912 properties_.store(hdr->Properties(), std::memory_order_relaxed);
917 if (!opts.read_isymbols) SetInputSymbols(
nullptr);
922 if (!opts.read_osymbols) SetOutputSymbols(
nullptr);
924 isymbols_.reset(opts.isymbols->Copy());
927 osymbols_.reset(opts.osymbols->Copy());
941 template <
class IFST,
class OFST>
942 void Cast(
const IFST &ifst, OFST *ofst) {
943 using OImpl =
typename OFST::Impl;
944 ofst->impl_ = std::shared_ptr<OImpl>(
945 ifst.impl_,
reinterpret_cast<OImpl *
>(ifst.impl_.get()));
954 std::ostringstream ostrm;
955 fst.
Write(ostrm, options);
972 std::istringstream istrm((std::string(s)));
bool Write(std::ostream &strm) const
void SetProperties(uint64_t props, uint64_t mask) const
void SetProperties(uint64_t props)
constexpr int32_t kFstMagicNumber
constexpr uint8_t kArcValueFlags
typename ArcMapFst< Arc, Arc, EncodeMapper< Arc > >::Arc Arc
void Cast(const F &, G *)
constexpr uint8_t kArcNoCache
static Fst * Read(std::istream &strm, const FstReadOptions &opts)
virtual size_t NumArcs(StateId) const =0
bool CompatProperties(uint64_t props1, uint64_t props2)
const SymbolTable * OutputSymbols() const
Fst< Arc > * StringToFst(std::string_view s)
void SetFlags(const char *usage, int *argc, char ***argv, bool remove_flags, const char *src="")
typename Arc::Weight Weight
virtual SymbolTable * Copy() const
constexpr uint8_t kArcFlags
constexpr uint64_t kError
virtual bool Write(std::ostream &strm, const FstWriteOptions &opts) const
size_t NumArcs(const Fst< Arc > &fst, typename Arc::StateId s)
virtual Weight Final(StateId) const =0
void SetOutputSymbols(const SymbolTable *osyms)
typename RandGenFst< FromArc, ToArc, Sampler >::Arc::StateId StateId
static Fst * Read(const std::string &source)
typename ArcMapFst< Arc, Arc, EncodeMapper< Arc > >::Arc Arc
typename FST::Arc::StateId StateId
constexpr uint8_t kArcILabelValue
const SymbolTable * osymbols
FstReadOptions(const std::string_view source="<unspecified>", const FstHeader *header=nullptr, const SymbolTable *isymbols=nullptr, const SymbolTable *osymbols=nullptr)
SymbolTable * InputSymbols()
const Arc & Value() const
static FstRegister< Arc > * GetRegister()
virtual size_t NumInputEpsilons(StateId) const =0
ArcIterator(const FST &fst, StateId s)
static void WriteFstHeader(const Fst< Arc > &fst, std::ostream &strm, const FstWriteOptions &opts, int version, std::string_view type, uint64_t properties, FstHeader *hdr)
void SetFlags(uint8_t flags, uint8_t mask)
size_t NumOutputEpsilons(const Fst< Arc > &fst, typename Arc::StateId s)
SymbolTable * OutputSymbols()
typename Arc::StateId StateId
virtual uint64_t Properties() const
static bool UpdateFstHeader(const Fst< Arc > &fst, std::ostream &strm, const FstWriteOptions &opts, int version, std::string_view type, uint64_t properties, FstHeader *hdr, size_t header_offset)
typename Arc::StateId StateId
std::string DebugString() const
uint64_t KnownProperties(uint64_t props)
constexpr uint8_t kArcOLabelValue
void WriteHeader(std::ostream &strm, const FstWriteOptions &opts, int version, FstHeader *hdr) const
typename S::Arc::Weight Weight
std::unique_ptr< StateIteratorBase< Arc > > base
std::unique_ptr< ArcIteratorBase< Arc > > base
void SetInputSymbols(const SymbolTable *isyms)
FstWriteOptions(std::string_view source="<unspecified>", bool write_header=true, bool write_isymbols=true, bool write_osymbols=true, bool align=FST_FLAGS_fst_align, bool stream_write=false)
const std::string & Type() const
virtual std::optional< StateId > NumStatesIfKnown() const
bool WriteFile(const std::string &source) const
static FileReadMode ReadMode(std::string_view mode)
constexpr uint8_t kArcWeightValue
FstImpl(const FstImpl< Arc > &impl)
Arc::Weight Final(const Fst< Arc > &fst, typename Arc::StateId s)
typename internal::SynchronizeFstImpl< A >::Arc Arc
virtual bool Write(const std::string &source) const
virtual const SymbolTable * InputSymbols() const =0
const SymbolTable * InputSymbols() const
constexpr uint8_t kArcNextStateValue
void SetType(std::string_view type)
virtual MatcherBase< Arc > * InitMatcher(MatchType match_type) const
typename Arc::StateId StateId
typename Arc::StateId StateId
const SymbolTable * isymbols
size_t NumInputEpsilons(const Fst< Arc > &fst, typename Arc::StateId s)
virtual size_t NumOutputEpsilons(StateId) const =0
std::string FstToString(const Fst< Arc > &fst, const FstWriteOptions &options=FstWriteOptions("FstToString"))
static SymbolTable * Read(std::istream &strm, std::string_view source)
virtual uint64_t Properties(uint64_t mask) const
void Destroy(ArcIterator< FST > *aiter, MemoryPool< ArcIterator< FST >> *pool)
FstImpl & operator=(const FstImpl &impl)
void UpdateProperties(uint64_t props, uint64_t mask)
typename S::Arc::StateId StateId
void SetProperties(uint64_t props, uint64_t mask)
std::atomic< uint64_t > properties_
virtual const SymbolTable * OutputSymbols() const =0
StateIterator(const FST &fst)