FST  openfst-1.7.1
OpenFst Library
util.cc
Go to the documentation of this file.
1 // See www.openfst.org for extensive documentation on this weighted
2 // finite-state transducer library.
3 //
4 // FST utility definitions.
5 
6 #include <fst/util.h>
7 #include <cctype>
8 #include <sstream>
9 #include <string>
10 #include <fst/flags.h>
11 #include <fst/log.h>
12 #include <fst/mapped-file.h>
13 
14 // Utility flag definitions
15 
16 DEFINE_bool(fst_error_fatal, true,
17  "FST errors are fatal; o.w. return objects flagged as bad: "
18  "e.g., FSTs: kError property set, FST weights: not a Member()");
19 
20 namespace fst {
21 
22 void SplitString(char *full, const char *delim, std::vector<char *> *vec,
23  bool omit_empty_strings) {
24  char *p = full;
25  while (p) {
26  if ((p = strpbrk(full, delim))) {
27  p[0] = '\0';
28  }
29  if (!omit_empty_strings || full[0] != '\0') vec->push_back(full);
30  if (p) full = p + 1;
31  }
32 }
33 
34 int64 StrToInt64(const string &s, const string &src, size_t nline,
35  bool allow_negative, bool *error) {
36  int64 n;
37  const char *cs = s.c_str();
38  char *p;
39  if (error) *error = false;
40  n = strtoll(cs, &p, 10);
41  if (p < cs + s.size() || (!allow_negative && n < 0)) {
42  FSTERROR() << "StrToInt64: Bad integer = " << s << "\", source = " << src
43  << ", line = " << nline;
44  if (error) *error = true;
45  return 0;
46  }
47  return n;
48 }
49 
50 void ConvertToLegalCSymbol(string *s) {
51  for (auto it = s->begin(); it != s->end(); ++it) {
52  if (!isalnum(*it)) {
53  *it = '_';
54  }
55  }
56 }
57 
58 // Skips over input characters to align to 'align' bytes. Returns false if can't
59 // align.
60 bool AlignInput(std::istream &strm) {
61  char c;
62  for (int i = 0; i < MappedFile::kArchAlignment; ++i) {
63  int64 pos = strm.tellg();
64  if (pos < 0) {
65  LOG(ERROR) << "AlignInput: Can't determine stream position";
66  return false;
67  }
68  if (pos % MappedFile::kArchAlignment == 0) break;
69  strm.read(&c, 1);
70  }
71  return true;
72 }
73 
74 // Write null output characters to align to 'align' bytes. Returns false if
75 // can't align.
76 bool AlignOutput(std::ostream &strm) {
77  for (int i = 0; i < MappedFile::kArchAlignment; ++i) {
78  int64 pos = strm.tellp();
79  if (pos < 0) {
80  LOG(ERROR) << "AlignOutput: Can't determine stream position";
81  return false;
82  }
83  if (pos % MappedFile::kArchAlignment == 0) break;
84  strm.write("", 1);
85  }
86  return true;
87 }
88 
89 int AlignBufferWithOutputStream(std::ostream &strm,
90  std::ostringstream &buffer) {
91  const auto strm_pos = strm.tellp();
92  if (strm_pos == -1) {
93  LOG(ERROR) << "Cannot determine stream position";
94  return -1;
95  }
96  const int stream_offset = strm_pos % MappedFile::kArchAlignment;
97  for (int i = 0; i < stream_offset; ++i) buffer.write("", 1);
98  return stream_offset;
99 }
100 
101 } // namespace fst
void ConvertToLegalCSymbol(string *s)
Definition: util.cc:50
DEFINE_bool(fst_error_fatal, true,"FST errors are fatal; o.w. return objects flagged as bad: ""e.g., FSTs: kError property set, FST weights: not a Member()")
#define LOG(type)
Definition: log.h:48
static constexpr int kArchAlignment
Definition: mapped-file.h:68
int64_t int64
Definition: types.h:27
#define FSTERROR()
Definition: util.h:35
void SplitString(char *line, const char *delim, std::vector< char * > *vec, bool omit_empty_strings)
Definition: util.cc:22
bool AlignOutput(std::ostream &strm)
Definition: util.cc:76
int AlignBufferWithOutputStream(std::ostream &strm, std::ostringstream &buffer)
Definition: util.cc:89
bool AlignInput(std::istream &strm)
Definition: util.cc:60
int64 StrToInt64(const string &s, const string &src, size_t nline, bool allow_negative, bool *error=nullptr)
Definition: util.cc:34