32 "Single non-white-space character delimiter inside sequences of " 33 "feature symbols and output symbols");
35 "Special symbol that designates an empty sequence");
37 DEFINE_string(start_symbol,
"<s>",
"Start of sentence symbol");
41 "Treat input model as a classifier instead of a tagger");
47 return FST_FLAGS_delimiter.size() == 1 &&
48 !std::isspace(FST_FLAGS_delimiter[0]);
52 bool okay = !FST_FLAGS_empty_symbol.empty();
53 for (
size_t i = 0; i < FST_FLAGS_empty_symbol.size(); ++i) {
54 char c = FST_FLAGS_empty_symbol[i];
55 if (std::isspace(c)) okay =
false;
61 const std::string &epsilon_symbol,
62 const std::string &unknown_symbol,
const std::string &vocab,
63 char **models,
int models_len,
const std::string &out,
64 const std::string &save_isymbols,
65 const std::string &save_fsymbols,
66 const std::string &save_osymbols) {
68 models_len, out, save_isymbols, save_fsymbols,
70 Apply<Operation<LinearCompileArgs>>(
"LinearCompileTpl", arc_type, &args);
77 std::istringstream strm(str);
79 while (strm >> buf) out->push_back(buf);
83 std::set<std::string> preds;
84 for (
int i = 0; i < models_len; ++i) {
85 std::ifstream in(models[i]);
86 if (!in)
LOG(FATAL) <<
"Failed to open " << models[i];
88 std::getline(in, line);
90 while (std::getline(in, line)) {
92 std::vector<std::string> fields;
94 if (fields.size() != 3)
95 LOG(FATAL) <<
"Wrong number of fields in source " << models[i]
96 <<
", line " << num_line;
97 preds.insert(fields[1]);
void SplitByWhitespace(const std::string &str, std::vector< std::string > *out)
REGISTER_FST_OPERATION_3ARCS(Compress, CompressArgs)
std::tuple< const std::string &, const std::string &, const std::string &, char **, int, const std::string &, const std::string &, const std::string &, const std::string & > LinearCompileArgs
DEFINE_string(delimiter,"|","Single non-white-space character delimiter inside sequences of ""feature symbols and output symbols")
bool ValidateEmptySymbol()
int ScanNumClasses(char **models, int models_len)
void LinearCompile(const std::string &arc_type, const std::string &epsilon_symbol, const std::string &unknown_symbol, const std::string &vocab, char **models, int models_len, const std::string &out, const std::string &save_isymbols, const std::string &save_fsymbols, const std::string &save_osymbols)
DEFINE_bool(classifier, false,"Treat input model as a classifier instead of a tagger")
void LinearCompileTpl(LinearCompileArgs *args)