如何摆脱带有spirit :: lex的令牌中的转义字符？

Question

我想标记我自己的SQL语法扩展.这涉及识别双引号字符串中的转义双引号.例如在MySQL中,这两个字符串标记是等价的:( """"第二个双引号充当转义字符)和'"'.我尝试了不同的东西,但我仍然坚持如何替换令牌的价值.

#include <boost/spirit/include/lex_lexertl.hpp>
namespace lex = boost::spirit::lex;

template <typename Lexer>
struct sql_tokens : lex::lexer<Lexer>
{
  sql_tokens()
  {
    string_quote_double = "\\\"";    // '"'

    this->self("INITIAL")
      = string_quote_double [ lex::_state = "STRING_DOUBLE" ] // how to also ignore + ctx.more()?
      | ...
      ;

    this->self("STRING_DOUBLE") 
      = lex::token_def<>("[^\\\"]*") // action: ignore + ctx.more()
      | lex::token_def<>("\\\"\\\"") // how to set token value to '"' ?
      | lex::token_def<>("\\\"") [ lex::_state = "INITIAL" ]
      ;
  }

  lex::token_def<> string_quote_double, ...;
};

那么如何将令牌的值设置为何"时""找到？

除此之外,我还有以下问题:我可以编写一个语义动作的仿函数来调用ctx.more()并同时忽略该标记(从而将"低级"标记组合成一个"高级"字符串标记).但是如何优雅地将它与lex :: _ state =".."结合起来？

Answer 1

编辑回应评论,见下文"更新"

我建议不要在词法分析器中解决这个问题.让lexer产生原始字符串:

template <typename Lexer>
    struct mylexer_t : lex::lexer<Lexer>
{
    mylexer_t()
    {
        string_quote_double = "\\\"([^\"]|\\\"\\\")*\\\"";

        this->self("INITIAL")
            = string_quote_double
            | lex::token_def<>("[ \t\r\n]") [ lex::_pass = lex::pass_flags::pass_ignore ]
            ;
    }

    lex::token_def<std::string> string_quote_double;
};

注意暴露像这样的令牌属性需要修改的令牌typedef:

typedef lex::lexertl::token<char const*, boost::mpl::vector<char, std::string> > token_type;
typedef lex::lexertl::actor_lexer<token_type> lexer_type;

解析器中的后处理:

template <typename Iterator> struct mygrammar_t
    : public qi::grammar<Iterator, std::vector<std::string>()>
{
    typedef mygrammar_t<Iterator> This;

    template <typename TokenDef>
        mygrammar_t(TokenDef const& tok) : mygrammar_t::base_type(start)
    {
        using namespace qi;

        string_quote_double %= tok.string_quote_double [ undoublequote ];
        start = *string_quote_double;

        BOOST_SPIRIT_DEBUG_NODES((start)(string_quote_double));
    }

  private:
    qi::rule<Iterator, std::vector<std::string>()> start;
    qi::rule<Iterator, std::string()> string_quote_double;
};

如您所见,undoubleqoute可以是任何满足Spirit语义操作标准的Phoenix演员.一个脑死亡的示例实现将是:

static bool undoublequote(std::string& val)
{
    auto outidx = 0;
    for(auto in = val.begin(); in!=val.end(); ++in) {
        switch(*in) {
            case '"': 
                if (++in == val.end()) { // eat the escape
                    // end of input reached
                    val.resize(outidx); // resize to effective chars
                    return true;
                }
                // fall through
            default:
                val[outidx++] = *in; // append the character
        }
    }

    return false; // not ended with double quote as expected
}

但我建议你写一个"正确的"去逃避者(我敢肯定,MySQL将允许\t,\r,\u001e甚至更古老的东西,以及).

我在旧答案中有一些更完整的样本:

• 去做
• 链接
• 这是一个搜索页面,其中包含许多使用Spirit的相关答案

UPDATE

实际上,正如您所指出的,将属性值规范化集成到词法分析器本身是相当容易的:

template <typename Lexer>
    struct mylexer_t : lex::lexer<Lexer>
{
    struct undoublequote_lex_type {
        template <typename, typename, typename, typename> struct result { typedef void type; };

        template <typename It, typename IdType, typename pass_flag, typename Ctx>
            void operator()(It& f, It& l, pass_flag& pass, IdType& id, Ctx& ctx) const {
                std::string raw(f,l);
                if (undoublequote(raw))
                    ctx.set_value(raw);
                else
                    pass = lex::pass_flags::pass_fail;
            }
    } undoublequote_lex;

    mylexer_t()
    {
        string_quote_double = "\\\"([^\"]|\\\"\\\")*\\\"";

        const static undoublequote_lex_type undoublequote_lex;
        this->self("INITIAL")
            = string_quote_double [ undoublequote_lex ]
            | lex::token_def<>("[ \t\r\n]") [ lex::_pass = lex::pass_flags::pass_ignore ]
            ;
    }

    lex::token_def<std::string> string_quote_double;
};

这将重用undoublequote上面显示的相同函数,但将其包装在undoublequote_lex_type满足Lexer语义操作标准的延迟可调用对象(或"多态函数")中.

这是一个完全可靠的概念证明:

//#include <boost/config/warning_disable.hpp>
//#define BOOST_SPIRIT_DEBUG_PRINT_SOME 80
//#define BOOST_SPIRIT_DEBUG // before including Spirit
#include <boost/spirit/include/lex_lexertl.hpp>
#include <boost/spirit/include/qi.hpp>
#include <fstream>
#ifdef MEMORY_MAPPED
#   include <boost/iostreams/device/mapped_file.hpp>
#endif
//#include <boost/spirit/include/lex_generate_static_lexertl.hpp>

namespace /*anon*/
{
    namespace phx=boost::phoenix;
    namespace qi =boost::spirit::qi;
    namespace lex=boost::spirit::lex;

    template <typename Lexer>
        struct mylexer_t : lex::lexer<Lexer>
    {
        mylexer_t()
        {
            string_quote_double = "\\\"([^\"]|\\\"\\\")*\\\"";

            this->self("INITIAL")
                = string_quote_double
                | lex::token_def<>("[ \t\r\n]") [ lex::_pass = lex::pass_flags::pass_ignore ]
                ;
        }

        lex::token_def<std::string> string_quote_double;
    };

    static bool undoublequote(std::string& val)
    {
        auto outidx = 0;
        for(auto in = val.begin(); in!=val.end(); ++in) {
            switch(*in) {
                case '"': 
                    if (++in == val.end()) { // eat the escape
                        // end of input reached
                        val.resize(outidx); // resize to effective chars
                        return true;
                    }
                    // fall through
                default:
                    val[outidx++] = *in; // append the character
            }
        }

        return false; // not ended with double quote as expected
    }

    template <typename Iterator> struct mygrammar_t
        : public qi::grammar<Iterator, std::vector<std::string>()>
    {
        typedef mygrammar_t<Iterator> This;

        template <typename TokenDef>
            mygrammar_t(TokenDef const& tok) : mygrammar_t::base_type(start)
        {
            using namespace qi;

            string_quote_double %= tok.string_quote_double [ undoublequote ];
            start = *string_quote_double;

            BOOST_SPIRIT_DEBUG_NODES((start)(string_quote_double));
        }

      private:
        qi::rule<Iterator, std::vector<std::string>()> start;
        qi::rule<Iterator, std::string()> string_quote_double;
    };
}

std::vector<std::string> do_test_parse(const std::string& v)
{
    char const *first = &v[0];
    char const *last = first+v.size();

    typedef lex::lexertl::token<char const*, boost::mpl::vector<char, std::string> > token_type;
    typedef lex::lexertl::actor_lexer<token_type> lexer_type;

    typedef mylexer_t<lexer_type>::iterator_type iterator_type;
    const static mylexer_t<lexer_type> mylexer;
    const static mygrammar_t<iterator_type> parser(mylexer);

    auto iter = mylexer.begin(first, last);
    auto end = mylexer.end();

    std::vector<std::string> data;
    bool r = qi::parse(iter, end, parser, data);

    r = r && (iter == end);

    if (!r)
        std::cerr << "parsing (" << iter->state() << ") failed at: '" << std::string(first, last) << "'\n";

    return data;
}

int main(int argc, const char *argv[])
{
    for (auto&& s : do_test_parse( "\"bla\"\"blo\""))
        std::cout << s << std::endl;
}