Boost::spirit (classic) primitives vs custom parsers

Question

I'm a beginner in Boost::spirit and I want to define grammar that parses TTCN language. (http://www.trex.informatik.uni-goettingen.de/trac/wiki/ttcn-3_4.5.1) I'm trying to define some rules for 'primitve' parsers like Alpha, AlphaNum to be faitful 1 to 1 to original grammar but obviously I do something wrong because grammar defined this way does not work. But when I use primite parsers in place of TTCN's it started to work.

Can someone tell why 'manually' defined rules does not work as expected ? How to fix it, because I would like to stick close to original grammar. Is it a begginer's code bug or something different ?

#define BOOST_SPIRIT_DEBUG

#include <boost/spirit/include/classic_symbols.hpp>
#include <boost/spirit/include/classic_tree_to_xml.hpp>
#include <boost/spirit/include/classic_position_iterator.hpp>
#include <boost/spirit/include/classic_core.hpp>
#include <boost/spirit/include/classic_parse_tree.hpp>
#include <boost/spirit/include/classic_ast.hpp>
#include <iostream>
#include <string>
#include <boost/spirit/home/classic/debug.hpp>
using namespace boost::spirit::classic;
using namespace std;
using namespace BOOST_SPIRIT_CLASSIC_NS;

typedef node_iter_data_factory<int> factory_t;
typedef position_iterator<std::string::iterator> pos_iterator_t;
typedef tree_match<pos_iterator_t, factory_t> parse_tree_match_t;
typedef parse_tree_match_t::const_tree_iterator iter_t;


struct ParseGrammar: public grammar<ParseGrammar>
{
      template<typename ScannerT>
      struct definition
      {
            definition(ParseGrammar const &)
            {
               KeywordImport = str_p("import");
               KeywordAll = str_p("all");
               SemiColon = ch_p(';');
               Underscore = ch_p('_');

               NonZeroNum = range_p('1','9');
               Num = ch_p('0') | NonZeroNum;
               UpperAlpha = range_p('A', 'Z');
               LowerAlpha = range_p('a', 'z');
               Alpha = UpperAlpha | LowerAlpha;
               AlphaNum = Alpha | Num;

               //this does not!
               Identifier = lexeme_d[Alpha >> *(AlphaNum | Underscore)];

               // Uncomment below line to make rule work
               // Identifier = lexeme_d[alpha_p >> *(alnum_p | Underscore)];

               Module = KeywordImport >> Identifier >> KeywordAll >> SemiColon;

               BOOST_SPIRIT_DEBUG_NODE(Module);
               BOOST_SPIRIT_DEBUG_NODE(KeywordImport);
               BOOST_SPIRIT_DEBUG_NODE(KeywordAll);
               BOOST_SPIRIT_DEBUG_NODE(Identifier);
               BOOST_SPIRIT_DEBUG_NODE(SemiColon);
            }

            rule<ScannerT> KeywordImport,KeywordAll,Module,Identifier,SemiColon;
            rule<ScannerT> Alpha,UpperAlpha,LowerAlpha,Underscore,Num,AlphaNum;
            rule<ScannerT> NonZeroNum;
            rule<ScannerT> const&
            start() const { return Module; }
      };
};

int main()
{
   ParseGrammar resolver;    //  Our parser
   BOOST_SPIRIT_DEBUG_NODE(resolver);

   string content = "import foobar all;";

   pos_iterator_t pos_begin(content.begin(), content.end());
   pos_iterator_t pos_end;

   tree_parse_info<pos_iterator_t, factory_t> info;
       info = ast_parse<factory_t>(pos_begin, pos_end, resolver, space_p);

   std::cout << "\ninfo.length : " << info.length << std::endl;
   std::cout << "info.full   : " << info.full << std::endl;

   if(info.full)
   {
      std::cout << "OK: Parsing succeeded\n\n";
   }
   else
   {
      int line = info.stop.get_position().line;
      int column = info.stop.get_position().column;
      std::cout << "-------------------------\n";
      std::cout << "ERROR: Parsing failed\n";
      std::cout << "stopped at: " << line  << ":" << column << "\n";
      std::cout << "-------------------------\n";
   }
   return 0;
}

Answer 1

I don't do Spirit Classic (which has been deprecated for some years now).

I can only assume you've mixed something up with skippers. Here's the thing translated into Spirit V2:

#define BOOST_SPIRIT_DEBUG
#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/support_line_pos_iterator.hpp>

namespace qi = boost::spirit::qi;

typedef boost::spirit::line_pos_iterator<std::string::const_iterator> pos_iterator_t;

template <typename Iterator = pos_iterator_t, typename Skipper = qi::space_type>
struct ParseGrammar: public qi::grammar<Iterator, Skipper>
{
    ParseGrammar() : ParseGrammar::base_type(Module)
    {
        using namespace qi;
        KeywordImport = lit("import");
        KeywordAll    = lit("all");
        SemiColon     = lit(';');

#if 1
        // this rule obviously works
        Identifier = lexeme [alpha >> *(alnum | '_')];
#else
        // this does too, but less efficiently

        Underscore    = lit('_');
        NonZeroNum    = char_('1','9');
        Num           = char_('0') | NonZeroNum;
        UpperAlpha    = char_('A', 'Z');
        LowerAlpha    = char_('a', 'z');
        Alpha         = UpperAlpha | LowerAlpha;
        AlphaNum      = Alpha | Num;

        Identifier = lexeme [Alpha >> *(AlphaNum | Underscore)];
#endif

        Module = KeywordImport >> Identifier >> KeywordAll >> SemiColon;

        BOOST_SPIRIT_DEBUG_NODES((Module)(KeywordImport)(KeywordAll)(Identifier)(SemiColon))
    }

    qi::rule<Iterator, Skipper> Module;
    qi::rule<Iterator> KeywordImport,KeywordAll,Identifier,SemiColon;
    qi::rule<Iterator> Alpha,UpperAlpha,LowerAlpha,Underscore,Num,AlphaNum;
    qi::rule<Iterator> NonZeroNum;
};

int main()
{
   std::string const content = "import \r\n\r\nfoobar\r\n\r\n all; bogus";

   pos_iterator_t first(content.begin()), iter=first, last(content.end());

   ParseGrammar<pos_iterator_t> resolver;    //  Our parser
   bool ok = phrase_parse(iter, last, resolver, qi::space);

   std::cout << std::boolalpha;
   std::cout << "\nok : " << ok << std::endl;
   std::cout << "full   : " << (iter == last) << std::endl;

   if(ok && iter==last)
   {
      std::cout << "OK: Parsing fully succeeded\n\n";
   }
   else
   {
      int line   = get_line(iter);
      int column = get_column(first, iter);
      std::cout << "-------------------------\n";
      std::cout << "ERROR: Parsing failed or not complete\n";
      std::cout << "stopped at: " << line  << ":" << column << "\n";
      std::cout << "remaining: '" << std::string(iter, last) << "'\n";
      std::cout << "-------------------------\n";
   }
   return 0;
}

I've added a little "bogus" at the end of input, so the output becomes a nicer demonstration:

<Module>
  <try>import \r\n\r\nfoobar\r\n\r</try>
  <KeywordImport>
    <try>import \r\n\r\nfoobar\r\n\r</try>
    <success> \r\n\r\nfoobar\r\n\r\n all;</success>
    <attributes>[]</attributes>
  </KeywordImport>
  <Identifier>
    <try>foobar\r\n\r\n all; bogu</try>
    <success>\r\n\r\n all; bogus</success>
    <attributes>[]</attributes>
  </Identifier>
  <KeywordAll>
    <try>all; bogus</try>
    <success>; bogus</success>
    <attributes>[]</attributes>
  </KeywordAll>
  <SemiColon>
    <try>; bogus</try>
    <success> bogus</success>
    <attributes>[]</attributes>
  </SemiColon>
  <success> bogus</success>
  <attributes>[]</attributes>
</Module>

ok : true
full   : false
-------------------------
ERROR: Parsing failed or not complete
stopped at: 3:8
remaining: 'bogus'
-------------------------

That all said, this is what I'd probably reduce it to:

template <typename Iterator, typename Skipper = qi::space_type>
struct ParseGrammar: public qi::grammar<Iterator, Skipper>
{
    ParseGrammar() : ParseGrammar::base_type(Module)
    {
        using namespace qi;

        Identifier = alpha >> *(alnum | '_');
        Module     = "import" >> Identifier >> "all" >> ';';

        BOOST_SPIRIT_DEBUG_NODES((Module)(Identifier))
    }

    qi::rule<Iterator, Skipper> Module;
    qi::rule<Iterator> Identifier;
};

As you can see, the Identifier rule is implicitely a lexeme because it doesn't declared to use a skipper.

See it Live on Coliru