//  Copyright (c) 2001-2009 Hartmut Kaiser
// 
//  Distributed under the Boost Software License, Version 1.0. (See accompanying 
//  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)

#if !defined(BOOST_SPIRIT_LEX_LEXERTL_TOKEN_FEB_10_2008_0751PM)
#define BOOST_SPIRIT_LEX_LEXERTL_TOKEN_FEB_10_2008_0751PM

#if defined(_MSC_VER) && (_MSC_VER >= 1020)
#pragma once      // MS compatible compilers support #pragma once
#endif

#include <boost/spirit/home/qi/detail/assign_to.hpp>
#include <boost/spirit/home/support/placeholders.hpp>
#include <boost/spirit/home/support/detail/lexer/generator.hpp>
#include <boost/spirit/home/support/detail/lexer/rules.hpp>
#include <boost/spirit/home/support/detail/lexer/consts.hpp>
#include <boost/fusion/include/vector.hpp>
#include <boost/fusion/include/at.hpp>
#include <boost/fusion/include/value_at.hpp>
#include <boost/detail/iterator.hpp>
#include <boost/variant.hpp>
#include <boost/mpl/vector.hpp>
#include <boost/mpl/insert.hpp>
#include <boost/mpl/begin.hpp>
#include <boost/mpl/bool.hpp>
#include <boost/mpl/identity.hpp>
#include <boost/mpl/if.hpp>
#include <boost/range/iterator_range.hpp>

namespace boost { namespace spirit { namespace lex 
{ 
    ///////////////////////////////////////////////////////////////////////////
    //
    //  The lexertl_token is the type of the objects returned by default by the 
    //  lexertl_iterator.
    //
    //    template parameters:
    //        Iterator        The type of the iterator used to access the
    //                        underlying character stream.
    //        AttributeTypes  A mpl sequence containing the types of all 
    //                        required different token values to be supported 
    //                        by this token type.
    //        HasState        A mpl::bool_ indicating, whether this token type
    //                        should support lexer states.
    //
    //  It is possible to use other token types with the spirit::lex 
    //  framework as well. If you plan to use a different type as your token 
    //  type, you'll need to expose the following things from your token type 
    //  to make it compatible with spirit::lex:
    //
    //    typedefs
    //        iterator_type   The type of the iterator used to access the
    //                        underlying character stream.
    //
    //        id_type         The type of the token id used.
    //
    //    methods
    //        default constructor
    //                        This should initialize the token as an end of 
    //                        input token.
    //        constructors    The prototype of the other required 
    //                        constructors should be:
    //
    //              token(int)
    //                        This constructor should initialize the token as 
    //                        an invalid token (not carrying any specific 
    //                        values)
    //
    //              where:  the int is used as a tag only and its value is 
    //                      ignored
    //
    //                        and:
    //
    //              token(std::size_t id, std::size_t state, 
    //                    iterator_type first, iterator_type last);
    //
    //              where:  id:           token id
    //                      state:        lexer state this token was matched in
    //                      first, last:  pair of iterators marking the matched 
    //                                    range in the underlying input stream 
    //
    //        accessors
    //              id()      return the token id of the matched input sequence
    //
    //              state()   return the lexer state this token was matched in
    //
    //              value()   return the token value
    //
    //  Additionally, you will have to implement a couple of helper functions
    //  in the same namespace as the token type: a comparison operator==() to 
    //  compare your token instances, a token_is_valid() function and different 
    //  construct() function overloads as described below.
    //
    ///////////////////////////////////////////////////////////////////////////
    template <
        typename Iterator = char const*, 
        typename AttributeTypes = mpl::vector0<>, 
        typename HasState = mpl::true_
    >
    struct lexertl_token;

    ///////////////////////////////////////////////////////////////////////////
    //  This specialization of the token type doesn't contain any item data and
    //  doesn't support working with lexer states.
    ///////////////////////////////////////////////////////////////////////////
    template <typename Iterator>
    struct lexertl_token<Iterator, omitted, mpl::false_>
    {
        typedef Iterator iterator_type;
        typedef mpl::false_ has_state;
        typedef std::size_t id_type;
        
        //  default constructed tokens correspond to EOI tokens
        lexertl_token() 
          : id_(boost::lexer::npos) 
        {}
        
        //  construct an invalid token
        lexertl_token(int)
          : id_(0) 
        {}
        
        lexertl_token(std::size_t id, std::size_t)
          : id_(id)
        {}
        
        lexertl_token(std::size_t id, std::size_t, 
                Iterator const&, Iterator const&)
          : id_(id)
        {}
        
        //  this default conversion operator is needed to allow the direct 
        //  usage of tokens in conjunction with the primitive parsers defined 
        //  in Qi
        operator std::size_t() const { return id_; }
        
        std::size_t id() const { return id_; }
        std::size_t state() const { return 0; }   // always '0' (INITIAL state)

    protected:
        std::size_t id_;         // token id, 0 if nothing has been matched
    };
    
    ///////////////////////////////////////////////////////////////////////////
    //  This specialization of the token type doesn't contain any item data but
    //  supports working with lexer states.
    ///////////////////////////////////////////////////////////////////////////
    template <typename Iterator>
    struct lexertl_token<Iterator, omitted, mpl::true_>
      : lexertl_token<Iterator, omitted, mpl::false_>
    {
    private:
        typedef lexertl_token<Iterator, omitted, mpl::false_> base_type;

    public:
        typedef Iterator iterator_type;
        typedef mpl::true_ has_state;

        //  default constructed tokens correspond to EOI tokens
        lexertl_token() 
          : state_(boost::lexer::npos) 
        {}
        
        //  construct an invalid token
        lexertl_token(int)
          : base_type(0), state_(boost::lexer::npos) 
        {}
        
        lexertl_token(std::size_t id, std::size_t state)
          : base_type(id, boost::lexer::npos), state_(state)
        {}
        
        lexertl_token(std::size_t id, std::size_t state, 
                Iterator const&, Iterator const&)
          : base_type(id, boost::lexer::npos), state_(state)
        {}
        
        std::size_t state() const { return state_; }

    protected:
        std::size_t state_;      // lexer state this token was matched in
    };
    
    ///////////////////////////////////////////////////////////////////////////
    //  The generic version of the lexertl_token type derives from the 
    //  specialization above and adds a single data member holding the item 
    //  data carried by the token instance.
    ///////////////////////////////////////////////////////////////////////////
    namespace detail
    {
        ///////////////////////////////////////////////////////////////////////
        //  Metafunction to calculate the type of the variant data item to be 
        //  stored with each token instance.
        //
        //  Note: The iterator pair needs to be the first type in the list of 
        //        types supported by the generated variant type (this is being 
        //        used to identify whether the stored data item in a particular 
        //        token instance needs to be converted from the pair of 
        //        iterators (see the first of the construct() functions below).
        ///////////////////////////////////////////////////////////////////////
        template <typename IteratorPair, typename AttributeTypes>
        struct token_value_typesequence
        {
            typedef typename 
                mpl::insert<
                    AttributeTypes, 
                    typename mpl::begin<AttributeTypes>::type, 
                    IteratorPair
                >::type
            sequence_type;
            typedef typename make_variant_over<sequence_type>::type type;
        };
        
        ///////////////////////////////////////////////////////////////////////
        //  The type of the data item stored with a token instance is defined 
        //  by the template parameter 'AttributeTypes' and may be:
        //  
        //     omitted:           no data item is stored with the token 
        //                        instance (this is handled by the 
        //                        specializations of the lexertl_token class
        //                        below)
        //     mpl::vector0<>:    each token instance stores a pair of 
        //                        iterators pointing to the matched input 
        //                        sequence
        //     mpl::vector<...>:  each token instance stores a variant being 
        //                        able to store the pair of iterators pointing 
        //                        to the matched input sequence, or any of the 
        //                        types a specified in the mpl::vector<>
        //
        //  All this is done to ensure the token type is as small (in terms 
        //  of its byte-size) as possible.
        ///////////////////////////////////////////////////////////////////////
        template <typename IteratorPair, typename AttributeTypes>
        struct token_value_type
        {
            typedef 
                typename mpl::eval_if<
                    is_same<AttributeTypes, mpl::vector0<> >,
                    mpl::identity<IteratorPair>,
                    token_value_typesequence<IteratorPair, AttributeTypes>
                >::type 
            type;
        };
    }

    template <typename Iterator, typename AttributeTypes, typename HasState>
    struct lexertl_token : lexertl_token<Iterator, omitted, HasState>
    {
    private: // precondition assertions
#if !BOOST_WORKAROUND(BOOST_MSVC, <= 1300)
        BOOST_STATIC_ASSERT((mpl::is_sequence<AttributeTypes>::value || 
                            is_same<AttributeTypes, omitted>::value));
#endif
        typedef lexertl_token<Iterator, omitted, HasState> base_type;
        
    protected: 
        //  If no additional token value types are given, the the token will 
        //  hold the plain pair of iterators pointing to the matched range
        //  in the underlying input sequence. Otherwise the token value is 
        //  stored as a variant and will again hold the pair of iterators but
        //  is able to hold any of the given data types as well. The conversion 
        //  from the iterator pair to the required data type is done when it is
        //  accessed for the first time.
        typedef iterator_range<Iterator> iterpair_type;
        typedef 
            typename detail::token_value_type<iterpair_type, AttributeTypes>::type 
        token_value_type;

    public:
        typedef Iterator iterator_type;

        //  default constructed tokens correspond to EOI tokens
        lexertl_token() 
          : value_(iterpair_type(iterator_type(), iterator_type())) 
        {}
        
        //  construct an invalid token
        lexertl_token(int)
          : base_type(0), value_(iterpair_type(iterator_type(), iterator_type())) 
        {}
        
        lexertl_token(std::size_t id, std::size_t state, 
                Iterator first, Iterator last)
          : base_type(id, state), value_(iterpair_type(first, last))
        {}

        token_value_type& value() { return value_; }
        token_value_type const& value() const { return value_; }
        
    protected:
        token_value_type value_; // token value, by default a pair of iterators
    };
    
    ///////////////////////////////////////////////////////////////////////////
    //  tokens are considered equal, if their id's match (these are unique)
    template <typename Iterator, typename AttributeTypes, typename HasState>
    inline bool 
    operator== (lexertl_token<Iterator, AttributeTypes, HasState> const& lhs, 
                lexertl_token<Iterator, AttributeTypes, HasState> const& rhs)
    {
        return lhs.id() == rhs.id();
    }
    
    ///////////////////////////////////////////////////////////////////////////
    //  This overload is needed by the multi_pass/functor_input_policy to 
    //  validate a token instance. It has to be defined in the same namespace 
    //  as the token class itself to allow ADL to find it.
    ///////////////////////////////////////////////////////////////////////////
    template <typename Iterator, typename AttributeTypes, typename HasState>
    inline bool 
    token_is_valid(lexertl_token<Iterator, AttributeTypes, HasState> const& t)
    {
        return 0 != t.id() && std::size_t(boost::lexer::npos) != t.id();
    }
    
    ///////////////////////////////////////////////////////////////////////////
    //  We have to provide overloads for the construct() function allowing
    //  to extract the needed value from the token. These overloads have to be
    //  defined in the same namespace as the token class itself to allow ADL to 
    //  find them.
    ///////////////////////////////////////////////////////////////////////////
    
    //  This is called from the parse function of token_def if the token_def
    //  has been defined to carry a special attribute type
    template <typename Attribute, typename Iterator, typename AttributeTypes,
        typename HasState>
    inline void construct(Attribute& attr, 
        lexertl_token<Iterator, AttributeTypes, HasState>& t)
    {
    //  The goal of this function is to avoid the conversion of the pair of
    //  iterators (to the matched character sequence) into the token value 
    //  of the required type being done more than once. For this purpose it 
    //  checks whether the stored value type is still the default one (pair 
    //  of iterators) and if yes, replaces the pair of iterators with the 
    //  converted value to be returned from subsequent calls.
        
        if (0 == t.value().which()) {
        //  first access to the token value
            typedef iterator_range<Iterator> iterpair_type;
            iterpair_type const& ip = get<iterpair_type>(t.value());
            
        // Interestingly enough we use the assign_to() framework defined in 
        // Spirit.Qi allowing to convert the pair of iterators to almost any 
        // required type (assign_to(), if available, uses the standard Spirit 
        // parsers to do the conversion, and falls back to boost::lexical_cast
        // otherwise).
            qi::detail::assign_to(ip.begin(), ip.end(), attr);

        //  If you get an error during the compilation of the following 
        //  assignment expression, you probably forgot to list one or more 
        //  types used as token value types (in your token_def<...> 
        //  definitions) in your definition of the token class. I.e. any token 
        //  value type used for a token_def<...> definition has to be listed 
        //  during the declaration of the token type to use. For instance let's 
        //  assume we have two token_def's:
        //
        //      token_def<int> number; number = "...";
        //      token_def<std::string> identifier; identifier = "...";
        //
        //  Then you'll have to use the following token type definition 
        //  (assuming you are using the lexertl_token class):
        //
        //      typedef mpl::vector<int, std::string> token_values;
        //      typedef lexertl_token<base_iter_type, token_values> token_type;
        //
        //  where: base_iter_type is the iterator type used to expose the 
        //         underlying input stream.
        //
        //  This token_type has to be used as the second template parameter 
        //  to the lexer class:
        //
        //      typedef lexertl_lexer<base_iter_type, token_type> lexer_type;
        //
        //  again, assuming you're using the lexertl_lexer<> template for your 
        //  tokenization.
        
            t.value() = attr;   // re-assign value
        }
        else {
        // reuse the already assigned value
            qi::detail::assign_to(get<Attribute>(t.value()), attr);
        }
    }
    
    //  This is called from the parse function of token_def if the token type
    //  has no special attribute type assigned 
    template <typename Attribute, typename Iterator, typename HasState>
    inline void construct(Attribute& attr, 
        lexertl_token<Iterator, mpl::vector0<>, HasState>& t)
    {
    //  The default type returned by the token_def parser component (if it
    //  has no token value type assigned) is the pair of iterators to the 
    //  matched character sequence.
        
        qi::detail::assign_to(t.value().begin(), t.value().end(), attr);
    }
    
    //  This is called from the parse function of token_def if the token type
    //  has been explicitly omitted (i.e. no attribute value is used), which
    //  essentially means that every attribute gets initialized using default 
    //  constructed values.
    template <typename Attribute, typename Iterator, typename HasState>
    inline void
    construct(Attribute& attr, lexertl_token<Iterator, omitted, HasState>& t)
    {
    }
    
    //  This is called from the parse function of token_set or lexer_def_
    template <typename Iterator, typename AttributeTypes, typename HasState>
    inline void
    construct(fusion::vector<std::size_t, iterator_range<Iterator> >& attr,
        lexertl_token<Iterator, AttributeTypes, HasState> const& t)
    {
    //  The type returned by the token_set and lexer_def_ parser components
    //  is a fusion::vector containing the token id of the matched token 
    //  and the pair of iterators to the matched character sequence.
        
        typedef iterator_range<Iterator> iterpair_type;
        typedef 
            fusion::vector<std::size_t, iterator_range<Iterator> > 
        attribute_type;
        
        iterpair_type const& ip = get<iterpair_type>(t.value());
        attr = attribute_type(t.id(), get<iterpair_type>(t.value()));
    }

}}}

#endif
