pub use self::BinOpToken::*;
pub use self::Nonterminal::*;
pub use self::DelimToken::*;
pub use self::IdentStyle::*;
pub use self::Lit::*;
pub use self::Token::*;
use ast;
use ext::mtwt;
use ptr::P;
use util::interner::{RcStr, StrInterner};
use util::interner;
use serialize::{Decodable, Decoder, Encodable, Encoder};
use std::fmt;
use std::mem;
use std::ops::Deref;
use std::path::BytesContainer;
use std::rc::Rc;
#[allow(non_camel_case_types)]
#[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Show, Copy)]
pub enum BinOpToken {
Plus,
Minus,
Star,
Slash,
Percent,
Caret,
And,
Or,
Shl,
Shr,
}
#[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Show, Copy)]
pub enum DelimToken {
Paren,
Bracket,
Brace,
}
#[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Show, Copy)]
pub enum IdentStyle {
ModName,
Plain,
}
#[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Show, Copy)]
pub enum SpecialMacroVar {
CrateMacroVar,
}
impl SpecialMacroVar {
pub fn as_str(self) -> &'static str {
match self {
SpecialMacroVar::CrateMacroVar => "crate",
}
}
}
#[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Show, Copy)]
pub enum Lit {
Byte(ast::Name),
Char(ast::Name),
Integer(ast::Name),
Float(ast::Name),
Str_(ast::Name),
StrRaw(ast::Name, uint),
Binary(ast::Name),
BinaryRaw(ast::Name, uint),
}
impl Lit {
pub fn short_name(&self) -> &'static str {
match *self {
Byte(_) => "byte",
Char(_) => "char",
Integer(_) => "integer",
Float(_) => "float",
Str_(_) | StrRaw(..) => "str",
Binary(_) | BinaryRaw(..) => "binary str"
}
}
}
#[allow(non_camel_case_types)]
#[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Show)]
pub enum Token {
Eq,
Lt,
Le,
EqEq,
Ne,
Ge,
Gt,
AndAnd,
OrOr,
Not,
Tilde,
BinOp(BinOpToken),
BinOpEq(BinOpToken),
At,
Dot,
DotDot,
DotDotDot,
Comma,
Semi,
Colon,
ModSep,
RArrow,
LArrow,
FatArrow,
Pound,
Dollar,
Question,
OpenDelim(DelimToken),
CloseDelim(DelimToken),
Literal(Lit, Option<ast::Name>),
Ident(ast::Ident, IdentStyle),
Underscore,
Lifetime(ast::Ident),
Interpolated(Nonterminal),
DocComment(ast::Name),
MatchNt(ast::Ident, ast::Ident, IdentStyle, IdentStyle),
SubstNt(ast::Ident, IdentStyle),
SpecialVarNt(SpecialMacroVar),
Whitespace,
Comment,
Shebang(ast::Name),
Eof,
}
impl Token {
pub fn can_begin_expr(&self) -> bool {
match *self {
OpenDelim(_) => true,
Ident(_, _) => true,
Underscore => true,
Tilde => true,
Literal(_, _) => true,
Not => true,
BinOp(Minus) => true,
BinOp(Star) => true,
BinOp(And) => true,
BinOp(Or) => true, OrOr => true, AndAnd => true, DotDot => true, ModSep => true,
Interpolated(NtExpr(..)) => true,
Interpolated(NtIdent(..)) => true,
Interpolated(NtBlock(..)) => true,
Interpolated(NtPath(..)) => true,
_ => false,
}
}
pub fn is_lit(&self) -> bool {
match *self {
Literal(_, _) => true,
_ => false,
}
}
pub fn is_ident(&self) -> bool {
match *self {
Ident(_, _) => true,
_ => false,
}
}
pub fn is_path(&self) -> bool {
match *self {
Interpolated(NtPath(..)) => true,
_ => false,
}
}
#[allow(non_upper_case_globals)]
pub fn is_plain_ident(&self) -> bool {
match *self {
Ident(_, Plain) => true,
_ => false,
}
}
pub fn is_lifetime(&self) -> bool {
match *self {
Lifetime(..) => true,
_ => false,
}
}
pub fn is_mutability(&self) -> bool {
self.is_keyword(keywords::Mut) ||
self.is_keyword(keywords::Const)
}
pub fn to_binop(&self) -> Option<ast::BinOp> {
match *self {
BinOp(Star) => Some(ast::BiMul),
BinOp(Slash) => Some(ast::BiDiv),
BinOp(Percent) => Some(ast::BiRem),
BinOp(Plus) => Some(ast::BiAdd),
BinOp(Minus) => Some(ast::BiSub),
BinOp(Shl) => Some(ast::BiShl),
BinOp(Shr) => Some(ast::BiShr),
BinOp(And) => Some(ast::BiBitAnd),
BinOp(Caret) => Some(ast::BiBitXor),
BinOp(Or) => Some(ast::BiBitOr),
Lt => Some(ast::BiLt),
Le => Some(ast::BiLe),
Ge => Some(ast::BiGe),
Gt => Some(ast::BiGt),
EqEq => Some(ast::BiEq),
Ne => Some(ast::BiNe),
AndAnd => Some(ast::BiAnd),
OrOr => Some(ast::BiOr),
_ => None,
}
}
#[allow(non_upper_case_globals)]
pub fn is_keyword(&self, kw: keywords::Keyword) -> bool {
match *self {
Ident(sid, Plain) => kw.to_name() == sid.name,
_ => false,
}
}
pub fn is_keyword_allow_following_colon(&self, kw: keywords::Keyword) -> bool {
match *self {
Ident(sid, _) => { kw.to_name() == sid.name }
_ => { false }
}
}
#[allow(non_upper_case_globals)]
pub fn is_any_keyword(&self) -> bool {
match *self {
Ident(sid, Plain) => {
let n = sid.name;
n == SELF_KEYWORD_NAME
|| n == STATIC_KEYWORD_NAME
|| n == SUPER_KEYWORD_NAME
|| STRICT_KEYWORD_START <= n
&& n <= RESERVED_KEYWORD_FINAL
},
_ => false
}
}
#[allow(non_upper_case_globals)]
pub fn is_strict_keyword(&self) -> bool {
match *self {
Ident(sid, Plain) => {
let n = sid.name;
n == SELF_KEYWORD_NAME
|| n == STATIC_KEYWORD_NAME
|| n == SUPER_KEYWORD_NAME
|| STRICT_KEYWORD_START <= n
&& n <= STRICT_KEYWORD_FINAL
},
Ident(sid, ModName) => {
let n = sid.name;
n != SELF_KEYWORD_NAME
&& n != SUPER_KEYWORD_NAME
&& STRICT_KEYWORD_START <= n
&& n <= STRICT_KEYWORD_FINAL
}
_ => false,
}
}
#[allow(non_upper_case_globals)]
pub fn is_reserved_keyword(&self) -> bool {
match *self {
Ident(sid, Plain) => {
let n = sid.name;
RESERVED_KEYWORD_START <= n
&& n <= RESERVED_KEYWORD_FINAL
},
_ => false,
}
}
pub fn mtwt_eq(&self, other : &Token) -> bool {
match (self, other) {
(&Ident(id1,_), &Ident(id2,_)) | (&Lifetime(id1), &Lifetime(id2)) =>
mtwt::resolve(id1) == mtwt::resolve(id2),
_ => *self == *other
}
}
}
#[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash)]
pub enum Nonterminal {
NtItem(P<ast::Item>),
NtBlock(P<ast::Block>),
NtStmt(P<ast::Stmt>),
NtPat(P<ast::Pat>),
NtExpr(P<ast::Expr>),
NtTy(P<ast::Ty>),
NtIdent(Box<ast::Ident>, IdentStyle),
NtMeta(P<ast::MetaItem>),
NtPath(Box<ast::Path>),
NtTT(P<ast::TokenTree>), }
impl fmt::Show for Nonterminal {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match *self {
NtItem(..) => f.pad("NtItem(..)"),
NtBlock(..) => f.pad("NtBlock(..)"),
NtStmt(..) => f.pad("NtStmt(..)"),
NtPat(..) => f.pad("NtPat(..)"),
NtExpr(..) => f.pad("NtExpr(..)"),
NtTy(..) => f.pad("NtTy(..)"),
NtIdent(..) => f.pad("NtIdent(..)"),
NtMeta(..) => f.pad("NtMeta(..)"),
NtPath(..) => f.pad("NtPath(..)"),
NtTT(..) => f.pad("NtTT(..)"),
}
}
}
macro_rules! first {
( $first:expr, $( $remainder:expr, )* ) => ( $first )
}
macro_rules! last {
( $first:expr, $( $remainder:expr, )+ ) => ( last!( $( $remainder, )+ ) );
( $first:expr, ) => ( $first )
}
macro_rules! declare_special_idents_and_keywords {(
// So now, in these rules, why is each definition parenthesised?
// Answer: otherwise we get a spurious local ambiguity bug on the "}"
pub mod special_idents {
$( ($si_name:expr, $si_static:ident, $si_str:expr); )*
}
pub mod keywords {
'strict:
$( ($sk_name:expr, $sk_variant:ident, $sk_str:expr); )*
'reserved:
$( ($rk_name:expr, $rk_variant:ident, $rk_str:expr); )*
}
) => {
static STRICT_KEYWORD_START: ast::Name = first!($( ast::Name($sk_name), )*);
static STRICT_KEYWORD_FINAL: ast::Name = last!($( ast::Name($sk_name), )*);
static RESERVED_KEYWORD_START: ast::Name = first!($( ast::Name($rk_name), )*);
static RESERVED_KEYWORD_FINAL: ast::Name = last!($( ast::Name($rk_name), )*);
pub mod special_idents {
use ast;
$(
#[allow(non_upper_case_globals)]
pub const $si_static: ast::Ident = ast::Ident {
name: ast::Name($si_name),
ctxt: 0,
};
)*
}
pub mod special_names {
use ast;
$(
#[allow(non_upper_case_globals)]
pub const $si_static: ast::Name = ast::Name($si_name);
)*
}
pub mod keywords {
pub use self::Keyword::*;
use ast;
#[derive(Copy)]
pub enum Keyword {
$( $sk_variant, )*
$( $rk_variant, )*
}
impl Keyword {
pub fn to_name(&self) -> ast::Name {
match *self {
$( $sk_variant => ast::Name($sk_name), )*
$( $rk_variant => ast::Name($rk_name), )*
}
}
}
}
fn mk_fresh_ident_interner() -> IdentInterner {
let mut init_vec = Vec::new();
$(init_vec.push($si_str);)*
$(init_vec.push($sk_str);)*
$(init_vec.push($rk_str);)*
interner::StrInterner::prefill(&init_vec[])
}
}}
pub const SELF_KEYWORD_NAME: ast::Name = ast::Name(SELF_KEYWORD_NAME_NUM);
const STATIC_KEYWORD_NAME: ast::Name = ast::Name(STATIC_KEYWORD_NAME_NUM);
const SUPER_KEYWORD_NAME: ast::Name = ast::Name(SUPER_KEYWORD_NAME_NUM);
pub const SELF_KEYWORD_NAME_NUM: u32 = 1;
const STATIC_KEYWORD_NAME_NUM: u32 = 2;
const SUPER_KEYWORD_NAME_NUM: u32 = 3;
declare_special_idents_and_keywords! {
pub mod special_idents {
(0, invalid, "");
(super::SELF_KEYWORD_NAME_NUM, self_, "self");
(super::STATIC_KEYWORD_NAME_NUM, statik, "static");
(super::SUPER_KEYWORD_NAME_NUM, super_, "super");
(4, static_lifetime, "'static");
(5, tt, "tt");
(6, matchers, "matchers");
(7, clownshoe_abi, "__rust_abi");
(8, opaque, "<opaque>");
(9, unnamed_field, "<unnamed_field>");
(10, type_self, "Self");
(11, prelude_import, "prelude_import");
(12, FullRange, "FullRange");
}
pub mod keywords {
'strict:
(13, As, "as");
(14, Break, "break");
(15, Crate, "crate");
(16, Else, "else");
(17, Enum, "enum");
(18, Extern, "extern");
(19, False, "false");
(20, Fn, "fn");
(21, For, "for");
(22, If, "if");
(23, Impl, "impl");
(24, In, "in");
(25, Let, "let");
(26, Loop, "loop");
(27, Match, "match");
(28, Mod, "mod");
(29, Move, "move");
(30, Mut, "mut");
(31, Pub, "pub");
(32, Ref, "ref");
(33, Return, "return");
(super::STATIC_KEYWORD_NAME_NUM, Static, "static");
(super::SELF_KEYWORD_NAME_NUM, Self, "self");
(34, Struct, "struct");
(super::SUPER_KEYWORD_NAME_NUM, Super, "super");
(35, True, "true");
(36, Trait, "trait");
(37, Type, "type");
(38, Unsafe, "unsafe");
(39, Use, "use");
(40, Virtual, "virtual");
(41, While, "while");
(42, Continue, "continue");
(43, Proc, "proc");
(44, Box, "box");
(45, Const, "const");
(46, Where, "where");
'reserved:
(47, Alignof, "alignof");
(48, Be, "be");
(49, Offsetof, "offsetof");
(50, Priv, "priv");
(51, Pure, "pure");
(52, Sizeof, "sizeof");
(53, Typeof, "typeof");
(54, Unsized, "unsized");
(55, Yield, "yield");
(56, Do, "do");
(57, Abstract, "abstract");
(58, Final, "final");
(59, Override, "override");
(60, Macro, "macro");
}
}
pub type IdentInterner = StrInterner;
pub fn get_ident_interner() -> Rc<IdentInterner> {
thread_local!(static KEY: Rc<::parse::token::IdentInterner> = {
Rc::new(mk_fresh_ident_interner())
});
KEY.with(|k| k.clone())
}
pub fn reset_ident_interner() {
let interner = get_ident_interner();
interner.reset(mk_fresh_ident_interner());
}
#[derive(Clone, PartialEq, Hash, PartialOrd, Eq, Ord)]
pub struct InternedString {
string: RcStr,
}
impl InternedString {
#[inline]
pub fn new(string: &'static str) -> InternedString {
InternedString {
string: RcStr::new(string),
}
}
#[inline]
fn new_from_rc_str(string: RcStr) -> InternedString {
InternedString {
string: string,
}
}
#[inline]
pub fn get<'a>(&'a self) -> &'a str {
&self.string[]
}
}
impl Deref for InternedString {
type Target = str;
fn deref(&self) -> &str { &*self.string }
}
impl BytesContainer for InternedString {
fn container_as_bytes<'a>(&'a self) -> &'a [u8] {
unsafe {
let this = self.get();
mem::transmute::<&[u8],&[u8]>(this.container_as_bytes())
}
}
}
impl fmt::Show for InternedString {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
fmt::String::fmt(self, f)
}
}
impl fmt::String for InternedString {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "{}", &self.string[])
}
}
impl<'a> PartialEq<&'a str> for InternedString {
#[inline(always)]
fn eq(&self, other: & &'a str) -> bool {
PartialEq::eq(&self.string[], *other)
}
#[inline(always)]
fn ne(&self, other: & &'a str) -> bool {
PartialEq::ne(&self.string[], *other)
}
}
impl<'a> PartialEq<InternedString > for &'a str {
#[inline(always)]
fn eq(&self, other: &InternedString) -> bool {
PartialEq::eq(*self, &other.string[])
}
#[inline(always)]
fn ne(&self, other: &InternedString) -> bool {
PartialEq::ne(*self, &other.string[])
}
}
impl Decodable for InternedString {
fn decode<D: Decoder>(d: &mut D) -> Result<InternedString, D::Error> {
Ok(get_name(get_ident_interner().intern(&try!(d.read_str())[])))
}
}
impl Encodable for InternedString {
fn encode<S: Encoder>(&self, s: &mut S) -> Result<(), S::Error> {
s.emit_str(&self.string[])
}
}
#[inline]
pub fn get_name(name: ast::Name) -> InternedString {
let interner = get_ident_interner();
InternedString::new_from_rc_str(interner.get(name))
}
#[inline]
pub fn get_ident(ident: ast::Ident) -> InternedString {
get_name(ident.name)
}
#[inline]
pub fn intern_and_get_ident(s: &str) -> InternedString {
get_name(intern(s))
}
#[inline]
pub fn intern(s: &str) -> ast::Name {
get_ident_interner().intern(s)
}
#[inline]
pub fn gensym(s: &str) -> ast::Name {
get_ident_interner().gensym(s)
}
#[inline]
pub fn str_to_ident(s: &str) -> ast::Ident {
ast::Ident::new(intern(s))
}
#[inline]
pub fn gensym_ident(s: &str) -> ast::Ident {
ast::Ident::new(gensym(s))
}
pub fn fresh_name(src: &ast::Ident) -> ast::Name {
let interner = get_ident_interner();
interner.gensym_copy(src.name)
}
pub fn fresh_mark() -> ast::Mrk {
gensym("mark").uint() as u32
}
#[cfg(test)]
mod test {
use super::*;
use ast;
use ext::mtwt;
fn mark_ident(id : ast::Ident, m : ast::Mrk) -> ast::Ident {
ast::Ident { name: id.name, ctxt:mtwt::apply_mark(m, id.ctxt) }
}
#[test] fn mtwt_token_eq_test() {
assert!(Gt.mtwt_eq(&Gt));
let a = str_to_ident("bac");
let a1 = mark_ident(a,92);
assert!(Ident(a, ModName).mtwt_eq(&Ident(a1, Plain)));
}
}