From 754759688bd964397d09b5f689fcf8d155a8136b Mon Sep 17 00:00:00 2001 From: cgswords Date: Wed, 29 Jun 2016 11:55:10 -0700 Subject: [PATCH] Preliminary implementation for TokenStreams and TokenSlices, including unit tests and associated operations. --- src/libsyntax/tokenstream.rs | 1160 ++++++++++++++++++++++++++++++++-- 1 file changed, 1121 insertions(+), 39 deletions(-) diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index f0f0a7bc580d3..0ad09fd0f7dfb 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -8,18 +8,34 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -//! # Token Trees -//! TokenTrees are syntactic forms for dealing with tokens. The description below is -//! more complete; in short a TokenTree is a single token, a delimited sequence of token -//! trees, or a sequence with repetition for list splicing as part of macro expansion. +//! # Token Streams +//! +//! TokenStreams represent syntactic objects before they are converted into ASTs. +//! A `TokenStream` is, roughly speaking, a sequence (eg stream) of `TokenTree`s, +//! which are themselves either a single Token, a Delimited subsequence of tokens, +//! or a SequenceRepetition specifier (for the purpose of sequence generation during macro +//! expansion). +//! +//! A TokenStream also has a slice view, `TokenSlice`, that is analogous to `str` for +//! `String`: it allows the programmer to divvy up, explore, and otherwise partition a +//! TokenStream as borrowed subsequences. -use ast::{AttrStyle}; -use codemap::{Span}; +use ast::{self, AttrStyle, LitKind}; +use syntax_pos::{Span, DUMMY_SP, NO_EXPANSION}; +use codemap::Spanned; use ext::base; use ext::tt::macro_parser; use parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration}; use parse::lexer; -use parse::token; +use parse; +use parse::token::{self, Token, Lit, InternedString, Nonterminal}; +use parse::token::Lit as TokLit; + +use std::fmt; +use std::mem; +use std::ops::Index; +use std::ops; +use std::iter::*; use std::rc::Rc; @@ -56,6 +72,11 @@ impl Delimited { pub fn close_tt(&self) -> TokenTree { TokenTree::Token(self.close_span, self.close_token()) } + + /// Returns the token trees inside the delimiters. + pub fn subtrees(&self) -> &[TokenTree] { + &self.tts + } } /// A sequence of token trees @@ -91,7 +112,7 @@ pub enum KleeneOp { /// /// The RHS of an MBE macro is the only place `SubstNt`s are substituted. /// Nothing special happens to misnamed or misplaced `SubstNt`s. -#[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug)] +#[derive(Debug, Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash)] pub enum TokenTree { /// A single token Token(Span, token::Token), @@ -99,9 +120,7 @@ pub enum TokenTree { Delimited(Span, Rc), // This only makes sense in MBE macros. - /// A kleene-style repetition sequence with a span - // FIXME(eddyb) #12938 Use DST. Sequence(Span, Rc), } @@ -111,28 +130,22 @@ impl TokenTree { TokenTree::Token(_, token::DocComment(name)) => { match doc_comment_style(&name.as_str()) { AttrStyle::Outer => 2, - AttrStyle::Inner => 3 + AttrStyle::Inner => 3, } } TokenTree::Token(_, token::SpecialVarNt(..)) => 2, TokenTree::Token(_, token::MatchNt(..)) => 3, - TokenTree::Delimited(_, ref delimed) => { - delimed.tts.len() + 2 - } - TokenTree::Sequence(_, ref seq) => { - seq.tts.len() - } - TokenTree::Token(..) => 0 + TokenTree::Delimited(_, ref delimed) => delimed.tts.len() + 2, + TokenTree::Sequence(_, ref seq) => seq.tts.len(), + TokenTree::Token(..) => 0, } } pub fn get_tt(&self, index: usize) -> TokenTree { match (self, index) { - (&TokenTree::Token(sp, token::DocComment(_)), 0) => { - TokenTree::Token(sp, token::Pound) - } + (&TokenTree::Token(sp, token::DocComment(_)), 0) => TokenTree::Token(sp, token::Pound), (&TokenTree::Token(sp, token::DocComment(name)), 1) - if doc_comment_style(&name.as_str()) == AttrStyle::Inner => { + if doc_comment_style(&name.as_str()) == AttrStyle::Inner => { TokenTree::Token(sp, token::Not) } (&TokenTree::Token(sp, token::DocComment(name)), _) => { @@ -140,16 +153,19 @@ impl TokenTree { // Searches for the occurrences of `"#*` and returns the minimum number of `#`s // required to wrap the text. - let num_of_hashes = stripped.chars().scan(0, |cnt, x| { - *cnt = if x == '"' { - 1 - } else if *cnt != 0 && x == '#' { - *cnt + 1 - } else { - 0 - }; - Some(*cnt) - }).max().unwrap_or(0); + let num_of_hashes = stripped.chars() + .scan(0, |cnt, x| { + *cnt = if x == '"' { + 1 + } else if *cnt != 0 && x == '#' { + *cnt + 1 + } else { + 0 + }; + Some(*cnt) + }) + .max() + .unwrap_or(0); TokenTree::Delimited(sp, Rc::new(Delimited { delim: token::Bracket, @@ -181,24 +197,24 @@ impl TokenTree { TokenTree::Token(sp, token::Ident(kind))]; v[index].clone() } - (&TokenTree::Sequence(_, ref seq), _) => { - seq.tts[index].clone() - } - _ => panic!("Cannot expand a token tree") + (&TokenTree::Sequence(_, ref seq), _) => seq.tts[index].clone(), + _ => panic!("Cannot expand a token tree"), } } /// Returns the `Span` corresponding to this token tree. pub fn get_span(&self) -> Span { match *self { - TokenTree::Token(span, _) => span, + TokenTree::Token(span, _) => span, TokenTree::Delimited(span, _) => span, - TokenTree::Sequence(span, _) => span, + TokenTree::Sequence(span, _) => span, } } /// Use this token tree as a matcher to parse given tts. - pub fn parse(cx: &base::ExtCtxt, mtch: &[TokenTree], tts: &[TokenTree]) + pub fn parse(cx: &base::ExtCtxt, + mtch: &[TokenTree], + tts: &[TokenTree]) -> macro_parser::NamedParseResult { // `None` is because we're not interpolating let arg_rdr = lexer::new_tt_reader_with_doc_flag(&cx.parse_sess().span_diagnostic, @@ -208,5 +224,1071 @@ impl TokenTree { true); macro_parser::parse(cx.parse_sess(), cx.cfg(), arg_rdr, mtch) } + + /// Check if this TokenTree is equal to the other, regardless of span information. + pub fn eq_unspanned(&self, other: &TokenTree) -> bool { + match (self, other) { + (&TokenTree::Token(_, ref tk), &TokenTree::Token(_, ref tk2)) => tk == tk2, + (&TokenTree::Delimited(_, ref dl), &TokenTree::Delimited(_, ref dl2)) => { + (*dl).delim == (*dl2).delim && dl.tts.len() == dl2.tts.len() && + { + for (tt1, tt2) in dl.tts.iter().zip(dl2.tts.iter()) { + if !tt1.eq_unspanned(tt2) { + return false; + } + } + true + } + } + (_, _) => false, + } + } + + /// Retrieve the TokenTree's span. + pub fn span(&self) -> Span { + match *self { + TokenTree::Token(sp, _) | + TokenTree::Delimited(sp, _) | + TokenTree::Sequence(sp, _) => sp, + } + } + + /// Indicates if the stream is a token that is equal to the provided token. + pub fn eq_token(&self, t: Token) -> bool { + match *self { + TokenTree::Token(_, ref tk) => *tk == t, + _ => false, + } + } + + /// Indicates if the token is an identifier. + pub fn is_ident(&self) -> bool { + self.maybe_ident().is_some() + } + + /// Returns an identifier. + pub fn maybe_ident(&self) -> Option { + match *self { + TokenTree::Token(_, Token::Ident(t)) => Some(t.clone()), + TokenTree::Delimited(_, ref dl) => { + let tts = dl.subtrees(); + if tts.len() != 1 { + return None; + } + tts[0].maybe_ident() + } + _ => None, + } + } + + /// Returns a Token literal. + pub fn maybe_lit(&self) -> Option { + match *self { + TokenTree::Token(_, Token::Literal(l, _)) => Some(l.clone()), + TokenTree::Delimited(_, ref dl) => { + let tts = dl.subtrees(); + if tts.len() != 1 { + return None; + } + tts[0].maybe_lit() + } + _ => None, + } + } + + /// Returns an AST string literal. + pub fn maybe_str(&self) -> Option { + match *self { + TokenTree::Token(sp, Token::Literal(Lit::Str_(s), _)) => { + let l = LitKind::Str(token::intern_and_get_ident(&parse::str_lit(&s.as_str())), + ast::StrStyle::Cooked); + Some(Spanned { + node: l, + span: sp, + }) + } + TokenTree::Token(sp, Token::Literal(Lit::StrRaw(s, n), _)) => { + let l = LitKind::Str(token::intern_and_get_ident(&parse::raw_str_lit(&s.as_str())), + ast::StrStyle::Raw(n)); + Some(Spanned { + node: l, + span: sp, + }) + } + _ => None, + } + } +} + +/// #Token Streams +/// +/// TokenStreams are a syntactic abstraction over TokenTrees. The goal is for procedural +/// macros to work over TokenStreams instead of arbitrary syntax. For now, however, we +/// are going to cut a few corners (i.e., use some of the AST structure) when we need to +/// for backwards compatibility. + +/// TokenStreams are collections of TokenTrees that represent a syntactic structure. The +/// struct itself shouldn't be directly manipulated; the internal structure is not stable, +/// and may be changed at any time in the future. The operators will not, however (except +/// for signatures, later on). +#[derive(Eq,Clone,Hash,RustcEncodable,RustcDecodable)] +pub struct TokenStream { + pub span: Span, + pub tts: Vec, +} + +impl fmt::Debug for TokenStream { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + if self.tts.len() == 0 { + write!(f, "([empty")?; + } else { + write!(f, "([")?; + write!(f, "{:?}", self.tts[0])?; + + for tt in self.tts.iter().skip(1) { + write!(f, ",{:?}", tt)?; + } + } + write!(f, "|")?; + self.span.fmt(f)?; + write!(f, "])") + } +} + +/// Checks if two TokenStreams are equivalent (including spans). For unspanned +/// equality, see `eq_unspanned`. +impl PartialEq for TokenStream { + fn eq(&self, other: &TokenStream) -> bool { + self.tts == other.tts + } +} + +// NB this will disregard gaps. if we have [a|{2,5} , b|{11,13}], the resultant span +// will be at {2,13}. Without finer-grained span structures, however, this seems to be +// our only recourse. +// FIXME Do something smarter to compute the expansion id. +fn covering_span(trees: &[TokenTree]) -> Span { + // disregard any dummy spans we have + let trees = trees.iter().filter(|t| t.span() != DUMMY_SP).collect::>(); + + // if we're out of spans, stop + if trees.len() < 1 { + return DUMMY_SP; + } + + // set up the initial values + let fst_span = trees[0].span(); + + let mut lo_span = fst_span.lo; + let mut hi_span = fst_span.hi; + let mut expn_id = fst_span.expn_id; + + // compute the spans iteratively + for t in trees.iter().skip(1) { + let sp = t.span(); + if sp.lo < lo_span { + lo_span = sp.lo; + } + if hi_span < sp.hi { + hi_span = sp.hi; + } + if expn_id != sp.expn_id { + expn_id = NO_EXPANSION; + } + } + + Span { + lo: lo_span, + hi: hi_span, + expn_id: expn_id, + } +} + +/// TokenStream operators include basic destructuring, boolean operations, `maybe_...` +/// operations, and `maybe_..._prefix` operations. Boolean operations are straightforward, +/// indicating information about the structure of the stream. The `maybe_...` operations +/// return `Some<...>` if the tokenstream contains the appropriate item. +/// +/// Similarly, the `maybe_..._prefix` operations potentially return a +/// partially-destructured stream as a pair where the first element is the expected item +/// and the second is the remainder of the stream. As anb example, +/// +/// `maybe_path_prefix("a::b::c(a,b,c).foo()") -> (a::b::c, "(a,b,c).foo()")` +impl TokenStream { + /// Convert a vector of `TokenTree`s into a `TokenStream`. + pub fn from_tts(trees: Vec) -> TokenStream { + let span = covering_span(&trees); + TokenStream { + tts: trees, + span: span, + } + } + + /// Copies all of the TokenTrees from the TokenSlice, appending them to the stream. + pub fn append_stream(mut self, ts2: &TokenSlice) { + for tt in ts2.iter() { + self.tts.push(tt.clone()); + } + self.span = covering_span(&self.tts[..]); + } + + /// Manually change a TokenStream's span. + pub fn respan(self, span: Span) -> TokenStream { + TokenStream { + tts: self.tts, + span: span, + } + } + + /// Construct a TokenStream from an ast literal. + pub fn from_ast_lit_str(lit: ast::Lit) -> Option { + match lit.node { + LitKind::Str(val, _) => { + let val = TokLit::Str_(token::intern(&val)); + Some(TokenStream::from_tts(vec![TokenTree::Token(lit.span, + Token::Literal(val, None))])) + } + _ => None, + } + + } + + /// Convert a vector of TokenTrees into a parentheses-delimited TokenStream. + pub fn as_paren_delimited_stream(tts: Vec) -> TokenStream { + let new_sp = covering_span(&tts); + + let new_delim = Rc::new(Delimited { + delim: token::DelimToken::Paren, + open_span: DUMMY_SP, + tts: tts, + close_span: DUMMY_SP, + }); + + TokenStream::from_tts(vec![TokenTree::Delimited(new_sp, new_delim)]) + } + + /// Convert an interned string into a one-element TokenStream. + pub fn from_interned_string_as_ident(s: InternedString) -> TokenStream { + TokenStream::from_tts(vec![TokenTree::Token(DUMMY_SP, + Token::Ident(token::str_to_ident(&s[..])))]) + } } +/// TokenSlices are 'views' of `TokenStream's; they fit the same role as `str`s do for +/// `String`s. In general, most TokenStream manipulations will be refocusing their internal +/// contents by taking a TokenSlice and then using indexing and the provided operators. +#[derive(PartialEq, Eq, Debug)] +pub struct TokenSlice([TokenTree]); + +impl ops::Deref for TokenStream { + type Target = TokenSlice; + + fn deref(&self) -> &TokenSlice { + let tts: &[TokenTree] = &*self.tts; + unsafe { mem::transmute(tts) } + } +} + +impl TokenSlice { + /// Convert a borrowed TokenTree slice into a borrowed TokenSlice. + fn from_tts(tts: &[TokenTree]) -> &TokenSlice { + unsafe { mem::transmute(tts) } + } + + /// Indicates whether the `TokenStream` is empty. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Return the `TokenSlice`'s length. + pub fn len(&self) -> usize { + self.0.len() + } + + /// Check equality versus another TokenStream, ignoring span information. + pub fn eq_unspanned(&self, other: &TokenSlice) -> bool { + if self.len() != other.len() { + return false; + } + for (tt1, tt2) in self.iter().zip(other.iter()) { + if !tt1.eq_unspanned(tt2) { + return false; + } + } + true + } + + /// Compute a span that covers the entire TokenSlice (eg, one wide enough to include + /// the entire slice). If the inputs share expansion identification, it is preserved. + /// If they do not, it is discarded. + pub fn covering_span(&self) -> Span { + covering_span(&self.0) + } + + /// Indicates where the stream is of the form `= `, where `` is a continued + /// `TokenStream`. + pub fn is_assignment(&self) -> bool { + self.maybe_assignment().is_some() + } + + /// Returns the RHS of an assigment. + pub fn maybe_assignment(&self) -> Option<&TokenSlice> { + if !(self.len() > 1) { + return None; + } + + Some(&self[1..]) + } + + /// Indicates where the stream is a single, delimited expression (e.g., `(a,b,c)` or + /// `{a,b,c}`). + pub fn is_delimited(&self) -> bool { + self.maybe_delimited().is_some() + } + + /// Returns the inside of the delimited term as a new TokenStream. + pub fn maybe_delimited(&self) -> Option<&TokenSlice> { + if !(self.len() == 1) { + return None; + } + + match self[0] { + TokenTree::Delimited(_, ref rc) => Some(TokenSlice::from_tts(&*rc.tts)), + _ => None, + } + } + + /// Returns a list of `TokenSlice`s if the stream is a delimited list, breaking the + /// stream on commas. + pub fn maybe_comma_list(&self) -> Option> { + let maybe_tts = self.maybe_delimited(); + + let ts: &TokenSlice; + match maybe_tts { + Some(t) => { + ts = t; + } + None => { + return None; + } + } + + let splits: Vec<&TokenSlice> = ts.split(|x| match *x { + TokenTree::Token(_, Token::Comma) => true, + _ => false, + }) + .filter(|x| x.len() > 0) + .collect(); + + Some(splits) + } + + /// Returns a Nonterminal if it is Interpolated. + pub fn maybe_interpolated_nonterminal(&self) -> Option { + if !(self.len() == 1) { + return None; + } + + match self[0] { + TokenTree::Token(_, Token::Interpolated(ref nt)) => Some(nt.clone()), + _ => None, + } + } + + /// Indicates if the stream is exactly one identifier. + pub fn is_ident(&self) -> bool { + self.maybe_ident().is_some() + } + + /// Returns an identifier + pub fn maybe_ident(&self) -> Option { + if !(self.len() == 1) { + return None; + } + + let tok = if let Some(tts) = self.maybe_delimited() { + if tts.len() != 1 { + return None; + } + &tts[0] + } else { + &self[0] + }; + + match *tok { + TokenTree::Token(_, Token::Ident(t)) => Some(t), + _ => None, + } + } + + /// Indicates if the stream is exactly one literal + pub fn is_lit(&self) -> bool { + self.maybe_lit().is_some() + } + + /// Returns a literal + pub fn maybe_lit(&self) -> Option { + if !(self.len() == 1) { + return None; + } + + let tok = if let Some(tts) = self.maybe_delimited() { + if tts.len() != 1 { + return None; + } + &tts[0] + } else { + &self[0] + }; + + match *tok { + TokenTree::Token(_, Token::Literal(l, _)) => Some(l), + _ => None, + } + } + + /// Returns an AST string literal if the TokenStream is either a normal ('cooked') or + /// raw string literal. + pub fn maybe_str(&self) -> Option { + if !(self.len() == 1) { + return None; + } + + match self[0] { + TokenTree::Token(sp, Token::Literal(Lit::Str_(s), _)) => { + let l = LitKind::Str(token::intern_and_get_ident(&parse::str_lit(&s.as_str())), + ast::StrStyle::Cooked); + Some(Spanned { + node: l, + span: sp, + }) + } + TokenTree::Token(sp, Token::Literal(Lit::StrRaw(s, n), _)) => { + let l = LitKind::Str(token::intern_and_get_ident(&parse::raw_str_lit(&s.as_str())), + ast::StrStyle::Raw(n)); + Some(Spanned { + node: l, + span: sp, + }) + } + _ => None, + } + } + + /// This operation extracts the path prefix , returning an AST path struct and the remainder + /// of the stream (if it finds one). To be more specific, a tokenstream that has a valid, + /// non-global path as a prefix (eg `foo(bar, baz)`, `foo::bar(bar)`, but *not* + /// `::foo::bar(baz)`) will yield the path and the remaining tokens (as a slice). The previous + /// examples will yield + /// `Some((Path { segments = vec![foo], ... }, [(bar, baz)]))`, + /// `Some((Path { segments = vec![foo, bar] }, [(baz)]))`, + /// and `None`, respectively. + pub fn maybe_path_prefix(&self) -> Option<(ast::Path, &TokenSlice)> { + let mut segments: Vec = Vec::new(); + + let path: Vec<&TokenTree> = self.iter() + .take_while(|x| x.is_ident() || x.eq_token(Token::ModSep)) + .collect::>(); + + let path_size = path.len(); + if path_size == 0 { + return None; + } + + let cov_span = self[..path_size].covering_span(); + let rst = &self[path_size..]; + + let fst_id = path[0]; + + if let Some(id) = fst_id.maybe_ident() { + segments.push(ast::PathSegment { + identifier: id, + parameters: ast::PathParameters::none(), + }); + } else { + return None; + } + + // Let's use a state machine to parse out the rest. + enum State { + Mod, // Expect a `::`, or return None otherwise. + Ident, // Expect an ident, or return None otherwise. + } + let mut state = State::Mod; + + for p in &path[1..] { + match state { + State::Mod => { + // State 0: ['::' -> state 1, else return None] + if p.eq_token(Token::ModSep) { + state = State::Ident; + } else { + return None; + } + } + State::Ident => { + // State 1: [ident -> state 0, else return None] + if let Some(id) = p.maybe_ident() { + segments.push(ast::PathSegment { + identifier: id, + parameters: ast::PathParameters::none(), + }); + state = State::Mod; + } else { + return None; + } + } + } + } + + let path = ast::Path { + span: cov_span, + global: false, + segments: segments, + }; + Some((path, rst)) + } + + /// Returns an iterator over a TokenSlice (as a sequence of TokenStreams). + fn iter(&self) -> Iter { + Iter { vs: self } + } + + /// Splits a TokenSlice based on the provided `&TokenTree -> bool` predicate. + fn split

(&self, pred: P) -> Split

+ where P: FnMut(&TokenTree) -> bool + { + Split { + vs: self, + pred: pred, + finished: false, + } + } +} + +pub struct Iter<'a> { + vs: &'a TokenSlice, +} + +impl<'a> Iterator for Iter<'a> { + type Item = &'a TokenTree; + + fn next(&mut self) -> Option<&'a TokenTree> { + if self.vs.is_empty() { + return None; + } + + let ret = Some(&self.vs[0]); + self.vs = &self.vs[1..]; + ret + } +} + +pub struct Split<'a, P> + where P: FnMut(&TokenTree) -> bool +{ + vs: &'a TokenSlice, + pred: P, + finished: bool, +} + +impl<'a, P> Iterator for Split<'a, P> + where P: FnMut(&TokenTree) -> bool +{ + type Item = &'a TokenSlice; + + fn next(&mut self) -> Option<&'a TokenSlice> { + if self.finished { + return None; + } + + match self.vs.iter().position(|x| (self.pred)(x)) { + None => { + self.finished = true; + Some(&self.vs[..]) + } + Some(idx) => { + let ret = Some(&self.vs[..idx]); + self.vs = &self.vs[idx + 1..]; + ret + } + } + } +} + +impl Index for TokenStream { + type Output = TokenTree; + + fn index(&self, index: usize) -> &TokenTree { + Index::index(&**self, index) + } +} + +impl ops::Index> for TokenStream { + type Output = TokenSlice; + + fn index(&self, index: ops::Range) -> &TokenSlice { + Index::index(&**self, index) + } +} + +impl ops::Index> for TokenStream { + type Output = TokenSlice; + + fn index(&self, index: ops::RangeTo) -> &TokenSlice { + Index::index(&**self, index) + } +} + +impl ops::Index> for TokenStream { + type Output = TokenSlice; + + fn index(&self, index: ops::RangeFrom) -> &TokenSlice { + Index::index(&**self, index) + } +} + +impl ops::Index for TokenStream { + type Output = TokenSlice; + + fn index(&self, _index: ops::RangeFull) -> &TokenSlice { + Index::index(&**self, _index) + } +} + +impl Index for TokenSlice { + type Output = TokenTree; + + fn index(&self, index: usize) -> &TokenTree { + &self.0[index] + } +} + +impl ops::Index> for TokenSlice { + type Output = TokenSlice; + + fn index(&self, index: ops::Range) -> &TokenSlice { + TokenSlice::from_tts(&self.0[index]) + } +} + +impl ops::Index> for TokenSlice { + type Output = TokenSlice; + + fn index(&self, index: ops::RangeTo) -> &TokenSlice { + TokenSlice::from_tts(&self.0[index]) + } +} + +impl ops::Index> for TokenSlice { + type Output = TokenSlice; + + fn index(&self, index: ops::RangeFrom) -> &TokenSlice { + TokenSlice::from_tts(&self.0[index]) + } +} + +impl ops::Index for TokenSlice { + type Output = TokenSlice; + + fn index(&self, _index: ops::RangeFull) -> &TokenSlice { + TokenSlice::from_tts(&self.0[_index]) + } +} + + +#[cfg(test)] +mod tests { + use super::*; + use ast; + use syntax_pos::{Span, BytePos, NO_EXPANSION, DUMMY_SP}; + use parse::token::{self, str_to_ident, Token, Lit}; + use util::parser_testing::string_to_tts; + use std::rc::Rc; + + fn sp(a: u32, b: u32) -> Span { + Span { + lo: BytePos(a), + hi: BytePos(b), + expn_id: NO_EXPANSION, + } + } + + #[test] + fn test_is_empty() { + let test0 = TokenStream::from_tts(Vec::new()); + let test1 = TokenStream::from_tts(vec![TokenTree::Token(sp(0, 1), + Token::Ident(str_to_ident("a")))]); + let test2 = TokenStream::from_tts(string_to_tts("foo(bar::baz)".to_string())); + + assert_eq!(test0.is_empty(), true); + assert_eq!(test1.is_empty(), false); + assert_eq!(test2.is_empty(), false); + } + + #[test] + fn test_is_delimited() { + let test0 = TokenStream::from_tts(string_to_tts("foo(bar::baz)".to_string())); + let test1 = TokenStream::from_tts(string_to_tts("(bar::baz)".to_string())); + let test2 = TokenStream::from_tts(string_to_tts("(foo,bar,baz)".to_string())); + let test3 = TokenStream::from_tts(string_to_tts("(foo,bar,baz)(zab,rab,oof)".to_string())); + let test4 = TokenStream::from_tts(string_to_tts("(foo,bar,baz)foo".to_string())); + let test5 = TokenStream::from_tts(string_to_tts("".to_string())); + + assert_eq!(test0.is_delimited(), false); + assert_eq!(test1.is_delimited(), true); + assert_eq!(test2.is_delimited(), true); + assert_eq!(test3.is_delimited(), false); + assert_eq!(test4.is_delimited(), false); + assert_eq!(test5.is_delimited(), false); + } + + #[test] + fn test_is_assign() { + let test0 = TokenStream::from_tts(string_to_tts("= bar::baz".to_string())); + let test1 = TokenStream::from_tts(string_to_tts("= \"5\"".to_string())); + let test2 = TokenStream::from_tts(string_to_tts("= 5".to_string())); + let test3 = TokenStream::from_tts(string_to_tts("(foo = 10)".to_string())); + let test4 = TokenStream::from_tts(string_to_tts("= (foo,bar,baz)".to_string())); + let test5 = TokenStream::from_tts(string_to_tts("".to_string())); + + assert_eq!(test0.is_assignment(), true); + assert_eq!(test1.is_assignment(), true); + assert_eq!(test2.is_assignment(), true); + assert_eq!(test3.is_assignment(), false); + assert_eq!(test4.is_assignment(), true); + assert_eq!(test5.is_assignment(), false); + } + + #[test] + fn test_is_lit() { + let test0 = TokenStream::from_tts(string_to_tts("\"foo\"".to_string())); + let test1 = TokenStream::from_tts(string_to_tts("5".to_string())); + let test2 = TokenStream::from_tts(string_to_tts("foo".to_string())); + let test3 = TokenStream::from_tts(string_to_tts("foo::bar".to_string())); + let test4 = TokenStream::from_tts(string_to_tts("foo(bar)".to_string())); + + assert_eq!(test0.is_lit(), true); + assert_eq!(test1.is_lit(), true); + assert_eq!(test2.is_lit(), false); + assert_eq!(test3.is_lit(), false); + assert_eq!(test4.is_lit(), false); + } + + #[test] + fn test_is_ident() { + let test0 = TokenStream::from_tts(string_to_tts("\"foo\"".to_string())); + let test1 = TokenStream::from_tts(string_to_tts("5".to_string())); + let test2 = TokenStream::from_tts(string_to_tts("foo".to_string())); + let test3 = TokenStream::from_tts(string_to_tts("foo::bar".to_string())); + let test4 = TokenStream::from_tts(string_to_tts("foo(bar)".to_string())); + + assert_eq!(test0.is_ident(), false); + assert_eq!(test1.is_ident(), false); + assert_eq!(test2.is_ident(), true); + assert_eq!(test3.is_ident(), false); + assert_eq!(test4.is_ident(), false); + } + + #[test] + fn test_maybe_assignment() { + let test0_input = TokenStream::from_tts(string_to_tts("= bar::baz".to_string())); + let test1_input = TokenStream::from_tts(string_to_tts("= \"5\"".to_string())); + let test2_input = TokenStream::from_tts(string_to_tts("= 5".to_string())); + let test3_input = TokenStream::from_tts(string_to_tts("(foo = 10)".to_string())); + let test4_input = TokenStream::from_tts(string_to_tts("= (foo,bar,baz)".to_string())); + let test5_input = TokenStream::from_tts(string_to_tts("".to_string())); + + let test0 = test0_input.maybe_assignment(); + let test1 = test1_input.maybe_assignment(); + let test2 = test2_input.maybe_assignment(); + let test3 = test3_input.maybe_assignment(); + let test4 = test4_input.maybe_assignment(); + let test5 = test5_input.maybe_assignment(); + + let test0_expected = TokenStream::from_tts(vec![TokenTree::Token(sp(2, 5), + token::Ident(str_to_ident("bar"))), + TokenTree::Token(sp(5, 7), token::ModSep), + TokenTree::Token(sp(7, 10), + token::Ident(str_to_ident("baz")))]); + assert_eq!(test0, Some(&test0_expected[..])); + + let test1_expected = TokenStream::from_tts(vec![TokenTree::Token(sp(2, 5), + token::Literal(Lit::Str_(token::intern("5")), None))]); + assert_eq!(test1, Some(&test1_expected[..])); + + let test2_expected = TokenStream::from_tts(vec![TokenTree::Token( sp(2,3) + , token::Literal( + Lit::Integer( + token::intern(&(5.to_string()))), + None))]); + assert_eq!(test2, Some(&test2_expected[..])); + + assert_eq!(test3, None); + + + let test4_tts = vec![TokenTree::Token(sp(3, 6), token::Ident(str_to_ident("foo"))), + TokenTree::Token(sp(6, 7), token::Comma), + TokenTree::Token(sp(7, 10), token::Ident(str_to_ident("bar"))), + TokenTree::Token(sp(10, 11), token::Comma), + TokenTree::Token(sp(11, 14), token::Ident(str_to_ident("baz")))]; + + let test4_expected = TokenStream::from_tts(vec![TokenTree::Delimited(sp(2, 15), + Rc::new(Delimited { + delim: token::DelimToken::Paren, + open_span: sp(2, 3), + tts: test4_tts, + close_span: sp(14, 15), + }))]); + assert_eq!(test4, Some(&test4_expected[..])); + + assert_eq!(test5, None); + + } + + #[test] + fn test_maybe_delimited() { + let test0_input = TokenStream::from_tts(string_to_tts("foo(bar::baz)".to_string())); + let test1_input = TokenStream::from_tts(string_to_tts("(bar::baz)".to_string())); + let test2_input = TokenStream::from_tts(string_to_tts("(foo,bar,baz)".to_string())); + let test3_input = TokenStream::from_tts(string_to_tts("(foo,bar,baz)(zab,rab)" + .to_string())); + let test4_input = TokenStream::from_tts(string_to_tts("(foo,bar,baz)foo".to_string())); + let test5_input = TokenStream::from_tts(string_to_tts("".to_string())); + + let test0 = test0_input.maybe_delimited(); + let test1 = test1_input.maybe_delimited(); + let test2 = test2_input.maybe_delimited(); + let test3 = test3_input.maybe_delimited(); + let test4 = test4_input.maybe_delimited(); + let test5 = test5_input.maybe_delimited(); + + assert_eq!(test0, None); + + let test1_expected = TokenStream::from_tts(vec![TokenTree::Token(sp(1, 4), + token::Ident(str_to_ident("bar"))), + TokenTree::Token(sp(4, 6), token::ModSep), + TokenTree::Token(sp(6, 9), + token::Ident(str_to_ident("baz")))]); + assert_eq!(test1, Some(&test1_expected[..])); + + let test2_expected = TokenStream::from_tts(vec![TokenTree::Token(sp(1, 4), + token::Ident(str_to_ident("foo"))), + TokenTree::Token(sp(4, 5), token::Comma), + TokenTree::Token(sp(5, 8), + token::Ident(str_to_ident("bar"))), + TokenTree::Token(sp(8, 9), token::Comma), + TokenTree::Token(sp(9, 12), + token::Ident(str_to_ident("baz")))]); + assert_eq!(test2, Some(&test2_expected[..])); + + assert_eq!(test3, None); + + assert_eq!(test4, None); + + assert_eq!(test5, None); + } + + #[test] + fn test_maybe_comma_list() { + let test0_input = TokenStream::from_tts(string_to_tts("foo(bar::baz)".to_string())); + let test1_input = TokenStream::from_tts(string_to_tts("(bar::baz)".to_string())); + let test2_input = TokenStream::from_tts(string_to_tts("(foo,bar,baz)".to_string())); + let test3_input = TokenStream::from_tts(string_to_tts("(foo::bar,bar,baz)".to_string())); + let test4_input = TokenStream::from_tts(string_to_tts("(foo,bar,baz)(zab,rab)" + .to_string())); + let test5_input = TokenStream::from_tts(string_to_tts("(foo,bar,baz)foo".to_string())); + let test6_input = TokenStream::from_tts(string_to_tts("".to_string())); + // The following is supported behavior! + let test7_input = TokenStream::from_tts(string_to_tts("(foo,bar,)".to_string())); + + let test0 = test0_input.maybe_comma_list(); + let test1 = test1_input.maybe_comma_list(); + let test2 = test2_input.maybe_comma_list(); + let test3 = test3_input.maybe_comma_list(); + let test4 = test4_input.maybe_comma_list(); + let test5 = test5_input.maybe_comma_list(); + let test6 = test6_input.maybe_comma_list(); + let test7 = test7_input.maybe_comma_list(); + + assert_eq!(test0, None); + + let test1_stream = TokenStream::from_tts(vec![TokenTree::Token(sp(1, 4), + token::Ident(str_to_ident("bar"))), + TokenTree::Token(sp(4, 6), token::ModSep), + TokenTree::Token(sp(6, 9), + token::Ident(str_to_ident("baz")))]); + + let test1_expected: Vec<&TokenSlice> = vec![&test1_stream[..]]; + assert_eq!(test1, Some(test1_expected)); + + let test2_foo = TokenStream::from_tts(vec![TokenTree::Token(sp(1, 4), + token::Ident(str_to_ident("foo")))]); + let test2_bar = TokenStream::from_tts(vec![TokenTree::Token(sp(5, 8), + token::Ident(str_to_ident("bar")))]); + let test2_baz = TokenStream::from_tts(vec![TokenTree::Token(sp(9, 12), + token::Ident(str_to_ident("baz")))]); + let test2_expected: Vec<&TokenSlice> = vec![&test2_foo[..], &test2_bar[..], &test2_baz[..]]; + assert_eq!(test2, Some(test2_expected)); + + let test3_path = TokenStream::from_tts(vec![TokenTree::Token(sp(1, 4), + token::Ident(str_to_ident("foo"))), + TokenTree::Token(sp(4, 6), token::ModSep), + TokenTree::Token(sp(6, 9), + token::Ident(str_to_ident("bar")))]); + let test3_bar = TokenStream::from_tts(vec![TokenTree::Token(sp(10, 13), + token::Ident(str_to_ident("bar")))]); + let test3_baz = TokenStream::from_tts(vec![TokenTree::Token(sp(14, 17), + token::Ident(str_to_ident("baz")))]); + let test3_expected: Vec<&TokenSlice> = + vec![&test3_path[..], &test3_bar[..], &test3_baz[..]]; + assert_eq!(test3, Some(test3_expected)); + + assert_eq!(test4, None); + + assert_eq!(test5, None); + + assert_eq!(test6, None); + + + let test7_expected: Vec<&TokenSlice> = vec![&test2_foo[..], &test2_bar[..]]; + assert_eq!(test7, Some(test7_expected)); + } + + // pub fn maybe_ident(&self) -> Option + #[test] + fn test_maybe_ident() { + let test0 = TokenStream::from_tts(string_to_tts("\"foo\"".to_string())).maybe_ident(); + let test1 = TokenStream::from_tts(string_to_tts("5".to_string())).maybe_ident(); + let test2 = TokenStream::from_tts(string_to_tts("foo".to_string())).maybe_ident(); + let test3 = TokenStream::from_tts(string_to_tts("foo::bar".to_string())).maybe_ident(); + let test4 = TokenStream::from_tts(string_to_tts("foo(bar)".to_string())).maybe_ident(); + + assert_eq!(test0, None); + assert_eq!(test1, None); + assert_eq!(test2, Some(str_to_ident("foo"))); + assert_eq!(test3, None); + assert_eq!(test4, None); + } + + // pub fn maybe_lit(&self) -> Option + #[test] + fn test_maybe_lit() { + let test0 = TokenStream::from_tts(string_to_tts("\"foo\"".to_string())).maybe_lit(); + let test1 = TokenStream::from_tts(string_to_tts("5".to_string())).maybe_lit(); + let test2 = TokenStream::from_tts(string_to_tts("foo".to_string())).maybe_lit(); + let test3 = TokenStream::from_tts(string_to_tts("foo::bar".to_string())).maybe_lit(); + let test4 = TokenStream::from_tts(string_to_tts("foo(bar)".to_string())).maybe_lit(); + + assert_eq!(test0, Some(Lit::Str_(token::intern("foo")))); + assert_eq!(test1, Some(Lit::Integer(token::intern(&(5.to_string()))))); + assert_eq!(test2, None); + assert_eq!(test3, None); + assert_eq!(test4, None); + } + + #[test] + fn test_maybe_path_prefix() { + let test0_input = TokenStream::from_tts(string_to_tts("foo(bar::baz)".to_string())); + let test1_input = TokenStream::from_tts(string_to_tts("(bar::baz)".to_string())); + let test2_input = TokenStream::from_tts(string_to_tts("(foo,bar,baz)".to_string())); + let test3_input = TokenStream::from_tts(string_to_tts("foo::bar(bar,baz)".to_string())); + + let test0 = test0_input.maybe_path_prefix(); + let test1 = test1_input.maybe_path_prefix(); + let test2 = test2_input.maybe_path_prefix(); + let test3 = test3_input.maybe_path_prefix(); + + let test0_tts = vec![TokenTree::Token(sp(4, 7), token::Ident(str_to_ident("bar"))), + TokenTree::Token(sp(7, 9), token::ModSep), + TokenTree::Token(sp(9, 12), token::Ident(str_to_ident("baz")))]; + + let test0_stream = TokenStream::from_tts(vec![TokenTree::Delimited(sp(3, 13), + Rc::new(Delimited { + delim: token::DelimToken::Paren, + open_span: sp(3, 4), + tts: test0_tts, + close_span: sp(12, 13), + }))]); + + let test0_expected = Some((ast::Path::from_ident(sp(0, 3), str_to_ident("foo")), + &test0_stream[..])); + assert_eq!(test0, test0_expected); + + assert_eq!(test1, None); + assert_eq!(test2, None); + + let test3_path = ast::Path { + span: sp(0, 8), + global: false, + segments: vec![ast::PathSegment { + identifier: str_to_ident("foo"), + parameters: ast::PathParameters::none(), + }, + ast::PathSegment { + identifier: str_to_ident("bar"), + parameters: ast::PathParameters::none(), + }], + }; + + let test3_tts = vec![TokenTree::Token(sp(9, 12), token::Ident(str_to_ident("bar"))), + TokenTree::Token(sp(12, 13), token::Comma), + TokenTree::Token(sp(13, 16), token::Ident(str_to_ident("baz")))]; + + let test3_stream = TokenStream::from_tts(vec![TokenTree::Delimited(sp(8, 17), + Rc::new(Delimited { + delim: token::DelimToken::Paren, + open_span: sp(8, 9), + tts: test3_tts, + close_span: sp(16, 17), + }))]); + let test3_expected = Some((test3_path, &test3_stream[..])); + assert_eq!(test3, test3_expected); + } + + #[test] + fn test_as_paren_delimited_stream() { + let test0 = TokenStream::as_paren_delimited_stream(string_to_tts("foo,bar,".to_string())); + let test1 = TokenStream::as_paren_delimited_stream(string_to_tts("baz(foo,bar)" + .to_string())); + + let test0_tts = vec![TokenTree::Token(sp(0, 3), token::Ident(str_to_ident("foo"))), + TokenTree::Token(sp(3, 4), token::Comma), + TokenTree::Token(sp(4, 7), token::Ident(str_to_ident("bar"))), + TokenTree::Token(sp(7, 8), token::Comma)]; + let test0_stream = TokenStream::from_tts(vec![TokenTree::Delimited(sp(0, 8), + Rc::new(Delimited { + delim: token::DelimToken::Paren, + open_span: DUMMY_SP, + tts: test0_tts, + close_span: DUMMY_SP, + }))]); + + assert_eq!(test0, test0_stream); + + + let test1_tts = vec![TokenTree::Token(sp(4, 7), token::Ident(str_to_ident("foo"))), + TokenTree::Token(sp(7, 8), token::Comma), + TokenTree::Token(sp(8, 11), token::Ident(str_to_ident("bar")))]; + + let test1_parse = vec![TokenTree::Token(sp(0, 3), token::Ident(str_to_ident("baz"))), + TokenTree::Delimited(sp(3, 12), + Rc::new(Delimited { + delim: token::DelimToken::Paren, + open_span: sp(3, 4), + tts: test1_tts, + close_span: sp(11, 12), + }))]; + + let test1_stream = TokenStream::from_tts(vec![TokenTree::Delimited(sp(0, 12), + Rc::new(Delimited { + delim: token::DelimToken::Paren, + open_span: DUMMY_SP, + tts: test1_parse, + close_span: DUMMY_SP, + }))]); + + assert_eq!(test1, test1_stream); + } + +}