Trait yap::Tokens

pub trait Tokens: Iterator + Sized {
    type Location: TokenLocation + PartialEq + Debug + Clone;

    fn location(&self) -> Self::Location;
    fn set_location(&mut self, location: Self::Location);
    fn is_at_location(&self, location: &Self::Location) -> bool;

    fn with_context<C>(self, context: C) -> WithContext<Self, C>  { ... }
    fn with_context_mut<C>(&mut self, context: C) -> WithContextMut<&mut Self, C>  { ... }
    fn slice(
        &mut self,
        from: Self::Location,
        to: Self::Location
    ) -> Slice<'_, Self> { ... }
    fn offset(&self) -> usize { ... }
    fn peek(&mut self) -> Option<Self::Item> { ... }
    fn token<I>(&mut self, t: I) -> bool
    where
        Self::Item: PartialEq,
        I: Borrow<Self::Item>,
    { ... }
    fn tokens<It>(&mut self, ts: It) -> bool
    where
        Self::Item: PartialEq,
        It: IntoIterator,
        It::Item: Borrow<Self::Item>,
    { ... }
    fn one_of_tokens<It>(&mut self, ts: It) -> Option<Self::Item>
    where
        Self::Item: PartialEq,
        It: IntoIterator,
        It::Item: Borrow<Self::Item>,
    { ... }
    fn tokens_while<F>(&mut self, f: F) -> TokensWhile<'_, Self, F>
    where
        F: FnMut(&Self::Item) -> bool,
    { ... }
    fn skip_tokens_while<F>(&mut self, f: F) -> usize
    where
        F: FnMut(&Self::Item) -> bool,
    { ... }
    fn many<F, Output>(&mut self, parser: F) -> Many<'_, Self, F>
    where
        F: FnMut(&mut Self) -> Option<Output>,
    { ... }
    fn many_err<F, Output, E>(&mut self, parser: F) -> ManyErr<'_, Self, F>
    where
        F: FnMut(&mut Self) -> Result<Output, E>,
    { ... }
    fn skip_many<F>(&mut self, parser: F) -> usize
    where
        F: FnMut(&mut Self) -> bool,
    { ... }
    fn skip_many1<F, E, Ignored>(&mut self, parser: F) -> Result<usize, E>
    where
        F: FnMut(&mut Self) -> Result<Ignored, E>,
    { ... }
    fn sep_by<F, S, Output>(
        &mut self,
        parser: F,
        separator: S
    ) -> SepBy<'_, Self, F, S>
    where
        F: FnMut(&mut Self) -> Option<Output>,
        S: FnMut(&mut Self) -> bool,
    { ... }
    fn sep_by_err<F, S, E, Output>(
        &mut self,
        parser: F,
        separator: S
    ) -> SepByErr<'_, Self, F, S>
    where
        F: FnMut(&mut Self) -> Result<Output, E>,
        S: FnMut(&mut Self) -> bool,
    { ... }
    fn sep_by_all<F, S, Output>(
        &mut self,
        parser: F,
        separator: S
    ) -> SepByAll<'_, Self, F, S, Output>
    where
        F: FnMut(&mut Self) -> Option<Output>,
        S: FnMut(&mut Self) -> Option<Output>,
    { ... }
    fn sep_by_all_err<F, S, Output, E>(
        &mut self,
        parser: F,
        separator: S
    ) -> SepByAllErr<'_, Self, F, S, Output>
    where
        F: FnMut(&mut Self) -> Result<Output, E>,
        S: FnMut(&mut Self) -> Option<Output>,
    { ... }
    fn surrounded_by<F, S, Output>(&mut self, parser: F, surrounding: S) -> Output
    where
        F: FnMut(&mut Self) -> Output,
        S: FnMut(&mut Self),
    { ... }
    fn optional<F, Output>(&mut self, f: F) -> Option<Output>
    where
        F: FnMut(&mut Self) -> Option<Output>,
    { ... }
    fn skip_optional<F>(&mut self, f: F)
    where
        F: FnMut(&mut Self),
    { ... }
}

The tokens trait builds on the Iterator trait, and adds a bunch of useful methods for parsing tokens from the underlying iterable type.

Required Associated Types

type Location: TokenLocation + PartialEq + Debug + Clone

An object which can be used to reset the token stream to some position.

Required Methods

fn location(&self) -> Self::Location

Return a “location” pointer. This can be passed to Tokens::set_location to set the tokens location back to the state at the time it was handed out. If the crate::TokenLocation trait is in scope, you can also call the crate::TokenLocation::offset() method on it to obtain the current offset.

Example

use yap::{ Tokens, IntoTokens, TokenLocation };

let mut s = "abcde".into_tokens();

let location = s.location();

assert_eq!(s.next().unwrap(), 'a');
assert_eq!(s.location().offset(), 1);
assert_eq!(s.next().unwrap(), 'b');
assert_eq!(s.location().offset(), 2);

s.set_location(location);

assert_eq!(s.next().unwrap(), 'a');
assert_eq!(s.location().offset(), 1);
assert_eq!(s.next().unwrap(), 'b');
assert_eq!(s.location().offset(), 2);

fn set_location(&mut self, location: Self::Location)

Set the tokens to the location provided. See Tokens::location.

fn is_at_location(&self, location: &Self::Location) -> bool

Return true if the current cursor location matches the location given, or false otherwise.

Example

use yap::{ Tokens, IntoTokens };

let mut s = "abc".into_tokens();
let location = s.location();
assert_eq!(s.is_at_location(&location), true);
s.next();
assert_eq!(s.is_at_location(&location), false);
s.set_location(location);
assert_eq!(s.is_at_location(&location), true);

Provided Methods

fn with_context<C>(self, context: C) -> WithContext<Self, C>

Attach some context to your tokens. The returned struct, WithContext, also implements Tokens, and so has can be used in much the same way. Since this consumes your tokens, it’s better suited to permanent context that you’d like throughout the parsing.

See Tokens::with_context_mut for a version that’s easier to attach temporary context with.

Example

use yap::{ Tokens, IntoTokens, types::WithContext };

fn skip_digits(toks: &mut WithContext<impl Tokens<Item=char>, usize>) {
    let n_skipped = toks.skip_tokens_while(|c| c.is_digit(10));
    *toks.context_mut() += n_skipped;
}

let mut tokens = "123abc456".into_tokens().with_context(0usize);

skip_digits(&mut tokens);
tokens.skip_tokens_while(|c| c.is_alphabetic());
skip_digits(&mut tokens);

assert_eq!(*tokens.context(), 6);

fn with_context_mut<C>(&mut self, context: C) -> WithContextMut<&mut Self, C>

Unlike Tokens::with_context, which consumes the tokens, this borrows them mutably, allowing it to be used when you only have a mutable reference to tokens (which is a common function signature to use), and making it better suited to attaching temporary contexts.

Be aware that if you attach context in a function called recursively, the type checker may shout at you for contructing a type like WithContextMut<WithContextMut<WithContextMut<..>>>. In these cases, you can “break the cycle” by removing the original WithContextMut by using crate::types::WithContextMut::into_parts() before wrapping the tokens in a new context for the recursive call.

Example

use yap::{ Tokens, IntoTokens };

fn count_digit_comma_calls(toks: &mut impl Tokens<Item=char>) -> (u8, u8) {
    let mut counts = (0u8, 0u8);
    toks.with_context_mut(&mut counts).sep_by(
        |t| {
            t.context_mut().0 += 1;
            let n_skipped = t.skip_tokens_while(|c| c.is_digit(10));
            if n_skipped == 0 { None } else { Some(()) }
        },
        |t| {
            t.context_mut().1 += 1;
            t.token(',')
        }
    ).last();
    counts
}

let n: usize = 0;
let mut tokens = "123,4,56,1,34,1".into_tokens();

let (digits, seps) = count_digit_comma_calls(&mut tokens);

assert_eq!(tokens.remaining().len(), 0);
// digits parsed 6 times:
assert_eq!(digits, 6);
// Attempted to parse seps 6 times; failure on last ends it:
assert_eq!(seps, 6);

fn slice(&mut self, from: Self::Location, to: Self::Location) -> Slice<'_, Self>

Return a slice of tokens starting at the to location provided and ending just prior to the from location provided (ie equivalent to the range to..from).

The slice returned from implements Iterator and Tokens, so you can use the full range of parsing functions on it, or simply collect up the slice of tokens as you wish.

Note: the slice returned from this prevents the original tokens from being used until it’s dropped, and resets the original tokens to their current location on Drop. if you std::mem::forget it, the original token location will equal whatever the slice location was when it was forgotten.

Example

use yap::{ Tokens, IntoTokens };

let mut s = "abcdefghijklmnop".into_tokens();

(0..5).for_each(|_| { s.next(); });
let from = s.location();
(0..5).for_each(|_| { s.next(); });
let to = s.location();

assert_eq!(s.next(), Some('k'));
assert_eq!(s.next(), Some('l'));

// Iterating the from..to range given:
let vals: String = s.slice(from.clone(), to.clone()).collect();
assert_eq!(&*vals, "fghij");

// After the above is dropped, we can continue
// from where we left off:
assert_eq!(s.next(), Some('m'));
assert_eq!(s.next(), Some('n'));

// We can iterate this range again as we please:
let vals: String = s.slice(from, to).collect();
assert_eq!(&*vals, "fghij");

// And the original remains unaffected..
assert_eq!(s.next(), Some('o'));
assert_eq!(s.next(), Some('p'));

fn offset(&self) -> usize

Return the current offset into the tokens that we’ve parsed up to so far. The exact meaning of this can vary by implementation; when parsing slices, it is index of the slice item we’ve consumed up to, and when parsing &str’s it is the number of bytes (not characters) consumed so far.

Example

use yap::{ Tokens, IntoTokens };

let mut s = "abc".into_tokens();
assert_eq!(s.offset(), 0);
s.next();
assert_eq!(s.offset(), 1);
s.next();
assert_eq!(s.offset(), 2);

fn peek(&mut self) -> Option<Self::Item>

Return the next item in the input without consuming it.

Prefer this to using the peekable iterator method, which consumes the tokens, and internally keeps hold of the peeked state itself.

Example

use yap::{ Tokens, IntoTokens };

let mut s = "abc".into_tokens();
assert_eq!(s.peek(), Some('a'));
assert_eq!(s.peek(), Some('a'));

fn token<I>(&mut self, t: I) -> boolwhere
    Self::Item: PartialEq,
    I: Borrow<Self::Item>,

Expect a specific token to be next. If the token is not found, the iterator is not advanced.

Example

use yap::{ Tokens, IntoTokens };

let mut s = "abc".into_tokens();
assert_eq!(s.token(&'a'), true);
assert_eq!(s.token(&'b'), true);
assert_eq!(s.token('z'), false);
assert_eq!(s.token('y'), false);
assert_eq!(s.token('c'), true);

fn tokens<It>(&mut self, ts: It) -> boolwhere
    Self::Item: PartialEq,
    It: IntoIterator,
    It::Item: Borrow<Self::Item>,

Expect a specific set of tokens to be next. If the tokens are not found, the iterator is not advanced. Anything that implements IntoIterator with an Item type that can be borrowed to produce &Item can be provided as an input to this.

Example

use yap::{ Tokens, IntoTokens };

let mut s = "abcdef".into_tokens();

assert_eq!(s.tokens("abc".chars()), true);
assert_eq!(s.remaining(), "def");

assert_eq!(s.tokens("de".chars()), true);
assert_eq!(s.remaining(), "f");

fn one_of_tokens<It>(&mut self, ts: It) -> Option<Self::Item>where
    Self::Item: PartialEq,
    It: IntoIterator,
    It::Item: Borrow<Self::Item>,

Return the first token that matches the tokens provided, or None if none of them match.

Example

use yap::{ Tokens, IntoTokens };

let mut s = "abcdef".into_tokens();

assert_eq!(s.one_of_tokens("abc".chars()), Some('a'));
assert_eq!(s.one_of_tokens("abc".chars()), Some('b'));
assert_eq!(s.one_of_tokens("abc".chars()), Some('c'));
assert_eq!(s.one_of_tokens("abc".chars()), None);
assert_eq!(s.remaining(), "def");

fn tokens_while<F>(&mut self, f: F) -> TokensWhile<'_, Self, F>where
    F: FnMut(&Self::Item) -> bool,

Return an iterator that will consume tokens until the provided function returns false.

Example

use yap::{ Tokens, IntoTokens };

let mut s = "12345abc".into_tokens();
let digits: String = s.tokens_while(|c| c.is_numeric()).collect();
assert_eq!(&*digits, "12345");
assert_eq!(s.remaining(), "abc");

fn skip_tokens_while<F>(&mut self, f: F) -> usizewhere
    F: FnMut(&Self::Item) -> bool,

Iterate over the tokens until the provided function returns false on one. Only consume the tokens that the function returned true for, and ignore them.

Example

use yap::{ Tokens, IntoTokens };

let mut s = "12345abc".into_tokens();
let n_skipped = s.skip_tokens_while(|c| c.is_numeric());

assert_eq!(n_skipped, 5);
assert_eq!(s.remaining(), "abc");

fn many<F, Output>(&mut self, parser: F) -> Many<'_, Self, F>where
    F: FnMut(&mut Self) -> Option<Output>,

Returns an iterator that, on each iteration, attempts to run the provided parser on the remaining tokens. If the parser returns None, no tokens will be consumed.

Example

use yap::{ Tokens, IntoTokens };

fn parse_digit_pair(tokens: &mut impl Tokens<Item=char>) -> Option<u32> {
    let d1 = tokens.next()?;
    let d2 = tokens.next()?;
    // Return the result of adding the 2 digits we saw:
    Some(d1.to_digit(10)? + d2.to_digit(10)?)
}

let mut s = "12345abcde".into_tokens();
let digits: Vec<u32> = s.many(|t| parse_digit_pair(t)).collect();

assert_eq!(digits, vec![3, 7]);
assert_eq!(s.remaining(), "5abcde");

fn many_err<F, Output, E>(&mut self, parser: F) -> ManyErr<'_, Self, F>where
    F: FnMut(&mut Self) -> Result<Output, E>,

Returns an iterator that, on each iteration, attempts to run the provided parser on the remaining tokens. If the parser returns an error, no tokens will be consumed and the error will be returned as the final iteration.

Example

use yap::{ Tokens, IntoTokens };

#[derive(Debug, PartialEq)]
enum Err { NotEnoughTokens, NotADigit(char) }
fn parse_digit_pair(tokens: &mut impl Tokens<Item=char>) -> Result<u32, Err> {
    let n1 = tokens.next()
        .ok_or(Err::NotEnoughTokens)
        .and_then(|c| c.to_digit(10).ok_or(Err::NotADigit(c)))?;
    let n2 = tokens.next()
        .ok_or(Err::NotEnoughTokens)
        .and_then(|c| c.to_digit(10).ok_or(Err::NotADigit(c)))?;
    Ok(n1 + n2)
}

let mut s = "12345abcde".into_tokens();
let mut digits_iter = s.many_err(|t| parse_digit_pair(t));

assert_eq!(digits_iter.next(), Some(Ok(3)));
assert_eq!(digits_iter.next(), Some(Ok(7)));
assert_eq!(digits_iter.next(), Some(Err(Err::NotADigit('a'))));
assert_eq!(digits_iter.next(), None);
assert_eq!(s.remaining(), "5abcde");

fn skip_many<F>(&mut self, parser: F) -> usizewhere
    F: FnMut(&mut Self) -> bool,

Ignore 0 or more instances of some parser.

Example

use yap::{ Tokens, IntoTokens };

struct ABC;
fn parse_abc(tokens: &mut impl Tokens<Item=char>) -> Option<ABC> {
    let a = tokens.next()?;
    let b = tokens.next()?;
    let c = tokens.next()?;
    if a == 'a' && b == 'b' && c == 'c' {
        Some(ABC)
    } else {
        None
    }
}

let mut s = "abcabcababab".into_tokens();
s.skip_many(|t| parse_abc(t).is_some());

assert_eq!(s.remaining(), "ababab");

fn skip_many1<F, E, Ignored>(&mut self, parser: F) -> Result<usize, E>where
    F: FnMut(&mut Self) -> Result<Ignored, E>,

Ignore 1 or more instances of some parser. If the provided parser fails immediately, return the error that it produced.

Example

use yap::{ Tokens, IntoTokens };

struct ABC;
fn parse_abc(tokens: &mut impl Tokens<Item=char>) -> Option<ABC> {
    let a = tokens.next()?;
    let b = tokens.next()?;
    let c = tokens.next()?;
    if a == 'a' && b == 'b' && c == 'c' {
        Some(ABC)
    } else {
        None
    }
}

let mut s = "abcabcabcxyz".into_tokens();
let skipped = s.skip_many1(|t| parse_abc(t).ok_or("aaah"));

assert_eq!(skipped, Ok(3));
assert_eq!(s.remaining(), "xyz");

let mut s = "ababababcabc".into_tokens();
let skipped = s.skip_many1(|t| parse_abc(t).ok_or("aaah"));

assert_eq!(skipped, Err("aaah"));
assert_eq!(s.remaining(), "ababababcabc");

fn sep_by<F, S, Output>(
    &mut self,
    parser: F,
    separator: S
) -> SepBy<'_, Self, F, S>where
    F: FnMut(&mut Self) -> Option<Output>,
    S: FnMut(&mut Self) -> bool,

Return an iterator that parses anything matching the parser function, and expects to parse something matching the separator function between each one.

Example

use yap::{ Tokens, IntoTokens };

fn parse_digit(tokens: &mut impl Tokens<Item=char>) -> Option<u32> {
    let c = tokens.next()?;
    c.to_digit(10)
}

let mut s = "1,2,3,4,abc".into_tokens();
let digits: Vec<u32> = s.sep_by(|t| parse_digit(t), |t| t.token(',')).collect();
assert_eq!(digits, vec![1,2,3,4]);
assert_eq!(s.remaining(), ",abc");

fn sep_by_err<F, S, E, Output>(
    &mut self,
    parser: F,
    separator: S
) -> SepByErr<'_, Self, F, S>where
    F: FnMut(&mut Self) -> Result<Output, E>,
    S: FnMut(&mut Self) -> bool,

Return an iterator that parses anything matching the parser function, and expects to parse something matching the separator function between each one. Unlike Tokens::sep_by, this accepts parsers that return Results, and returns the result on each iteration. Once an error is hit, None is returned thereafter.

Example

use yap::{ Tokens, IntoTokens };

#[derive(Debug, PartialEq)]
enum Err { NoMoreTokens, NotADigit(char) }

fn parse_digit(tokens: &mut impl Tokens<Item=char>) -> Result<u32, Err> {
    let c = tokens.next().ok_or(Err::NoMoreTokens)?;
    c.to_digit(10).ok_or(Err::NotADigit(c))
}

let mut s = "1,2,a,1,2,3".into_tokens();
let mut digits_iter = s.sep_by_err(|t| parse_digit(t), |t| t.token(','));
assert_eq!(digits_iter.next(), Some(Ok(1)));
assert_eq!(digits_iter.next(), Some(Ok(2)));
assert_eq!(digits_iter.next(), Some(Err(Err::NotADigit('a'))));
assert_eq!(digits_iter.next(), None);
assert_eq!(s.remaining(), ",a,1,2,3");

fn sep_by_all<F, S, Output>(
    &mut self,
    parser: F,
    separator: S
) -> SepByAll<'_, Self, F, S, Output>where
    F: FnMut(&mut Self) -> Option<Output>,
    S: FnMut(&mut Self) -> Option<Output>,

Returns an iterator that parses anything matching the parser function, and expects to parse something matching the separator function between each one. The iterator returns the output from both the parser and separator function, which means that they are expected to return the same type.

Example

use yap::{ Tokens, IntoTokens };

#[derive(PartialEq,Debug)]
enum Op { Plus, Minus, Divide }
#[derive(PartialEq,Debug)]
enum OpOrDigit { Op(Op), Digit(u32) }

fn parse_op(tokens: &mut impl Tokens<Item=char>) -> Option<Op> {
    match tokens.next()? {
        '-' => Some(Op::Minus),
        '+' => Some(Op::Plus),
        '/' => Some(Op::Divide),
        _ => None
    }
}

fn parse_digit(tokens: &mut impl Tokens<Item=char>) -> Option<u32> {
    let c = tokens.next()?;
    c.to_digit(10)
}

let mut s = "1+2/3-4+abc".into_tokens();
let output: Vec<_> = s.sep_by_all(
    |t| parse_digit(t).map(OpOrDigit::Digit),
    |t| parse_op(t).map(OpOrDigit::Op)
).collect();

assert_eq!(output, vec![
    OpOrDigit::Digit(1),
    OpOrDigit::Op(Op::Plus),
    OpOrDigit::Digit(2),
    OpOrDigit::Op(Op::Divide),
    OpOrDigit::Digit(3),
    OpOrDigit::Op(Op::Minus),
    OpOrDigit::Digit(4),
]);
assert_eq!(s.remaining(), "+abc");

fn sep_by_all_err<F, S, Output, E>(
    &mut self,
    parser: F,
    separator: S
) -> SepByAllErr<'_, Self, F, S, Output>where
    F: FnMut(&mut Self) -> Result<Output, E>,
    S: FnMut(&mut Self) -> Option<Output>,

Similar to Tokens::sep_by_all, except that the iterator returned also hands back the first error encountered when attempting to run our parser.

Example

use yap::{ Tokens, IntoTokens };

#[derive(PartialEq,Debug)]
enum Op { Plus, Minus, Divide }
#[derive(PartialEq,Debug)]
enum OpOrDigit { Op(Op), Digit(u32) }
#[derive(Debug, PartialEq)]
enum Err { NoMoreTokens, NotADigit(char) }

fn parse_op(tokens: &mut impl Tokens<Item=char>) -> Option<Op> {
    match tokens.next()? {
        '-' => Some(Op::Minus),
        '+' => Some(Op::Plus),
        '/' => Some(Op::Divide),
        _ => None
    }
}

fn parse_digit(tokens: &mut impl Tokens<Item=char>) -> Result<u32, Err> {
    let c = tokens.next().ok_or(Err::NoMoreTokens)?;
    c.to_digit(10).ok_or(Err::NotADigit(c))
}

let mut s = "1+2/3-4+abc".into_tokens();
let output: Vec<_> = s.sep_by_all_err(
    |t| parse_digit(t).map(OpOrDigit::Digit),
    |t| parse_op(t).map(OpOrDigit::Op)
).collect();

assert_eq!(output, vec![
    Ok(OpOrDigit::Digit(1)),
    Ok(OpOrDigit::Op(Op::Plus)),
    Ok(OpOrDigit::Digit(2)),
    Ok(OpOrDigit::Op(Op::Divide)),
    Ok(OpOrDigit::Digit(3)),
    Ok(OpOrDigit::Op(Op::Minus)),
    Ok(OpOrDigit::Digit(4)),
    Err(Err::NotADigit('a'))
]);
assert_eq!(s.remaining(), "+abc");

fn surrounded_by<F, S, Output>(&mut self, parser: F, surrounding: S) -> Outputwhere
    F: FnMut(&mut Self) -> Output,
    S: FnMut(&mut Self),

Parse some tokens that are optionally surrounded by the result of a surrounding parser.

Example

use yap::{ Tokens, IntoTokens };

let mut s = "   hello    ".into_tokens();

let hello: String = s.surrounded_by(
    |t| t.tokens_while(|c| c.is_ascii_alphabetic()).collect(),
    |t| { t.skip_tokens_while(|c| c.is_ascii_whitespace()); }
);

assert_eq!(&*hello, "hello");
assert_eq!(s.remaining(), "");

fn optional<F, Output>(&mut self, f: F) -> Option<Output>where
    F: FnMut(&mut Self) -> Option<Output>,

Attempt to parse some output from the tokens. If the function returns None, no tokens will be consumed. Else, return whatever the function produced.

Example

use yap::{ Tokens, IntoTokens };

let mut s = "foobar".into_tokens();

let res = s.optional(|s| {
    let a = s.next();
    let b = s.next();
    if a == b {
        Some("yay")
    } else {
        None
    }
});

// nothing consumed since None returned from fn
assert_eq!(s.remaining(), "foobar");
assert_eq!(res, None);

let res = s.optional(|s| {
    let a = s.next()?;
    let b = s.next()?;
    Some((a, b))
});

// 2 chars consumed since Some returned from fn
assert_eq!(s.remaining(), "obar");
assert_eq!(res, Some(('f', 'o')));

fn skip_optional<F>(&mut self, f: F)where
    F: FnMut(&mut Self),

Run a parser against some tokens, and don’t care whether it succeeded or how much input it consumed.

Example

use yap::{ Tokens, IntoTokens };

let mut s = "   helloworld".into_tokens();

fn parse_whitespace(t: &mut impl Tokens<Item=char>) {
    t.skip_tokens_while(|c| c.is_ascii_whitespace());
}

s.skip_optional(|t| parse_whitespace(t));
let is_hello = s.tokens("hello".chars());
s.skip_optional(|t| parse_whitespace(t));
let world: String = s.tokens_while(|c| c.is_ascii_alphabetic()).collect();

// assert_eq!(is_hello, true);
// assert_eq!(&*world, "world");

Implementors

impl<'a> Tokens for StrTokens<'a>

type Location = StrTokensLocation

impl<'a, Item> Tokens for SliceTokens<'a, Item>

type Location = SliceTokensLocation

impl<T, C> Tokens for WithContext<T, C>where
    T: Tokens,

type Location = <T as Tokens>::Location

impl<T, C> Tokens for WithContextMut<&mut T, C>where
    T: Tokens,

type Location = <T as Tokens>::Location