Copyright	(C) 2026 - Eitan Chatav
License	BSD-style (see the file LICENSE)
Maintainer	Eitan Chatav <eitan.chatav@gmail.com>
Stability	provisional
Portability	non-portable
Safe Haskell	Safe-Inferred
Language	Haskell2010

Control.Lens.Grammar.Token

Contents

Tokenized
Categorized

Description

Synopsis

class Categorized token => Tokenized token p | p -> token where
- anyToken :: p
- token :: token -> p
- oneOf :: Foldable f => f token -> p
- notOneOf :: Foldable f => f token -> p
- asIn :: Categorize token -> p
- notAsIn :: Categorize token -> p
satisfy :: (Tokenized a (p a a), Choice p, Cochoice p) => (a -> Bool) -> p a a
tokens :: (Foldable f, Tokenized a (p a a), Monoidal p, Choice p, AsEmpty s, Cons s s a a) => f a -> p s s
class (Ord token, Ord (Categorize token), Enum (Categorize token)) => Categorized token where
- type Categorize token
- categorize :: token -> Categorize token
data GeneralCategory
- = UppercaseLetter
- | LowercaseLetter
- | TitlecaseLetter
- | ModifierLetter
- | OtherLetter
- | NonSpacingMark
- | SpacingCombiningMark
- | EnclosingMark
- | DecimalNumber
- | LetterNumber
- | OtherNumber
- | ConnectorPunctuation
- | DashPunctuation
- | OpenPunctuation
- | ClosePunctuation
- | InitialQuote
- | FinalQuote
- | OtherPunctuation
- | MathSymbol
- | CurrencySymbol
- | ModifierSymbol
- | OtherSymbol
- | Space
- | LineSeparator
- | ParagraphSeparator
- | Control
- | Format
- | Surrogate
- | PrivateUse
- | NotAssigned

Tokenized

class Categorized token => Tokenized token p | p -> token where Source #

Tokenized combinators for constructing lexical tokens.

Minimal complete definition

anyToken

Methods

anyToken :: p Source #

Any single token.

token :: token -> p Source #

A single specified token.

default token :: (p ~ q token token, Choice q, Cochoice q) => token -> p Source #

oneOf :: Foldable f => f token -> p Source #

A single token which is oneOf a set.

default oneOf :: (p ~ q token token, Choice q, Cochoice q, Foldable f) => f token -> p Source #

notOneOf :: Foldable f => f token -> p Source #

A single token which is notOneOf a set.

default notOneOf :: (p ~ q token token, Choice q, Cochoice q, Foldable f) => f token -> p Source #

asIn :: Categorize token -> p Source #

A single token which is asIn a category.

default asIn :: (p ~ q token token, Choice q, Cochoice q) => Categorize token -> p Source #

notAsIn :: Categorize token -> p Source #

A single token which is notAsIn a category.

default notAsIn :: (p ~ q token token, Choice q, Cochoice q) => Categorize token -> p Source #

Instances

Instances details

Tokenized Char RegBnf Source #
Instance details Defined in Control.Lens.Grammar Methods anyToken :: RegBnf Source # token :: Char -> RegBnf Source # oneOf :: Foldable f => f Char -> RegBnf Source # notOneOf :: Foldable f => f Char -> RegBnf Source # asIn :: Categorize Char -> RegBnf Source # notAsIn :: Categorize Char -> RegBnf Source #
Tokenized Char RegString Source #
Instance details Defined in Control.Lens.Grammar Methods anyToken :: RegString Source # token :: Char -> RegString Source # oneOf :: Foldable f => f Char -> RegString Source # notOneOf :: Foldable f => f Char -> RegString Source # asIn :: Categorize Char -> RegString Source # notAsIn :: Categorize Char -> RegString Source #
(Ord rule, Tokenized token rule) => Tokenized token (Bnf rule) Source #
Instance details Defined in Control.Lens.Grammar.BackusNaur Methods anyToken :: Bnf rule Source # token :: token -> Bnf rule Source # oneOf :: Foldable f => f token -> Bnf rule Source # notOneOf :: Foldable f => f token -> Bnf rule Source # asIn :: Categorize token -> Bnf rule Source # notAsIn :: Categorize token -> Bnf rule Source #
Categorized token => Tokenized token (TokenTest token) Source #
Instance details Defined in Control.Lens.Grammar.Boole Methods anyToken :: TokenTest token Source # token :: token -> TokenTest token Source # oneOf :: Foldable f => f token -> TokenTest token Source # notOneOf :: Foldable f => f token -> TokenTest token Source # asIn :: Categorize token -> TokenTest token Source # notAsIn :: Categorize token -> TokenTest token Source #
Categorized token => Tokenized token (RegEx token) Source #
Instance details Defined in Control.Lens.Grammar.Kleene Methods anyToken :: RegEx token Source # token :: token -> RegEx token Source # oneOf :: Foldable f => f token -> RegEx token Source # notOneOf :: Foldable f => f token -> RegEx token Source # asIn :: Categorize token -> RegEx token Source # notAsIn :: Categorize token -> RegEx token Source #
Categorized token => Tokenized token (RegExam token alg) Source #
Instance details Defined in Control.Lens.Grammar.Kleene Methods anyToken :: RegExam token alg Source # token :: token -> RegExam token alg Source # oneOf :: Foldable f => f token -> RegExam token alg Source # notOneOf :: Foldable f => f token -> RegExam token alg Source # asIn :: Categorize token -> RegExam token alg Source # notAsIn :: Categorize token -> RegExam token alg Source #
Categorized token => Tokenized token (token -> Bool) Source #
Instance details Defined in Control.Lens.Grammar.Token Methods anyToken :: token -> Bool Source # token :: token -> token -> Bool Source # oneOf :: Foldable f => f token -> token -> Bool Source # notOneOf :: Foldable f => f token -> token -> Bool Source # asIn :: Categorize token -> token -> Bool Source # notAsIn :: Categorize token -> token -> Bool Source #
Tokenized token k => Tokenized token (Grammor k a b) Source #
Instance details Defined in Data.Profunctor.Grammar Methods anyToken :: Grammor k a b Source # token :: token -> Grammor k a b Source # oneOf :: Foldable f => f token -> Grammor k a b Source # notOneOf :: Foldable f => f token -> Grammor k a b Source # asIn :: Categorize token -> Grammor k a b Source # notAsIn :: Categorize token -> Grammor k a b Source #
(Categorized a, a ~ Item s, IsList s, Cons s s a a, Snoc s s a a, Filterable m, Alternative m, Monad m) => Tokenized a (Parsor s m a a) Source #
Instance details Defined in Data.Profunctor.Grammar Methods anyToken :: Parsor s m a a Source # token :: a -> Parsor s m a a Source # oneOf :: Foldable f => f a -> Parsor s m a a Source # notOneOf :: Foldable f => f a -> Parsor s m a a Source # asIn :: Categorize a -> Parsor s m a a Source # notAsIn :: Categorize a -> Parsor s m a a Source #
(Categorized a, a ~ Item s, IsList s, Cons s s a a, Filterable m, Alternative m, Monad m) => Tokenized a (Printor s m a a) Source #
Instance details Defined in Data.Profunctor.Grammar Methods anyToken :: Printor s m a a Source # token :: a -> Printor s m a a Source # oneOf :: Foldable f => f a -> Printor s m a a Source # notOneOf :: Foldable f => f a -> Printor s m a a Source # asIn :: Categorize a -> Printor s m a a Source # notAsIn :: Categorize a -> Printor s m a a Source #

satisfy :: (Tokenized a (p a a), Choice p, Cochoice p) => (a -> Bool) -> p a a Source #

A single token that satisfies a predicate.

tokens :: (Foldable f, Tokenized a (p a a), Monoidal p, Choice p, AsEmpty s, Cons s s a a) => f a -> p s s Source #

A specified stream of tokens.

Categorized

class (Ord token, Ord (Categorize token), Enum (Categorize token)) => Categorized token where Source #

Categorized provides a type family Categorize and a function to categorize tokens into disjoint categories.

>>> :kind! Categorize Char
Categorize Char :: *
= GeneralCategory

>>> categorize 'a'
LowercaseLetter

Minimal complete definition

Nothing

Associated Types

type Categorize token Source #

type Categorize token = ()

Methods

categorize :: token -> Categorize token Source #

default categorize :: Categorize token ~ () => token -> Categorize token Source #

Instances

Instances details

Categorized Word8 Source #
Instance details Defined in Control.Lens.Grammar.Token Associated Types type Categorize Word8 Source # Methods categorize :: Word8 -> Categorize Word8 Source #
Categorized () Source #
Instance details Defined in Control.Lens.Grammar.Token Associated Types type Categorize () Source # Methods categorize :: () -> Categorize () Source #
Categorized Char Source #
Instance details Defined in Control.Lens.Grammar.Token Associated Types type Categorize Char Source # Methods categorize :: Char -> Categorize Char Source #

data GeneralCategory #

Unicode General Categories (column 2 of the UnicodeData table) in the order they are listed in the Unicode standard (the Unicode Character Database, in particular).

Examples

Expand

Basic usage:

>>> :t OtherLetter
OtherLetter :: GeneralCategory

Eq instance:

>>> UppercaseLetter == UppercaseLetter
True
>>> UppercaseLetter == LowercaseLetter
False

Ord instance:

>>> NonSpacingMark <= MathSymbol
True

Enum instance:

>>> enumFromTo ModifierLetter SpacingCombiningMark
[ModifierLetter,OtherLetter,NonSpacingMark,SpacingCombiningMark]

Read instance:

>>> read "DashPunctuation" :: GeneralCategory
DashPunctuation
>>> read "17" :: GeneralCategory
*** Exception: Prelude.read: no parse

Show instance:

>>> show EnclosingMark
"EnclosingMark"

Bounded instance:

>>> minBound :: GeneralCategory
UppercaseLetter
>>> maxBound :: GeneralCategory
NotAssigned

Ix instance:

>>> import Data.Ix ( index )
>>> index (OtherLetter,Control) FinalQuote
12
>>> index (OtherLetter,Control) Format
*** Exception: Error in array index

Constructors

UppercaseLetter	Lu: Letter, Uppercase
LowercaseLetter	Ll: Letter, Lowercase
TitlecaseLetter	Lt: Letter, Titlecase
ModifierLetter	Lm: Letter, Modifier
OtherLetter	Lo: Letter, Other
NonSpacingMark	Mn: Mark, Non-Spacing
SpacingCombiningMark	Mc: Mark, Spacing Combining
EnclosingMark	Me: Mark, Enclosing
DecimalNumber	Nd: Number, Decimal
LetterNumber	Nl: Number, Letter
OtherNumber	No: Number, Other
ConnectorPunctuation	Pc: Punctuation, Connector
DashPunctuation	Pd: Punctuation, Dash
OpenPunctuation	Ps: Punctuation, Open
ClosePunctuation	Pe: Punctuation, Close
InitialQuote	Pi: Punctuation, Initial quote
FinalQuote	Pf: Punctuation, Final quote
OtherPunctuation	Po: Punctuation, Other
MathSymbol	Sm: Symbol, Math
CurrencySymbol	Sc: Symbol, Currency
ModifierSymbol	Sk: Symbol, Modifier
OtherSymbol	So: Symbol, Other
Space	Zs: Separator, Space
LineSeparator	Zl: Separator, Line
ParagraphSeparator	Zp: Separator, Paragraph
Control	Cc: Other, Control
Format	Cf: Other, Format
Surrogate	Cs: Other, Surrogate
PrivateUse	Co: Other, Private Use
NotAssigned	Cn: Other, Not Assigned

Instances

Instances details

Bounded GeneralCategory	Since: base-2.1
Instance details Defined in GHC.Unicode Methods minBound :: GeneralCategory # maxBound :: GeneralCategory #
Enum GeneralCategory	Since: base-2.1
Instance details Defined in GHC.Unicode Methods succ :: GeneralCategory -> GeneralCategory # pred :: GeneralCategory -> GeneralCategory # toEnum :: Int -> GeneralCategory # fromEnum :: GeneralCategory -> Int # enumFrom :: GeneralCategory -> [GeneralCategory] # enumFromThen :: GeneralCategory -> GeneralCategory -> [GeneralCategory] # enumFromTo :: GeneralCategory -> GeneralCategory -> [GeneralCategory] # enumFromThenTo :: GeneralCategory -> GeneralCategory -> GeneralCategory -> [GeneralCategory] #
Generic GeneralCategory
Instance details Defined in GHC.Generics Associated Types type Rep GeneralCategory :: Type -> Type # Methods from :: GeneralCategory -> Rep GeneralCategory x # to :: Rep GeneralCategory x -> GeneralCategory #
Ix GeneralCategory	Since: base-2.1
Instance details Defined in GHC.Unicode Methods range :: (GeneralCategory, GeneralCategory) -> [GeneralCategory] # index :: (GeneralCategory, GeneralCategory) -> GeneralCategory -> Int # unsafeIndex :: (GeneralCategory, GeneralCategory) -> GeneralCategory -> Int # inRange :: (GeneralCategory, GeneralCategory) -> GeneralCategory -> Bool # rangeSize :: (GeneralCategory, GeneralCategory) -> Int # unsafeRangeSize :: (GeneralCategory, GeneralCategory) -> Int #
Read GeneralCategory	Since: base-2.1
Instance details Defined in GHC.Read Methods readsPrec :: Int -> ReadS GeneralCategory # readList :: ReadS [GeneralCategory] # readPrec :: ReadPrec GeneralCategory # readListPrec :: ReadPrec [GeneralCategory] #
Show GeneralCategory	Since: base-2.1
Instance details Defined in GHC.Unicode Methods showsPrec :: Int -> GeneralCategory -> ShowS # show :: GeneralCategory -> String # showList :: [GeneralCategory] -> ShowS #
Eq GeneralCategory	Since: base-2.1
Instance details Defined in GHC.Unicode Methods (==) :: GeneralCategory -> GeneralCategory -> Bool # (/=) :: GeneralCategory -> GeneralCategory -> Bool #
Ord GeneralCategory	Since: base-2.1
Instance details Defined in GHC.Unicode Methods compare :: GeneralCategory -> GeneralCategory -> Ordering # (<) :: GeneralCategory -> GeneralCategory -> Bool # (<=) :: GeneralCategory -> GeneralCategory -> Bool # (>) :: GeneralCategory -> GeneralCategory -> Bool # (>=) :: GeneralCategory -> GeneralCategory -> Bool # max :: GeneralCategory -> GeneralCategory -> GeneralCategory # min :: GeneralCategory -> GeneralCategory -> GeneralCategory #
type Rep GeneralCategory	Since: base-4.15.0.0
Instance details Defined in GHC.Generics type Rep GeneralCategory = D1 ('MetaData "GeneralCategory" "GHC.Unicode" "base" 'False) ((((C1 ('MetaCons "UppercaseLetter" 'PrefixI 'False) (U1 :: Type -> Type) :+: (C1 ('MetaCons "LowercaseLetter" 'PrefixI 'False) (U1 :: Type -> Type) :+: C1 ('MetaCons "TitlecaseLetter" 'PrefixI 'False) (U1 :: Type -> Type))) :+: ((C1 ('MetaCons "ModifierLetter" 'PrefixI 'False) (U1 :: Type -> Type) :+: C1 ('MetaCons "OtherLetter" 'PrefixI 'False) (U1 :: Type -> Type)) :+: (C1 ('MetaCons "NonSpacingMark" 'PrefixI 'False) (U1 :: Type -> Type) :+: C1 ('MetaCons "SpacingCombiningMark" 'PrefixI 'False) (U1 :: Type -> Type)))) :+: (((C1 ('MetaCons "EnclosingMark" 'PrefixI 'False) (U1 :: Type -> Type) :+: C1 ('MetaCons "DecimalNumber" 'PrefixI 'False) (U1 :: Type -> Type)) :+: (C1 ('MetaCons "LetterNumber" 'PrefixI 'False) (U1 :: Type -> Type) :+: C1 ('MetaCons "OtherNumber" 'PrefixI 'False) (U1 :: Type -> Type))) :+: ((C1 ('MetaCons "ConnectorPunctuation" 'PrefixI 'False) (U1 :: Type -> Type) :+: C1 ('MetaCons "DashPunctuation" 'PrefixI 'False) (U1 :: Type -> Type)) :+: (C1 ('MetaCons "OpenPunctuation" 'PrefixI 'False) (U1 :: Type -> Type) :+: C1 ('MetaCons "ClosePunctuation" 'PrefixI 'False) (U1 :: Type -> Type))))) :+: (((C1 ('MetaCons "InitialQuote" 'PrefixI 'False) (U1 :: Type -> Type) :+: (C1 ('MetaCons "FinalQuote" 'PrefixI 'False) (U1 :: Type -> Type) :+: C1 ('MetaCons "OtherPunctuation" 'PrefixI 'False) (U1 :: Type -> Type))) :+: ((C1 ('MetaCons "MathSymbol" 'PrefixI 'False) (U1 :: Type -> Type) :+: C1 ('MetaCons "CurrencySymbol" 'PrefixI 'False) (U1 :: Type -> Type)) :+: (C1 ('MetaCons "ModifierSymbol" 'PrefixI 'False) (U1 :: Type -> Type) :+: C1 ('MetaCons "OtherSymbol" 'PrefixI 'False) (U1 :: Type -> Type)))) :+: (((C1 ('MetaCons "Space" 'PrefixI 'False) (U1 :: Type -> Type) :+: C1 ('MetaCons "LineSeparator" 'PrefixI 'False) (U1 :: Type -> Type)) :+: (C1 ('MetaCons "ParagraphSeparator" 'PrefixI 'False) (U1 :: Type -> Type) :+: C1 ('MetaCons "Control" 'PrefixI 'False) (U1 :: Type -> Type))) :+: ((C1 ('MetaCons "Format" 'PrefixI 'False) (U1 :: Type -> Type) :+: C1 ('MetaCons "Surrogate" 'PrefixI 'False) (U1 :: Type -> Type)) :+: (C1 ('MetaCons "PrivateUse" 'PrefixI 'False) (U1 :: Type -> Type) :+: C1 ('MetaCons "NotAssigned" 'PrefixI 'False) (U1 :: Type -> Type))))))