Your IP : 18.119.166.34

Current Path : /lib64/python3.8/lib2to3/pgen2/__pycache__/
Upload File :
Current File : //lib64/python3.8/lib2to3/pgen2/__pycache__/tokenize.cpython-38.opt-2.pyc

U

e5d?R�
@s�dZdZddlZddlZddlmZmZddlTddlm	Z	dd	�e
e	�D�d
ddgZ[	zeWne
k
rzeZYnXd
d�Zdd�Zdd�Zdd�ZdZdZeede�ee�ZdZdZdZdZedd�Zeeeee�ZdZedd �ee�Zd!eZeee�Zed"ed#�Z ee ee�Z!d$Z"d%Z#d&Z$d'Z%d(Z&ee&d)e&d*�Z'ee&d+e&d,�Z(ed-d.d/d0d1d2d3d4d5�	Z)d6Z*ed7d8d9�Z+ee)e*e+�Z,ee!e,e(e�Z-ee-Z.ee&d:ed;d�e&d<ed=d��Z/edee'�Z0eee0e!e,e/e�Z1e2ej3e.e1e$e%f�\Z4Z5Z6Z7ed>d?d@dA�ed>d?dBdC�BdDdEdFdGdHdIhBZ8e�3e"�e�3e#�e6e7dJ�dKdL�e8D�dMdL�e8D�dNdL�e8D��Z9d)d*hdOdP�e8D�BdQdP�e8D�BZ:d;d=hdRdP�e8D�BdSdP�e8D�BZ;dTZ<GdUdV�dVe=�Z>GdWdX�dXe=�Z?dYdZ�Z@e@fd[d
�ZAd\d]�ZBGd^d_�d_�ZCe�3d`ejD�ZEe�3daejD�ZFdbdc�ZGddde�ZHdfd�ZIdgd�ZJeKdhk�r�ddlLZLeMeLjN�dk�r�eAeOeLjNd�jP�neAeLjQjP�dS)izKa-Ping Yee <ping@lfw.org>z@GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro�N)�BOM_UTF8�lookup)�*�)�tokencCsg|]}|ddkr|�qS)r�_�)�.0�xrr�./usr/lib64/python3.8/lib2to3/pgen2/tokenize.py�
<listcomp>%sr�tokenize�generate_tokens�
untokenizecGsdd�|�dS)N�(�|�))�join��choicesrrr�group0�rcGst|�dS)Nr�rrrrr�any1rrcGst|�dS)N�?rrrrr�maybe2rrcst�fdd��D��S)Nc3s4|],}�dD]}|��|��kr||VqqdS))�N)�casefold)r	r
�y��lrr�	<genexpr>4s

z _combinations.<locals>.<genexpr>)�setrrrr�
_combinations3s�r#z[ \f\t]*z	#[^\r\n]*z\\\r?\nz\w+z0[bB]_?[01]+(?:_[01]+)*z(0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?z0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?z[1-9]\d*(?:_\d+)*[lL]?z0[lL]?z[eE][-+]?\d+(?:_\d+)*z\d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?z\.\d+(?:_\d+)*z\d+(?:_\d+)*z\d+(?:_\d+)*[jJ]z[jJ]z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z'(?:[uUrRbBfF]|[rR][fFbB]|[fFbBuU][rR])?�'''�"""z'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"z\*\*=?z>>=?z<<=?z<>z!=z//=?z->z[+\-*/%&@|^=<>]=?�~z[][(){}]z\r?\nz:=z[:;.,`@]z'[^\n'\\]*(?:\\.[^\n'\\]*)*�'z"[^\n"\\]*(?:\\.[^\n"\\]*)*�"�r�R�f�F�b�B�u�UZurZuRZUrZUR)r'r(r$r%cCsi|]}|�d�t�qS�r$)�single3prog�r	�prefixrrr�
<dictcomp>ysr5cCsi|]}|�d�t�qS�r%)�double3progr3rrrr5zscCsi|]
}|d�qS�Nrr3rrrr5{scCsh|]}|�d��qSr1rr3rrr�	<setcomp>sr9cCsh|]}|�d��qSr6rr3rrrr9�scCsh|]}|�d��qS)r'rr3rrrr9�scCsh|]}|�d��qS)r(rr3rrrr9�s�c@seZdZdS)�
TokenErrorN��__name__�
__module__�__qualname__rrrrr;�sr;c@seZdZdS)�StopTokenizingNr<rrrrr@�sr@c		Cs4|\}}|\}}td||||t|t|�f�dS)Nz%d,%d-%d,%d:	%s	%s)�print�tok_name�repr)	�typerZxxx_todo_changemeZxxx_todo_changeme1�lineZsrowZscolZerowZecolrrr�
printtoken�s
�rFcCs(zt||�Wntk
r"YnXdSr8)�
tokenize_loopr@)�readline�
tokeneaterrrrr
�s
cCst|�D]}||�qdSr8)r)rHrIZ
token_inforrrrG�srGc@s,eZdZdd�Zdd�Zdd�Zdd�Zd	S)
�UntokenizercCsg|_d|_d|_dS)Nrr)�tokens�prev_row�prev_col)�selfrrr�__init__�szUntokenizer.__init__cCs*|\}}||j}|r&|j�d|�dS)N� )rMrK�append)rN�start�row�col�
col_offsetrrr�add_whitespace�s
zUntokenizer.add_whitespacecCs�|D]p}t|�dkr$|�||�qv|\}}}}}|�|�|j�|�|\|_|_|ttfkr|jd7_d|_qd�	|j�S)N�rrr)
�len�compatrVrKrQrLrM�NEWLINE�NLr)rN�iterable�t�tok_typerrR�endrErrrr�s
zUntokenizer.untokenizec	Cs�d}g}|jj}|\}}|ttfkr,|d7}|ttfkr<d}|D]�}|dd�\}}|ttttfkrl|d7}|tkr�|�|�q@n>|t	kr�|�
�q@n*|ttfkr�d}n|r�|r�||d�d}||�q@dS)NFrPTrW���)rKrQ�NAME�NUMBERrZr[�ASYNC�AWAIT�INDENT�DEDENT�pop)	rNrr\�	startline�indents�toks_append�toknum�tokval�tokrrrrY�s0
zUntokenizer.compatN)r=r>r?rOrVrrYrrrrrJ�srJz&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s^[ \t\f]*(?:[#\r\n]|$)cCsH|dd����dd�}|dks*|�d�r.dS|dks@|�d�rDdS|S)	N�r�-�utf-8zutf-8-)zlatin-1�
iso-8859-1ziso-latin-1)zlatin-1-ziso-8859-1-ziso-latin-1-rq)�lower�replace�
startswith)�orig_enc�encrrr�_get_normal_name�s�rwcs�d�d}d}�fdd�}�fdd�}|�}|�t�rHd�|dd�}d	}|sT|gfS||�}|rj||gfSt�|�s~||gfS|�}|s�||gfS||�}|r�|||gfS|||gfS)
NFrpcs(z��WStk
r"t�YSXdSr8)�
StopIteration�bytesr)rHrr�read_or_stopsz%detect_encoding.<locals>.read_or_stopcs�z|�d�}Wntk
r$YdSXt�|�}|s8dSt|�d��}zt|�}Wn tk
rrtd|��YnX�r�|j	dkr�td��|d7}|S)N�asciirzunknown encoding: rpzencoding problem: utf-8z-sig)
�decode�UnicodeDecodeError�	cookie_re�matchrwrr�LookupError�SyntaxError�name)rE�line_stringr�encoding�codec)�	bom_foundrr�find_cookies"

z$detect_encoding.<locals>.find_cookieT�z	utf-8-sig)rtr�blank_rer)rHr��defaultrzr��first�secondr)r�rHr�detect_encoding�s0




r�cCst�}|�|�Sr8)rJr)r\�utrrrr:sccstd}}}d\}}d}dg}d}d}	d}
d}z
|�}Wntk
rPd}YnX|d}dt|�}
}|�r2|s|td|��|�|�}|r�|�d�}
}t||d|�|||f||fVd\}}d}nd|�r|dd�dk�r|d	d�d
k�rt||||t|�f|fVd}d}q.n||}||}q.�nB|dk�r\|�s\|�sL�qd}|
|k�r�||
dk�rr|d}n8||
dk�r�|tdt}n||
d
k�r�d}n�q�|
d}
�qP|
|k�rĐq|�r�|Vd}||
dk�r�||
dk�rT||
d��d�}|
t|�}t	|||
f||
t|�f|fVt
||d�||f|t|�f|fVq.t
t	f||
dk||
d�||
f|t|�f|fVq.||dk�r�|�|�t|d|
�|df||
f|fV||dk�r4||k�r�t
dd||
|f��|dd�}|	�r|
|dk�rd}	d}d}
td||
f||
f|fV�q�|	�rt|�rt|
|dk�rtd}	d}d}
n|�sptd|df��d}|
|kr.t�||
�}|�r�|�d�\}}||f||f|}}}
|||�||}}|tjk�s�|dk�r�|dk�r�t||||fV�q|dk�rJt}|dk�rt
}n
|	�r&d}|�r6|Vd}|||||fV�q|dk�rx|�rd|Vd}t	||||fV�q|tk�r�t|}|�||
�}|�r�|�d�}
|||
�}|�r�|Vd}t||||
f|fVn||f}||d�}|}q.�q|tk�s$|dd�tk�s$|dd�tk�r�|ddk�rx||f}t|�p\t|d�p\t|d}||d�d}}|}q.n |�r�|Vd}t||||fV�q|���rr|dk�r�|	�r�|dk�r�tnt||||fV�qtt||||f}|dk�r�|�s�|}�qt|dk�rZ|�rZ|dtk�rZ|ddk�rZd}	|d}
t|d|d|d|dfVd}|�rj|Vd}|Vnz|dk�r�|�r�|Vd}t
||||
f|fVd}nF|dk�r�|d}n|d k�r�|d}|�r�|Vd}t||||fVn(t||
||
f||
df|fV|
d}
�qtq.|�r,|Vd}|dd�D]}td|df|dfdfV�q8td|df|dfdfVdS)!Nr)rrFrrzEOF in multi-line string���z\
���z\
rP�	�z#
�#z
r`z3unindent does not match any outer indentation levelz
<tokenize>zEOF in multi-line statement�.TrWr��
)�async�awaitr��def��\z([{z)]})rxrXr;rr_�STRING�
ERRORTOKEN�tabsize�rstrip�COMMENTr[rQre�IndentationErrorrf�
pseudoprog�span�stringZdigitsrbrZ�
triple_quoted�endprogs�
single_quoted�isidentifierrcrdra�OP�	ENDMARKER)rH�lnum�parenlev�	continued�contstr�needcont�contlineriZstashedZ	async_defZasync_def_indentZasync_def_nlrE�pos�max�strstart�endprog�endmatchr_�column�
comment_tokenZnl_pos�pseudomatchrR�spos�eposr�initial�newlinerm�indentrrrrOs�



�*
�


�
�
�
 

���





��
�

�

�
��




��__main__)R�
__author__�__credits__r��re�codecsrrZlib2to3.pgen2.tokenrr�dir�__all__ry�	NameError�strrrrr#�
Whitespace�Comment�Ignore�Name�	Binnumber�	Hexnumber�	Octnumber�	Decnumber�	Intnumber�Exponent�
Pointfloat�Expfloat�Floatnumber�
Imagnumber�Number�Single�Double�Single3�Double3Z
_litprefix�Triple�StringZOperatorZBracket�Special�Funny�
PlainToken�Token�ContStr�PseudoExtras�PseudoToken�map�compileZ	tokenprogr�r2r7Z_strprefixesr�r�r�r��	Exceptionr;r@rFr
rGrJ�ASCIIr~r�rwr�rrr=�sysrX�argv�openrH�stdinrrrr�<module>s���


�����
������������8Ib

?>