Your IP : 18.224.38.170


Current Path : /lib64/python3.8/__pycache__/
Upload File :
Current File : //lib64/python3.8/__pycache__/tokenize.cpython-38.opt-2.pyc

U

e5d�d�@sDdZdZddlmZddlmZmZddlZddl	m
Z
ddlZddl
Z
ddlZddlTddlmZe
�d	e
j�Ze
�d
e
j�ZddlZejddd
ddgZ[Gdd�de�dd��Zdd�Zdd�Zdd�ZdZdZeede�ee�ZdZdZdZ dZ!dZ"eee e!e"�Z#d Z$ed!d"�ee$�Z%d#e$Z&ee%e&�Z'ed$e'd%�Z(ee(e'e#�Z)d&d'�Z*d(d)�Z+ee*��Z,d*Z-d+Z.d,Z/d-Z0ee,d.e,d/�Z1ee,d0e,d1�Z2ee3e
j4e5ed2d3���Z6ed4e6�Z7ee)e7e2e�Z8ee8Z9ee,d5ed6d�e,d7ed8d��Z:ed9ee1�Z;eee;e)e7e:e�Z<iZ=e*�D]6Z>e-e=e>d6<e.e=e>d8<e/e=e>d.<e0e=e>d/<�q(e?�Z@e?�ZAe*�D]JZBeBd8eBd6fD]ZCe@�DeC��q�eBd/eBd.fD]ZCeA�DeC��q��qrd:ZEGd;d<�d<eF�ZGGd=d>�d>eF�ZHGd?d@�d@�ZIdAd�ZJdBdC�ZKdDd
�ZLdEdF�ZdGd�ZMdHdI�ZNdJd�ZOdKdL�ZPeQdMk�r@eP�dS)NzKa-Ping Yee <ping@lfw.org>zpGvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro, Raymond Hettinger, Trent Nelson, Michael Foord�)�open)�lookup�BOM_UTF8N)�
TextIOWrapper)�*)�EXACT_TOKEN_TYPESz&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s^[ \t\f]*(?:[#\r\n]|$)�tokenize�generate_tokens�detect_encoding�
untokenize�	TokenInfoc@s eZdZdd�Zedd��ZdS)rcCs$d|jt|jf}d|j|d�S)Nz%d (%s)z8TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r))�type)r
�tok_name�_replace)�self�annotated_type�r� /usr/lib64/python3.8/tokenize.py�__repr__.s
�zTokenInfo.__repr__cCs(|jtkr|jtkrt|jS|jSdS�N)r
�OP�stringr�rrrr�
exact_type3s
zTokenInfo.exact_typeN)�__name__�
__module__�__qualname__r�propertyrrrrrr-sztype string start end linecGsdd�|�dS)N�(�|�))�join��choicesrrr�group:�r$cGst|�dS)Nr�r$r"rrr�any;r%r'cGst|�dS)N�?r&r"rrr�maybe<r%r)z[ \f\t]*z	#[^\r\n]*z\\\r?\nz\w+z0[xX](?:_?[0-9a-fA-F])+z0[bB](?:_?[01])+z0[oO](?:_?[0-7])+z(?:0(?:_?0)*|[1-9](?:_?[0-9])*)z[eE][-+]?[0-9](?:_?[0-9])*z)[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?z\.[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*[jJ]z[jJ]cCs^ddddddg}dh}|D]>}t�|�D].}tjdd	�|D��D]}|�d�|��q@q(q|S)
N�b�r�u�f�br�fr�cSsg|]}||��f�qSr)�upper)�.0�crrr�
<listcomp>^sz(_all_string_prefixes.<locals>.<listcomp>)�
_itertools�permutations�product�addr!)�_valid_string_prefixes�result�prefix�tr,rrr�_all_string_prefixesSsr=cCst�|tj�Sr)�re�compile�UNICODE)�exprrrr�_compilebsrBz[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z'''z"""z'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"T)�reversez\r?\nz'[^\n'\\]*(?:\\.[^\n'\\]*)*�'z"[^\n"\\]*(?:\\.[^\n"\\]*)*�"z
\\\r?\n|\Z�c@seZdZdS)�
TokenErrorN�rrrrrrrrG�srGc@seZdZdS)�StopTokenizingNrHrrrrrI�srIc@s,eZdZdd�Zdd�Zdd�Zdd�Zd	S)
�UntokenizercCsg|_d|_d|_d|_dS)N�r)�tokens�prev_row�prev_col�encodingrrrr�__init__�szUntokenizer.__init__cCs�|\}}||jks&||jkr>||jkr>td�|||j|j���||j}|rb|j�d|�d|_||j}|r�|j�d|�dS)Nz+start ({},{}) precedes previous end ({},{})�\
r� )rMrN�
ValueError�formatrL�append)r�start�row�col�
row_offset�
col_offsetrrr�add_whitespace�s�

zUntokenizer.add_whitespacecCs6t|�}g}d}|D�]}t|�dkr8|�||��q*|\}}}}	}
|tkrV||_q|tkrd�q*|tkrz|�|�qnl|tkr�|�	�|	\|_
|_qnL|tt
fkr�d}n:|r�|r�|d}|dt|�kr�|j�|�t|�|_d}|�|�|j�|�|	\|_
|_|tt
fkr|j
d7_
d|_qd�|j�S)NF�T���rKrr0)�iter�len�compat�ENCODINGrO�	ENDMARKER�INDENTrU�DEDENT�poprMrN�NEWLINE�NLrLr[r!)r�iterable�it�indents�	startliner<�tok_type�tokenrV�end�line�indentrrrr�sF



zUntokenizer.untokenizec
Cs�g}|jj}|dttfk}d}t�|g|�D]�}|dd�\}}	|tkrR|	|_q.|tt	fkrf|	d7}	|t
kr�|rzd|	}	d}nd}|tkr�|�|	�q.n>|tkr�|�
�q.n*|ttfkr�d}n|r�|r�||d�d}||	�q.dS)NrFr\rRTr])rLrUrfrgr5�chainrarO�NAME�NUMBER�STRINGrcrdre)
rrmrhrj�toks_appendrk�
prevstring�tok�toknum�tokvalrrrr`�s8
zUntokenizer.compatN)rrrrPr[rr`rrrrrJ�s
%rJcCs*t�}|�|�}|jdk	r&|�|j�}|Sr)rJrrO�encode)rh�ut�outrrrrs


cCsH|dd����dd�}|dks*|�d�r.dS|dks@|�d�rDdS|S)	N��_�-�utf-8zutf-8-)zlatin-1�
iso-8859-1ziso-latin-1)zlatin-1-ziso-8859-1-ziso-latin-1-r�)�lower�replace�
startswith)�orig_enc�encrrr�_get_normal_names�r�cs�z�jj�Wntk
r$d�YnXd�d}d}�fdd�}��fdd�}|�}|�t�rpd�|dd�}d	}|s||gfS||�}|r�||gfSt�|�s�||gfS|�}|s�||gfS||�}|r�|||gfS|||gfS)
NFr�cs$z��WStk
rYdSXdS�Nr%)�
StopIterationr��readlinerr�read_or_stop?sz%detect_encoding.<locals>.read_or_stopcs�z|�d�}Wn4tk
rBd}�dk	r6d�|��}t|��YnXt�|�}|sVdSt|�d��}zt|�}Wn:t	k
r��dkr�d|}nd��|�}t|��YnX�r�|dkr؈dkr�d}n
d���}t|��|d	7}|S)
Nr�z'invalid or missing encoding declarationz{} for {!r}rKzunknown encoding: zunknown encoding for {!r}: {}zencoding problem: utf-8z encoding problem for {!r}: utf-8z-sig)
�decode�UnicodeDecodeErrorrT�SyntaxError�	cookie_re�matchr�r$r�LookupError)ro�line_string�msgr�rO�codec)�	bom_found�filenamerr�find_cookieEs8

�
z$detect_encoding.<locals>.find_cookieT��	utf-8-sig)�__self__�name�AttributeErrorr�r�blank_rer�)r�rO�defaultr�r��first�secondr)r�r�r�rr
's8
&




cCsXt|d�}z2t|j�\}}|�d�t||dd�}d|_|WS|���YnXdS)N�rbrT)�line_bufferingr+)�
_builtin_openr
r��seekr�mode�close)r��bufferrO�lines�textrrrr�s

rcCs6t|�\}}t�d�}t�|t|d�|�}t|j|�Sr�)r
r5�repeatrqr^�	_tokenize�__next__)r�rO�consumed�empty�rl_genrrrr�s
ccsld}}}d}d\}}d}dg}	|dk	rH|dkr6d}tt|ddd�Vd}
d}z|}
|�}Wntk
rvd}YnX|dk	r�|�|�}|d	7}dt|�}}
|�rp|s�td
|��|�|�}|�r|�d�}}tt||d|�|||f||�Vd\}}d}nf|�rZ|dd�dk�rZ|d
d�dk�rZtt	||||t|�f|�Vd}d}qPn||}||}qP�n�|dk�r|�s|�s��q�d}||
k�r�||dk�r�|d	7}n8||dk�r�|t
d	t
}n||dk�r�d}n�q�|d	7}�q�||
k�r�q�||dk�r�||dk�r^||d��d�}tt|||f||t|�f|�V|t|�7}tt
||d�||f|t|�f|�VqP||	dk�r�|	�|�tt|d|�|df||f|�V||	dk�r.||	k�r�tdd|||f��|	dd�}	ttd||f||f|�V�q�n|�s*td|df��d}||
krPtt��||�}|�r�|�d	�\}}||f||f|}}}||k�r��q.|||�||}}||k�s�|dk�r�|dk�r�|dk�r�tt||||�V�q�|dk�r|dk�r�tt
||||�Vntt||||�V�q�|dk�r2tt||||�V�q�|tk�r�tt|�}|�||�}|�r�|�d�}|||�}tt||||f|�Vn||f}||d�}|}qP�q�|tk�s�|dd�tk�s�|dd�tk�rF|ddk�r2||f}tt�|��pt�|d	��pt�|d��}||d�d	}}|}qPntt||||�Vnf|���rdtt||||�VnH|dk�rtd	}n8|dk�r�|d	7}n|d k�r�|d	8}tt||||�Vn*tt	||||f||d	f|�V|d	7}�q.qP|
�r |
ddk�r ttd|d	t|
�f|d	t|
�d	fd�V|	d	d�D] }ttd|df|dfd�V�q,ttd|df|dfd�VdS)!Nr�
0123456789)r0rr�r�)rrr0r%rKzEOF in multi-line string���rQ���z\
rR�	�z#
�#z
r]z3unindent does not match any outer indentation levelz
<tokenize>zEOF in multi-line statement�.z...r\r��
�\z([{z)]})rrar�r�r_rGr�rnrt�
ERRORTOKEN�tabsize�rstrip�COMMENTrgrUrc�IndentationErrorrdrB�PseudoToken�spanrsrf�
triple_quoted�endpats�
single_quoted�get�isidentifierrrrrb)r�rO�lnum�parenlev�	continued�numchars�contstr�needcont�contlinerj�	last_linero�pos�max�strstart�endprog�endmatchrn�column�
comment_token�pseudomatchrV�spos�eposrm�initialrprrrr��s<




�*

�


�
�
"

� 

���





����






�.r�cCs
t|d�Sr)r�r�rrrr	dsc

s$ddl}dd��d�fdd�	}|jdd�}|jdd	d
dd�|jd
ddddd�|��}z�|jr�|j}t|d��}tt|j��}W5QRXnd}t	t
jjd�}|D]>}|j}|j
r�|j}d|j|j}	td|	t||jf�q�W�n8tk
�r6}
z0|
jddd�\}}||
jd|||f�W5d}
~
XYn�tk
�r|}
z(|
jd\}}||
jd|||f�W5d}
~
XYn�tk
�r�}
z||
|�W5d}
~
XYnxtk
�r�}
z||
�W5d}
~
XYnNtk
�r�td�Yn2tk
�r}
z�d|
��W5d}
~
XYnXdS)NrcSstj�|�tj�d�dS)Nr�)�sys�stderr�write)�messagerrr�perrorpszmain.<locals>.perrorcsR|r"|f||f}�d|�n"|r8�d||f�n�d|�t�d�dS)Nz%s:%d:%d: error: %sz
%s: error: %sz	error: %srK)r��exit)r�r��location�args�r�rr�errortszmain.<locals>.errorzpython -m tokenize)�progr�r(zfilename.pyz'the file to tokenize; defaults to stdin)�dest�nargs�metavar�helpz-ez--exact�exact�
store_truez(display token names using the exact type)r��actionr�r�z<stdin>z%d,%d-%d,%d:z%-20s%-15s%-15rrKr�zinterrupted
zunexpected error: %s)NN)�argparse�ArgumentParser�add_argument�
parse_argsr�r��listrr�r�r��stdinr
r�rrVrn�printrrr�r�rGr��OSError�KeyboardInterrupt�	Exception)
r�r��parserr�r�r-rLrm�
token_type�token_range�errror�rr�r�mainlsT���&&r��__main__)R�
__author__�__credits__�builtinsrr��codecsrr�collections�ior�	itertoolsr5r>r�rmrr?�ASCIIr�r��__all__�
namedtuplerr$r'r)�
Whitespace�Comment�Ignore�Name�	Hexnumber�	Binnumber�	Octnumber�	Decnumber�	Intnumber�Exponent�
Pointfloat�Expfloat�Floatnumber�
Imagnumber�Numberr=rB�StringPrefix�Single�Double�Single3�Double3�Triple�String�map�escape�sorted�Special�Funny�
PlainToken�Token�ContStr�PseudoExtrasr�r��_prefix�setr�r�r<r,r8r�r�rGrIrJrr�r
rr�r	r�rrrrr�<module>s�
�
��

�
���

_]8=


?>