Your IP : 18.220.43.27

Current Path : /opt/alt/python35/lib64/python3.5/__pycache__/
Upload File :
Current File : //opt/alt/python35/lib64/python3.5/__pycache__/tokenize.cpython-35.opt-2.pyc



���]�l�X@s4dZdZddlmZddlmZmZddlZddl	m
Z
ddlmZddl
Z
ddlZddlTe
jd	e
j�Ze
jd
e
j�ZddlZejddd
ddddgZ[eZdee<edZdee<edZdee<ed7Zdedededededede de!de"de#de$d e%d!e&d"e'd#e(d$e)d%e*d&e+d'e,d(e-d)e.d*e/d+e0d,e1d-e2d.e3d/e4d0e5d1e6d2e7d3e8d4e9d5e:d6e;d7e<d8e=d9e>d:e?d;e@d<eAd=eBd>eCd?eDd@eEi,ZFGdAd�dejGddB��ZHdCdD�ZIdEdF�ZJdGdH�ZKdIZLdJZMeLeJdKeL�eKeM�ZNdLZOdMZPdNZQdOZRdPZSeIePeQeReS�ZTdQZUeIdRdS�eKeU�ZVdTeUZWeIeVeW�ZXeIdUeXdV�ZYeIeYeXeT�ZZdWZ[dXZ\dYZ]dZZ^d[Z_eIe[d\e[d]�Z`eIe[d^e[d_�ZaeId`dadbd*dcddded-�ZbdfZceIdgdhdi�ZdeIebeced�ZeeIeZeeeaeO�ZfeNefZgeIe[djeIdkdK�e[dleIdmdK��ZheIdneMe`�ZieLeIeieZeeeheO�Zjdodp�Zkdke\dme]d\e^d]e_dqe^dre_dse^dte_due^dve_dwe^dxe_dye^dze_d{e^d|e_d}e^d~e_de^d�e_d�e^d�e_d�e^d�e_d�e^d�e_d�e^d�e_d�e^d�e_d�e^d�e_d�dd�dd�dd�dd�dd�di&ZliZmxd�D]Znenemen<qCWiZoxd�D]Zneneoen<qdWd�ZpGd�d��d�eq�ZrGd�d��d�eq�ZsGd�d��d��Ztd�d�Zud�d��Zvd�d
�Zwd�d��Zd�d�Zxd�d��Zyd�d��Zzd�d��Z{e|d�kr0e{�dS)�zKa-Ping Yee <ping@lfw.org>zpGvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro, Raymond Hettinger, Trent Nelson, Michael Foord�)�open)�lookup�BOM_UTF8N)�
TextIOWrapper)�chain)�*z&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s^[ \t\f]*(?:[#\r\n]|$)�COMMENT�tokenize�detect_encoding�NL�
untokenize�ENCODING�	TokenInfo����(�)�[�]�:�,�;�+�-r�/�|�&�<�>�=�.�%�{�}z==z!=z<=z>=�~�^z<<z>>z**z+=z-=z*=z/=z%=z&=z|=z^=z<<=z>>=z**=z//z//=�@z@=c@s.eZdZdd�Zedd��ZdS)rcCs.d|jt|jf}d|jd|�S)Nz%d (%s)z8TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)�type)r(�tok_name�_replace)�self�annotated_type�r-�-/opt/alt/python35/lib64/python3.5/tokenize.py�__repr__cszTokenInfo.__repr__cCs4|jtkr)|jtkr)t|jS|jSdS)N)r(�OP�string�EXACT_TOKEN_TYPES)r+r-r-r.�
exact_typehszTokenInfo.exact_typeN)�__name__�
__module__�__qualname__r/�propertyr3r-r-r-r.rbsztype string start end linecGsddj|�dS)Nrrr)�join)�choicesr-r-r.�grouposr:cGst|�dS)Nr)r:)r9r-r-r.�anypsr;cGst|�dS)N�?)r:)r9r-r-r.�maybeqsr=z[ \f\t]*z	#[^\r\n]*z\\\r?\nz\w+z0[xX][0-9a-fA-F]+z
0[bB][01]+z0[oO][0-7]+z(?:0+|[1-9][0-9]*)z[eE][-+]?[0-9]+z[0-9]+\.[0-9]*z\.[0-9]+z[0-9]+z
[0-9]+[jJ]z[jJ]z(?:[bB][rR]?|[rR][bB]?|[uU])?z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""�'''�"""z'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"z\*\*=?z>>=?z<<=?z//=?z->z[+\-*/%&@|^=<>]=?z[][(){}]z\r?\nz\.\.\.z[:;.,@]z'[^\n'\\]*(?:\\.[^\n'\\]*)*�'z"[^\n"\\]*(?:\\.[^\n"\\]*)*�"z
\\\r?\n|\ZcCstj|tj�S)N)�re�compile�UNICODE)�exprr-r-r.�_compile�srF�r'''�r"""�b'''�b"""�R'''�R"""�B'''�B"""�br'''�br"""�bR'''�bR"""�Br'''�Br"""�BR'''�BR"""�rb'''�rb"""�Rb'''�Rb"""�rB'''�rB"""�RB'''�RB"""�u'''�u"""�U'''�U"""�r�R�b�B�u�U�r'�r"�R'�R"�b'�b"�B'�B"�br'�br"�Br'�Br"�bR'�bR"�BR'�BR"�rb'�rb"�rB'�rB"�Rb'�Rb"�RB'�RB"�u'�u"�U'�U"�c@seZdZdS)�
TokenErrorN)r4r5r6r-r-r-r.r��sr�c@seZdZdS)�StopTokenizingN)r4r5r6r-r-r-r.r��sr�c@s@eZdZdd�Zdd�Zdd�Zdd�Zd	S)
�UntokenizercCs(g|_d|_d|_d|_dS)Nrr)�tokens�prev_row�prev_col�encoding)r+r-r-r.�__init__�s			zUntokenizer.__init__cCs�|\}}||jks9||jkr]||jkr]tdj|||j|j���||j}|r�|jjd|�d|_||j}|r�|jjd|�dS)Nz+start ({},{}) precedes previous end ({},{})z\
r� )r�r��
ValueError�formatr��append)r+�start�row�col�
row_offset�
col_offsetr-r-r.�add_whitespace�s-	
	
zUntokenizer.add_whitespacecCs�t|�}g}d}x}|D]u}t|�dkrH|j||�P|\}}}}	}
|tkru||_q|tkr�P|tkr�|j|�qn�|tkr�|j	�|	\|_
|_qnl|tt
fkr�d}nQ|r;|r;|d}|dt|�kr5|jj|�t|�|_d}|j|�|jj|�|	\|_
|_|tt
fkr|j
d7_
d|_qWdj|j�S)NFrTrr����)�iter�len�compatr
r��	ENDMARKER�INDENTr��DEDENT�popr�r��NEWLINErr�r�r8)r+�iterable�it�indents�	startline�t�tok_type�tokenr��end�line�indentr-r-r.r�sF
	

	


zUntokenizer.untokenizec
CsJg}|jj}|dttfk}d}xt|g|�D]}|dd�\}}	|tkru|	|_qA|ttt	t
fkr�|	d7}	|tkr�|r�d|	}	d}nd}|tkr�|j|	�qAnW|t
kr�|j�qAn;|ttfkrd}n |r8|r8||d�d}||	�qAWdS)NrFrr�Trr�)r�r�r�rrr
r��NAME�NUMBER�ASYNC�AWAIT�STRINGr�r�r�)
r+r�r�r��toks_appendr��
prevstring�tok�toknum�tokvalr-r-r.r�s8	

	

	zUntokenizer.compatN)r4r5r6r�r�rr�r-r-r-r.r��s
%r�cCs=t�}|j|�}|jdk	r9|j|j�}|S)N)r�rr��encode)r��ut�outr-r-r.r?s
	cCsd|dd�j�jdd�}|dks=|jd�rAdS|dks\|jd
�r`dS|S)N��_rzutf-8zutf-8-�latin-1�
iso-8859-1�iso-latin-1�latin-1-�iso-8859-1-�iso-latin-1-)r�r�r�)r�r�r�)�lower�replace�
startswith)�orig_enc�encr-r-r.�_get_normal_nameZs"r�cs1y�jj�Wntk
r*d�YnXd�d}d}�fdd�}��fdd�}|�}|jt�r�d�|dd�}d	}|s�|gfS||�}|r�||gfStj|�s�||gfS|�}|s�||gfS||�}|r!|||gfS|||gfS)
NFzutf-8cs(y��SWntk
r#dSYnXdS)N�)�
StopIterationr-)�readliner-r.�read_or_stop}s
z%detect_encoding.<locals>.read_or_stopcs2y|jd�}WnBtk
rWd}�dk	rGdj|��}t|��YnXtj|�}|sqdSt|jd��}yt|�}WnIt	k
r��dkr�d|}ndj�|�}t|��YnX�r.|dkr$�dkr	d}ndj��}t|��|d	7}|S)
Nzutf-8z'invalid or missing encoding declarationz{} for {!r}rzunknown encoding: zunknown encoding for {!r}: {}zencoding problem: utf-8z encoding problem for {!r}: utf-8z-sig)
�decode�UnicodeDecodeErrorr��SyntaxError�	cookie_re�matchr�r:r�LookupError)r��line_string�msgr�r��codec)�	bom_found�filenamer-r.�find_cookie�s6


			
z$detect_encoding.<locals>.find_cookieTrz	utf-8-sig)�__self__�name�AttributeErrorr�r�blank_rer�)r�r��defaultr�r��first�secondr-)r�r�r�r.r
es8
&	


	
cCsst|d�}yHt|j�\}}|jd�t||dd�}d|_|SWn|j��YnXdS)N�rbr�line_bufferingTrc)�
_builtin_openr
r��seekr�mode�close)r��bufferr��lines�textr-r-r.r�s
	
rcCs_ddlm}m}t|�\}}t|d�}|d�}t||||�j|�S)Nr)r�repeatr�)�	itertoolsrr�r
r��	_tokenize�__next__)r�rr�r��consumed�rl_gen�emptyr-r-r.r	�s
c cs�	d}}}d}d!\}}d}dg}	d}
d}d}d}
|dk	r||dkred}tt|d"d#d�Vx�y
|�}Wntk
r�d}YnX|dk	r�|j|�}|d7}dt|�}}|r�|s�td	|��|j|�}|rf|jd�}}tt||d|�|||f||�Vd$\}}d}q�|r�|d%d�dkr�|d&d�d
kr�tt	||||t|�f|�Vd}d}qq�||}||}qn�|dkr�|r�|sPd}xv||kr�||dkr7|d7}n?||dkr\|t
dt
}n||dkrud}nP|d7}qW||kr�P||dkr�||dkr=||d�jd�}|t|�}tt|||f||t|�f|�Vtt
||d�||f|t|�f|�Vqtt
tf||dk||d�||f|t|�f|�Vq||	d'kr�|	j|�tt|d|�|df||f|�Vx�||	d(kri||	krtdd|||f��|	dd)�}	|rC||	d*krCd}d}
d}ttd||f||f|�Vq�W|r�|
r�||	d+kr�d}d}
d}n!|s�td|df��d}xo||kr-	tt�j||�}|r�|jd�\}}||f||f|}}}||kr-q�|||�||}}||ksx|dkr�|dkr�|dkr�tt||||�Vq*	|dkr�|
r�|
Vd}
|dkr�tt
||||�Vq�tt||||�V|r�d}
q*	|dkr2|
r|
Vd}
tt||||�Vq*	|tkr�tt|�}|j||�}|r�|jd�}|||�}tt||||f|�Vq�||f}||d�}|}Pq*	|tks|dd
�tks|dd�tkr�|d,dkrl||f}tt|pEt|dpEt|d
�}||d�d}}|}Pq�tt||||�Vq*	|j�r�|d-kr�|r�t|dkr�tnt||||�Vq�tt||||�}|dkr|
r|}
q�|dkrm|
rm|
jtkrm|
jdkrmd}|	d.}tt|
j|
j |
j|
j!�Vd}
|
r~|
Vd}
|Vq*	|dkr�d}q*	|dkr�|d7}n|d kr�|d8}|
r�|
Vd}
tt"||||�Vq�tt	||||f||df|�V|d7}q�WqW|
rC	|
Vd}
x;|	dd�D])}ttd|df|dfd�VqT	Wtt#d|df|dfd�VdS)/Nr�
0123456789r�Fz	utf-8-sigzutf-8r�rzEOF in multi-line stringrz\
rz\
r��	�z#
�#z
z3unindent does not match any outer indentation levelz
<tokenize>zEOF in multi-line statementr!z...T�
�async�await�def�\z([{z)]})r�r)rr)rr)r�r������r�r�r�r�r�r�)r�r�r�)$rr
r�r�r�r�r�r�r��
ERRORTOKEN�tabsize�rstriprrr�r��IndentationErrorr�rF�PseudoToken�spanr�r��
triple_quoted�endpats�
single_quoted�isidentifierr�r�r�r(r1r�r�r0r�) r�r��lnum�parenlev�	continued�numchars�contstr�needcont�contliner��stashed�	async_def�async_def_indent�async_def_nlr��pos�max�strstart�endprog�endmatchr��column�
comment_token�nl_pos�pseudomatchr��spos�eposr��initialr�r�r-r-r.r��sh	


	2



		$#' 
-'	 $	 

	


'r�cCs
t|d�S)N)r�)r�r-r-r.�generate_tokens�src
s�ddl}dd��dd�fdd�}|jdd�}|jdd	d
ddd
dd�|jdddddddd�|j�}y�|jr�|j}t|d��}tt|j��}WdQRXnd}t	t
jjd�}x[|D]S}|j}|j
r|j}d|j|j}	td|	t||jf�q�WWnbtk
r�}
z?|
jddd�\}}||
jd|||f�WYdd}
~
Xntk
r}
z5|
jd\}}||
jd|||f�WYdd}
~
Xn�tk
r8}
z||
|�WYdd}
~
Xntk
rf}
z||
�WYdd}
~
XnQtk
r�td�Yn6tk
r�}
z�d|
��WYdd}
~
XnXdS)NrcSst|dtj�dS)N�file)�print�sys�stderr)�messager-r-r.�perror�szmain.<locals>.perrorcsg|r+|f||f}�d|�n+|rH�d||f�n�d|�tjd�dS)Nz%s:%d:%d: error: %sz
%s: error: %sz	error: %sr)r�exit)rr��location�args)r r-r.�error�szmain.<locals>.error�progzpython -m tokenize�destr��nargsr<�metavarzfilename.py�helpz'the file to tokenize; defaults to stdinz-ez--exact�exact�action�
store_truez(display token names using the exact typer�z<stdin>z%d,%d-%d,%d:z%-20s%-15s%-15rrrzinterrupted
zunexpected error: %s)�argparse�ArgumentParser�add_argument�
parse_argsr�r��listr	r�r�r�stdinr(r*r3r�r�rr)r1r�r#r�r��OSError�KeyboardInterrupt�	Exception)
r-r$�parserr#r��fr�r��
token_type�token_range�errr�rr-)r r.�main�sN		
			 //
r;�__main__)r>r?rGrHrKrLrIrJrMrNrOrPrSrTrQrRrUrVrWrXr[r\rYrZr]r^r_r`rarb)r@rArirjrkrlrmrnrorprqrrrsrtrurvrwrxryrzr{r|r}r~rr�r�r�r�r�)}�
__author__�__credits__�builtinsrr��codecsrr�collections�iorr�rrBrr�rC�ASCIIr�r��__all__�N_TOKENSrr)rr
�LPAR�RPAR�LSQB�RSQB�COLON�COMMA�SEMI�PLUS�MINUS�STAR�SLASH�VBAR�AMPER�LESS�GREATER�EQUAL�DOT�PERCENT�LBRACE�RBRACE�EQEQUAL�NOTEQUAL�	LESSEQUAL�GREATEREQUAL�TILDE�
CIRCUMFLEX�	LEFTSHIFT�
RIGHTSHIFT�
DOUBLESTAR�	PLUSEQUAL�MINEQUAL�	STAREQUAL�
SLASHEQUAL�PERCENTEQUAL�
AMPEREQUAL�	VBAREQUAL�CIRCUMFLEXEQUAL�LEFTSHIFTEQUAL�RIGHTSHIFTEQUAL�DOUBLESTAREQUAL�DOUBLESLASH�DOUBLESLASHEQUAL�AT�ATEQUALr2�
namedtuplerr:r;r=�
Whitespace�Comment�Ignore�Name�	Hexnumber�	Binnumber�	Octnumber�	Decnumber�	Intnumber�Exponent�
Pointfloat�Expfloat�Floatnumber�
Imagnumber�Number�StringPrefix�Single�Double�Single3�Double3�Triple�String�Operator�Bracket�Special�Funny�
PlainToken�Token�ContStr�PseudoExtrasr�rFrr�r�rr�r5r�r�r�rr�r
r	r�rr;r4r-r-r-r.�<module>sB






"



	




_]�<
?>