a
R�fe � @ s> d Z dZddlmZ ddlmZmZ ddlZddl m
Z
ddl Z ddl
Z
ddlZddlT ddlmZ e
�d e
j�Ze
�d
e
j�ZddlZejg d � Z[G d d
� d
e�d
d��Zdd� Zdd� Zdd� ZdZdZ eede � ee � Z dZ dZ dZ dZ!d Z"ee e e!e"�Z#d Z$ed d �ee$� Z%d e$ Z&ee%e&�Z'ed!e'd" �Z(ee(e'e#�Z)d#d$� Z*d%d&� Z+ee*� � Z,d'Z-d(Z.d)Z/d*Z0ee,d+ e,d, �Z1ee,d- e,d. �Z2ee3e
j4e5ed/d0��� Z6ed1e6�Z7ee)e7e2e �Z8e e8 Z9ee,d2 ed3d� e,d4 ed5d� �Z:ed6e e1�Z;eee;e)e7e:e � Z<i Z=e*� D ]6Z>e-e=e>d3 < e.e=e>d5 < e/e=e>d+ < e0e=e>d, < �q"e?� Z@e?� ZAe*� D ]JZBeBd5 eBd3 fD ]ZCe@�DeC� �q�eBd, eBd+ fD ]ZCeA�DeC� �q��qld7ZEG d8d9� d9eF�ZGG d:d;� d;eF�ZHG d<d=� d=�ZId>d?� ZJd@dA� ZKdBdC� ZLdDdE� ZdFdG� ZMdHdI� ZNdJdK� ZOdLdM� ZPeQdNk�r:eP� dS )OzKa-Ping Yee <ping@lfw.org>zpGvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro, Raymond Hettinger, Trent Nelson, Michael Foord� )�open)�lookup�BOM_UTF8N)�
TextIOWrapper)�*)�EXACT_TOKEN_TYPESz&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s ^[ \t\f]*(?:[#\r\n]|$))�tokenize�generate_tokens�detect_encoding�
untokenize� TokenInfoc @ s e Zd Zdd� Zedd� �ZdS )r c C s$ d| j t| j f }d| j|d� S )Nz%d (%s)z8TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r))�type)r
�tok_name�_replace)�self�annotated_type� r �-/opt/alt/python39/lib64/python3.9/tokenize.py�__repr__. s
�zTokenInfo.__repr__c C s( | j tkr | jtv r t| j S | j S d S �N)r
�OP�stringr �r r r r �
exact_type3 s
zTokenInfo.exact_typeN)�__name__�
__module__� __qualname__r �propertyr r r r r r - s r ztype string start end linec G s dd� | � d S )N�(�|�))�join��choicesr r r �group: � r$ c G s t | � d S )Nr �r$ r" r r r �any; r% r' c G s t | � d S )N�?r& r" r r r �maybe< r% r) z[ \f\t]*z #[^\r\n]*z\\\r?\nz\w+z0[xX](?:_?[0-9a-fA-F])+z0[bB](?:_?[01])+z0[oO](?:_?[0-7])+z (?:0(?:_?0)*|[1-9](?:_?[0-9])*)z[eE][-+]?[0-9](?:_?[0-9])*z)[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?z\.[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*[jJ]z[jJ]c C sV g d�} dh}| D ]>}t �|�D ].}t jdd� |D �� D ]}|�d�|�� q8q q|S )N)�b�r�u�f�br�fr� c S s g | ]}||� � f�qS r )�upper)�.0�cr r r �
<listcomp>^ r% z(_all_string_prefixes.<locals>.<listcomp>)�
_itertools� permutations�product�addr! )�_valid_string_prefixes�result�prefix�tr, r r r �_all_string_prefixesS s r= c C s t �| t j�S r )�re�compile�UNICODE)�exprr r r �_compileb s rB z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z'''z"""z '[^\n'\\]*(?:\\.[^\n'\\]*)*'z "[^\n"\\]*(?:\\.[^\n"\\]*)*"T)�reversez\r?\nz'[^\n'\\]*(?:\\.[^\n'\\]*)*�'z"[^\n"\\]*(?:\\.[^\n"\\]*)*�"z
\\\r?\n|\Z� c @ s e Zd ZdS )�
TokenErrorN�r r r r r r r rG � r% rG c @ s e Zd ZdS )�StopTokenizingNrH r r r r rI � r% rI c @ s, e Zd Zdd� Zdd� Zdd� Zdd� Zd S )
� Untokenizerc C s g | _ d| _d| _d | _d S )N� r )�tokens�prev_row�prev_col�encodingr r r r �__init__� s zUntokenizer.__init__c C s� |\}}|| j k s&|| j kr>|| jk r>td�||| j | j���|| j }|rb| j�d| � d| _|| j }|r�| j�d| � d S )Nz+start ({},{}) precedes previous end ({},{})�\
r � )rM rN �
ValueError�formatrL �append)r �start�row�col�
row_offset�
col_offsetr r r �add_whitespace� s �
zUntokenizer.add_whitespacec C s6 t |�}g }d}|D �]}t|�dkr8| �||� �q*|\}}}} }
|tkrV|| _q|tkrd �q*|tkrz|�|� qnl|tkr�|� � | \| _
| _ qnL|t t
fv r�d}n:|r�|r�|d } |d t| �kr�| j�| � t| �| _ d}| �|� | j�|� | \| _
| _ |t t
fv r| j
d7 _
d| _ qd�| j�S )NF� T���rK r r0 )�iter�len�compat�ENCODINGrO � ENDMARKER�INDENTrU �DEDENT�poprM rN �NEWLINE�NLrL r[ r! ) r �iterable�it�indents� startliner< �tok_type�tokenrV �end�line�indentr r r r � sF
zUntokenizer.untokenizec
C s� g }| j j}|d ttfv }d}t�|g|�D ]�}|d d� \}} |tkrR| | _q.|tt fv rf| d7 } |t
kr�|rzd| } d}nd}|t kr�|�| � q.n>|t kr�|�
� q.n*|ttfv r�d}n|r�|r�||d � d}|| � q.d S )Nr Fr\ rR Tr] )rL rU rf rg r5 �chainra rO �NAME�NUMBER�STRINGrc rd re )
r rm rh rj � toks_appendrk �
prevstring�tok�toknum�tokvalr r r r` � s8
zUntokenizer.compatN)r r r rP r[ r r` r r r r rJ � s
%rJ c C s* t � }|�| �}|jd ur&|�|j�}|S r )rJ r rO �encode)rh �ut�outr r r r s
r c C sH | d d� � � �dd�}|dks*|�d�r.dS |dv s@|�d�rDdS | S ) N� �_�-�utf-8zutf-8-)zlatin-1�
iso-8859-1z iso-latin-1)zlatin-1-z iso-8859-1-z iso-latin-1-r� )�lower�replace�
startswith)�orig_enc�encr r r �_get_normal_name s �r� c s� z �j j�W n ty" d �Y n0 d� d }d}�fdd�}� �fdd�}|� }|�t�rnd� |dd � }d }|sz|g fS ||�}|r�||gfS t�|�s�||gfS |� }|s�||gfS ||�}|r�|||gfS |||gfS )
NFr� c s" z� � W S t y Y dS 0 d S �Nr% )�
StopIterationr ��readliner r � read_or_stop? s z%detect_encoding.<locals>.read_or_stopc s� z| � d�}W n2 ty@ d}�d ur4d�|��}t|��Y n0 t�|�}|sTd S t|�d��}z t|�}W n8 t y� �d u r�d| }n d��|�}t|��Y n0 � r�|dkrԈd u r�d}n
d���}t|��|d 7 }|S )
Nr� z'invalid or missing encoding declarationz {} for {!r}rK zunknown encoding: z unknown encoding for {!r}: {}zencoding problem: utf-8z encoding problem for {!r}: utf-8z-sig)
�decode�UnicodeDecodeErrorrT � SyntaxError� cookie_re�matchr� r$ r � LookupError)ro � line_string�msgr� rO �codec)� bom_found�filenamer r � find_cookieE s8
�
z$detect_encoding.<locals>.find_cookieT� � utf-8-sig)�__self__�name�AttributeErrorr� r �blank_rer� )r� rO �defaultr� r� �first�secondr )r� r� r� r r
' s8
&
r
c C sX t | d�}z2t|j�\}}|�d� t||dd�}d|_|W S |�� � Y n0 d S )N�rbr T)�line_bufferingr+ )�
_builtin_openr
r� �seekr �mode�close)r� �bufferrO �lines�textr r r r � s
r c C s6 t | �\}}t�d�}t�|t| d�|�}t|j|�S r� )r
r5 �repeatrq r^ � _tokenize�__next__)r� rO �consumed�empty�rl_genr r r r � s
r c c sz d } }}d}d\}}d }dg} |d urH|dkr6d}t t|ddd�V d}
d} z| }
| � } W n tyt d} Y n0 |d ur�| �|�} |d 7 }dt| � } }
|�rn| s�td
|��|�| �}|�r|�d� } }t t|| d |� |||f|| �V d\}}d }nf|�rX| d d � d k�rX| d
d � dk�rXt t || ||t| �f|�V d}d }qPn|| }|| }qP�n�|dk�r|�s| �s��q�d}| |
k �r�| | dk�r�|d 7 }n8| | dk�r�|t
d t
}n| | dk�r�d}n�q�| d 7 } �q�| |
k�r �q�| | dv �r�| | dk�r\| | d � � d�}t t ||| f|| t|� f| �V | t|�7 } t t
| | d � || f|t| �f| �V qP|| d k�r�| �|� t t| d | � |df|| f| �V || d k �r,|| v�r�tdd|| | f��| d d� } t td|| f|| f| �V �q�n| �s(td|df��d}| |
k rPtt��| | �}|�r�|�d �\}}||f||f| }}} ||k�r~�q,| ||� | | }}||v �s�|dk�r�|dk�r�|dk�r�t t|||| �V �q�|dv �r|dk�r�t t
|||| �V nt t|||| �V �q�|dk�r0t t |||| �V �q�|tv �r�tt| �}|�| | �}|�r�|�d�} | || � }t t|||| f| �V n||f}| |d � }| }qP�q�|tv �s�|d d� tv �s�|d d � tv �rD|d d k�r0||f}tt�|��pt�|d ��pt�|d ��}| |d � d }}| }qPnt t|||| �V nf|�� �rbt t |||| �V nH|d k�rrd }n8|d v �r�|d 7 }n|d v �r�|d 8 }t t |||| �V n*t t | | || f|| d f| �V | d 7 } �q,qP|
�r.|
d dv�r.|
� � � d��s.t td|d t|
�f|d t|
�d fd�V | d d � D ] }t td|df|dfd�V �q:t t d|df|dfd�V d S )!Nr �
0123456789)r0 r r� r� )r r r0 r% rK zEOF in multi-line string���rQ ���z\
rR � � z#
�#z
r] z3unindent does not match any outer indentation levelz
<tokenize>zEOF in multi-line statement�.z...r\ r� �
�\z([{z)]})!r ra r� r� r_ rG r� rn rt �
ERRORTOKEN�tabsize�rstrip�COMMENTrg rU rc �IndentationErrorrd rB � PseudoToken�spanrs rf �
triple_quoted�endpats�
single_quoted�get� isidentifierrr r �stripr� rb )r� rO �lnum�parenlev� continued�numchars�contstr�needcont�contlinerj � last_linero �pos�max�strstart�endprog�endmatchrn �column�
comment_token� pseudomatchrV �spos�eposrm �initialrp r r r r� � s&
�*
�
� �
"
�
���
�� � �
�$. r� c C s
t | d �S r )r� r� r r r r d s r c
s@ dd l } dd� � d� fdd� }| jdd�}|jdd d
d d � |jd
ddddd� |�� }z�|jr�|j}t|d�� }tt|j��}W d � q�1 s�0 Y nd}t t
j jd �}|D ]>}|j }|j
r�|j}d|j|j } td| t| |jf � q�W �n@ t�yL }
z8|
jd dd� \} } ||
jd || | f� W Y d }
~
n�d }
~
0 t�y� }
z0|
jd \} } ||
jd || | f� W Y d }
~
n�d }
~
0 t�y� }
z||
|� W Y d }
~
n�d }
~
0 t�y� }
z||
� W Y d }
~
nVd }
~
0 t�y td� Y n4 t�y: }
z� d|
� � W Y d }
~
n
d }
~
0 0 d S ) Nr c S s t j�| � t j�d� d S )Nr� )�sys�stderr�write)�messager r r �perrorp s zmain.<locals>.perrorc sR |r"|f| | f }� d| � n"|r8� d|| f � n � d| � t �d� d S )Nz%s:%d:%d: error: %sz
%s: error: %sz error: %srK )r� �exit)r� r� �location�args�r� r r �errort s zmain.<locals>.errorzpython -m tokenize)�progr� r( z filename.pyz'the file to tokenize; defaults to stdin)�dest�nargs�metavar�helpz-ez--exact�exact�
store_truez(display token names using the exact type)r� �actionr� r� z<stdin>z %d,%d-%d,%d:z%-20s%-15s%-15rrK r� z interrupted
zunexpected error: %s)NN)�argparse�ArgumentParser� add_argument�
parse_argsr� r� �listr r� r� r� �stdinr
r� r rV rn �printr r r� r� rG r� �OSError�KeyboardInterrupt� Exception)
r� r� �parserr� r� r- rL rm �
token_type� token_range�errro r� r r� r �mainl sT � � .�** r� �__main__)R�
__author__� __credits__�builtinsr r� �codecsr r � collections�ior � itertoolsr5 r> r� rm r r? �ASCIIr� r� �__all__�
namedtupler r$ r' r) �
Whitespace�Comment�Ignore�Name� Hexnumber� Binnumber� Octnumber� Decnumber� Intnumber�Exponent�
Pointfloat�Expfloat� Floatnumber�
Imagnumber�Numberr= rB � StringPrefix�Single�Double�Single3�Double3�Triple�String�map�escape�sorted�Special�Funny�
PlainToken�Token�ContStr� PseudoExtrasr� r� �_prefix�setr� r� r< r, r8 r� r� rG rI rJ r r� r
r r� r r� r r r r r �<module> s�
��
�
���
_ ] 8=