o
6��fAe � @ sR d Z dZddlmZ ddlmZmZ ddlZddl Z ddl
m Z ddl Z
ddlZddlZddlT ddlmZ e�d ej�Ze�d
ej�ZddlZejg d � Z[G d d
� d
e�d
d��Zdd� Zdd� Zdd� ZdZ dZ e ede � ee � Z dZ dZ dZ!dZ"d Z#ee e!e"e#�Z$d Z%ed d �ee%� Z&d e% Z'ee&e'�Z(ed!e(d" �Z)ee)e(e$�Z*d#d$� Z+e j,d%d&� �Z-ee+� � Z.d'Z/d(Z0d)Z1d*Z2ee.d+ e.d, �Z3ee.d- e.d. �Z4ee5ej6e7ed/d0��� Z8ed1e8�Z9ee*e9e4e �Z:e e: Z;ee.d2 ed3d� e.d4 ed5d� �Z<ed6e e3�Z=e ee=e*e9e<e � Z>i Z?e+� D ]Z@e/e?e@d3 < e0e?e@d5 < e1e?e@d+ < e2e?e@d, < �qeA� ZBeA� ZCe+� D ]%ZDeDd5 eDd3 fD ]ZEeB�FeE� �qHeDd, eDd+ fD ]ZEeC�FeE� �qY�q>d7ZGG d8d9� d9eH�ZIG d:d;� d;eH�ZJG d<d=� d=�ZKd>d?� ZLd@dA� ZMdBdC� ZNdDdE� ZdFdG� ZOdHdI� ZPdJdK� ZQdLdM� ZReSdNk�r�eR� dS dS )OzKa-Ping Yee <ping@lfw.org>zpGvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro, Raymond Hettinger, Trent Nelson, Michael Foord� )�open)�lookup�BOM_UTF8N)�
TextIOWrapper)�*)�EXACT_TOKEN_TYPESz&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s ^[ \t\f]*(?:[#\r\n]|$))�tokenize�generate_tokens�detect_encoding�
untokenize� TokenInfoc @ s e Zd Zdd� Zedd� �ZdS )r c C s$ d| j t| j f }d| j|d� S )Nz%d (%s)z8TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r))�type)r
�tok_name�_replace)�self�annotated_type� r �//opt/alt/python310/lib64/python3.10/tokenize.py�__repr__/ s
�zTokenInfo.__repr__c C s$ | j tkr| jtv rt| j S | j S �N)r
�OP�stringr �r r r r �
exact_type4 s
zTokenInfo.exact_typeN)�__name__�
__module__� __qualname__r �propertyr r r r r r . s r ztype string start end linec G s dd� | � d S )N�(�|�))�join��choicesr r r �group; s r$ c G � t | � d S )Nr �r$ r" r r r �any< � r' c G r% )N�?r& r" r r r �maybe= r( r* z[ \f\t]*z #[^\r\n]*z\\\r?\nz\w+z0[xX](?:_?[0-9a-fA-F])+z0[bB](?:_?[01])+z0[oO](?:_?[0-7])+z (?:0(?:_?0)*|[1-9](?:_?[0-9])*)z[eE][-+]?[0-9](?:_?[0-9])*z)[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?z\.[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*[jJ]z[jJ]c C sV g d�} dh}| D ] }t �|�D ]}t jdd� |D �� D ]
}|�d�|�� q qq |S )N)�b�r�u�f�br�fr� c S s g | ]}||� � f�qS r )�upper)�.0�cr r r �
<listcomp>_ s z(_all_string_prefixes.<locals>.<listcomp>)�
_itertools� permutations�product�addr! )�_valid_string_prefixes�result�prefix�tr- r r r �_all_string_prefixesT s ��r> c C s t �| t j�S r )�re�compile�UNICODE)�exprr r r �_compilec s rC z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z'''z"""z '[^\n'\\]*(?:\\.[^\n'\\]*)*'z "[^\n"\\]*(?:\\.[^\n"\\]*)*"T)�reversez\r?\nz'[^\n'\\]*(?:\\.[^\n'\\]*)*�'z"[^\n"\\]*(?:\\.[^\n"\\]*)*�"z
\\\r?\n|\Z� c @ � e Zd ZdS )�
TokenErrorN�r r r r r r r rI � r( rI c @ rH )�StopTokenizingNrJ r r r r rK � r( rK c @ s, e Zd Zdd� Zdd� Zdd� Zdd� Zd S )
� Untokenizerc C s g | _ d| _d| _d | _d S )N� r )�tokens�prev_row�prev_col�encodingr r r r �__init__� s
zUntokenizer.__init__c C s� |\}}|| j k s|| j kr || jk r td�||| j | j���|| j }|r1| j�d| � d| _|| j }|rB| j�d| � d S d S )Nz+start ({},{}) precedes previous end ({},{})�\
r � )rO rP �
ValueError�formatrN �append)r �start�row�col�
row_offset�
col_offsetr r r �add_whitespace� s �
�zUntokenizer.add_whitespacec C s, t |�}g }d}|D ]�}t|�dkr| �||� nv|\}}}} }
|tkr)|| _q
|tkr/ na|tkr9|�|� q
|tkrH|� � | \| _
| _ q
|t t
fv rQd}n |rn|rn|d } |d t| �krl| j�| � t| �| _ d}| �|� | j�|� | \| _
| _ |t t
fv r�| j
d7 _
d| _ q
d�| j�S )NF� T���rM r r1 )�iter�len�compat�ENCODINGrQ � ENDMARKER�INDENTrW �DEDENT�poprO rP �NEWLINE�NLrN r] r! ) r �iterable�it�indents� startliner= �tok_type�tokenrX �end�line�indentr r r r � sH
� zUntokenizer.untokenizec
C s� g }| j j}|d ttfv }d}t�|g|�D ]W}|d d� \}} |tkr)| | _q|tt fv r3| d7 } |t
kr@|r=d| } d}nd}|t krL|�| � q|t krU|�
� q|ttfv r^d}n |rj|rj||d � d}|| � qd S )Nr Fr^ rT Tr_ )rN rW rh ri r6 �chainrc rQ �NAME�NUMBER�STRINGre rf rg )
r ro rj rl � toks_appendrm �
prevstring�tok�toknum�tokvalr r r rb � s:
�zUntokenizer.compatN)r r r rR r] r rb r r r r rL � s
%rL c C s, t � }|�| �}|jd ur|�|j�}|S r )rL r rQ �encode)rj �ut�outr r r r s
r c C sJ | d d� � � �dd�}|dks|�d�rdS |dv s!|�d�r#dS | S ) N� �_�-�utf-8zutf-8-)zlatin-1�
iso-8859-1z iso-latin-1)zlatin-1-z iso-8859-1-z iso-latin-1-r� )�lower�replace�
startswith)�orig_enc�encr r r �_get_normal_name s �r� c s� z�j j�W n ty d �Y nw d� d }d}�fdd�}� �fdd�}|� }|�t�r8d� |dd � }d }|s>|g fS ||�}|rI||gfS t�|�sS||gfS |� }|s]||gfS ||�}|ri|||gfS |||gfS )
NFr� c s z� � W S t y Y dS w �N� )�
StopIterationr ��readliner r � read_or_stopA s
�z%detect_encoding.<locals>.read_or_stopc s� z| � d�}W n ty d}�d urd�|��}t|��w t�|�}|s(d S t|�d��}zt|�}W n t yR �d u rHd| }t|��d��|�}t|��w � rp|dkrl�d u rcd}t|��d���}t|��|d 7 }|S )
Nr� z'invalid or missing encoding declarationz {} for {!r}rM zunknown encoding: z unknown encoding for {!r}: {}zencoding problem: utf-8z encoding problem for {!r}: utf-8z-sig)
�decode�UnicodeDecodeErrorrV � SyntaxError� cookie_re�matchr� r$ r � LookupError)rq � line_string�msgr� rQ �codec)� bom_found�filenamer r � find_cookieG s@ �
���
�z$detect_encoding.<locals>.find_cookieT� � utf-8-sig)�__self__�name�AttributeErrorr� r �blank_rer� )r� rQ �defaultr� r� �first�secondr )r� r� r� r r
) s< � &
r
c C sP t | d�}zt|j�\}}|�d� t||dd�}d|_|W S |�� � )N�rbr T)�line_bufferingr, )�
_builtin_openr
r� �seekr �mode�close)r� �bufferrQ �lines�textr r r r � s
r c C s8 t | �\}}t�d�}t�|t| d�|�}t|j|�S r� )r
r6 �repeatrs r` � _tokenize�__next__)r� rQ �consumed�empty�rl_genr r r r � s
r c c sl � d } }}d}d\}}d }dg} |d ur%|dkr d}t t|ddd�V d}
d} z| }
| � } W n ty< d} Y nw |d urF| �|�} |d
7 }dt| �} }
|r�| sZtd |��|�| �}|r�|�d� } }t t|| d |� |||f|| �V d\}}d }�n|r�| d d � d
kr�| dd � dkr�t t || ||t| �f|�V d}d }q)|| }|| }q)|dk�r�|�s�| s��n)d}| |
k r�| | dkr�|d
7 }n| | dkr�|t
d
t
}n
| | dkr�d}nn| d
7 } | |
k s�| |
kr��n�| | dv �r8| | dk�r$| | d � � d�}t t ||| f|| t|� f| �V | t|�7 } t t
| | d � || f|t| �f| �V q)|| d k�rU| �|� t t| d | � |df|| f| �V || d k �r�|| v�rjtdd|| | f��| d d� } t td|| f|| f| �V || d k �s\n | �s�td|df��d}| |
k �r�tt��| | �}|�r�|�d
�\}}||f||f|}}} ||k�r��q�| ||� | | }}||v �s�|dk�r�|dk�r�|dk�r�t t|||| �V n�|dv �r|dk�r�t t
|||| �V n�t t|||| �V n�|dk�rt t |||| �V n�|tv �rItt| �}|�| | �}|�r<|�d�} | || � }t t|||| f| �V n�||f}| |d � }| }n�|tv �s`|d d � tv �s`|d d � tv �r�|d d k�r�||f}tt�|��p�t�|d
��p�t�|d ��}| |d � d
}}| }nXt t|||| �V nI|�� �r�t t |||| �V n:|d k�r�d
}n2|d v �r�|d
7 }n |d!v �r�|d
8 }t t |||| �V nt t | | || f|| d
f| �V | d
7 } | |
k �s�q*|
�r|
d dv�r|
� � � d��st td|d
t|
�f|d
t|
�d
fd�V | d
d � D ]}t td|df|dfd�V �qt t d|df|dfd�V d S )"Nr �
0123456789)r1 r r� r� )r r r1 r� TrM zEOF in multi-line string���rS ���z\
rT � � z#
�#z
r_ z3unindent does not match any outer indentation levelz
<tokenize>zEOF in multi-line statement�.z...r^ r� �
�\z([{z)]})!r rc r� r� ra rI r� rp rv �
ERRORTOKEN�tabsize�rstrip�COMMENTri rW re �IndentationErrorrf rC � PseudoToken�spanru rh �
triple_quoted�endpats�
single_quoted�get� isidentifierrt r �stripr� rd )r� rQ �lnum�parenlev� continued�numchars�contstr�needcont�contlinerl � last_linerq �pos�max�strstart�endprog�endmatchrp �column�
comment_token� pseudomatchrX �spos�eposro �initialrr r r r r� � s* �
�
�$
�
�
� �
"
� ��
� �
�
�� $#. r� c C s t | d �S r )r� r� r r r r f s
r c
s2 dd l } dd� � d� fdd� }| jdd�}|jdd d
d d � |jd
ddddd� |�� }zQ|jrP|j}t|d��}tt|j��}W d � n1 sJw Y n d}t t
j jd �}|D ] }|j }|j
rf|j}d|j|j } td| t| |jf � q[W d S ty� }
z |
jd dd� \} } ||
jd || | f� W Y d }
~
d S d }
~
w ty� }
z|
jd \} } ||
jd || | f� W Y d }
~
d S d }
~
w ty� }
z ||
|� W Y d }
~
d S d }
~
w ty� }
z ||
� W Y d }
~
d S d }
~
w t�y td� Y d S t�y }
z� d|
� � d }
~
ww ) Nr c S s t j�| � t j�d� d S )Nr� )�sys�stderr�write)�messager r r �perrorr s zmain.<locals>.perrorc sR |r|f| | f }� d| � n|r � d|| f � n� d| � t �d� d S )Nz%s:%d:%d: error: %sz
%s: error: %sz error: %srM )r� �exit)r� r� �location�args�r� r r �errorv s zmain.<locals>.errorzpython -m tokenize)�progr� r) z filename.pyz'the file to tokenize; defaults to stdin)�dest�nargs�metavar�helpz-ez--exact�exact�
store_truez(display token names using the exact type)r� �actionr� r� z<stdin>z %d,%d-%d,%d:z%-20s%-15s%-15rrM r� z interrupted
zunexpected error: %s)NN)�argparse�ArgumentParser� add_argument�
parse_argsr� r� �listr r� r� r� �stdinr
r� r rX rp �printr r r� r� rI r� �OSError�KeyboardInterrupt� Exception)
r� r� �parserr� r� r. rN ro �
token_type� token_range�errrq r� r r� r �mainn sf � � ����$�$��� ��r� �__main__)T�
__author__� __credits__�builtinsr r� �codecsr r � collections� functools�ior � itertoolsr6 r? r� ro r r@ �ASCIIr� r� �__all__�
namedtupler r$ r' r* �
Whitespace�Comment�Ignore�Name� Hexnumber� Binnumber� Octnumber� Decnumber� Intnumber�Exponent�
Pointfloat�Expfloat� Floatnumber�
Imagnumber�Numberr> � lru_cacherC � StringPrefix�Single�Double�Single3�Double3�Triple�String�map�escape�sorted�Special�Funny�
PlainToken�Token�ContStr� PseudoExtrasr� r� �_prefix�setr� r� r= r- r9 r� r� rI rK rL r r� r
r r� r r� r r r r r �<module> s�
��
�
���
�_ ] 8
=
�