Current File : //usr/local/apps/python3/lib/python3.11/test/__pycache__/test_tokenize.cpython-311.opt-1.pyc
�

�Ke9���R�ddlmZddlmZddlmZmZmZmZmZm	Z	m
Z
mZmZm
Z
mZmZmZmZmZmZmZddlmZmZddlZddlmZddlmZmZddlmZm Z ddlmZdd	l!m"Z"m#Z#ddl$Z$ddl%Z%d
�Z&Gd�de��Z'Gd
�de'��Z(d�Z)Gd�de��Z*Gd�de��Z+Gd�de��Z,Gd�de��Z-Gd�de��Z.Gd�de��Z/Gd�de��Z0Gd�de��Z1Gd �d!ej��Z2e3d"krej4��dSdS)#�)�support)�	os_helper)�tokenize�	_tokenize�
untokenize�NUMBER�NAME�OP�STRING�	ENDMARKER�ENCODING�tok_name�detect_encoding�open�Untokenizer�generate_tokens�NEWLINE�!_generate_tokens_from_c_tokenizer�DEDENT)�BytesIO�StringION)�dedent)�TestCase�mock)�VALID_UNDERSCORE_LITERALS�INVALID_UNDERSCORE_LITERALS)�run_test_script�make_scriptc�$�g}t|�����}|ddv}|D]_\}}}}}	|tkrnK|r|tkr
|d|kr�/t|}|�d|d�d|d�d|�d|�����`|S)N���z
r�    �10� �13)�len�
splitlinesrrr�append)
�token_generator�
source_string�result�	num_lines�missing_trailing_nl�type�token�start�end�lines
          �</usr/local/apps/python3/lib/python3.11/test/test_tokenize.py�stringify_tokens_from_sourcer3s���
�F��M�,�,�.�.�/�/�I�'��+�6�9��)8�C�C�%��e�U�C���9����E��	�4�7�?�?�s�1�v��7J�7J����~���
�
�A�T�A�A�A�u�A�A�A�E�A�A�C�A�A�B�B�B�B��M�c��eZdZd�Zd�Zd�Zd�Zd�Zd�Zd�Z	d�Z
d	�Zd
�Zd�Z
d�Zd
�Zd�Zd�Zd�Zd�Zd�Zd�Zd�ZdS)�TokenizeTestc�
�t|�d����}tt|j��|��}|�|dg|������z��dS)N�utf-8z*    ENCODING   'utf-8'       (0, 0) (0, 0))r�encoder3r�readline�assertEqual�rstripr&��self�s�expected�fr*s     r2�check_tokenizezTokenizeTest.check_tokenize-s~��
�A�H�H�W�%�%�&�&��-�h�q�z�.B�.B�A�F�F������F�G�!���*�*�5�5�7�7�8�	9�	9�	9�	9�	9r4c�$�td�d����}tt|j����}|�|djt��|�|djt��dS)N�xr8���r )	rr9�listrr:r;r-rr)r>rA�tokenss   r2�test_implicit_newlinez"TokenizeTest.test_implicit_newline6sr��
�C�J�J�w�'�'�(�(���h�q�z�*�*�+�+���������'�2�2�2��������)�4�4�4�4�4r4c��|�dd��|�dd��d}t|��j}|�td��5t|��D]}�	ddd��dS#1swxYwYdS)Nz1 + 1z�    NUMBER     '1'           (1, 0) (1, 1)
    OP         '+'           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    z3if False:
    # NL
    
    True = False # NEWLINE
af    NAME       'if'          (1, 0) (1, 2)
    NAME       'False'       (1, 3) (1, 8)
    OP         ':'           (1, 8) (1, 9)
    NEWLINE    '\n'          (1, 9) (1, 10)
    COMMENT    '# NL'        (2, 4) (2, 8)
    NL         '\n'          (2, 8) (2, 9)
    NL         '\n'          (3, 4) (3, 5)
    INDENT     '    '        (4, 0) (4, 4)
    NAME       'True'        (4, 4) (4, 8)
    OP         '='           (4, 9) (4, 10)
    NAME       'False'       (4, 11) (4, 16)
    COMMENT    '# NEWLINE'   (4, 17) (4, 26)
    NEWLINE    '\n'          (4, 26) (4, 27)
    DEDENT     ''            (5, 0) (5, 0)
    sdef k(x):
    x += 2
  x += 5
z3unindent does not match any outer indentation level)rBrr:�assertRaisesRegex�IndentationErrorr)r>�indent_error_filer:�toks    r2�
test_basiczTokenizeTest.test_basic>s�����G�&�		�		�		�
	
���;�=�		�		�		�&��
�,�-�-�6��
�
#�
#�$4�%>�?�?�	�	� ��)�)�
�
���
�	�	�	�	�	�	�	�	�	�	�	�	����	�	�	�	�	�	s�A?�?B�Bc��|�dd��|�dd��|�dd��|�dd��|�d	d
��|�dd��|�d
d��|�dd��|�dd��dS)N�0xff <= 255z�    NUMBER     '0xff'        (1, 0) (1, 4)
    OP         '<='          (1, 5) (1, 7)
    NUMBER     '255'         (1, 8) (1, 11)
    �0b10 <= 255z�    NUMBER     '0b10'        (1, 0) (1, 4)
    OP         '<='          (1, 5) (1, 7)
    NUMBER     '255'         (1, 8) (1, 11)
    �0o123 <= 0O123z�    NUMBER     '0o123'       (1, 0) (1, 5)
    OP         '<='          (1, 6) (1, 8)
    NUMBER     '0O123'       (1, 9) (1, 14)
    �1234567 > ~0x15z�    NUMBER     '1234567'     (1, 0) (1, 7)
    OP         '>'           (1, 8) (1, 9)
    OP         '~'           (1, 10) (1, 11)
    NUMBER     '0x15'        (1, 11) (1, 15)
    �2134568 != 1231515z�    NUMBER     '2134568'     (1, 0) (1, 7)
    OP         '!='          (1, 8) (1, 10)
    NUMBER     '1231515'     (1, 11) (1, 18)
    �(-124561-1) & 200000000ac    OP         '('           (1, 0) (1, 1)
    OP         '-'           (1, 1) (1, 2)
    NUMBER     '124561'      (1, 2) (1, 8)
    OP         '-'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         ')'           (1, 10) (1, 11)
    OP         '&'           (1, 12) (1, 13)
    NUMBER     '200000000'   (1, 14) (1, 23)
    �0xdeadbeef != -1z�    NUMBER     '0xdeadbeef'  (1, 0) (1, 10)
    OP         '!='          (1, 11) (1, 13)
    OP         '-'           (1, 14) (1, 15)
    NUMBER     '1'           (1, 15) (1, 16)
    �0xdeadc0de & 12345z�    NUMBER     '0xdeadc0de'  (1, 0) (1, 10)
    OP         '&'           (1, 11) (1, 12)
    NUMBER     '12345'       (1, 13) (1, 18)
    �0xFF & 0x15 | 1234z�    NUMBER     '0xFF'        (1, 0) (1, 4)
    OP         '&'           (1, 5) (1, 6)
    NUMBER     '0x15'        (1, 7) (1, 11)
    OP         '|'           (1, 12) (1, 13)
    NUMBER     '1234'        (1, 14) (1, 18)
    �rB�r>s r2�test_intzTokenizeTest.test_intcs:�����M�,�		�		�		�
	
���M�,�		�		�		�
	
���,�/�		�		�		�
	
���-�0�		�		�		�	
���0�3�		�		�		�
	
���5�	8�			�			�			�	
���.�1�		�		�		�	
���0�3�		�		�		�
	
���0�3�		�		�		�		�		r4c��|�dd��|�dd��|�dd��|�dd��dS)	Nzx = 0z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '0'           (1, 4) (1, 5)
    zx = 0xfffffffffffz�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '0xfffffffffff' (1, 4) (1, 17)
    zx = 123141242151251616110z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '123141242151251616110' (1, 4) (1, 25)
    zx = -15921590215012591z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    OP         '-'           (1, 4) (1, 5)
    NUMBER     '15921590215012591' (1, 5) (1, 22)
    rYrZs r2�	test_longzTokenizeTest.test_long�s������G�&�		�		�		�
	
���/�2�		�		�		�
	
���7�:�		�		�		�
	
���4�7�		�		�		�		�		r4c�:�|�dd��|�dd��|�dd��|�dd��|�d	d
��|�dd��|�d
d��dS)N�x = 3.14159z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3.14159'     (1, 4) (1, 11)
    �x = 314159.z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '314159.'     (1, 4) (1, 11)
    �x = .314159z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '.314159'     (1, 4) (1, 11)
    �x = 3e14159z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3e14159'     (1, 4) (1, 11)
    �	x = 3E123z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3E123'       (1, 4) (1, 9)
    �
x+y = 3e-1230z�    NAME       'x'           (1, 0) (1, 1)
    OP         '+'           (1, 1) (1, 2)
    NAME       'y'           (1, 2) (1, 3)
    OP         '='           (1, 4) (1, 5)
    NUMBER     '3e-1230'     (1, 6) (1, 13)
    �x = 3.14e159z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3.14e159'    (1, 4) (1, 12)
    rYrZs r2�
test_floatzTokenizeTest.test_float�s������M�,�		�		�		�
	
���M�,�		�		�		�
	
���M�,�		�		�		�
	
���M�,�		�		�		�
	
���K�*�		�		�		�
	
���O�.�		�		�		�	
���N�-�		�		�		�		�		r4c��d�}tD]&}d|vr�|�||��|���'tD]!}|�||��|���"dS)Nc��t|�d����}t|j��D]\}}}}}|tkr|cS�dS)Nr8z
invalid token)rr9rr:r)r?rA�toktyper.r/r0r1s       r2�number_tokenz;TokenizeTest.test_underscore_literals.<locals>.number_token�s]�������)�)�*�*�A�4<�Q�Z�4H�4H�
!�
!�0����s�D��f�$�$� �L�L�L�%�"�?r4�()rr;r�assertNotEqual)r>rj�lits   r2�test_underscore_literalsz%TokenizeTest.test_underscore_literals�s���	#�	#�	#�-�	5�	5�C��c�z�z�����\�\�#�.�.��4�4�4�4�.�	8�	8�C������S� 1� 1�3�7�7�7�7�	8�	8r4c�~�|�dd��|�dd��|�dd��|�dd��|�d	d
��|�dd��|�d
d��|�dd��|�dd��|�dd��|�dd��|�dd��|�dd��|�dd��|�dd��|�dd ��|�d!d"��|�d#d$��|�d%d&��|�d'd(��|�d)d*��|�d+d,��|�d-d.��|�d/d0��|�d1d2��|�d3d4��dS)5N�x = ''; y = ""a5    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     "''"          (1, 4) (1, 6)
    OP         ';'           (1, 6) (1, 7)
    NAME       'y'           (1, 8) (1, 9)
    OP         '='           (1, 10) (1, 11)
    STRING     '""'          (1, 12) (1, 14)
    �x = '"'; y = "'"a6    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     '\'"\''       (1, 4) (1, 7)
    OP         ';'           (1, 7) (1, 8)
    NAME       'y'           (1, 9) (1, 10)
    OP         '='           (1, 11) (1, 12)
    STRING     '"\'"'        (1, 13) (1, 16)
    �x = "doesn't "shrink", does it"z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     '"doesn\'t "' (1, 4) (1, 14)
    NAME       'shrink'      (1, 14) (1, 20)
    STRING     '", does it"' (1, 20) (1, 31)
    �x = 'abc' + 'ABC'z�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     "'abc'"       (1, 4) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    STRING     "'ABC'"       (1, 12) (1, 17)
    �y = "ABC" + "ABC"z�    NAME       'y'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     '"ABC"'       (1, 4) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    STRING     '"ABC"'       (1, 12) (1, 17)
    �%x = r'abc' + r'ABC' + R'ABC' + R'ABC'a�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     "r'abc'"      (1, 4) (1, 10)
    OP         '+'           (1, 11) (1, 12)
    STRING     "r'ABC'"      (1, 13) (1, 19)
    OP         '+'           (1, 20) (1, 21)
    STRING     "R'ABC'"      (1, 22) (1, 28)
    OP         '+'           (1, 29) (1, 30)
    STRING     "R'ABC'"      (1, 31) (1, 37)
    �%y = r"abc" + r"ABC" + R"ABC" + R"ABC"a�    NAME       'y'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     'r"abc"'      (1, 4) (1, 10)
    OP         '+'           (1, 11) (1, 12)
    STRING     'r"ABC"'      (1, 13) (1, 19)
    OP         '+'           (1, 20) (1, 21)
    STRING     'R"ABC"'      (1, 22) (1, 28)
    OP         '+'           (1, 29) (1, 30)
    STRING     'R"ABC"'      (1, 31) (1, 37)
    �u'abc' + U'abc'z�    STRING     "u'abc'"      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     "U'abc'"      (1, 9) (1, 15)
    �u"abc" + U"abc"z�    STRING     'u"abc"'      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     'U"abc"'      (1, 9) (1, 15)
    �b'abc' + B'abc'z�    STRING     "b'abc'"      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     "B'abc'"      (1, 9) (1, 15)
    �b"abc" + B"abc"z�    STRING     'b"abc"'      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     'B"abc"'      (1, 9) (1, 15)
    �%br'abc' + bR'abc' + Br'abc' + BR'abc'a;    STRING     "br'abc'"     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     "bR'abc'"     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     "Br'abc'"     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     "BR'abc'"     (1, 30) (1, 37)
    �%br"abc" + bR"abc" + Br"abc" + BR"abc"a;    STRING     'br"abc"'     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     'bR"abc"'     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     'Br"abc"'     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     'BR"abc"'     (1, 30) (1, 37)
    �%rb'abc' + rB'abc' + Rb'abc' + RB'abc'a;    STRING     "rb'abc'"     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     "rB'abc'"     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     "Rb'abc'"     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     "RB'abc'"     (1, 30) (1, 37)
    �%rb"abc" + rB"abc" + Rb"abc" + RB"abc"a;    STRING     'rb"abc"'     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     'rB"abc"'     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     'Rb"abc"'     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     'RB"abc"'     (1, 30) (1, 37)
    �"a\
de\
fg"�3    STRING     '"a\\\nde\\\nfg"' (1, 0) (3, 3)
    �u"a\
de"�/    STRING     'u"a\\\nde"'  (1, 0) (2, 3)
    �rb"a\
d"�/    STRING     'rb"a\\\nd"'  (1, 0) (2, 2)
    �
"""a\
b"""�0    STRING     '"""a\\\nb"""' (1, 0) (2, 4)
    �u"""a\
b"""�1    STRING     'u"""a\\\nb"""' (1, 0) (2, 4)
    �rb"""a\
b\
c"""�7    STRING     'rb"""a\\\nb\\\nc"""' (1, 0) (3, 4)
    �f"abc"�/    STRING     'f"abc"'      (1, 0) (1, 6)
    �	fR"a{b}c"�/    STRING     'fR"a{b}c"'   (1, 0) (1, 9)
    �
f"""abc"""�0    STRING     'f"""abc"""'  (1, 0) (1, 10)
    �f"abc\
def"�1    STRING     'f"abc\\\ndef"' (1, 0) (2, 4)
    �Rf"abc\
def"�2    STRING     'Rf"abc\\\ndef"' (1, 0) (2, 4)
    rYrZs r2�test_stringzTokenizeTest.test_string�s������.�1�		�		�		�	
���1�4�		�		�		�	
���A�D�		�		�		�	
���/�2�		�		�		�	
���/�2�		�		�		�	
���C�
F�
		�
		�
		�	
���C�
F�
		�
		�
		�	
���-�0�		�		�		�
	
���-�0�		�		�		�	
���-�0�		�		�		�
	
���-�0�		�		�		�
	
���C�F�		�		�		�	
���C�F�		�		�		�	
���C�F�		�		�		�	
���C�F�		�		�		�	
�����		�		�		�
	
�����		�		�		�	
�����		�		�		�	
�����		�		�		�	
�����		�		�		�	
�����		�		�		�
	
���H�'�		�		�		�	
���K�*�		�		�		�	
���L�+�		�		�		�	
�����		�		�		�	
�����		�		�		�		�		r4c��|�dd��|�dd��|�dd��dS)N�!def d22(a, b, c=2, d=2, *k): passa    NAME       'def'         (1, 0) (1, 3)
    NAME       'd22'         (1, 4) (1, 7)
    OP         '('           (1, 7) (1, 8)
    NAME       'a'           (1, 8) (1, 9)
    OP         ','           (1, 9) (1, 10)
    NAME       'b'           (1, 11) (1, 12)
    OP         ','           (1, 12) (1, 13)
    NAME       'c'           (1, 14) (1, 15)
    OP         '='           (1, 15) (1, 16)
    NUMBER     '2'           (1, 16) (1, 17)
    OP         ','           (1, 17) (1, 18)
    NAME       'd'           (1, 19) (1, 20)
    OP         '='           (1, 20) (1, 21)
    NUMBER     '2'           (1, 21) (1, 22)
    OP         ','           (1, 22) (1, 23)
    OP         '*'           (1, 24) (1, 25)
    NAME       'k'           (1, 25) (1, 26)
    OP         ')'           (1, 26) (1, 27)
    OP         ':'           (1, 27) (1, 28)
    NAME       'pass'        (1, 29) (1, 33)
    �def d01v_(a=1, *k, **w): passa�    NAME       'def'         (1, 0) (1, 3)
    NAME       'd01v_'       (1, 4) (1, 9)
    OP         '('           (1, 9) (1, 10)
    NAME       'a'           (1, 10) (1, 11)
    OP         '='           (1, 11) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    OP         ','           (1, 13) (1, 14)
    OP         '*'           (1, 15) (1, 16)
    NAME       'k'           (1, 16) (1, 17)
    OP         ','           (1, 17) (1, 18)
    OP         '**'          (1, 19) (1, 21)
    NAME       'w'           (1, 21) (1, 22)
    OP         ')'           (1, 22) (1, 23)
    OP         ':'           (1, 23) (1, 24)
    NAME       'pass'        (1, 25) (1, 29)
    �&def d23(a: str, b: int=3) -> int: passa�    NAME       'def'         (1, 0) (1, 3)
    NAME       'd23'         (1, 4) (1, 7)
    OP         '('           (1, 7) (1, 8)
    NAME       'a'           (1, 8) (1, 9)
    OP         ':'           (1, 9) (1, 10)
    NAME       'str'         (1, 11) (1, 14)
    OP         ','           (1, 14) (1, 15)
    NAME       'b'           (1, 16) (1, 17)
    OP         ':'           (1, 17) (1, 18)
    NAME       'int'         (1, 19) (1, 22)
    OP         '='           (1, 22) (1, 23)
    NUMBER     '3'           (1, 23) (1, 24)
    OP         ')'           (1, 24) (1, 25)
    OP         '->'          (1, 26) (1, 28)
    NAME       'int'         (1, 29) (1, 32)
    OP         ':'           (1, 32) (1, 33)
    NAME       'pass'        (1, 34) (1, 38)
    rYrZs r2�
test_functionzTokenizeTest.test_function�sr�����?�B�		�		�		�,	
���;�>�		�		�		�"	
���D�G�		�		�		�		�		r4c�2�|�dd��dS)N�Xif 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != 1 and 5 in 1 not in 1 is 1 or 5 is not 1: passa�    NAME       'if'          (1, 0) (1, 2)
    NUMBER     '1'           (1, 3) (1, 4)
    OP         '<'           (1, 5) (1, 6)
    NUMBER     '1'           (1, 7) (1, 8)
    OP         '>'           (1, 9) (1, 10)
    NUMBER     '1'           (1, 11) (1, 12)
    OP         '=='          (1, 13) (1, 15)
    NUMBER     '1'           (1, 16) (1, 17)
    OP         '>='          (1, 18) (1, 20)
    NUMBER     '5'           (1, 21) (1, 22)
    OP         '<='          (1, 23) (1, 25)
    NUMBER     '0x15'        (1, 26) (1, 30)
    OP         '<='          (1, 31) (1, 33)
    NUMBER     '0x12'        (1, 34) (1, 38)
    OP         '!='          (1, 39) (1, 41)
    NUMBER     '1'           (1, 42) (1, 43)
    NAME       'and'         (1, 44) (1, 47)
    NUMBER     '5'           (1, 48) (1, 49)
    NAME       'in'          (1, 50) (1, 52)
    NUMBER     '1'           (1, 53) (1, 54)
    NAME       'not'         (1, 55) (1, 58)
    NAME       'in'          (1, 59) (1, 61)
    NUMBER     '1'           (1, 62) (1, 63)
    NAME       'is'          (1, 64) (1, 66)
    NUMBER     '1'           (1, 67) (1, 68)
    NAME       'or'          (1, 69) (1, 71)
    NUMBER     '5'           (1, 72) (1, 73)
    NAME       'is'          (1, 74) (1, 76)
    NAME       'not'         (1, 77) (1, 80)
    NUMBER     '1'           (1, 81) (1, 82)
    OP         ':'           (1, 82) (1, 83)
    NAME       'pass'        (1, 84) (1, 88)
    rYrZs r2�test_comparisonzTokenizeTest.test_comparison��0�����M�!O�"		�"		�"		�"		�"		r4c�2�|�dd��dS)Nzx = 1 << 1 >> 5a6    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    OP         '<<'          (1, 6) (1, 8)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '>>'          (1, 11) (1, 13)
    NUMBER     '5'           (1, 14) (1, 15)
    rYrZs r2�
test_shiftzTokenizeTest.test_shift�s,�����-�0�		�		�		�		�		r4c�2�|�dd��dS)N�%x = 1 - y + 15 - 1 + 0x124 + z + a[5]a$    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    OP         '-'           (1, 6) (1, 7)
    NAME       'y'           (1, 8) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    NUMBER     '15'          (1, 12) (1, 14)
    OP         '-'           (1, 15) (1, 16)
    NUMBER     '1'           (1, 17) (1, 18)
    OP         '+'           (1, 19) (1, 20)
    NUMBER     '0x124'       (1, 21) (1, 26)
    OP         '+'           (1, 27) (1, 28)
    NAME       'z'           (1, 29) (1, 30)
    OP         '+'           (1, 31) (1, 32)
    NAME       'a'           (1, 33) (1, 34)
    OP         '['           (1, 34) (1, 35)
    NUMBER     '5'           (1, 35) (1, 36)
    OP         ']'           (1, 36) (1, 37)
    rYrZs r2�
test_additivezTokenizeTest.test_additive��-�����C�F�		�		�		�		�		r4c�2�|�dd��dS)N�x = 1//1*1/5*12%0x12@42a�    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    OP         '//'          (1, 5) (1, 7)
    NUMBER     '1'           (1, 7) (1, 8)
    OP         '*'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '/'           (1, 10) (1, 11)
    NUMBER     '5'           (1, 11) (1, 12)
    OP         '*'           (1, 12) (1, 13)
    NUMBER     '12'          (1, 13) (1, 15)
    OP         '%'           (1, 15) (1, 16)
    NUMBER     '0x12'        (1, 16) (1, 20)
    OP         '@'           (1, 20) (1, 21)
    NUMBER     '42'          (1, 21) (1, 23)
    rYrZs r2�test_multiplicativez TokenizeTest.test_multiplicative�,�����5�8�		�		�		�		�		r4c�^�|�dd��|�dd��dS)N�~1 ^ 1 & 1 |1 ^ -1a�    OP         '~'           (1, 0) (1, 1)
    NUMBER     '1'           (1, 1) (1, 2)
    OP         '^'           (1, 3) (1, 4)
    NUMBER     '1'           (1, 5) (1, 6)
    OP         '&'           (1, 7) (1, 8)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '|'           (1, 11) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    OP         '^'           (1, 14) (1, 15)
    OP         '-'           (1, 16) (1, 17)
    NUMBER     '1'           (1, 17) (1, 18)
    �-1*1/1+1*1//1 - ---1**1aH    OP         '-'           (1, 0) (1, 1)
    NUMBER     '1'           (1, 1) (1, 2)
    OP         '*'           (1, 2) (1, 3)
    NUMBER     '1'           (1, 3) (1, 4)
    OP         '/'           (1, 4) (1, 5)
    NUMBER     '1'           (1, 5) (1, 6)
    OP         '+'           (1, 6) (1, 7)
    NUMBER     '1'           (1, 7) (1, 8)
    OP         '*'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '//'          (1, 10) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    OP         '-'           (1, 14) (1, 15)
    OP         '-'           (1, 16) (1, 17)
    OP         '-'           (1, 17) (1, 18)
    OP         '-'           (1, 18) (1, 19)
    NUMBER     '1'           (1, 19) (1, 20)
    OP         '**'          (1, 20) (1, 22)
    NUMBER     '1'           (1, 22) (1, 23)
    rYrZs r2�
test_unaryzTokenizeTest.test_unary*sN�����0�3�		�		�		�	
���5�8�		�		�		�		�		r4c�2�|�dd��dS)N�/import sys, time
x = sys.modules['time'].time()a�    NAME       'import'      (1, 0) (1, 6)
    NAME       'sys'         (1, 7) (1, 10)
    OP         ','           (1, 10) (1, 11)
    NAME       'time'        (1, 12) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    NAME       'x'           (2, 0) (2, 1)
    OP         '='           (2, 2) (2, 3)
    NAME       'sys'         (2, 4) (2, 7)
    OP         '.'           (2, 7) (2, 8)
    NAME       'modules'     (2, 8) (2, 15)
    OP         '['           (2, 15) (2, 16)
    STRING     "'time'"      (2, 16) (2, 22)
    OP         ']'           (2, 22) (2, 23)
    OP         '.'           (2, 23) (2, 24)
    NAME       'time'        (2, 24) (2, 28)
    OP         '('           (2, 28) (2, 29)
    OP         ')'           (2, 29) (2, 30)
    rYrZs r2�
test_selectorzTokenizeTest.test_selectorO�-�����N�Q�		�		�		�		�		r4c�2�|�dd��dS)N� @staticmethod
def foo(x,y): passa    OP         '@'           (1, 0) (1, 1)
    NAME       'staticmethod' (1, 1) (1, 13)
    NEWLINE    '\n'          (1, 13) (1, 14)
    NAME       'def'         (2, 0) (2, 3)
    NAME       'foo'         (2, 4) (2, 7)
    OP         '('           (2, 7) (2, 8)
    NAME       'x'           (2, 8) (2, 9)
    OP         ','           (2, 9) (2, 10)
    NAME       'y'           (2, 10) (2, 11)
    OP         ')'           (2, 11) (2, 12)
    OP         ':'           (2, 12) (2, 13)
    NAME       'pass'        (2, 14) (2, 18)
    rYrZs r2�test_methodzTokenizeTest.test_methode�-�����?�
B�
		�
		�
		�
		�
		r4c�2�|�dd��dS)Nzdef f():
	if x
        	passa_    NAME       'def'         (1, 0) (1, 3)
    NAME       'f'           (1, 4) (1, 5)
    OP         '('           (1, 5) (1, 6)
    OP         ')'           (1, 6) (1, 7)
    OP         ':'           (1, 7) (1, 8)
    NEWLINE    '\n'          (1, 8) (1, 9)
    INDENT     '\t'          (2, 0) (2, 1)
    NAME       'if'          (2, 1) (2, 3)
    NAME       'x'           (2, 4) (2, 5)
    NEWLINE    '\n'          (2, 5) (2, 6)
    INDENT     '        \t'  (3, 0) (3, 9)
    NAME       'pass'        (3, 9) (3, 13)
    DEDENT     ''            (4, 0) (4, 0)
    DEDENT     ''            (4, 0) (4, 0)
    rYrZs r2�	test_tabszTokenizeTest.test_tabsvs.�����-�/�		�		�		�		�		r4c�2�|�dd��dS)Nu!Örter = 'places'
grün = 'green'u7    NAME       'Örter'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    STRING     "'places'"    (1, 8) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    NAME       'grün'        (2, 0) (2, 4)
    OP         '='           (2, 5) (2, 6)
    STRING     "'green'"     (2, 7) (2, 14)
    rYrZs r2�test_non_ascii_identifiersz'TokenizeTest.test_non_ascii_identifiers�s-�����@�C�		�		�		�		�		r4c�2�|�dd��dS)N�#Örter = u'places'
grün = U'green'u7    NAME       'Örter'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    STRING     "u'places'"   (1, 8) (1, 17)
    NEWLINE    '\n'          (1, 17) (1, 18)
    NAME       'grün'        (2, 0) (2, 4)
    OP         '='           (2, 5) (2, 6)
    STRING     "U'green'"    (2, 7) (2, 15)
    rYrZs r2�test_unicodezTokenizeTest.test_unicode��-�����B�E�		�		�		�		�		r4c��|�dd��|�dd��|�dd��|�dd��|�d	d
��|�dd��|�d
d��|�dd��|�dd��|�dd��|�dd��|�dd��|�dd��|�dd��|�dd��|�dd ��|�d!d"��|�d#d$��|�d%d&��|�d'd(��|�d)d*��dS)+N�	async = 1z�    NAME       'async'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    NUMBER     '1'           (1, 8) (1, 9)
    �a = (async = 1)a8    NAME       'a'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    OP         '('           (1, 4) (1, 5)
    NAME       'async'       (1, 5) (1, 10)
    OP         '='           (1, 11) (1, 12)
    NUMBER     '1'           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    �async()z�    NAME       'async'       (1, 0) (1, 5)
    OP         '('           (1, 5) (1, 6)
    OP         ')'           (1, 6) (1, 7)
    �class async(Bar):passa<    NAME       'class'       (1, 0) (1, 5)
    NAME       'async'       (1, 6) (1, 11)
    OP         '('           (1, 11) (1, 12)
    NAME       'Bar'         (1, 12) (1, 15)
    OP         ')'           (1, 15) (1, 16)
    OP         ':'           (1, 16) (1, 17)
    NAME       'pass'        (1, 17) (1, 21)
    �class async:passz�    NAME       'class'       (1, 0) (1, 5)
    NAME       'async'       (1, 6) (1, 11)
    OP         ':'           (1, 11) (1, 12)
    NAME       'pass'        (1, 12) (1, 16)
    �	await = 1z�    NAME       'await'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    NUMBER     '1'           (1, 8) (1, 9)
    �	foo.asyncz�    NAME       'foo'         (1, 0) (1, 3)
    OP         '.'           (1, 3) (1, 4)
    NAME       'async'       (1, 4) (1, 9)
    �async for a in b: passa;    NAME       'async'       (1, 0) (1, 5)
    NAME       'for'         (1, 6) (1, 9)
    NAME       'a'           (1, 10) (1, 11)
    NAME       'in'          (1, 12) (1, 14)
    NAME       'b'           (1, 15) (1, 16)
    OP         ':'           (1, 16) (1, 17)
    NAME       'pass'        (1, 18) (1, 22)
    �async with a as b: passa<    NAME       'async'       (1, 0) (1, 5)
    NAME       'with'        (1, 6) (1, 10)
    NAME       'a'           (1, 11) (1, 12)
    NAME       'as'          (1, 13) (1, 15)
    NAME       'b'           (1, 16) (1, 17)
    OP         ':'           (1, 17) (1, 18)
    NAME       'pass'        (1, 19) (1, 23)
    �	async.fooz�    NAME       'async'       (1, 0) (1, 5)
    OP         '.'           (1, 5) (1, 6)
    NAME       'foo'         (1, 6) (1, 9)
    �asyncz/    NAME       'async'       (1, 0) (1, 5)
    �async
#comment
awaitz�    NAME       'async'       (1, 0) (1, 5)
    NEWLINE    '\n'          (1, 5) (1, 6)
    COMMENT    '#comment'    (2, 0) (2, 8)
    NL         '\n'          (2, 8) (2, 9)
    NAME       'await'       (3, 0) (3, 5)
    �async
...
awaitz�    NAME       'async'       (1, 0) (1, 5)
    NEWLINE    '\n'          (1, 5) (1, 6)
    OP         '...'         (2, 0) (2, 3)
    NEWLINE    '\n'          (2, 3) (2, 4)
    NAME       'await'       (3, 0) (3, 5)
    �async
awaitz�    NAME       'async'       (1, 0) (1, 5)
    NEWLINE    '\n'          (1, 5) (1, 6)
    NAME       'await'       (2, 0) (2, 5)
    �
foo.async + 1z�    NAME       'foo'         (1, 0) (1, 3)
    OP         '.'           (1, 3) (1, 4)
    NAME       'async'       (1, 4) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    NUMBER     '1'           (1, 12) (1, 13)
    �async def foo(): passa;    NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    OP         ':'           (1, 15) (1, 16)
    NAME       'pass'        (1, 17) (1, 21)
    �Nasync def foo():
  def foo(await):
    await = 1
  if 1:
    await
async += 1
a�    NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    OP         ':'           (1, 15) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    INDENT     '  '          (2, 0) (2, 2)
    NAME       'def'         (2, 2) (2, 5)
    NAME       'foo'         (2, 6) (2, 9)
    OP         '('           (2, 9) (2, 10)
    NAME       'await'       (2, 10) (2, 15)
    OP         ')'           (2, 15) (2, 16)
    OP         ':'           (2, 16) (2, 17)
    NEWLINE    '\n'          (2, 17) (2, 18)
    INDENT     '    '        (3, 0) (3, 4)
    NAME       'await'       (3, 4) (3, 9)
    OP         '='           (3, 10) (3, 11)
    NUMBER     '1'           (3, 12) (3, 13)
    NEWLINE    '\n'          (3, 13) (3, 14)
    DEDENT     ''            (4, 2) (4, 2)
    NAME       'if'          (4, 2) (4, 4)
    NUMBER     '1'           (4, 5) (4, 6)
    OP         ':'           (4, 6) (4, 7)
    NEWLINE    '\n'          (4, 7) (4, 8)
    INDENT     '    '        (5, 0) (5, 4)
    NAME       'await'       (5, 4) (5, 9)
    NEWLINE    '\n'          (5, 9) (5, 10)
    DEDENT     ''            (6, 0) (6, 0)
    DEDENT     ''            (6, 0) (6, 0)
    NAME       'async'       (6, 0) (6, 5)
    OP         '+='          (6, 6) (6, 8)
    NUMBER     '1'           (6, 9) (6, 10)
    NEWLINE    '\n'          (6, 10) (6, 11)
    �)async def foo():
  async for i in 1: passa�    NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    OP         ':'           (1, 15) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    INDENT     '  '          (2, 0) (2, 2)
    NAME       'async'       (2, 2) (2, 7)
    NAME       'for'         (2, 8) (2, 11)
    NAME       'i'           (2, 12) (2, 13)
    NAME       'in'          (2, 14) (2, 16)
    NUMBER     '1'           (2, 17) (2, 18)
    OP         ':'           (2, 18) (2, 19)
    NAME       'pass'        (2, 20) (2, 24)
    DEDENT     ''            (3, 0) (3, 0)
    �async def foo(async): awaitah    NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    NAME       'async'       (1, 14) (1, 19)
    OP         ')'           (1, 19) (1, 20)
    OP         ':'           (1, 20) (1, 21)
    NAME       'await'       (1, 22) (1, 27)
    �@def f():

  def baz(): pass
  async def bar(): pass

  await = 2a�    NAME       'def'         (1, 0) (1, 3)
    NAME       'f'           (1, 4) (1, 5)
    OP         '('           (1, 5) (1, 6)
    OP         ')'           (1, 6) (1, 7)
    OP         ':'           (1, 7) (1, 8)
    NEWLINE    '\n'          (1, 8) (1, 9)
    NL         '\n'          (2, 0) (2, 1)
    INDENT     '  '          (3, 0) (3, 2)
    NAME       'def'         (3, 2) (3, 5)
    NAME       'baz'         (3, 6) (3, 9)
    OP         '('           (3, 9) (3, 10)
    OP         ')'           (3, 10) (3, 11)
    OP         ':'           (3, 11) (3, 12)
    NAME       'pass'        (3, 13) (3, 17)
    NEWLINE    '\n'          (3, 17) (3, 18)
    NAME       'async'       (4, 2) (4, 7)
    NAME       'def'         (4, 8) (4, 11)
    NAME       'bar'         (4, 12) (4, 15)
    OP         '('           (4, 15) (4, 16)
    OP         ')'           (4, 16) (4, 17)
    OP         ':'           (4, 17) (4, 18)
    NAME       'pass'        (4, 19) (4, 23)
    NEWLINE    '\n'          (4, 23) (4, 24)
    NL         '\n'          (5, 0) (5, 1)
    NAME       'await'       (6, 2) (6, 7)
    OP         '='           (6, 8) (6, 9)
    NUMBER     '2'           (6, 10) (6, 11)
    DEDENT     ''            (7, 0) (7, 0)
    �Fasync def f():

  def baz(): pass
  async def bar(): pass

  await = 2a    NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'f'           (1, 10) (1, 11)
    OP         '('           (1, 11) (1, 12)
    OP         ')'           (1, 12) (1, 13)
    OP         ':'           (1, 13) (1, 14)
    NEWLINE    '\n'          (1, 14) (1, 15)
    NL         '\n'          (2, 0) (2, 1)
    INDENT     '  '          (3, 0) (3, 2)
    NAME       'def'         (3, 2) (3, 5)
    NAME       'baz'         (3, 6) (3, 9)
    OP         '('           (3, 9) (3, 10)
    OP         ')'           (3, 10) (3, 11)
    OP         ':'           (3, 11) (3, 12)
    NAME       'pass'        (3, 13) (3, 17)
    NEWLINE    '\n'          (3, 17) (3, 18)
    NAME       'async'       (4, 2) (4, 7)
    NAME       'def'         (4, 8) (4, 11)
    NAME       'bar'         (4, 12) (4, 15)
    OP         '('           (4, 15) (4, 16)
    OP         ')'           (4, 16) (4, 17)
    OP         ':'           (4, 17) (4, 18)
    NAME       'pass'        (4, 19) (4, 23)
    NEWLINE    '\n'          (4, 23) (4, 24)
    NL         '\n'          (5, 0) (5, 1)
    NAME       'await'       (6, 2) (6, 7)
    OP         '='           (6, 8) (6, 9)
    NUMBER     '2'           (6, 10) (6, 11)
    DEDENT     ''            (7, 0) (7, 0)
    rYrZs r2�
test_asynczTokenizeTest.test_async�s������K�*�		�		�		�	
���-�0�		�		�		�	
���I�(�		�		�		�	
���3�6�		�		�		�	
���.�1�		�		�		�	
���K�*�		�		�		�	
���K�*�		�		�		�	
���4�7�		�		�		�	
���5�8�		�		�		�	
���K�*�		�		�		�	
���G�&�		�		�		�	
���4�7�		�		�		�	
���/�2�		�		�		�	
���N�-�		�		�		�	
���O�.�		�		�		�	
���3�6�		�		�		�	
����#�*		�*		�*		�X	
�����		�		�		�*	
���=�	@�			�			�			�	
�����
#		�#		�#		�J	
�����
$		�$		�$		�$		�$		r4N)�__name__�
__module__�__qualname__rBrHrNr[r]rfrnr�r�r�r�r�r�r�r�r�r�r�r�r��r4r2r6r6&sO������9�9�9�5�5�5�#�#�#�J7	�7	�7	�r	�	�	�0&	�&	�&	�P
8�
8�
8�d	�d	�d	�L:	�:	�:	�x$	�$	�$	�L
	�
	�
	�	�	�	�.	�	�	�(#	�#	�#	�J	�	�	�,	�	�	�"	�	�	�*
	�
	�
	�
	�
	�
	�P	�P	�P	�P	�P	r4r6c��eZdZd�ZdS)�GenerateTokensTestc���t|��}tt|j��|��}|�||��������dS�N)rr3rr:r;r<r&r=s     r2rBz!GenerateTokensTest.check_tokenize�sZ��
�Q�K�K��-�o�a�j�.I�.I�1�M�M���������!2�!2�!=�!=�!?�!?�@�@�@�@�@r4N)r�r�r�rBr�r4r2r�r��s(������A�A�A�A�Ar4r�c	��g}tt|�d����j��}|D]q\}}}}}|tkrGd|vrC|�tdftdftt|��ftdfg���Z|�
||f���rt|���d��S)Nr8�.�Decimalrk�))
rrr9r:r�extendr	r
r�reprr'r�decode)r?r*�g�toknum�tokval�_s      r2�decistmtr��s���
�F������'�*�*�+�+�4�5�5�A�$%�	,�	,�����1�a��V����v�
�
��M�M��y�!��S�	���f���&��S�	�	�
�
�
�
�
�M�M�6�6�*�+�+�+�+��f���$�$�W�-�-�-r4c��eZdZd�ZdS)�TestMiscc�.�ddlm}d}|�t|��d��|�tt
|����d��|�t
t|����|d����dS)Nr)r�z+21.3e-5*-.1234/81.7z8+Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7')z-3.2171603427[0-9]*e-0+7z!-3.217160342717258261933904529E-7)�decimalr�r;r��assertRegexr��eval)r>r�r?s   r2�
test_decistmtzTestMisc.test_decistmt�s���	$�#�#�#�#�#�"������!���S�	U�	U�	U�	
����d�1�g�g���(B�C�C�C�	
����h�q�k�k�*�*� ��!D�E�E�	G�	G�	G�	G�	Gr4N)r�r�r�r�r�r4r2r�r��s(������G�G�G�G�Gr4r�c�6�eZdZdZd�Zd�Zd�Zd�Zd�Zd�Z	dS)	�TestTokenizerAdheresToPep0263zU
    Test that tokenizer adheres to the coding behaviour stipulated in PEP 0263.
    c���tj�tj�t��|��}t
�|t|d����dS)N�rb)�os�path�join�dirname�__file__�
TestRoundtrip�check_roundtripr)r>�filenamer�s   r2�	_testFilez'TestTokenizerAdheresToPep0263._testFile�sK���w�|�|�B�G�O�O�H�5�5�x�@�@���%�%�d�D��t�,<�,<�=�=�=�=�=r4c�4�d}|�|��dS)Nz9tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txt�r��r>rAs  r2�'test_utf8_coding_cookie_and_no_utf8_bomzETestTokenizerAdheresToPep0263.test_utf8_coding_cookie_and_no_utf8_bom����G�����q�����r4c�L�d}|�t|j|��dS)a�
        As per PEP 0263, if a file starts with a utf-8 BOM signature, the only
        allowed encoding for the comment is 'utf-8'.  The text file used in
        this test starts with a BOM signature, but specifies latin1 as the
        coding, so verify that a SyntaxError is raised, which matches the
        behaviour of the interpreter when it encounters a similar condition.
        z8tokenize_tests-latin1-coding-cookie-and-utf8-bom-sig.txtN��assertRaises�SyntaxErrorr�r�s  r2�&test_latin1_coding_cookie_and_utf8_bomzDTestTokenizerAdheresToPep0263.test_latin1_coding_cookie_and_utf8_bom�s*��
G�����+�t�~�q�9�9�9�9�9r4c�4�d}|�|��dS)Nz9tokenize_tests-no-coding-cookie-and-utf8-bom-sig-only.txtr�r�s  r2�"test_no_coding_cookie_and_utf8_bomz@TestTokenizerAdheresToPep0263.test_no_coding_cookie_and_utf8_bom�rr4c�4�d}|�|��dS)Nz6tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txtr�r�s  r2�$test_utf8_coding_cookie_and_utf8_bomzBTestTokenizerAdheresToPep0263.test_utf8_coding_cookie_and_utf8_boms��D�����q�����r4c��|�t|jd��|�t|jd��dS)Nz
bad_coding.pyzbad_coding2.pyrrZs r2�test_bad_coding_cookiez4TestTokenizerAdheresToPep0263.test_bad_coding_cookies=�����+�t�~��G�G�G����+�t�~�7G�H�H�H�H�Hr4N)
r�r�r��__doc__r�r�rrr	rr�r4r2r�r��s}��������>�>�>����	:�	:�	:�������I�I�I�I�Ir4r�c��eZdZd�Zd�ZdS)�
Test_Tokenizec�����d}|�d���d���fd�}tt|d�����dd�}dg}|�||d	��dS)
N�"ЉЊЈЁЂ"r8Fc����sd��SdS�NTr4r�)�firstr1s��r2r:zNTest_Tokenize.test__tokenize_decodes_with_specified_encoding.<locals>.readlines����
������sr4��encoding�rE��r)rr)r�rzbytes not decoded with encoding)r9rFrr;)r>�literalr:rG�expected_tokensrr1s     @@r2�.test__tokenize_decodes_with_specified_encodingz<Test_Tokenize.test__tokenize_decodes_with_specified_encodings����� ���~�~�g�&�&����	�	�	�	�	�	��i��7�;�;�;�<�<�Q�r�T�B��N�O�������:�	<�	<�	<�	<�	<r4c����d�d���fd�}tt|d�����dd�}dg}|�||d��dS)NrFc����sd��SdSrr�)rrs��r2r:zQTest_Tokenize.test__tokenize_does_not_decode_with_encoding_none.<locals>.readline s����
������sr4rrErz*string not tokenized when encoding is None)rFrr;)r>r:rGrrrs    @@r2�1test__tokenize_does_not_decode_with_encoding_nonez?Test_Tokenize.test__tokenize_does_not_decode_with_encoding_nones����� ����	�	�	�	�	�	��i��4�8�8�8�9�9�#�2�#�>��N�O�������E�	G�	G�	G�	G�	Gr4N)r�r�r�rrr�r4r2rr	s7������<�<�<�$G�G�G�G�Gr4rc��eZdZd�Zd�Zd�Zd�Zd�Zd�Zd�Z	d�Z
d	�Zd
�Zd�Z
d�Zd
�Zd�Zd�Zd�Zd�Zd�Zd�Zd�ZdS)�TestDetectEncodingc����d���fd�}|S)Nrc�V���t���krt���}�dz
�|S�Nr)r%�
StopIteration)r1�index�liness ��r2r:z1TestDetectEncoding.get_readline.<locals>.readline3s1�����E�
�
�"�"�#�#���<�D��Q�J�E��Kr4r�)r>r'r:r&s ` @r2�get_readlinezTestDetectEncoding.get_readline1s.������	�	�	�	�	�	��r4c���d}t|�|����\}}|�|d��|�|t|dd�����dS)N)�# something
�print(something)
�do_something(else)
r8�)rr(r;rF�r>r'r�consumed_liness    r2�test_no_bom_no_encoding_cookiez1TestDetectEncoding.test_no_bom_no_encoding_cookie<sk��
��
$3�4�3D�3D�U�3K�3K�#L�#L� ��.�����7�+�+�+������e�B�Q�B�i���9�9�9�9�9r4c��d}t|�|����\}}|�|d��|�|ddg��dS)N)s# something
r+r,�	utf-8-sigr*r+�rr(r;r.s    r2�test_bom_no_cookiez%TestDetectEncoding.test_bom_no_cookieFss��
��
$3�4�3D�3D�U�3K�3K�#L�#L� ��.�����;�/�/�/�����*�,A�B�	D�	D�	D�	D�	Dr4c��d}t|�|����\}}|�|d��|�|dg��dS)N)�# -*- coding: latin-1 -*-
r+r,�
iso-8859-1r6r3r.s    r2�test_cookie_first_line_no_bomz0TestDetectEncoding.test_cookie_first_line_no_bomQsa��
��
$3�4�3D�3D�U�3K�3K�#L�#L� ��.�����<�0�0�0�����*H�)I�J�J�J�J�Jr4c��d}t|�|����\}}|�|d��|�|dg��dS)N)s# coding=utf-8
r+r,r2s# coding=utf-8
r3r.s    r2�&test_matched_bom_and_cookie_first_linez9TestDetectEncoding.test_matched_bom_and_cookie_first_line[sa��
��
$3�4�3D�3D�U�3K�3K�#L�#L� ��.�����;�/�/�/�����*=�)>�?�?�?�?�?r4c�v�d}|�|��}|�tt|��dS)N)s## vim: set fileencoding=ascii :
r+r,�r(rrr�r>r'r:s   r2�<test_mismatched_bom_and_cookie_first_line_raises_syntaxerrorzOTestDetectEncoding.test_mismatched_bom_and_cookie_first_line_raises_syntaxerrores<��
��
�$�$�U�+�+�����+���A�A�A�A�Ar4c��d}t|�|����\}}|�|d��ddg}|�||��dS)N)�
#! something
� # vim: set fileencoding=ascii :
r+r,�asciir@rAr3�r>r'rr/r@s     r2�test_cookie_second_line_no_bomz1TestDetectEncoding.test_cookie_second_line_no_bomnsh��
��$3�4�3D�3D�U�3K�3K�#L�#L� ��.�����7�+�+�+�%�'K�L�������2�2�2�2�2r4c��d}t|�|����\}}|�|d��|�|ddg��dS)N)�#! something
�f# coding=utf-8
r+r,r2r@rGr3r.s    r2�'test_matched_bom_and_cookie_second_linez:TestDetectEncoding.test_matched_bom_and_cookie_second_linezss��
��$3�4�3D�3D�U�3K�3K�#L�#L� ��.�����;�/�/�/�����+�-A�B�	D�	D�	D�	D�	Dr4c�v�d}|�|��}|�tt|��dS)N)rFrAr+r,r<r=s   r2�=test_mismatched_bom_and_cookie_second_line_raises_syntaxerrorzPTestDetectEncoding.test_mismatched_bom_and_cookie_second_line_raises_syntaxerror�s<��
���$�$�U�+�+�����+���A�A�A�A�Ar4c��d}t|�|����\}}|�|d��dg}|�||��dS)N)�print('£')
�%# vim: set fileencoding=iso8859-15 :
�
print('€')
r8rLr3rCs     r2�/test_cookie_second_line_noncommented_first_linezBTestDetectEncoding.test_cookie_second_line_noncommented_first_line�se��
��
$3�4�3D�3D�U�3K�3K�#L�#L� ��.�����7�+�+�+�*�+�������2�2�2�2�2r4c��d}t|�|����\}}|�|d��ddg}|�||��dS)N)�
#print('£')
rMrN�
iso8859-15rQrMr3rCs     r2�,test_cookie_second_line_commented_first_linez?TestDetectEncoding.test_cookie_second_line_commented_first_line�sh��
��
$3�4�3D�3D�U�3K�3K�#L�#L� ��.�����<�0�0�0�+�-V�W�������2�2�2�2�2r4c��d}t|�|����\}}|�|d��ddg}|�||��dS)N)�
rMrNrRrUrMr3rCs     r2�(test_cookie_second_line_empty_first_linez;TestDetectEncoding.test_cookie_second_line_empty_first_line�sh��
��
$3�4�3D�3D�U�3K�3K�#L�#L� ��.�����<�0�0�0��D�E�������2�2�2�2�2r4c��d}|D]y}dD]t}|�d|��}dd|�d��zdzdd	f}|�|��}t|��\}}|�|d
���u�zdS)N)zlatin-1r7ziso-latin-1zlatin-1-unixziso-8859-1-unixziso-latin-1-mac��-r�rY�#!/usr/bin/python
�
# coding: rBrUsprint(things)
sdo_something += 4
r7��replacer9r(rr;�	r>�	encodingsr�rep�encr'�rl�foundr/s	         r2�test_latin1_normalizationz,TestDetectEncoding.test_latin1_normalization�s���;�	�!�		6�		6�H�!�
6�
6���&�&�s�C�0�0��/�&����G�)<�)<�<�u�D�+�/�1���&�&�u�-�-��(7��(;�(;�%��~�� � ���5�5�5�5�
6�		6�		6r4c�v�d}|�|��}|�tt|��dS)N�s
print("�")r<r=s   r2�test_syntaxerror_latin1z*TestDetectEncoding.test_syntaxerror_latin1�s<�����$�$�U�+�+�����+���A�A�A�A�Ar4c��d}|D]x}dD]s}|�d|��}dd|�d��zdzdf}|�|��}t|��\}}|�|d	���t�ydS)
N)r8z	utf-8-macz
utf-8-unixrXrYrZr[rBrUs1 + 3
r8r\r^s	         r2�test_utf8_normalizationz*TestDetectEncoding.test_utf8_normalization�s���8�	�!�	1�	1�H�!�
1�
1���&�&�s�C�0�0��/�&����G�)<�)<�<�u�D�#�%���&�&�u�-�-��(7��(;�(;�%��~�� � ���0�0�0�0�
1�	1�	1r4c�
�|�d��}t|��\}}|�|d��|�|dg��t|�d����\}}|�|d��|�|g��|�d��}t|��\}}|�|d��|�|dg��|�d��}t|��\}}|�|d��|�|g��|�d��}|�tt|��dS)	N)r+r8r+r�)sprint(something)
r2)s)s# coding: bad
)r(rr;rr�r>r:rr/s    r2�test_short_filesz#TestDetectEncoding.test_short_files�s~���$�$�%=�>�>��#2�8�#<�#<� ��.�����7�+�+�+�����*?�)@�A�A�A�#2�4�3D�3D�R�3H�3H�#I�#I� ��.�����7�+�+�+������,�,�,��$�$�%I�J�J��#2�8�#<�#<� ��.�����;�/�/�/�����*?�)@�A�A�A��$�$�%7�8�8��#2�8�#<�#<� ��.�����;�/�/�/������,�,�,��$�$�%:�;�;�����+���A�A�A�A�Ar4c��|�d��}t|��\}}|�|d��|�|dg��dS)N)�print("#coding=fake")r8rn)r(rr;rks    r2�test_false_encodingz&TestDetectEncoding.test_false_encoding�s^���$�$�%@�A�A��#2�8�#<�#<� ��.�����7�+�+�+�����*B�)C�D�D�D�D�Dr4c��tjdz}|�tj|��dD]�}t	|d|���5}td|z|���td|���ddd��n#1swxYwYt
|��5}|�|j|��|�|j	d��ddd��n#1swxYwY��t	|dd	���5}td|���ddd��n#1swxYwYt
|��5}|�|jd	��|�|j	d��ddd��dS#1swxYwYdS)
Nz.py)ziso-8859-15r8�wrz# coding: %s��fileuprint('euro:€')�rr2)
r�TESTFN�
addCleanup�unlinkr�print�
tokenize_openr;r�mode)r>r�r�fps    r2�	test_openzTestDetectEncoding.test_open�sy���#�e�+�����	�(�(�3�3�3�1�	/�	/�H��h��h�7�7�7�
7�2��n�x�/�b�9�9�9�9��,�2�6�6�6�6�
7�
7�
7�
7�
7�
7�
7�
7�
7�
7�
7����
7�
7�
7�
7��x�(�(�
/�B�� � ���h�7�7�7�� � ���#�.�.�.�
/�
/�
/�
/�
/�
/�
/�
/�
/�
/�
/����
/�
/�
/�
/��
�(�C�+�
6�
6�
6�	3�"��(�r�2�2�2�2�	3�	3�	3�	3�	3�	3�	3�	3�	3�	3�	3����	3�	3�	3�	3�
�8�
$�
$�	+�����R�[�+�6�6�6����R�W�c�*�*�*�	+�	+�	+�	+�	+�	+�	+�	+�	+�	+�	+�	+����	+�	+�	+�	+�	+�	+sH�&A8�8A<	�?A<	�7C�C	�C	�3D�D�D�+7E/�/E3�6E3c���d}d�G�fd�d��}|�t��5|�|��}|`t|j��ddd��n#1swxYwY|�td�|����5|�|��}t|j��ddd��dS#1swxYwYdS)N�some_file_pathrfc� ��eZdZd�Z�fd�ZdS)�;TestDetectEncoding.test_filename_in_exception.<locals>.Bunkc�0�||_||_d|_dS)Nr)�name�_lines�_index)r>r'r�s   r2�__init__zDTestDetectEncoding.test_filename_in_exception.<locals>.Bunk.__init__s�� ��	�#�������r4c���|jt���krt��|j}|xjdz
c_|Sr$)r�r%r%)r>r1r's  �r2r:zDTestDetectEncoding.test_filename_in_exception.<locals>.Bunk.readlines=����;�#�e�*�*�,�,�'�'��T�[�)�����q� ����r4N)r�r�r�r�r:)r's�r2�Bunkr�s=�������
 �
 �
 �

�
�
�
�
�
�
r4r�z.*{})rrr�rr:rJ�format)r>r�r��insr's    @r2�test_filename_in_exceptionz-TestDetectEncoding.test_filename_in_exception	s{�������	�	�	�	�	�	�	�	�	�	��
�
�{�
+�
+�	*�	*��$�u�d�#�#�C����C�L�)�)�)�		*�	*�	*�	*�	*�	*�	*�	*�	*�	*�	*����	*�	*�	*�	*�
�
#�
#�K����t�1D�1D�
E�
E�	*�	*��$�u�d�#�#�C��C�L�)�)�)�	*�	*�	*�	*�	*�	*�	*�	*�	*�	*�	*�	*����	*�	*�	*�	*�	*�	*s#�#A�A"�%A"�!C�C	�C	c��td��}tjd|���5|�tt
d��ddd��n#1swxYwY|�|j��dS)Ns#coding:xxxztokenize._builtin_open)�return_value�foobar)rr�patchrrry�
assertTrue�closed)r>�ms  r2�test_open_errorz"TestDetectEncoding.test_open_error%s����N�#�#��
�Z�0�q�
A�
A�
A�	D�	D����k�=�(�C�C�C�	D�	D�	D�	D�	D�	D�	D�	D�	D�	D�	D����	D�	D�	D�	D������!�!�!�!�!s�"A�A�AN)r�r�r�r(r0r4r8r:r>rDrHrJrOrSrVrdrgrirlror|r�r�r�r4r2r!r!/s[������	�	�	�:�:�:�	D�	D�	D�K�K�K�@�@�@�B�B�B�
3�
3�
3�
D�
D�
D�B�B�B�	3�	3�	3�	3�	3�	3�	3�	3�	3�
6�
6�
6�B�B�B�1�1�1�B�B�B�.E�E�E�+�+�+�(*�*�*�8"�"�"�"�"r4r!c�2�eZdZd�Zd�Zd�Zd�Zd�Zd�ZdS)�TestTokenizec�h��	�
�ddl}t���	d�
�	fd�}�
fd�}d��fd�}|j}|j}||_||_	t|��}|�t|��gd���||_||_n#||_||_wxYw|��
�	��dS)Nrc����ddgfS)N�first�secondr�)r:rs �r2�mock_detect_encodingz8TestTokenize.test_tokenize.<locals>.mock_detect_encoding3s����h�	�2�2�2r4c�V��|�g}	|��}|r|�|���#|Sr�)r')r:r�out�	next_line�
encoding_useds    �r2�mock__tokenizez2TestTokenize.test_tokenize.<locals>.mock__tokenize6s?���$�M��C�
�$�H�J�J�	����J�J�y�)�)�)���
r4c�`���dz
��dkrdSt������S)Nr�r4)�strr9)�counters�r2�
mock_readlinez1TestTokenize.test_tokenize.<locals>.mock_readlineBs2����q�L�G��!�|�|��s��w�<�<�&�&�(�(�(r4)r�r��1�2�3�4)r�objectrrr;rF)r>�tokenize_moduler�r�r��orig_detect_encoding�orig__tokenize�resultsr�rr�s        @@@r2�
test_tokenizezTestTokenize.test_tokenize/s$�����*�*�*�*��8�8���
�	3�	3�	3�	3�	3�		�		�		�		�		���	)�	)�	)�	)�	)� /�>��(�2��*>��'�$2��!�	7��}�-�-�G����T�'�]�]�J�J�J�
L�
L�
L�/C�O�+�(6�O�%�%��/C�O�+�(6�O�%�6�6�6�6������1�1�1�1�1s�4B	�	Bc	��g}td��D]+}|�d�|������,|�d��d�|��}t	tt
|�d����j����}|�	|dj
d��dS)Ni�zdef i{i}(): return {i})�i�OK�
r8���)�ranger'r�r�rFrrr9r:r;�string)r>�bufr��tokss    r2�test_oneline_defszTestTokenize.test_oneline_defsWs������s���	=�	=�A��J�J�/�6�6��6�;�;�<�<�<�<��
�
�4�����i�i��n�n���H�W�S�Z�Z��%8�%8�9�9�B�C�C�D�D������b���$�/�/�/�/�/r4c	��ttt|�d����j����}t|��}|�t|��d|z��|�t|djtt��t|��D]B}|�t||dzjt||���C|�t|d|zjttj��|�t|d|zjttj
��dS)Nr8rrrr-)rFrrr9r:r%r;r�
exact_typer
r�r.rr)r>�opstr�optypesrG�num_optypesr�s      r2�assertExactTypeEqualz!TestTokenize.assertExactTypeEqualcsM���h�w�u�|�|�G�'<�'<�=�=�F�G�G�H�H���'�l�l������V���a�+�o�6�6�6�����&��)�"6�7�!�(�+�	-�	-�	-��{�#�#�	3�	3�A����X�f�Q��U�m�&>�?�%�g�a�j�1�
3�
3�
3�
3�����&��[��"9�"D�E�!�%�-�0�	2�	2�	2�����&��[��"9�"D�E�!�%�/�2�	4�	4�	4�	4�	4r4c��
�|�dtjtj��|�dtjtj��|�dtj��|�dtj��|�dtj��|�dtj	��|�dtj
��|�dtj��|�d	tj��|�d
tj
��|�dtj��|�dtj��|�d
tj��|�dtj��|�dtj��|�dtj��|�dtjtj��|�dtj��|�dtj��|�dtj��|�dtj��|�dtj��|�dtj��|�dtj��|�dtj��|�dtj��|�dtj��|�dtj ��|�dtj!��|�dtj"��|�dtj#��|�d tj$��|�d!tj%��|�d"tj&��|�d"tj&��|�d#tj'��|�d$tj(��|�d%tj)��|�d&tj*��|�d'tj+��|�d(tj,��|�d)tj-��|�d*tj.��|�d+tj/��|�d,tj0��|�d-tbtjtdtj	tbtjtdtjtbtjtd��|�d.tjtj2tjtj2tjtj2tj��|�d/tjtjtj1tjtj2tj��dS)0Nz()z[]�:�,�;�+rY�*�/�|�&�<�>�=r��%z{}z==z!=z<=z>=�~�^z<<z>>z**z+=z-=z*=z/=z%=z&=z|=z^=z<<=z>>=z**=z//z//=z:=z...z->�@z@=za**2+b**2==c**2z	{1, 2, 3}z
^(x & 0x1))3r�r.�LPAR�RPAR�LSQB�RSQB�COLON�COMMA�SEMI�PLUS�MINUS�STAR�SLASH�VBAR�AMPER�LESS�GREATER�EQUAL�DOT�PERCENT�LBRACE�RBRACE�EQEQUAL�NOTEQUAL�	LESSEQUAL�GREATEREQUAL�TILDE�
CIRCUMFLEX�	LEFTSHIFT�
RIGHTSHIFT�
DOUBLESTAR�	PLUSEQUAL�MINEQUAL�	STAREQUAL�
SLASHEQUAL�PERCENTEQUAL�
AMPEREQUAL�	VBAREQUAL�CIRCUMFLEXEQUAL�LEFTSHIFTEQUAL�RIGHTSHIFTEQUAL�DOUBLESTAREQUAL�DOUBLESLASH�DOUBLESLASHEQUAL�
COLONEQUAL�ELLIPSIS�RARROW�AT�ATEQUALr	rrZs r2�test_exact_typezTestTokenize.test_exact_typeqs���!�!�$��
�E�J�?�?�?��!�!�$��
�E�J�?�?�?��!�!�#�u�{�3�3�3��!�!�#�u�{�3�3�3��!�!�#�u�z�2�2�2��!�!�#�u�z�2�2�2��!�!�#�u�{�3�3�3��!�!�#�u�z�2�2�2��!�!�#�u�{�3�3�3��!�!�#�u�z�2�2�2��!�!�#�u�{�3�3�3��!�!�#�u�z�2�2�2��!�!�#�u�}�5�5�5��!�!�#�u�{�3�3�3��!�!�#�u�y�1�1�1��!�!�#�u�}�5�5�5��!�!�$���e�l�C�C�C��!�!�$��
�6�6�6��!�!�$���7�7�7��!�!�$���8�8�8��!�!�$��(:�;�;�;��!�!�#�u�{�3�3�3��!�!�#�u�'7�8�8�8��!�!�$���8�8�8��!�!�$��(8�9�9�9��!�!�$��(8�9�9�9��!�!�$���8�8�8��!�!�$���7�7�7��!�!�$���8�8�8��!�!�$��(8�9�9�9��!�!�$��(:�;�;�;��!�!�$��(8�9�9�9��!�!�$���8�8�8��!�!�$��(=�>�>�>��!�!�$��(=�>�>�>��!�!�%��)=�>�>�>��!�!�%��)>�?�?�?��!�!�%��)>�?�?�?��!�!�$��(9�:�:�:��!�!�%��)?�@�@�@��!�!�$��(8�9�9�9��!�!�%���8�8�8��!�!�$���5�5�5��!�!�#�u�x�0�0�0��!�!�$��
�6�6�6��!�!�"3�"&��(8�&�"'�*�"&��(8�&�"'�-�"&��(8�&�	B�	B�	B�	
�!�!�+�"'�,�"'�,���"'�,���"'�,�"'�,�	0�	0�	0�	
�!�!�,�"'�"2�"'�*�"'�*�e�k�5�<�"'�*�		.�	.�	.�	.�	.r4c�F�|�dtj��dS)Nz@          )r�r.r�rZs r2�%test_pathological_trailing_whitespacez2TestTokenize.test_pathological_trailing_whitespace�s ���!�!�-���:�:�:�:�:r4c	��d}tjtjtjtjtjtjg}ttt|�
d����j����}|�t|djtt��t!d��D]B}|�t||dzjt||���C|�t|djttj��dS)Nzb = 1

#testr8r�rr )r.r	r�rr�NL�COMMENTrFrrr9r:r;rr�r
r�r)r>�sourcerrGr�s     r2�5test_comment_at_the_end_of_the_source_without_newlinezBTestTokenize.test_comment_at_the_end_of_the_source_without_newline�s���!�� �:�u�{�E�L�%�-�QV�QY�[`�[h�i���h�w�v�}�}�W�'=�'=�>�>�G�H�H�I�I������&��)�"6�7��(�9K�L�L�L��q���	_�	_�A����X�f�Q��U�m�&>�?��/�Z[�J\�A]�^�^�^�^�����&��*�"7�8�(�5�?�:S�T�T�T�T�Tr4N)	r�r�r�r�r�r�r�r�rr�r4r2r�r�-su������&2�&2�&2�P	0�	0�	0�4�4�4�?.�?.�?.�B;�;�;�	U�	U�	U�	U�	Ur4r�c� �eZdZd�Zd�Zd�ZdS)�UntokenizeTestc�Z�t��}d|_d|_|�t��5}|�d��ddd��n#1swxYwY|�|jjdd��|�t|jd��dS)Nr-)rrrz'start (1,3) precedes previous end (2,2))r-r)	r�prev_row�prev_colr�
ValueError�add_whitespacer;�	exception�args)r>�u�cms   r2�test_bad_input_orderz#UntokenizeTest.test_bad_input_order�s����M�M����
���
�
�
�
�z�
*�
*�	$�b�
���U�#�#�#�	$�	$�	$�	$�	$�	$�	$�	$�	$�	$�	$����	$�	$�	$�	$������*�1�-�9�	;�	;�	;�	
���*�a�&6��>�>�>�>�>s�A�A� Ac�V�t��}d|_d|_g|_|�d��|�|jdg��d|_|�d��|�|jgd���t�|d��dS)Nr)r-r�\
r-)�r)rz\
\
r!za
  b
    c
  \
  c
)rr	r
rGrr;r�r�)r>rs  r2�test_backslash_continuationz*UntokenizeTest.test_backslash_continuation�s����M�M����
���
����	���� � � ������F�8�,�,�,���
�	���� � � ������#?�#?�#?�@�@�@��%�%�d�,H�I�I�I�I�Ir4c�p�t��}tdf}tdf|g}|�|t	g����|�|jdg��t��}|�|�t	|g����d��t��}|�|�t	|����d��|�|jd��|�tt	|����d��dS)N�Hellor8zHello sHello )	rr	r
�compat�iterr;rGrr)r>rr.rGs    r2�test_iter_compatzUntokenizeTest.test_iter_compat�s����M�M���w����W�%�u�-��	�����R���!�!�!������H�:�.�.�.��M�M��������d�E�7�m�m�4�4�h�?�?�?��M�M��������d�6�l�l�3�3�X�>�>�>������W�-�-�-�����D��L�L�1�1�9�=�=�=�=�=r4N)r�r�r�rrrr�r4r2rr�sD������
?�
?�
?�J�J�J�>�>�>�>�>r4rc�>�eZdZd�Zd�Zd�Zd�Zd�Zd�Zd�Z	d�Z
d	S)
r�c���t|t��r|�d��}n(|���}|���t|�d�����j}tt|����}d�|D��}t|��}t|�d�����j}d�t|��D��}|�||��t|��}	t|	�d�����j}
d�t|
��D��}|�||��dS)a)
        Test roundtrip for `untokenize`. `f` is an open file or a string.
        The source code in f is tokenized to both 5- and 2-tuples.
        Both sequences are converted back to source code via
        tokenize.untokenize(), and the latter tokenized again to 2-tuples.
        The test fails if the 3 pair tokenizations do not match.

        When untokenize bugs are fixed, untokenize with 5-tuples should
        reproduce code that does not contain a backslash continuation
        following spaces.  A proper test should test this.
        r8T)�keependsc�"�g|]}|dd���
S�Nr-r���.0rMs  r2�
<listcomp>z1TestRoundtrip.check_roundtrip.<locals>.<listcomp>s ��.�.�.�s�3�r��r�7�.�.�.r4c�"�g|]}|dd���
Srr�r s  r2r"z1TestRoundtrip.check_roundtrip.<locals>.<listcomp>� ��@�@�@�S��R�a�R��@�@�@r4c�"�g|]}|dd���
Srr�r s  r2r"z1TestRoundtrip.check_roundtrip.<locals>.<listcomp>	r$r4N)�
isinstancer�r9�read�closerr&�__next__rFrrr;)r>rA�coder:�tokens5�tokens2�bytes_from2�	readline2�
tokens2_from2�bytes_from5�	readline5�
tokens2_from5s            r2r�zTestRoundtrip.check_roundtrip�sM���a����	��8�8�G�$�$�D�D��6�6�8�8�D�
�G�G�I�I�I�������6�6�7�7�@���x��)�)�*�*��.�.�g�.�.�.�� ��)�)����/�/��/�>�>�?�?�H�	�@�@�H�Y�,?�,?�@�@�@�
������0�0�0� ��)�)����/�/��/�>�>�?�?�H�	�@�@�H�Y�,?�,?�@�@�@�
������0�0�0�0�0r4c���|�d��|�d��|�d��tjd��}t|d��5}|�|��ddd��n#1swxYwY|�d��|�d��|�d��|�d	��dS)
Nzif x == 1:
    print(x)
z # This is a comment
# This also
zif x == 1 : 
  print(x)
�tokenize_tests.txtr�zoif x == 1:
    # A comment by itself.
    print(x) # Comment here, too.
    # Another comment.
after_if = True
zPif (x # The comments need to go in the right place
    == 1):
    print('x==1')
z�class Test: # A comment here
  # A comment with weird indent
  after_com = 5
  def x(m): return m*5 # a one liner
  def y(m): # A whitespace after the colon
     return y*4 # 3-space indent
zttry: import somemodule
except ImportError: # comment
    print('Can not import' # comment2
)else:   print('Loaded')
)r�r�findfiler)r>�fnrAs   r2�test_roundtripzTestRoundtrip.test_roundtripsj��	
���.�	/�	/�	/����-�	.�	.�	.�	
���,�	-�	-�	-�
�
�2�
3�
3��
�"�d�^�^�	$�q�� � ��#�#�#�	$�	$�	$�	$�	$�	$�	$�	$�	$�	$�	$����	$�	$�	$�	$����1�	2�	2�	2�
	
���3�	4�	4�	4�	
���B�	C�	C�	C�	
���9�	:�	:�	:�	:�	:s�$B�B
�
B
c�0�|�d��dS)Nzna = (3,4, 
5,6)
y = [3, 4,
5]
z = {'a': 5,
'b':15, 'c':True}
x = len(y) + 5 - a[
3] - a[2]
+ len(z) - z[
'b']
�r�rZs r2�test_continuationzTestRoundtrip.test_continuation3s)�����	&�		'�		'�		'�		'�		'r4c�Z�|�d��|�d��dS)Nz)x=1+\
1
# This is a comment\
# This also
z# Comment \
x = 0r9rZs r2rz)TestRoundtrip.test_backslash_continuation@sH�����-�	.�	.�	.�	
���%�	&�	&�	&�	&�	&r4c�0�|�d��dS)Nz'' ''r9rZs r2�test_string_concatenationz'TestRoundtrip.test_string_concatenationIs�����W�%�%�%�%�%r4c�p�ddl}ddl}tjd��}tj�|��ptj}|�tj�|�	|��d����}|�
tj�|d����dD]8}|�
tj�|d��|z���9tjd��s|�|d��}|D]�}tj
d	krtd
|��t|d��5}|�|���5|�|��ddd��n#1swxYwYddd��n#1swxYwY��dS)
Nrr4ztest*.pyztest_unicode_identifiers.py)�buffer�builtin�fileio�inspectr��platform�sysz
test_%s.py�cpu�
r-rr�rr)�glob�randomrr5r�r�r��curdirr��escape�remove�is_resource_enabled�sample�verboserxr�subTestr�)r>rGrHr6�tempdir�	testfilesrA�testfiles        r2�test_random_fileszTestRoundtrip.test_random_filesMs��	��������
�
�2�
3�
3���'�/�/�"�%�%�2�����I�I�b�g�l�l�4�;�;�w�+?�+?��L�L�M�M�	�	�������g�/L�M�M�N�N�N�T�	F�	F�A����R�W�\�\�'�<�@�@�1�D�E�E�E�E��*�5�1�1�	5��
�
�i��4�4�I�!�	,�	,�H���!�#�#��j�(�+�+�+��h��%�%�
,���\�\�x�\�0�0�,�,��(�(��+�+�+�,�,�,�,�,�,�,�,�,�,�,����,�,�,�,�
,�
,�
,�
,�
,�
,�
,�
,�
,�
,�
,����
,�
,�
,�
,��	,�	,s6�F*�1F�F*�F�F*�F�F*�*F.	�1F.	c���t|t��r|�d��}tt	t|��j�����d��S)Nr8)r&r�r9rrrr:r�)r>r*s  r2�	roundtripzTestRoundtrip.roundtripjsR���d�C� � �	(��;�;�w�'�'�D��(�7�4�=�=�#9�:�:�;�;�B�B�7�K�K�Kr4c���d}|�|���d��}|�|d|d��|�|��dS)z�
        Ensure that although whitespace might be mutated in a roundtrip,
        the semantic meaning of the indentation remains consistent.
        zif False:
	x=3
	x=3
r�rr-N)rU�splitr;r�)r>r*�	codeliness   r2�#test_indentation_semantics_retainedz1TestRoundtrip.test_indentation_semantics_retainedos`��
+���N�N�4�(�(�.�.�t�4�4�	�����1��y��|�4�4�4����T�"�"�"�"�"r4N)r�r�r�r�r7r:rr=rSrUrYr�r4r2r�r��s�������1�1�1�@%:�%:�%:�N'�'�'�&�&�&�&�&�&�,�,�,�:L�L�L�
#�#�#�#�#r4r�c�t�eZdZd�Zd�Zd�Zd�Zd�Zd�Zd�Z	d�Z
d	�Zd
�Zd�Z
d�Zd
�Zd�Zd�Zd�Zd�ZdS)�
CTokenizeTestc��|�|���5tt|��|��}|�||��������ddd��dS#1swxYwYdS)N)r)rOr3rr;r<r&)r>r?r@r*s    r2rBzCTokenizeTest.check_tokenize{s����\�\��\�
#�
#�	E�	E�1�1�!�4�4�a���F�
���V�X�_�_�%6�%6�%A�%A�%C�%C�D�D�D�		E�	E�	E�	E�	E�	E�	E�	E�	E�	E�	E�	E����	E�	E�	E�	E�	E�	Es�AA<�<B�Bc��|�dd��|�dd��|�dd��|�dd��|�d	d
��|�dd��|�d
d��|�dd��|�dd��dS)NrPz�    NUMBER     '0xff'        (1, 0) (1, 4)
    LESSEQUAL  '<='          (1, 5) (1, 7)
    NUMBER     '255'         (1, 8) (1, 11)
    rQz�    NUMBER     '0b10'        (1, 0) (1, 4)
    LESSEQUAL  '<='          (1, 5) (1, 7)
    NUMBER     '255'         (1, 8) (1, 11)
    rRz�    NUMBER     '0o123'       (1, 0) (1, 5)
    LESSEQUAL  '<='          (1, 6) (1, 8)
    NUMBER     '0O123'       (1, 9) (1, 14)
    rSz�    NUMBER     '1234567'     (1, 0) (1, 7)
    GREATER    '>'           (1, 8) (1, 9)
    TILDE      '~'           (1, 10) (1, 11)
    NUMBER     '0x15'        (1, 11) (1, 15)
    rTz�    NUMBER     '2134568'     (1, 0) (1, 7)
    NOTEQUAL   '!='          (1, 8) (1, 10)
    NUMBER     '1231515'     (1, 11) (1, 18)
    rUac    LPAR       '('           (1, 0) (1, 1)
    MINUS      '-'           (1, 1) (1, 2)
    NUMBER     '124561'      (1, 2) (1, 8)
    MINUS      '-'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    RPAR       ')'           (1, 10) (1, 11)
    AMPER      '&'           (1, 12) (1, 13)
    NUMBER     '200000000'   (1, 14) (1, 23)
    rVz�    NUMBER     '0xdeadbeef'  (1, 0) (1, 10)
    NOTEQUAL   '!='          (1, 11) (1, 13)
    MINUS      '-'           (1, 14) (1, 15)
    NUMBER     '1'           (1, 15) (1, 16)
    rWz�    NUMBER     '0xdeadc0de'  (1, 0) (1, 10)
    AMPER      '&'           (1, 11) (1, 12)
    NUMBER     '12345'       (1, 13) (1, 18)
    rXz�    NUMBER     '0xFF'        (1, 0) (1, 4)
    AMPER      '&'           (1, 5) (1, 6)
    NUMBER     '0x15'        (1, 7) (1, 11)
    VBAR       '|'           (1, 12) (1, 13)
    NUMBER     '1234'        (1, 14) (1, 18)
    rYrZs r2r[zCTokenizeTest.test_int�s:�����M�,�		�		�		�	
���M�,�		�		�		�	
���,�/�		�		�		�	
���-�0�		�		�		�	
���0�3�		�		�		�	
���5�	8�			�			�			�	
���.�1�		�		�		�	
���0�3�		�		�		�	
���0�3�		�		�		�		�		r4c�:�|�dd��|�dd��|�dd��|�dd��|�d	d
��|�dd��|�d
d��dS)Nr_z�    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '3.14159'     (1, 4) (1, 11)
    r`z�    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '314159.'     (1, 4) (1, 11)
    raz�    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '.314159'     (1, 4) (1, 11)
    rbz�    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '3e14159'     (1, 4) (1, 11)
    rcz�    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '3E123'       (1, 4) (1, 9)
    rdz�    NAME       'x'           (1, 0) (1, 1)
    PLUS       '+'           (1, 1) (1, 2)
    NAME       'y'           (1, 2) (1, 3)
    EQUAL      '='           (1, 4) (1, 5)
    NUMBER     '3e-1230'     (1, 6) (1, 13)
    rez�    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '3.14e159'    (1, 4) (1, 12)
    rYrZs r2rfzCTokenizeTest.test_float�s������M�,�		�		�		�	
���M�,�		�		�		�	
���M�,�		�		�		�	
���M�,�		�		�		�	
���K�*�		�		�		�	
���O�.�		�		�		�	
���N�-�		�		�		�		�		r4c�~�|�dd��|�dd��|�dd��|�dd��|�d	d
��|�dd��|�d
d��|�dd��|�dd��|�dd��|�dd��|�dd��|�dd��|�dd��|�dd��|�dd ��|�d!d"��|�d#d$��|�d%d&��|�d'd(��|�d)d*��|�d+d,��|�d-d.��|�d/d0��|�d1d2��|�d3d4��dS)5Nrpa5    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     "''"          (1, 4) (1, 6)
    SEMI       ';'           (1, 6) (1, 7)
    NAME       'y'           (1, 8) (1, 9)
    EQUAL      '='           (1, 10) (1, 11)
    STRING     '""'          (1, 12) (1, 14)
    rqa6    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     '\'"\''       (1, 4) (1, 7)
    SEMI       ';'           (1, 7) (1, 8)
    NAME       'y'           (1, 9) (1, 10)
    EQUAL      '='           (1, 11) (1, 12)
    STRING     '"\'"'        (1, 13) (1, 16)
    rrz�    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     '"doesn\'t "' (1, 4) (1, 14)
    NAME       'shrink'      (1, 14) (1, 20)
    STRING     '", does it"' (1, 20) (1, 31)
    rsz�    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     "'abc'"       (1, 4) (1, 9)
    PLUS       '+'           (1, 10) (1, 11)
    STRING     "'ABC'"       (1, 12) (1, 17)
    rtz�    NAME       'y'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     '"ABC"'       (1, 4) (1, 9)
    PLUS       '+'           (1, 10) (1, 11)
    STRING     '"ABC"'       (1, 12) (1, 17)
    rua�    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     "r'abc'"      (1, 4) (1, 10)
    PLUS       '+'           (1, 11) (1, 12)
    STRING     "r'ABC'"      (1, 13) (1, 19)
    PLUS       '+'           (1, 20) (1, 21)
    STRING     "R'ABC'"      (1, 22) (1, 28)
    PLUS       '+'           (1, 29) (1, 30)
    STRING     "R'ABC'"      (1, 31) (1, 37)
    rva�    NAME       'y'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     'r"abc"'      (1, 4) (1, 10)
    PLUS       '+'           (1, 11) (1, 12)
    STRING     'r"ABC"'      (1, 13) (1, 19)
    PLUS       '+'           (1, 20) (1, 21)
    STRING     'R"ABC"'      (1, 22) (1, 28)
    PLUS       '+'           (1, 29) (1, 30)
    STRING     'R"ABC"'      (1, 31) (1, 37)
    rwz�    STRING     "u'abc'"      (1, 0) (1, 6)
    PLUS       '+'           (1, 7) (1, 8)
    STRING     "U'abc'"      (1, 9) (1, 15)
    rxz�    STRING     'u"abc"'      (1, 0) (1, 6)
    PLUS       '+'           (1, 7) (1, 8)
    STRING     'U"abc"'      (1, 9) (1, 15)
    ryz�    STRING     "b'abc'"      (1, 0) (1, 6)
    PLUS       '+'           (1, 7) (1, 8)
    STRING     "B'abc'"      (1, 9) (1, 15)
    rzz�    STRING     'b"abc"'      (1, 0) (1, 6)
    PLUS       '+'           (1, 7) (1, 8)
    STRING     'B"abc"'      (1, 9) (1, 15)
    r{a;    STRING     "br'abc'"     (1, 0) (1, 7)
    PLUS       '+'           (1, 8) (1, 9)
    STRING     "bR'abc'"     (1, 10) (1, 17)
    PLUS       '+'           (1, 18) (1, 19)
    STRING     "Br'abc'"     (1, 20) (1, 27)
    PLUS       '+'           (1, 28) (1, 29)
    STRING     "BR'abc'"     (1, 30) (1, 37)
    r|a;    STRING     'br"abc"'     (1, 0) (1, 7)
    PLUS       '+'           (1, 8) (1, 9)
    STRING     'bR"abc"'     (1, 10) (1, 17)
    PLUS       '+'           (1, 18) (1, 19)
    STRING     'Br"abc"'     (1, 20) (1, 27)
    PLUS       '+'           (1, 28) (1, 29)
    STRING     'BR"abc"'     (1, 30) (1, 37)
    r}a;    STRING     "rb'abc'"     (1, 0) (1, 7)
    PLUS       '+'           (1, 8) (1, 9)
    STRING     "rB'abc'"     (1, 10) (1, 17)
    PLUS       '+'           (1, 18) (1, 19)
    STRING     "Rb'abc'"     (1, 20) (1, 27)
    PLUS       '+'           (1, 28) (1, 29)
    STRING     "RB'abc'"     (1, 30) (1, 37)
    r~a;    STRING     'rb"abc"'     (1, 0) (1, 7)
    PLUS       '+'           (1, 8) (1, 9)
    STRING     'rB"abc"'     (1, 10) (1, 17)
    PLUS       '+'           (1, 18) (1, 19)
    STRING     'Rb"abc"'     (1, 20) (1, 27)
    PLUS       '+'           (1, 28) (1, 29)
    STRING     'RB"abc"'     (1, 30) (1, 37)
    rr�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�rYrZs r2r�zCTokenizeTest.test_string�s������.�1�		�		�		�	
���1�4�		�		�		�	
���>�A�		�		�		�	
���/�2�		�		�		�	
���/�2�		�		�		�	
���C�
F�
		�
		�
		�	
���C�
F�
		�
		�
		�	
���-�0�		�		�		�	
���-�0�		�		�		�	
���-�0�		�		�		�	
���-�0�		�		�		�	
���C�F�		�		�		�	
���C�F�		�		�		�	
���C�F�		�		�		�	
���C�F�		�		�		�	
���-�0�		�		�		�	
���L�+�		�		�		�	
���L�+�		�		�		�	
�����		�		�		�	
�����		�		�		�	
�����		�		�		�	
���H�'�		�		�		�	
���K�*�		�		�		�	
���L�+�		�		�		�	
�����		�		�		�
	
�����		�		�		�		�		r4c��|�dd��|�dd��|�dd��dS)Nr�a    NAME       'def'         (1, 0) (1, 3)
    NAME       'd22'         (1, 4) (1, 7)
    LPAR       '('           (1, 7) (1, 8)
    NAME       'a'           (1, 8) (1, 9)
    COMMA      ','           (1, 9) (1, 10)
    NAME       'b'           (1, 11) (1, 12)
    COMMA      ','           (1, 12) (1, 13)
    NAME       'c'           (1, 14) (1, 15)
    EQUAL      '='           (1, 15) (1, 16)
    NUMBER     '2'           (1, 16) (1, 17)
    COMMA      ','           (1, 17) (1, 18)
    NAME       'd'           (1, 19) (1, 20)
    EQUAL      '='           (1, 20) (1, 21)
    NUMBER     '2'           (1, 21) (1, 22)
    COMMA      ','           (1, 22) (1, 23)
    STAR       '*'           (1, 24) (1, 25)
    NAME       'k'           (1, 25) (1, 26)
    RPAR       ')'           (1, 26) (1, 27)
    COLON      ':'           (1, 27) (1, 28)
    NAME       'pass'        (1, 29) (1, 33)
    r�a�    NAME       'def'         (1, 0) (1, 3)
    NAME       'd01v_'       (1, 4) (1, 9)
    LPAR       '('           (1, 9) (1, 10)
    NAME       'a'           (1, 10) (1, 11)
    EQUAL      '='           (1, 11) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    COMMA      ','           (1, 13) (1, 14)
    STAR       '*'           (1, 15) (1, 16)
    NAME       'k'           (1, 16) (1, 17)
    COMMA      ','           (1, 17) (1, 18)
    DOUBLESTAR '**'          (1, 19) (1, 21)
    NAME       'w'           (1, 21) (1, 22)
    RPAR       ')'           (1, 22) (1, 23)
    COLON      ':'           (1, 23) (1, 24)
    NAME       'pass'        (1, 25) (1, 29)
    r�a�    NAME       'def'         (1, 0) (1, 3)
    NAME       'd23'         (1, 4) (1, 7)
    LPAR       '('           (1, 7) (1, 8)
    NAME       'a'           (1, 8) (1, 9)
    COLON      ':'           (1, 9) (1, 10)
    NAME       'str'         (1, 11) (1, 14)
    COMMA      ','           (1, 14) (1, 15)
    NAME       'b'           (1, 16) (1, 17)
    COLON      ':'           (1, 17) (1, 18)
    NAME       'int'         (1, 19) (1, 22)
    EQUAL      '='           (1, 22) (1, 23)
    NUMBER     '3'           (1, 23) (1, 24)
    RPAR       ')'           (1, 24) (1, 25)
    RARROW     '->'          (1, 26) (1, 28)
    NAME       'int'         (1, 29) (1, 32)
    COLON      ':'           (1, 32) (1, 33)
    NAME       'pass'        (1, 34) (1, 38)
    rYrZs r2r�zCTokenizeTest.test_function�sr�����?�B�		�		�		�.	
���;�>�		�		�		�$	
���D�G�		�		�		�		�		r4c�2�|�dd��dS)Nr�a�    NAME       'if'          (1, 0) (1, 2)
    NUMBER     '1'           (1, 3) (1, 4)
    LESS       '<'           (1, 5) (1, 6)
    NUMBER     '1'           (1, 7) (1, 8)
    GREATER    '>'           (1, 9) (1, 10)
    NUMBER     '1'           (1, 11) (1, 12)
    EQEQUAL    '=='          (1, 13) (1, 15)
    NUMBER     '1'           (1, 16) (1, 17)
    GREATEREQUAL '>='          (1, 18) (1, 20)
    NUMBER     '5'           (1, 21) (1, 22)
    LESSEQUAL  '<='          (1, 23) (1, 25)
    NUMBER     '0x15'        (1, 26) (1, 30)
    LESSEQUAL  '<='          (1, 31) (1, 33)
    NUMBER     '0x12'        (1, 34) (1, 38)
    NOTEQUAL   '!='          (1, 39) (1, 41)
    NUMBER     '1'           (1, 42) (1, 43)
    NAME       'and'         (1, 44) (1, 47)
    NUMBER     '5'           (1, 48) (1, 49)
    NAME       'in'          (1, 50) (1, 52)
    NUMBER     '1'           (1, 53) (1, 54)
    NAME       'not'         (1, 55) (1, 58)
    NAME       'in'          (1, 59) (1, 61)
    NUMBER     '1'           (1, 62) (1, 63)
    NAME       'is'          (1, 64) (1, 66)
    NUMBER     '1'           (1, 67) (1, 68)
    NAME       'or'          (1, 69) (1, 71)
    NUMBER     '5'           (1, 72) (1, 73)
    NAME       'is'          (1, 74) (1, 76)
    NAME       'not'         (1, 77) (1, 80)
    NUMBER     '1'           (1, 81) (1, 82)
    COLON      ':'           (1, 82) (1, 83)
    NAME       'pass'        (1, 84) (1, 88)
    rYrZs r2r�zCTokenizeTest.test_comparison�r�r4c�2�|�dd��dS)Nr�a$    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    MINUS      '-'           (1, 6) (1, 7)
    NAME       'y'           (1, 8) (1, 9)
    PLUS       '+'           (1, 10) (1, 11)
    NUMBER     '15'          (1, 12) (1, 14)
    MINUS      '-'           (1, 15) (1, 16)
    NUMBER     '1'           (1, 17) (1, 18)
    PLUS       '+'           (1, 19) (1, 20)
    NUMBER     '0x124'       (1, 21) (1, 26)
    PLUS       '+'           (1, 27) (1, 28)
    NAME       'z'           (1, 29) (1, 30)
    PLUS       '+'           (1, 31) (1, 32)
    NAME       'a'           (1, 33) (1, 34)
    LSQB       '['           (1, 34) (1, 35)
    NUMBER     '5'           (1, 35) (1, 36)
    RSQB       ']'           (1, 36) (1, 37)
    rYrZs r2r�zCTokenizeTest.test_additiver�r4c�2�|�dd��dS)Nr�a�    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    DOUBLESLASH '//'          (1, 5) (1, 7)
    NUMBER     '1'           (1, 7) (1, 8)
    STAR       '*'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    SLASH      '/'           (1, 10) (1, 11)
    NUMBER     '5'           (1, 11) (1, 12)
    STAR       '*'           (1, 12) (1, 13)
    NUMBER     '12'          (1, 13) (1, 15)
    PERCENT    '%'           (1, 15) (1, 16)
    NUMBER     '0x12'        (1, 16) (1, 20)
    AT         '@'           (1, 20) (1, 21)
    NUMBER     '42'          (1, 21) (1, 23)
    rYrZs r2r�z!CTokenizeTest.test_multiplicative%r�r4c�^�|�dd��|�dd��dS)Nr�a�    TILDE      '~'           (1, 0) (1, 1)
    NUMBER     '1'           (1, 1) (1, 2)
    CIRCUMFLEX '^'           (1, 3) (1, 4)
    NUMBER     '1'           (1, 5) (1, 6)
    AMPER      '&'           (1, 7) (1, 8)
    NUMBER     '1'           (1, 9) (1, 10)
    VBAR       '|'           (1, 11) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    CIRCUMFLEX '^'           (1, 14) (1, 15)
    MINUS      '-'           (1, 16) (1, 17)
    NUMBER     '1'           (1, 17) (1, 18)
    r�aI    MINUS      '-'           (1, 0) (1, 1)
    NUMBER     '1'           (1, 1) (1, 2)
    STAR       '*'           (1, 2) (1, 3)
    NUMBER     '1'           (1, 3) (1, 4)
    SLASH      '/'           (1, 4) (1, 5)
    NUMBER     '1'           (1, 5) (1, 6)
    PLUS       '+'           (1, 6) (1, 7)
    NUMBER     '1'           (1, 7) (1, 8)
    STAR       '*'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    DOUBLESLASH '//'          (1, 10) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    MINUS      '-'           (1, 14) (1, 15)
    MINUS      '-'           (1, 16) (1, 17)
    MINUS      '-'           (1, 17) (1, 18)
    MINUS      '-'           (1, 18) (1, 19)
    NUMBER     '1'           (1, 19) (1, 20)
    DOUBLESTAR '**'          (1, 20) (1, 22)
    NUMBER     '1'           (1, 22) (1, 23)
    rYrZs r2r�zCTokenizeTest.test_unary9sN�����0�3�		�		�		�	
���5�8�		�		�		�		�		r4c�2�|�dd��dS)Nr�a�    NAME       'import'      (1, 0) (1, 6)
    NAME       'sys'         (1, 7) (1, 10)
    COMMA      ','           (1, 10) (1, 11)
    NAME       'time'        (1, 12) (1, 16)
    NEWLINE    ''            (1, 16) (1, 16)
    NAME       'x'           (2, 0) (2, 1)
    EQUAL      '='           (2, 2) (2, 3)
    NAME       'sys'         (2, 4) (2, 7)
    DOT        '.'           (2, 7) (2, 8)
    NAME       'modules'     (2, 8) (2, 15)
    LSQB       '['           (2, 15) (2, 16)
    STRING     "'time'"      (2, 16) (2, 22)
    RSQB       ']'           (2, 22) (2, 23)
    DOT        '.'           (2, 23) (2, 24)
    NAME       'time'        (2, 24) (2, 28)
    LPAR       '('           (2, 28) (2, 29)
    RPAR       ')'           (2, 29) (2, 30)
    rYrZs r2r�zCTokenizeTest.test_selector_r�r4c�2�|�dd��dS�Nr�a    AT         '@'           (1, 0) (1, 1)
    NAME       'staticmethod' (1, 1) (1, 13)
    NEWLINE    ''            (1, 13) (1, 13)
    NAME       'def'         (2, 0) (2, 3)
    NAME       'foo'         (2, 4) (2, 7)
    LPAR       '('           (2, 7) (2, 8)
    NAME       'x'           (2, 8) (2, 9)
    COMMA      ','           (2, 9) (2, 10)
    NAME       'y'           (2, 10) (2, 11)
    RPAR       ')'           (2, 11) (2, 12)
    COLON      ':'           (2, 12) (2, 13)
    NAME       'pass'        (2, 14) (2, 18)
    rYrZs r2r�zCTokenizeTest.test_methodur�r4c�2�|�dd��dSrgrYrZs r2r�zCTokenizeTest.test_tabs�r�r4c��|�dd��|�dd��|�dd��|�dd��|�d	d
��|�dd��|�d
d��|�dd��|�dd��|�dd��|�dd��|�dd��|�dd��|�dd��|�dd��|�dd ��|�d!d"��|�d#d$��|�d%d&��|�d'd(��|�d)d*��dS)+Nr�z�    ASYNC      'async'       (1, 0) (1, 5)
    EQUAL      '='           (1, 6) (1, 7)
    NUMBER     '1'           (1, 8) (1, 9)
    r�a8    NAME       'a'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    LPAR       '('           (1, 4) (1, 5)
    ASYNC      'async'       (1, 5) (1, 10)
    EQUAL      '='           (1, 11) (1, 12)
    NUMBER     '1'           (1, 13) (1, 14)
    RPAR       ')'           (1, 14) (1, 15)
    r�z�    ASYNC      'async'       (1, 0) (1, 5)
    LPAR       '('           (1, 5) (1, 6)
    RPAR       ')'           (1, 6) (1, 7)
    r�a<    NAME       'class'       (1, 0) (1, 5)
    ASYNC      'async'       (1, 6) (1, 11)
    LPAR       '('           (1, 11) (1, 12)
    NAME       'Bar'         (1, 12) (1, 15)
    RPAR       ')'           (1, 15) (1, 16)
    COLON      ':'           (1, 16) (1, 17)
    NAME       'pass'        (1, 17) (1, 21)
    r�z�    NAME       'class'       (1, 0) (1, 5)
    ASYNC      'async'       (1, 6) (1, 11)
    COLON      ':'           (1, 11) (1, 12)
    NAME       'pass'        (1, 12) (1, 16)
    r�z�    AWAIT      'await'       (1, 0) (1, 5)
    EQUAL      '='           (1, 6) (1, 7)
    NUMBER     '1'           (1, 8) (1, 9)
    r�z�    NAME       'foo'         (1, 0) (1, 3)
    DOT        '.'           (1, 3) (1, 4)
    ASYNC      'async'       (1, 4) (1, 9)
    r�a;    ASYNC      'async'       (1, 0) (1, 5)
    NAME       'for'         (1, 6) (1, 9)
    NAME       'a'           (1, 10) (1, 11)
    NAME       'in'          (1, 12) (1, 14)
    NAME       'b'           (1, 15) (1, 16)
    COLON      ':'           (1, 16) (1, 17)
    NAME       'pass'        (1, 18) (1, 22)
    r�a<    ASYNC      'async'       (1, 0) (1, 5)
    NAME       'with'        (1, 6) (1, 10)
    NAME       'a'           (1, 11) (1, 12)
    NAME       'as'          (1, 13) (1, 15)
    NAME       'b'           (1, 16) (1, 17)
    COLON      ':'           (1, 17) (1, 18)
    NAME       'pass'        (1, 19) (1, 23)
    r�z�    ASYNC      'async'       (1, 0) (1, 5)
    DOT        '.'           (1, 5) (1, 6)
    NAME       'foo'         (1, 6) (1, 9)
    r�z/    ASYNC      'async'       (1, 0) (1, 5)
    r�z�    ASYNC      'async'       (1, 0) (1, 5)
    NEWLINE    ''            (1, 5) (1, 5)
    AWAIT      'await'       (3, 0) (3, 5)
    r�z�    ASYNC      'async'       (1, 0) (1, 5)
    NEWLINE    ''            (1, 5) (1, 5)
    ELLIPSIS   '...'         (2, 0) (2, 3)
    NEWLINE    ''            (2, 3) (2, 3)
    AWAIT      'await'       (3, 0) (3, 5)
    r�z�    ASYNC      'async'       (1, 0) (1, 5)
    NEWLINE    ''            (1, 5) (1, 5)
    AWAIT      'await'       (2, 0) (2, 5)
    r�z�    NAME       'foo'         (1, 0) (1, 3)
    DOT        '.'           (1, 3) (1, 4)
    ASYNC      'async'       (1, 4) (1, 9)
    PLUS       '+'           (1, 10) (1, 11)
    NUMBER     '1'           (1, 12) (1, 13)
    r�a;    ASYNC      'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    LPAR       '('           (1, 13) (1, 14)
    RPAR       ')'           (1, 14) (1, 15)
    COLON      ':'           (1, 15) (1, 16)
    NAME       'pass'        (1, 17) (1, 21)
    r�a�    ASYNC      'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    LPAR       '('           (1, 13) (1, 14)
    RPAR       ')'           (1, 14) (1, 15)
    COLON      ':'           (1, 15) (1, 16)
    NEWLINE    ''            (1, 16) (1, 16)
    INDENT     ''            (2, -1) (2, -1)
    NAME       'def'         (2, 2) (2, 5)
    NAME       'foo'         (2, 6) (2, 9)
    LPAR       '('           (2, 9) (2, 10)
    AWAIT      'await'       (2, 10) (2, 15)
    RPAR       ')'           (2, 15) (2, 16)
    COLON      ':'           (2, 16) (2, 17)
    NEWLINE    ''            (2, 17) (2, 17)
    INDENT     ''            (3, -1) (3, -1)
    AWAIT      'await'       (3, 4) (3, 9)
    EQUAL      '='           (3, 10) (3, 11)
    NUMBER     '1'           (3, 12) (3, 13)
    NEWLINE    ''            (3, 13) (3, 13)
    DEDENT     ''            (4, -1) (4, -1)
    NAME       'if'          (4, 2) (4, 4)
    NUMBER     '1'           (4, 5) (4, 6)
    COLON      ':'           (4, 6) (4, 7)
    NEWLINE    ''            (4, 7) (4, 7)
    INDENT     ''            (5, -1) (5, -1)
    AWAIT      'await'       (5, 4) (5, 9)
    NEWLINE    ''            (5, 9) (5, 9)
    DEDENT     ''            (6, -1) (6, -1)
    DEDENT     ''            (6, -1) (6, -1)
    ASYNC      'async'       (6, 0) (6, 5)
    PLUSEQUAL  '+='          (6, 6) (6, 8)
    NUMBER     '1'           (6, 9) (6, 10)
    NEWLINE    ''            (6, 10) (6, 10)
    r�a�    ASYNC      'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    LPAR       '('           (1, 13) (1, 14)
    RPAR       ')'           (1, 14) (1, 15)
    COLON      ':'           (1, 15) (1, 16)
    NEWLINE    ''            (1, 16) (1, 16)
    INDENT     ''            (2, -1) (2, -1)
    ASYNC      'async'       (2, 2) (2, 7)
    NAME       'for'         (2, 8) (2, 11)
    NAME       'i'           (2, 12) (2, 13)
    NAME       'in'          (2, 14) (2, 16)
    NUMBER     '1'           (2, 17) (2, 18)
    COLON      ':'           (2, 18) (2, 19)
    NAME       'pass'        (2, 20) (2, 24)
    DEDENT     ''            (2, -1) (2, -1)
    r�ah    ASYNC      'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    LPAR       '('           (1, 13) (1, 14)
    ASYNC      'async'       (1, 14) (1, 19)
    RPAR       ')'           (1, 19) (1, 20)
    COLON      ':'           (1, 20) (1, 21)
    AWAIT      'await'       (1, 22) (1, 27)
    r�a~    NAME       'def'         (1, 0) (1, 3)
    NAME       'f'           (1, 4) (1, 5)
    LPAR       '('           (1, 5) (1, 6)
    RPAR       ')'           (1, 6) (1, 7)
    COLON      ':'           (1, 7) (1, 8)
    NEWLINE    ''            (1, 8) (1, 8)
    INDENT     ''            (3, -1) (3, -1)
    NAME       'def'         (3, 2) (3, 5)
    NAME       'baz'         (3, 6) (3, 9)
    LPAR       '('           (3, 9) (3, 10)
    RPAR       ')'           (3, 10) (3, 11)
    COLON      ':'           (3, 11) (3, 12)
    NAME       'pass'        (3, 13) (3, 17)
    NEWLINE    ''            (3, 17) (3, 17)
    ASYNC      'async'       (4, 2) (4, 7)
    NAME       'def'         (4, 8) (4, 11)
    NAME       'bar'         (4, 12) (4, 15)
    LPAR       '('           (4, 15) (4, 16)
    RPAR       ')'           (4, 16) (4, 17)
    COLON      ':'           (4, 17) (4, 18)
    NAME       'pass'        (4, 19) (4, 23)
    NEWLINE    ''            (4, 23) (4, 23)
    AWAIT      'await'       (6, 2) (6, 7)
    EQUAL      '='           (6, 8) (6, 9)
    NUMBER     '2'           (6, 10) (6, 11)
    DEDENT     ''            (6, -1) (6, -1)
    r�a�    ASYNC      'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'f'           (1, 10) (1, 11)
    LPAR       '('           (1, 11) (1, 12)
    RPAR       ')'           (1, 12) (1, 13)
    COLON      ':'           (1, 13) (1, 14)
    NEWLINE    ''            (1, 14) (1, 14)
    INDENT     ''            (3, -1) (3, -1)
    NAME       'def'         (3, 2) (3, 5)
    NAME       'baz'         (3, 6) (3, 9)
    LPAR       '('           (3, 9) (3, 10)
    RPAR       ')'           (3, 10) (3, 11)
    COLON      ':'           (3, 11) (3, 12)
    NAME       'pass'        (3, 13) (3, 17)
    NEWLINE    ''            (3, 17) (3, 17)
    ASYNC      'async'       (4, 2) (4, 7)
    NAME       'def'         (4, 8) (4, 11)
    NAME       'bar'         (4, 12) (4, 15)
    LPAR       '('           (4, 15) (4, 16)
    RPAR       ')'           (4, 16) (4, 17)
    COLON      ':'           (4, 17) (4, 18)
    NAME       'pass'        (4, 19) (4, 23)
    NEWLINE    ''            (4, 23) (4, 23)
    AWAIT      'await'       (6, 2) (6, 7)
    EQUAL      '='           (6, 8) (6, 9)
    NUMBER     '2'           (6, 10) (6, 11)
    DEDENT     ''            (6, -1) (6, -1)
    rYrZs r2r�zCTokenizeTest.test_async�s������K�*�		�		�		�	
���-�0�		�		�		�	
���I�(�		�		�		�	
���3�6�		�		�		�	
���.�1�		�		�		�	
���K�*�		�		�		�	
���K�*�		�		�		�	
���4�7�		�		�		�	
���5�8�		�		�		�	
���K�*�		�		�		�	
���G�&�		�		�		�	
���4�7�		�		�		�	
���/�2�		�		�		�	
���N�-�		�		�		�	
���O�.�		�		�		�	
���3�6�		�		�		�	
����#�*		�*		�*		�X	
���H�K�		�		�		�&	
���9�	<�			�			�			�	
�����
!		�!		�!		�F	
�����
"		�"		�"		�"		�"		r4c�2�|�dd��dS)Nr�u7    NAME       'Örter'       (1, 0) (1, 6)
    EQUAL      '='           (1, 7) (1, 8)
    STRING     "u'places'"   (1, 9) (1, 18)
    NEWLINE    ''            (1, 18) (1, 18)
    NAME       'grün'        (2, 0) (2, 5)
    EQUAL      '='           (2, 6) (2, 7)
    STRING     "U'green'"    (2, 8) (2, 16)
    rYrZs r2r�zCTokenizeTest.test_unicode�	r�r4c��d�}|�t|d��|�t|d��|�t|d��|�t|d��|�t|d��|�t|d��|�t|d��|�t|d	��|�t|d
��|�t|d��|�t|d��|�t|d
��|�t|d��|�t|d��|�t|d��|�t|d��|�t|d��|�t|d��|�t|d��|�t|d��|�t|d��|�t|d��|�t|d��|�t|d��|�t|d��|�t|d��|�t|d��|�t|d��|�t|d��|�t|d��dS)Nc�:�tt|����Sr�)rFr�r�s r2�
get_tokensz5CTokenizeTest.test_invalid_syntax.<locals>.get_tokens�	s���9�&�A�A�B�B�Br4z(1+2]z(1+2}z{1+2]�1_z1.2_�1e2_z1e+� u€�0b12�0b1_2�0b2�0b1_�0b�0o18�0o1_8�0o8�0o1_�0o�0x1_�0x�012z'sdfsdfz'''sdfsdf''a�((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((a))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))�])rr)r>rns  r2�test_invalid_syntaxz!CTokenizeTest.test_invalid_syntax�	s���	C�	C�	C�	
���+�z�7�;�;�;����+�z�7�;�;�;����+�z�7�;�;�;����+�z�4�8�8�8����+�z�6�:�:�:����+�z�6�:�:�:����+�z�5�9�9�9����+�z�6�:�:�:����+�z�5�9�9�9����+�z�6�:�:�:����+�z�7�;�;�;����+�z�5�9�9�9����+�z�6�:�:�:����+�z�4�8�8�8����+�z�6�:�:�:����+�z�7�;�;�;����+�z�5�9�9�9����+�z�6�:�:�:����+�z�4�8�8�8����+�z�6�:�:�:����+�z�4�8�8�8����+�z�4�8�8�8����+�z�5�9�9�9����+�z�6�:�:�:����+�z�6�:�:�:����+�z�5�9�9�9����+�z�9�=�=�=����+�z�=�A�A�A����+�z�3H�I�I�I����+�z�3�7�7�7�7�7r4c��d}d�}||dz
��}tt|����}|�|djt��t|dd��||��}tt|����}|�|djt��|�tt
|dd��dS)N�dc�t�d�d�t|��D����}|d|zdzz
}|S)N�c3�&K�|]}d|zdzV��
dS)�  z	if True:
Nr�)r!rDs  r2�	<genexpr>zICTokenizeTest.test_max_indent.<locals>.generate_source.<locals>.<genexpr>�	s+����O�O�1�d�Q�h�,�6�O�O�O�O�O�Or4r�zpass
)r�r�)�indentsrs  r2�generate_sourcez6CTokenizeTest.test_max_indent.<locals>.generate_source�	s@���W�W�O�O��g���O�O�O�O�O�F��d�W�n�x�/�/�F��Mr4rr z<string>�exec)	rFrr;r-r�compilerrrK)r>�	MAXINDENTr��validrG�invalids      r2�test_max_indentzCTokenizeTest.test_max_indent�	s����	�	�	�	�
 ��	�A�
�.�.���7��>�>�?�?���������&�1�1�1���z�6�*�*�*�!�/�)�,�,���7��@�@�A�A���������'�2�2�2�����g�w�
�F�	
�	
�	
�	
�	
r4c�4�d�}td��}|�|d��td��}|�||��||����td��}|�|d��td��}|�||��||����td��}|�|d	��td
��}|�||��||����dS)Nc�4�d�t|��D��S)Nc��g|]	^}}}||f��
Sr�r�)r!�kindr�r�s    r2r"zYCTokenizeTest.test_continuation_lines_indentation.<locals>.get_tokens.<locals>.<listcomp>�	s#��g�g�g�'9��f�q�T�6�N�g�g�gr4)rrms r2rnzECTokenizeTest.test_continuation_lines_indentation.<locals>.get_tokens�	s ��g�g�=^�_e�=f�=f�g�g�g�gr4z�
            def fib(n):
                \
            '''Print a Fibonacci series up to n.'''
                \
            a, b = 0, 1
        ae    NAME       'def'         (2, 0) (2, 3)
    NAME       'fib'         (2, 4) (2, 7)
    LPAR       '('           (2, 7) (2, 8)
    NAME       'n'           (2, 8) (2, 9)
    RPAR       ')'           (2, 9) (2, 10)
    COLON      ':'           (2, 10) (2, 11)
    NEWLINE    ''            (2, 11) (2, 11)
    INDENT     ''            (4, -1) (4, -1)
    STRING     "'''Print a Fibonacci series up to n.'''" (4, 0) (4, 39)
    NEWLINE    ''            (4, 39) (4, 39)
    NAME       'a'           (6, 0) (6, 1)
    COMMA      ','           (6, 1) (6, 2)
    NAME       'b'           (6, 3) (6, 4)
    EQUAL      '='           (6, 5) (6, 6)
    NUMBER     '0'           (6, 7) (6, 8)
    COMMA      ','           (6, 8) (6, 9)
    NUMBER     '1'           (6, 10) (6, 11)
    NEWLINE    ''            (6, 11) (6, 11)
    DEDENT     ''            (6, -1) (6, -1)
        zu
            def fib(n):
                '''Print a Fibonacci series up to n.'''
                a, b = 0, 1
        z>
            pass
                \

            pass
        z�    NAME       'pass'        (2, 0) (2, 4)
    NEWLINE    ''            (2, 4) (2, 4)
    NAME       'pass'        (5, 0) (5, 4)
    NEWLINE    ''            (5, 4) (5, 4)
        z+
            pass
            pass
        z�
            if x:
                y = 1
                \
                        \
                    \
                \
                foo = 1
        aj    NAME       'if'          (2, 0) (2, 2)
    NAME       'x'           (2, 3) (2, 4)
    COLON      ':'           (2, 4) (2, 5)
    NEWLINE    ''            (2, 5) (2, 5)
    INDENT     ''            (3, -1) (3, -1)
    NAME       'y'           (3, 4) (3, 5)
    EQUAL      '='           (3, 6) (3, 7)
    NUMBER     '1'           (3, 8) (3, 9)
    NEWLINE    ''            (3, 9) (3, 9)
    NAME       'foo'         (8, 4) (8, 7)
    EQUAL      '='           (8, 8) (8, 9)
    NUMBER     '1'           (8, 10) (8, 11)
    NEWLINE    ''            (8, 11) (8, 11)
    DEDENT     ''            (8, -1) (8, -1)
        zI
            if x:
                y = 1
                foo = 1
        )rrBr;)r>rnr*�code_no_conts    r2�#test_continuation_lines_indentationz1CTokenizeTest.test_continuation_lines_indentation�	s���	h�	h�	h���
�
��	
���D�#�	
�	
�	
�,��
�
��	
�����D�)�)�:�:�l�+C�+C�D�D�D���
�
��	
���D�#�	
�	
�	
���
�
��
	
�����D�)�)�:�:�l�+C�+C�D�D�D���
�
��	
���D�#�	
�	
�	
�"��
�
��	
�����D�)�)�:�:�l�+C�+C�D�D�D�D�Dr4N)r�r�r�rBr[rfr�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r4r2r[r[zs.������E�E�E�?	�?	�?	�B,	�,	�,	�\t	�t	�t	�l=	�=	�=	�~$	�$	�$	�L	�	�	�.	�	�	�($	�$	�$	�L	�	�	�,	�	�	�"	�	�	�"H	�H	�H	�T
	�
	�
	�&8�&8�&8�P
�
�
�(`E�`E�`E�`E�`Er4r[c��eZdZd�ZdS)�CTokenizerBufferTestsc��dddz�dddz��}tj��5}t|d|��}t|��ddd��dS#1swxYwYdS)Nz"        #coding: latin-1
        #�ai'z

        #i'�foo)r�temp_dirrr)r>�test_scriptr��	file_names    r2�!test_newline_at_the_end_of_bufferz7CTokenizerBufferTests.test_newline_at_the_end_of_bufferM
s����

�e�)����e�)�����
�
!�
!�	'�X�#�H�e�[�A�A�I��I�&�&�&�	'�	'�	'�	'�	'�	'�	'�	'�	'�	'�	'�	'����	'�	'�	'�	'�	'�	's�!A�A�AN)r�r�r�r�r�r4r2r�r�L
s#������
'�
'�
'�
'�
'r4r��__main__)5�testr�test.supportrrrrrr	r
rrr
rrrryrrrrr�iorr�unittest�textwraprrr�test.test_grammarrr�test.support.script_helperrrr�r.r3r6r�r�r�r�rr!r�rr�r[r�r��mainr�r4r2�<module>r�s3��������"�"�"�"�"�"�I�I�I�I�I�I�I�I�I�I�I�I�I�I�I�I�I�I�I�I�I�I�I�I�I�I�I�I�I�I�I�I�I�I�I�I�I�I�!� � � � � � � �����������#�#�#�#�#�#�#�#�<�<�<�<�<�<�<�<�"�"�"�"�"�"�C�C�C�C�C�C�C�C�	�	�	�	�������� M	�M	�M	�M	�M	�8�M	�M	�M	�^A�A�A�A�A��A�A�A�
.�
.�
.�G�G�G�G�G�x�G�G�G�."I�"I�"I�"I�"I�H�"I�"I�"I�J#G�#G�#G�#G�#G�H�#G�#G�#G�L{"�{"�{"�{"�{"��{"�{"�{"�|RU�RU�RU�RU�RU�8�RU�RU�RU�h&>�&>�&>�&>�&>�X�&>�&>�&>�RM#�M#�M#�M#�M#�H�M#�M#�M#�`OE�OE�OE�OE�OE�H�OE�OE�OE�d'�'�'�'�'�H�-�'�'�'��z����H�M�O�O�O�O�O��r4