'\nThis tokenizer has been copied from the ``tokenize.py`` standard library\ntokenizer. The reason was simple: The standard library tokenizer fails\nif the indentation is not right. To make it possible to do error recovery the\n    tokenizer needed to be rewritten.\n\nBasically this is a stripped down version of the standard library module, so\nyou can read the documentation there. Additionally we included some speed and\nmemory optimizations here.\n'
from __future__ import absolute_import
_D='(?:\\{\\{|\\}\\}|\\\\N\\{'
_C=False
_B=None
_A=True
import sys,re,itertools as _itertools
from codecs import BOM_UTF8
from typing import NamedTuple,Tuple,Iterator,Iterable,List,Dict,Pattern,Set
from parso.python.token import PythonTokenTypes
from parso.utils import split_lines,PythonVersionInfo,parse_version_string
MAX_UNICODE='\U0010ffff'
STRING=PythonTokenTypes.STRING
NAME=PythonTokenTypes.NAME
NUMBER=PythonTokenTypes.NUMBER
OP=PythonTokenTypes.OP
NEWLINE=PythonTokenTypes.NEWLINE
INDENT=PythonTokenTypes.INDENT
DEDENT=PythonTokenTypes.DEDENT
ENDMARKER=PythonTokenTypes.ENDMARKER
ERRORTOKEN=PythonTokenTypes.ERRORTOKEN
ERROR_DEDENT=PythonTokenTypes.ERROR_DEDENT
FSTRING_START=PythonTokenTypes.FSTRING_START
FSTRING_STRING=PythonTokenTypes.FSTRING_STRING
FSTRING_END=PythonTokenTypes.FSTRING_END
class TokenCollection(NamedTuple):pseudo_token:Pattern;single_quoted:Set[str];triple_quoted:Set[str];endpats:Dict[str,Pattern];whitespace:Pattern;fstring_pattern_map:Dict[str,str];always_break_tokens:Tuple[str]
BOM_UTF8_STRING=BOM_UTF8.decode('utf-8')
_token_collection_cache={}
def group(*B,capture=_C,**C):
	assert not C;A='('
	if not capture:A+='?:'
	return A+'|'.join(B)+')'
def maybe(*A):return group(*A)+'?'
def _all_string_prefixes(*,include_fstring=_C,only_fstring=_C):
	C=only_fstring
	def E(prefix):
		for A in _itertools.product(*[(A,A.upper())for A in prefix]):yield''.join(A)
	A=['b','r','u','br'];B={''}
	if include_fstring:
		D=['f','fr']
		if C:A=D;B=set()
		else:A+=D
	elif C:return set()
	for F in A:
		for G in _itertools.permutations(F):B.update(E(G))
	return B
def _compile(expr):return re.compile(expr,re.UNICODE)
def _get_token_collection(version_info):
	A=version_info
	try:return _token_collection_cache[tuple(A)]
	except KeyError:_token_collection_cache[tuple(A)]=B=_create_token_collection(A);return B
unicode_character_name='[A-Za-z0-9\\-]+(?: [A-Za-z0-9\\-]+)*'
fstring_string_single_line=_compile(_D+unicode_character_name+'\\}|\\\\(?:\\r\\n?|\\n)|\\\\[^\\r\\nN]|[^{}\\r\\n\\\\])+')
fstring_string_multi_line=_compile(_D+unicode_character_name+'\\}|\\\\[^N]|[^{}\\\\])+')
fstring_format_spec_single_line=_compile('(?:\\\\(?:\\r\\n?|\\n)|[^{}\\r\\n])+')
fstring_format_spec_multi_line=_compile('[^{}]+')
def _create_token_collection(version_info):
	L='\\\\(?:\\r\\n?|\\n)';C='"';D="'";E='"""';F="'''";M='[ \\f\\t]*';W=_compile(M);X='#[^\\r\\n]*';Y='([A-Za-z_0-9\x80-'+MAX_UNICODE+']+)';Z='0[xX](?:_?[0-9a-fA-F])+';a='0[bB](?:_?[01])+';b='0[oO](?:_?[0-7])+';c='(?:0(?:_?0)*|[1-9](?:_?[0-9])*)';d=group(Z,a,b,c);N='[eE][-+]?[0-9](?:_?[0-9])*';e=group('[0-9](?:_?[0-9])*\\.(?:[0-9](?:_?[0-9])*)?','\\.[0-9](?:_?[0-9])*')+maybe(N);f='[0-9](?:_?[0-9])*'+N;O=group(e,f);g=group('[0-9](?:_?[0-9])*[jJ]',O+'[jJ]');h=group(g,O,d);I=_all_string_prefixes();P=group(*I);Q=group(*_all_string_prefixes(include_fstring=_A));J=_all_string_prefixes(include_fstring=_A,only_fstring=_A);i=group(*J);j="(?:\\\\.|[^'\\\\])*'";k='(?:\\\\.|[^"\\\\])*"';l="(?:\\\\.|'(?!'')|[^'\\\\])*'''";m='(?:\\\\.|"(?!"")|[^"\\\\])*"""';n=group(Q+F,Q+E);o=group('\\*\\*=?','>>=?','<<=?','//=?','->','[+\\-*/%&@`|^!=<>]=?','~');p='[][(){}]';K=['\\.\\.\\.','\\r\\n?','\\n','[;.,@]']
	if version_info>=(3,8):K.insert(0,':=?')
	else:K.insert(0,':')
	q=group(*K);r=group(o,p,q);s=group(P+"'[^\\r\\n'\\\\]*(?:\\\\.[^\\r\\n'\\\\]*)*"+group(D,L),P+'"[^\\r\\n"\\\\]*(?:\\\\.[^\\r\\n"\\\\]*)*'+group(C,L));R=[X,n];S=C,D,E,F
	if J:R.append(i+group(*S))
	t=group('\\\\(?:\\r\\n?|\\n)|\\Z',*R);u=group(M,capture=_A)+group(t,h,r,s,Y,capture=_A);B={}
	for G in I:B[G+D]=_compile(j);B[G+C]=_compile(k);B[G+F]=_compile(l);B[G+E]=_compile(m)
	T=set();U=set();V={}
	for H in I:
		for A in(C,D):T.add(H+A)
		for A in(E,F):U.add(H+A)
	for H in J:
		for A in S:V[H+A]=A
	v=';','import','class','def','try','except','finally','while','with','return','continue','break','del','pass','global','assert','nonlocal';w=_compile(u);return TokenCollection(w,T,U,B,W,V,set(v))
class Token(NamedTuple):
	type:PythonTokenTypes;string:str;start_pos:Tuple[int,int];prefix:str
	@property
	def end_pos(self):
		A=self;B=split_lines(A.string)
		if len(B)>1:return A.start_pos[0]+len(B)-1,0
		else:return A.start_pos[0],A.start_pos[1]+len(A.string)
class PythonToken(Token):
	def __repr__(A):return'TokenInfo(type=%s, string=%r, start_pos=%r, prefix=%r)'%A._replace(type=A.type.name)
class FStringNode:
	def __init__(A,quote):A.quote=quote;A.parentheses_count=0;A.previous_lines='';A.last_string_start_pos=_B;A.format_spec_count=0
	def open_parentheses(A,character):A.parentheses_count+=1
	def close_parentheses(A,character):
		A.parentheses_count-=1
		if A.parentheses_count==0:A.format_spec_count=0
	def allow_multiline(A):return len(A.quote)==3
	def is_in_expr(A):return A.parentheses_count>A.format_spec_count
	def is_in_format_spec(A):return not A.is_in_expr()and A.format_spec_count
def _close_fstring_if_necessary(fstring_stack,string,line_nr,column,additional_prefix):
	E=fstring_stack;B=additional_prefix;C=string
	for(G,A)in enumerate(E):
		F=C.lstrip();D=len(C)-len(F)
		if F.startswith(A.quote):H=PythonToken(FSTRING_END,A.quote,(line_nr,column+D),prefix=B+C[:D]);B='';assert not A.previous_lines;del E[G:];return H,'',len(A.quote)+D
	return _B,B,0
def _find_fstring_string(endpats,fstring_stack,line,lnum,pos):
	E=fstring_stack;C=pos;B=E[-1];F=B.allow_multiline()
	if B.is_in_format_spec():
		if F:D=fstring_format_spec_multi_line
		else:D=fstring_format_spec_single_line
	elif F:D=fstring_string_multi_line
	else:D=fstring_string_single_line
	G=D.match(line,C)
	if G is _B:return B.previous_lines,C
	if not B.previous_lines:B.last_string_start_pos=lnum,C
	A=G.group(0)
	for H in E:
		I=endpats[H.quote].match(A)
		if I is not _B:A=I.group(0)[:-len(H.quote)]
	J=C;J+=len(A)
	if A.endswith('\n')or A.endswith('\r'):B.previous_lines+=A;A=''
	else:A=B.previous_lines+A
	return A,J
def tokenize(code,*,version_info,start_pos=(1,0)):'Generate tokens from a the source code (string).';A=split_lines(code,keepends=_A);return tokenize_lines(A,version_info=version_info,start_pos=start_pos)
def _print_tokens(func):
	'\n    A small helper function to help debug the tokenize_lines function.\n    '
	def A(*B,**C):
		for A in func(*B,**C):print(A);yield A
	return A
def tokenize_lines(lines,*,version_info,indents=_B,start_pos=(1,0),is_first_token=_A):
	'\n    A heavily modified Python standard library tokenizer.\n\n    Additionally to the default information, yields also the prefix of each\n    token. This idea comes from lib2to3. The prefix contains all information\n    that is irrelevant for the parser like newlines in parentheses or comments.\n    ';d='\r\n';e=is_first_token;V=start_pos;K=indents
	def Y(start):
		A=start
		while A<K[-1]:
			if A>K[-2]:yield PythonToken(ERROR_DEDENT,'',(L,A),'');K[-1]=A;break
			K.pop();yield PythonToken(DEDENT,'',G,'')
	f,Z,o,Q,g,h,p=_get_token_collection(version_info);M=0
	if K is _B:K=[0]
	R=0;q='0123456789';I='';T:0;W:0;X:0;O=_A;E='';H='';L=V[0]-1;D=[]
	for B in lines:
		L+=1;C=0;R=len(B)
		if e:
			if B.startswith(BOM_UTF8_STRING):H=BOM_UTF8_STRING;B=B[1:];R=len(B)
			B='^'*V[1]+B;C=V[1];R+=V[1];e=_C
		if I:
			U=X.match(B)
			if U:C=U.end(0);yield PythonToken(STRING,I+B[:C],W,E);I='';T=''
			else:I=I+B;T=T+B;continue
		while C<R:
			if D:
				P=D[-1]
				if not P.is_in_expr():
					i,C=_find_fstring_string(Q,D,B,L,C)
					if i:yield PythonToken(FSTRING_STRING,i,P.last_string_start_pos,prefix='');P.previous_lines='';continue
					if C==R:break
				r=B[C:];j,H,s=_close_fstring_if_necessary(D,r,L,C,H);C+=s
				if j is not _B:yield j;continue
			if D:
				a=B
				for t in D:
					b=t.quote;k=Q[b].match(B,C)
					if k is not _B:
						l=k.group(0)
						if len(l)-len(b)+C<len(a):a=B[:C]+l[:-len(b)]
				N=f.match(a,C)
			else:N=f.match(B,C)
			if N:
				E=H+N.group(1);H='';F,C=N.span(2);G=L,F;A=N.group(2)
				if A=='':assert E;H=E;break
				J=A[0]
			else:S=g.match(B,C);J=B[S.end()];F=S.end();G=L,F
			if O and J not in'\r\n#'and(J!='\\'or N is _B):
				O=_C
				if M==0 and not D:
					c=F
					if c>K[-1]:yield PythonToken(INDENT,'',G,'');K.append(c)
					yield from Y(c)
			if not N:
				S=g.match(B,C)
				if O and M==0 and not D:yield from Y(S.end())
				C=S.end();O=_C;yield PythonToken(ERRORTOKEN,B[C],(L,C),H+S.group(0));H='';C+=1;continue
			if J in q or J=='.'and A!='.'and A!='...':yield PythonToken(NUMBER,A,G,E)
			elif N.group(3)is not _B:
				if A in p and(D or M):
					D[:]=[];M=0;m=re.match('[ \\f\\t]*$',B[:F])
					if m is not _B:yield from Y(m.end())
				if A.isidentifier():yield PythonToken(NAME,A,G,E)
				else:yield from _split_illegal_unicode_name(A,G,E)
			elif J in d:
				if any(not A.allow_multiline()for A in D):D.clear()
				if not O and M==0 and not D:yield PythonToken(NEWLINE,A,G,E)
				else:H=E+A
				O=_A
			elif J=='#':
				assert not A.endswith('\n')and not A.endswith('\r')
				if D and D[-1].is_in_expr():yield PythonToken(ERRORTOKEN,J,G,E);C=F+1
				else:H=E+A
			elif A in o:
				X=Q[A];U=X.match(B,C)
				if U:C=U.end(0);A=B[F:C];yield PythonToken(STRING,A,G,E)
				else:W=G;I=B[F:];T=B;break
			elif J in Z or A[:2]in Z or A[:3]in Z:
				if A[-1]in d:W=L,F;X=Q.get(J)or Q.get(A[1])or Q.get(A[2]);I=B[F:];T=B;break
				else:yield PythonToken(STRING,A,G,E)
			elif A in h:D.append(FStringNode(h[A]));yield PythonToken(FSTRING_START,A,G,E)
			elif J=='\\'and B[F:]in('\\\n','\\\r\n','\\\r'):H+=E+B[F:];break
			else:
				if A in'([{':
					if D:D[-1].open_parentheses(A)
					else:M+=1
				elif A in')]}':
					if D:D[-1].close_parentheses(A)
					elif M:M-=1
				elif A.startswith(':')and D and D[-1].parentheses_count-D[-1].format_spec_count==1:D[-1].format_spec_count+=1;A=':';C=F+1
				yield PythonToken(OP,A,G,E)
	if I:
		yield PythonToken(ERRORTOKEN,I,W,E)
		if I.endswith('\n')or I.endswith('\r'):O=_A
	if D:
		P=D[-1]
		if P.previous_lines:yield PythonToken(FSTRING_STRING,P.previous_lines,P.last_string_start_pos,prefix='')
	n=L,R
	for u in K[1:]:K.pop();yield PythonToken(DEDENT,'',n,'')
	yield PythonToken(ENDMARKER,'',n,H)
def _split_illegal_unicode_name(token,start_pos,prefix):
	E=prefix;B=start_pos
	def F():return PythonToken(ERRORTOKEN if D else NAME,A,G,E)
	A='';D=_C;G=B
	for(H,C)in enumerate(token):
		if D:
			if C.isidentifier():yield F();A=C;D=_C;E='';G=B[0],B[1]+H
			else:A+=C
		else:
			I=A+C
			if I.isidentifier():A=I
			else:
				if A:yield F();E='';G=B[0],B[1]+H
				A=C;D=_A
	if A:yield F()
if __name__=='__main__':
	path=sys.argv[1]
	with open(path)as f:code=f.read()
	for token in tokenize(code,version_info=parse_version_string('3.10')):print(token)