B
    @)H[Ý1 ã               @   s¸   d dl mZmZmZ d dlmZ d dlmZ ddl	m
Z
 ddl	mZ ddl	mZmZ ddl	mZmZmZ dd	l	mZmZ dd
l	mZ ddlmZ ddlmZ eeƒZG dd„ deƒZdS )é    )Úabsolute_importÚdivisionÚunicode_literals)Úunichr)Údequeé   )ÚspaceCharacters)Úentities)ÚasciiLettersÚasciiUpper2Lower)ÚdigitsÚ	hexDigitsÚEOF)Ú
tokenTypesÚtagTokenTypes)ÚreplacementCharacters)ÚHTMLInputStream)ÚTriec                   sd  e Zd ZdZd–‡ fdd„	Zdd„ Zdd„ Zd—d
d„Zdd„ Zdd„ Z	dd„ Z
dd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zd d!„ Zd"d#„ Zd$d%„ Zd&d'„ Zd(d)„ Zd*d+„ Zd,d-„ Zd.d/„ Zd0d1„ Zd2d3„ Zd4d5„ Zd6d7„ Zd8d9„ Zd:d;„ Zd<d=„ Z d>d?„ Z!d@dA„ Z"dBdC„ Z#dDdE„ Z$dFdG„ Z%dHdI„ Z&dJdK„ Z'dLdM„ Z(dNdO„ Z)dPdQ„ Z*dRdS„ Z+dTdU„ Z,dVdW„ Z-dXdY„ Z.dZd[„ Z/d\d]„ Z0d^d_„ Z1d`da„ Z2dbdc„ Z3ddde„ Z4dfdg„ Z5dhdi„ Z6djdk„ Z7dldm„ Z8dndo„ Z9dpdq„ Z:drds„ Z;dtdu„ Z<dvdw„ Z=dxdy„ Z>dzd{„ Z?d|d}„ Z@d~d„ ZAd€d�„ ZBd‚dƒ„ ZCd„d…„ ZDd†d‡„ ZEdˆd‰„ ZFdŠd‹„ ZGdŒd�„ ZHdŽd�„ ZId�d‘„ ZJd’d“„ ZKd”d•„ ZL‡  ZMS )˜ÚHTMLTokenizera	   This class takes care of tokenizing HTML.

    * self.currentToken
      Holds the token that is currently being processed.

    * self.state
      Holds a reference to the method to be invoked... XXX

    * self.stream
      Points to HTMLInputStream object.
    Nc                sF   t |f|Ž| _|| _d| _g | _| j| _d| _d | _t	t
| ƒ ¡  d S )NF)r   ÚstreamÚparserÚ
escapeFlagÚlastFourCharsÚ	dataStateÚstateÚescapeÚcurrentTokenÚsuperr   Ú__init__)Úselfr   r   Úkwargs)Ú	__class__© ú`C:\Users\HIRONO~1\AppData\Local\Temp\pip-install-qz7kpjmn\pip\pip\_vendor\html5lib\_tokenizer.pyr   "   s    zHTMLTokenizer.__init__c             c   s\   t g ƒ| _xL|  ¡ rVx&| jjr:td | jj d¡dœV  qW x| jrR| j ¡ V  q>W qW dS )zä This is where the magic happens.

        We do our usually processing through the states and when we have a token
        to return we yield the token which pauses processing until the next token
        is requested.
        Ú
ParseErrorr   )ÚtypeÚdataN)r   Ú
tokenQueuer   r   Úerrorsr   ÚpopÚpopleft)r   r"   r"   r#   Ú__iter__1   s    


zHTMLTokenizer.__iter__c       	   %   C   s  t }d}|rt}d}g }| j ¡ }x(||krJ|tk	rJ| |¡ | j ¡ }q$W td |¡|ƒ}|tkrŒt| }| j	 t
d dd|idœ¡ �nbd|  kr d	ksªn |d
krÎd}| j	 t
d dd|idœ¡ �n d|  krädk�s†n d|  krüdk�s†n d|  k�rdk�s†n d|  k�r0dk�s†n |tddddddddddddd d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2d3d4d5d
g#ƒk�r¢| j	 t
d dd|idœ¡ yt|ƒ}W n> tk
�rì   |d6 }td|d? B ƒtd7|d8@ B ƒ }Y nX |d9k�r| j	 t
d d:d;œ¡ | j |¡ |S )<zøThis function returns either U+FFFD or the character based on the
        decimal or hexadecimal representation. It also discards ";" if present.
        If not present self.tokenQueue.append({"type": tokenTypes["ParseError"]}) is invoked.
        é
   é   Ú r$   z$illegal-codepoint-for-numeric-entityÚ	charAsInt)r%   r&   Údatavarsi Ø  iÿß  iÿÿ u   ï¿½r   é   é   é   é   éŸ   iÐý  iïý  é   iþÿ  iÿÿ  iþÿ iÿÿ iþÿ iÿÿ iþÿ iÿÿ iþÿ iÿÿ iþÿ iÿÿ iþÿ iÿÿ iþÿ iÿÿ iþÿ iÿÿ iþÿ	 iÿÿ	 iþÿ
 iÿÿ
 iþÿ iÿÿ iþÿ iÿÿ iþÿ iÿÿ iþÿ iÿÿ iþÿ iÿÿ iþÿ i   i Ü  iÿ  ú;z numeric-entity-without-semicolon)r%   r&   )r   r   r   Úcharr   ÚappendÚintÚjoinr   r'   r   Ú	frozensetÚchrÚ
ValueErrorÚunget)	r   ÚisHexÚallowedÚradixÚ	charStackÚcr/   r8   Úvr"   r"   r#   ÚconsumeNumberEntityA   s`    

&

z!HTMLTokenizer.consumeNumberEntityFc       	      C   sÆ  d}| j  ¡ g}|d tksB|d tddfksB|d k	rV||d krV| j  |d ¡ �n |d dk�rd}| | j  ¡ ¡ |d dkr˜d}| | j  ¡ ¡ |r¨|d tks¸|sÔ|d tkrÔ| j  |d ¡ |  |¡}n4| j	 t
d	 d
dœ¡ | j  | ¡ ¡ dd |¡ }�njx8|d tk	�rDt d |¡¡�s0P | | j  ¡ ¡ �qW y$t d |d d… ¡¡}t|ƒ}W n tk
�r„   d }Y nX |d k	�rB|d dk�r´| j	 t
d	 ddœ¡ |d dk�r|�r|| tk�sò|| tk�sò|| dk�r| j  | ¡ ¡ dd |¡ }n.t| }| j  | ¡ ¡ |d ||d … ¡7 }n4| j	 t
d	 ddœ¡ | j  | ¡ ¡ dd |¡ }|�r˜| jd d d  |7  < n*|tk�r¨d}nd}| j	 t
| |dœ¡ d S )Nú&r   ú<ú#Féÿÿÿÿ)ÚxÚXTr$   zexpected-numeric-entity)r%   r&   r.   r7   znamed-entity-without-semicolonú=zexpected-named-entityr&   r   ÚSpaceCharactersÚ
Characters)r   r8   r   r   r?   r9   r   r   rF   r'   r   r)   r;   ÚentitiesTrieÚhas_keys_with_prefixÚlongest_prefixÚlenÚKeyErrorr
   r	   r   )	r   ÚallowedCharÚfromAttributeÚoutputrC   ÚhexÚ
entityNameÚentityLengthÚ	tokenTyper"   r"   r#   ÚconsumeEntity‰   sf    





zHTMLTokenizer.consumeEntityc             C   s   | j |dd� dS )zIThis method replaces the need for "entityInAttributeValueState".
        T)rU   rV   N)r\   )r   rU   r"   r"   r#   ÚprocessEntityInAttributeÙ   s    z&HTMLTokenizer.processEntityInAttributec             C   sˆ   | j }|d tkrp|d  t¡|d< |d td krp|d rR| j td ddœ¡ |d rp| j td d	dœ¡ | j |¡ | j| _d
S )z®This method is a generic handler for emitting the tags. It also sets
        the state to "data" because that's what's needed after a token has been
        emitted.
        r%   ÚnameÚEndTagr&   r$   zattributes-in-end-tag)r%   r&   ÚselfClosingzself-closing-flag-on-end-tagN)	r   r   Ú	translater   r   r'   r9   r   r   )r   Útokenr"   r"   r#   ÚemitCurrentTokenÞ   s    

zHTMLTokenizer.emitCurrentTokenc             C   sÈ   | j  ¡ }|dkr| j| _n¨|dkr.| j| _n–|dkrd| j td ddœ¡ | j td ddœ¡ n`|tkrpdS |t	krž| j td	 || j  
t	d
¡ dœ¡ n&| j  
d¡}| j td || dœ¡ d
S )NrG   rH   ú r$   zinvalid-codepoint)r%   r&   rO   FrN   T)rG   rH   rd   )r   r8   ÚentityDataStater   ÚtagOpenStater'   r9   r   r   r   Ú
charsUntil)r   r&   Úcharsr"   r"   r#   r   ò   s&    



zHTMLTokenizer.dataStatec             C   s   |   ¡  | j| _dS )NT)r\   r   r   )r   r"   r"   r#   re     s    zHTMLTokenizer.entityDataStatec             C   sÈ   | j  ¡ }|dkr| j| _n¨|dkr.| j| _n–|tkr:dS |dkrp| j td ddœ¡ | j td d	dœ¡ nT|t	krž| j td
 || j  
t	d¡ dœ¡ n&| j  
d¡}| j td || dœ¡ dS )NrG   rH   Frd   r$   zinvalid-codepoint)r%   r&   rO   u   ï¿½rN   T)rG   rH   rd   )r   r8   ÚcharacterReferenceInRcdatar   ÚrcdataLessThanSignStater   r'   r9   r   r   rg   )r   r&   rh   r"   r"   r#   ÚrcdataState  s&    



zHTMLTokenizer.rcdataStatec             C   s   |   ¡  | j| _dS )NT)r\   rk   r   )r   r"   r"   r#   ri   1  s    z(HTMLTokenizer.characterReferenceInRcdatac             C   sˆ   | j  ¡ }|dkr| j| _nh|dkrR| j td ddœ¡ | j td ddœ¡ n2|tkr^dS | j  d	¡}| j td || dœ¡ d
S )NrH   rd   r$   zinvalid-codepoint)r%   r&   rO   u   ï¿½F)rH   rd   T)	r   r8   ÚrawtextLessThanSignStater   r'   r9   r   r   rg   )r   r&   rh   r"   r"   r#   ÚrawtextState6  s    


zHTMLTokenizer.rawtextStatec             C   sˆ   | j  ¡ }|dkr| j| _nh|dkrR| j td ddœ¡ | j td ddœ¡ n2|tkr^dS | j  d	¡}| j td || dœ¡ d
S )NrH   rd   r$   zinvalid-codepoint)r%   r&   rO   u   ï¿½F)rH   rd   T)	r   r8   ÚscriptDataLessThanSignStater   r'   r9   r   r   rg   )r   r&   rh   r"   r"   r#   ÚscriptDataStateH  s    


zHTMLTokenizer.scriptDataStatec             C   sr   | j  ¡ }|tkrdS |dkrL| j td ddœ¡ | j td ddœ¡ n"| j td || j  d¡ dœ¡ dS )	NFrd   r$   zinvalid-codepoint)r%   r&   rO   u   ï¿½T)r   r8   r   r'   r9   r   rg   )r   r&   r"   r"   r#   ÚplaintextStateZ  s    

zHTMLTokenizer.plaintextStatec             C   s  | j  ¡ }|dkr| j| _nì|dkr.| j| _nÚ|tkrVtd |g dddœ| _| j| _n²|dkr”| j	 
td dd	œ¡ | j	 
td
 dd	œ¡ | j| _nt|dkrÈ| j	 
td dd	œ¡ | j  |¡ | j| _n@| j	 
td dd	œ¡ | j	 
td
 dd	œ¡ | j  |¡ | j| _dS )Nú!ú/ÚStartTagF)r%   r^   r&   r`   ÚselfClosingAcknowledgedú>r$   z'expected-tag-name-but-got-right-bracket)r%   r&   rO   z<>ú?z'expected-tag-name-but-got-question-markzexpected-tag-namerH   T)r   r8   ÚmarkupDeclarationOpenStater   ÚcloseTagOpenStater
   r   r   ÚtagNameStater'   r9   r   r?   ÚbogusCommentState)r   r&   r"   r"   r#   rf   i  s6    









zHTMLTokenizer.tagOpenStatec             C   sÊ   | j  ¡ }|tkr0td |g ddœ| _| j| _n–|dkrX| j td ddœ¡ | j	| _nn|t
kr–| j td ddœ¡ | j td	 d
dœ¡ | j	| _n0| j td dd|idœ¡ | j  |¡ | j| _dS )Nr_   F)r%   r^   r&   r`   ru   r$   z*expected-closing-tag-but-got-right-bracket)r%   r&   z expected-closing-tag-but-got-eofrO   z</z!expected-closing-tag-but-got-charr&   )r%   r&   r0   T)r   r8   r
   r   r   ry   r   r'   r9   r   r   r?   rz   )r   r&   r"   r"   r#   rx   Œ  s(    





zHTMLTokenizer.closeTagOpenStatec             C   s°   | j  ¡ }|tkr| j| _n�|dkr.|  ¡  n~|tkrV| j t	d ddœ¡ | j
| _nV|dkrh| j| _nD|dkrš| j t	d ddœ¡ | jd  d	7  < n| jd  |7  < d
S )Nru   r$   zeof-in-tag-name)r%   r&   rr   rd   zinvalid-codepointr^   u   ï¿½T)r   r8   r   ÚbeforeAttributeNameStater   rc   r   r'   r9   r   r   ÚselfClosingStartTagStater   )r   r&   r"   r"   r#   ry   ¤  s"    






zHTMLTokenizer.tagNameStatec             C   sP   | j  ¡ }|dkr"d| _| j| _n*| j td ddœ¡ | j  |¡ | j	| _dS )Nrr   r.   rO   rH   )r%   r&   T)
r   r8   ÚtemporaryBufferÚrcdataEndTagOpenStater   r'   r9   r   r?   rk   )r   r&   r"   r"   r#   rj   º  s    

z%HTMLTokenizer.rcdataLessThanSignStatec             C   sX   | j  ¡ }|tkr*|  j|7  _| j| _n*| j td ddœ¡ | j  	|¡ | j
| _dS )NrO   z</)r%   r&   T)r   r8   r
   r}   ÚrcdataEndTagNameStater   r'   r9   r   r?   rk   )r   r&   r"   r"   r#   r~   Å  s    

z#HTMLTokenizer.rcdataEndTagOpenStatec             C   s   | j o| j d  ¡ | j ¡ k}| j ¡ }|tkrT|rTtd | jg ddœ| _ | j| _n¨|dkr€|r€td | jg ddœ| _ | j	| _n||dkr´|r´td | jg ddœ| _ |  
¡  | j| _nH|tkrÌ|  j|7  _n0| j td d| j d	œ¡ | j |¡ | j| _d
S )Nr^   r_   F)r%   r^   r&   r`   rr   ru   rO   z</)r%   r&   T)r   Úlowerr}   r   r8   r   r   r{   r   r|   rc   r   r
   r'   r9   r?   rk   )r   Úappropriater&   r"   r"   r#   r   Ð  s2    



z#HTMLTokenizer.rcdataEndTagNameStatec             C   sP   | j  ¡ }|dkr"d| _| j| _n*| j td ddœ¡ | j  |¡ | j	| _dS )Nrr   r.   rO   rH   )r%   r&   T)
r   r8   r}   ÚrawtextEndTagOpenStater   r'   r9   r   r?   rm   )r   r&   r"   r"   r#   rl   ì  s    

z&HTMLTokenizer.rawtextLessThanSignStatec             C   sX   | j  ¡ }|tkr*|  j|7  _| j| _n*| j td ddœ¡ | j  	|¡ | j
| _dS )NrO   z</)r%   r&   T)r   r8   r
   r}   ÚrawtextEndTagNameStater   r'   r9   r   r?   rm   )r   r&   r"   r"   r#   r‚   ÷  s    

z$HTMLTokenizer.rawtextEndTagOpenStatec             C   s   | j o| j d  ¡ | j ¡ k}| j ¡ }|tkrT|rTtd | jg ddœ| _ | j| _n¨|dkr€|r€td | jg ddœ| _ | j	| _n||dkr´|r´td | jg ddœ| _ |  
¡  | j| _nH|tkrÌ|  j|7  _n0| j td d| j d	œ¡ | j |¡ | j| _d
S )Nr^   r_   F)r%   r^   r&   r`   rr   ru   rO   z</)r%   r&   T)r   r€   r}   r   r8   r   r   r{   r   r|   rc   r   r
   r'   r9   r?   rm   )r   r�   r&   r"   r"   r#   rƒ     s2    



z$HTMLTokenizer.rawtextEndTagNameStatec             C   sx   | j  ¡ }|dkr"d| _| j| _nR|dkrJ| j td ddœ¡ | j| _n*| j td ddœ¡ | j  	|¡ | j
| _dS )	Nrr   r.   rq   rO   z<!)r%   r&   rH   T)r   r8   r}   ÚscriptDataEndTagOpenStater   r'   r9   r   ÚscriptDataEscapeStartStater?   ro   )r   r&   r"   r"   r#   rn     s    


z)HTMLTokenizer.scriptDataLessThanSignStatec             C   sX   | j  ¡ }|tkr*|  j|7  _| j| _n*| j td ddœ¡ | j  	|¡ | j
| _dS )NrO   z</)r%   r&   T)r   r8   r
   r}   ÚscriptDataEndTagNameStater   r'   r9   r   r?   ro   )r   r&   r"   r"   r#   r„   ,  s    

z'HTMLTokenizer.scriptDataEndTagOpenStatec             C   s   | j o| j d  ¡ | j ¡ k}| j ¡ }|tkrT|rTtd | jg ddœ| _ | j| _n¨|dkr€|r€td | jg ddœ| _ | j	| _n||dkr´|r´td | jg ddœ| _ |  
¡  | j| _nH|tkrÌ|  j|7  _n0| j td d| j d	œ¡ | j |¡ | j| _d
S )Nr^   r_   F)r%   r^   r&   r`   rr   ru   rO   z</)r%   r&   T)r   r€   r}   r   r8   r   r   r{   r   r|   rc   r   r
   r'   r9   r?   ro   )r   r�   r&   r"   r"   r#   r†   7  s2    



z'HTMLTokenizer.scriptDataEndTagNameStatec             C   sJ   | j  ¡ }|dkr2| j td ddœ¡ | j| _n| j  |¡ | j| _dS )Nú-rO   )r%   r&   T)	r   r8   r'   r9   r   ÚscriptDataEscapeStartDashStater   r?   ro   )r   r&   r"   r"   r#   r…   S  s    

z(HTMLTokenizer.scriptDataEscapeStartStatec             C   sJ   | j  ¡ }|dkr2| j td ddœ¡ | j| _n| j  |¡ | j| _dS )Nr‡   rO   )r%   r&   T)	r   r8   r'   r9   r   ÚscriptDataEscapedDashDashStater   r?   ro   )r   r&   r"   r"   r#   rˆ   ]  s    

z,HTMLTokenizer.scriptDataEscapeStartDashStatec             C   s¶   | j  ¡ }|dkr2| j td ddœ¡ | j| _n€|dkrD| j| _nn|dkrz| j td ddœ¡ | j td ddœ¡ n8|tkrŒ| j	| _n&| j  
d	¡}| j td || dœ¡ d
S )Nr‡   rO   )r%   r&   rH   rd   r$   zinvalid-codepointu   ï¿½)rH   r‡   rd   T)r   r8   r'   r9   r   ÚscriptDataEscapedDashStater   Ú"scriptDataEscapedLessThanSignStater   r   rg   )r   r&   rh   r"   r"   r#   ÚscriptDataEscapedStateg  s"    




z$HTMLTokenizer.scriptDataEscapedStatec             C   s¶   | j  ¡ }|dkr2| j td ddœ¡ | j| _n€|dkrD| j| _nn|dkr‚| j td ddœ¡ | j td ddœ¡ | j| _n0|t	kr”| j
| _n| j td |dœ¡ | j| _d	S )
Nr‡   rO   )r%   r&   rH   rd   r$   zinvalid-codepointu   ï¿½T)r   r8   r'   r9   r   r‰   r   r‹   rŒ   r   r   )r   r&   r"   r"   r#   rŠ   {  s"    






z(HTMLTokenizer.scriptDataEscapedDashStatec             C   sÖ   | j  ¡ }|dkr*| j td ddœ¡ n¨|dkr<| j| _n–|dkrd| j td ddœ¡ | j| _nn|dkr¢| j td ddœ¡ | j td d	dœ¡ | j| _n0|t	kr´| j
| _n| j td |dœ¡ | j| _d
S )Nr‡   rO   )r%   r&   rH   ru   rd   r$   zinvalid-codepointu   ï¿½T)r   r8   r'   r9   r   r‹   r   ro   rŒ   r   r   )r   r&   r"   r"   r#   r‰   �  s&    






z,HTMLTokenizer.scriptDataEscapedDashDashStatec             C   s‚   | j  ¡ }|dkr"d| _| j| _n\|tkrT| j td d| dœ¡ || _| j	| _n*| j td ddœ¡ | j  
|¡ | j| _dS )Nrr   r.   rO   rH   )r%   r&   T)r   r8   r}   Ú scriptDataEscapedEndTagOpenStater   r
   r'   r9   r   Ú scriptDataDoubleEscapeStartStater?   rŒ   )r   r&   r"   r"   r#   r‹   ¥  s    


z0HTMLTokenizer.scriptDataEscapedLessThanSignStatec             C   sP   | j  ¡ }|tkr"|| _| j| _n*| j td ddœ¡ | j  	|¡ | j
| _dS )NrO   z</)r%   r&   T)r   r8   r
   r}   Ú scriptDataEscapedEndTagNameStater   r'   r9   r   r?   rŒ   )r   r&   r"   r"   r#   r�   ´  s    

z.HTMLTokenizer.scriptDataEscapedEndTagOpenStatec             C   s   | j o| j d  ¡ | j ¡ k}| j ¡ }|tkrT|rTtd | jg ddœ| _ | j| _n¨|dkr€|r€td | jg ddœ| _ | j	| _n||dkr´|r´td | jg ddœ| _ |  
¡  | j| _nH|tkrÌ|  j|7  _n0| j td d| j d	œ¡ | j |¡ | j| _d
S )Nr^   r_   F)r%   r^   r&   r`   rr   ru   rO   z</)r%   r&   T)r   r€   r}   r   r8   r   r   r{   r   r|   rc   r   r
   r'   r9   r?   rŒ   )r   r�   r&   r"   r"   r#   r�   ¿  s2    



z.HTMLTokenizer.scriptDataEscapedEndTagNameStatec             C   s˜   | j  ¡ }|ttdƒB krR| j td |dœ¡ | j ¡ dkrH| j	| _
q”| j| _
nB|tkr€| j td |dœ¡ |  j|7  _n| j  |¡ | j| _
dS )N)rr   ru   rO   )r%   r&   ÚscriptT)r   r8   r   r<   r'   r9   r   r}   r€   ÚscriptDataDoubleEscapedStater   rŒ   r
   r?   )r   r&   r"   r"   r#   rŽ   Û  s    


z.HTMLTokenizer.scriptDataDoubleEscapeStartStatec             C   sÒ   | j  ¡ }|dkr2| j td ddœ¡ | j| _nœ|dkrZ| j td ddœ¡ | j| _nt|dkr�| j td ddœ¡ | j td ddœ¡ n>|tkr¸| j td d	dœ¡ | j	| _n| j td |dœ¡ d
S )Nr‡   rO   )r%   r&   rH   rd   r$   zinvalid-codepointu   ï¿½zeof-in-script-in-scriptT)
r   r8   r'   r9   r   Ú scriptDataDoubleEscapedDashStater   Ú(scriptDataDoubleEscapedLessThanSignStater   r   )r   r&   r"   r"   r#   r‘   ë  s$    





z*HTMLTokenizer.scriptDataDoubleEscapedStatec             C   sâ   | j  ¡ }|dkr2| j td ddœ¡ | j| _n¬|dkrZ| j td ddœ¡ | j| _n„|dkr˜| j td ddœ¡ | j td ddœ¡ | j| _nF|t	krÀ| j td d	dœ¡ | j
| _n| j td |dœ¡ | j| _d
S )Nr‡   rO   )r%   r&   rH   rd   r$   zinvalid-codepointu   ï¿½zeof-in-script-in-scriptT)r   r8   r'   r9   r   Ú$scriptDataDoubleEscapedDashDashStater   r“   r‘   r   r   )r   r&   r"   r"   r#   r’      s(    







z.HTMLTokenizer.scriptDataDoubleEscapedDashStatec             C   s  | j  ¡ }|dkr*| j td ddœ¡ nÔ|dkrR| j td ddœ¡ | j| _n¬|dkrz| j td ddœ¡ | j| _n„|dkr¸| j td ddœ¡ | j td d	dœ¡ | j| _nF|t	krà| j td d
dœ¡ | j
| _n| j td |dœ¡ | j| _dS )Nr‡   rO   )r%   r&   rH   ru   rd   r$   zinvalid-codepointu   ï¿½zeof-in-script-in-scriptT)r   r8   r'   r9   r   r“   r   ro   r‘   r   r   )r   r&   r"   r"   r#   r”     s,    







z2HTMLTokenizer.scriptDataDoubleEscapedDashDashStatec             C   sP   | j  ¡ }|dkr8| j td ddœ¡ d| _| j| _n| j  |¡ | j	| _dS )Nrr   rO   )r%   r&   r.   T)
r   r8   r'   r9   r   r}   ÚscriptDataDoubleEscapeEndStater   r?   r‘   )r   r&   r"   r"   r#   r“   0  s    

z6HTMLTokenizer.scriptDataDoubleEscapedLessThanSignStatec             C   s˜   | j  ¡ }|ttdƒB krR| j td |dœ¡ | j ¡ dkrH| j	| _
q”| j| _
nB|tkr€| j td |dœ¡ |  j|7  _n| j  |¡ | j| _
dS )N)rr   ru   rO   )r%   r&   r�   T)r   r8   r   r<   r'   r9   r   r}   r€   rŒ   r   r‘   r
   r?   )r   r&   r"   r"   r#   r•   ;  s    


z,HTMLTokenizer.scriptDataDoubleEscapeEndStatec             C   s0  | j  ¡ }|tkr$| j  td¡ �n|tkrJ| jd  |dg¡ | j| _nâ|dkr\|  	¡  nÐ|dkrn| j
| _n¾|dkrª| j td dd	œ¡ | jd  |dg¡ | j| _n‚|d
kræ| j td dd	œ¡ | jd  ddg¡ | j| _nF|tk�r| j td dd	œ¡ | j| _n| jd  |dg¡ | j| _dS )NTr&   r.   ru   rr   )ú'ú"rM   rH   r$   z#invalid-character-in-attribute-name)r%   r&   rd   zinvalid-codepointu   ï¿½z#expected-attribute-name-but-got-eof)r   r8   r   rg   r
   r   r9   ÚattributeNameStater   rc   r|   r'   r   r   r   )r   r&   r"   r"   r#   r{   K  s6    










z&HTMLTokenizer.beforeAttributeNameStatec             C   sä  | j  ¡ }d}d}|dkr&| j| _�n.|tkr\| jd d d  || j  td¡ 7  < d}nø|dkrjd}nê|tkr|| j| _nØ|dkrŽ| j	| _nÆ|d	krÌ| j
 td
 ddœ¡ | jd d d  d7  < d}nˆ|dk�r| j
 td
 ddœ¡ | jd d d  |7  < d}nH|tk�r6| j
 td
 ddœ¡ | j| _n| jd d d  |7  < d}|�rà| jd d d  t¡| jd d d< xP| jd d d… D ]:\}}| jd d d |k�r”| j
 td
 ddœ¡ P �q”W |�rà|  ¡  dS )NTFrM   r&   rJ   r   ru   rr   rd   r$   zinvalid-codepoint)r%   r&   u   ï¿½)r–   r—   rH   z#invalid-character-in-attribute-namezeof-in-attribute-namezduplicate-attribute)r   r8   ÚbeforeAttributeValueStater   r
   r   rg   r   ÚafterAttributeNameStater|   r'   r9   r   r   r   ra   r   rc   )r   r&   ÚleavingThisStateÚ	emitTokenr^   Ú_r"   r"   r#   r˜   i  sR    








&
z HTMLTokenizer.attributeNameStatec             C   sD  | j  ¡ }|tkr$| j  td¡ �n|dkr8| j| _�n|dkrJ|  ¡  nö|tkrp| jd  	|dg¡ | j
| _nÐ|dkr‚| j| _n¾|dkr¾| j 	td d	d
œ¡ | jd  	ddg¡ | j
| _n‚|dkrú| j 	td dd
œ¡ | jd  	|dg¡ | j
| _nF|tk�r$| j 	td dd
œ¡ | j| _n| jd  	|dg¡ | j
| _dS )NTrM   ru   r&   r.   rr   rd   r$   zinvalid-codepoint)r%   r&   u   ï¿½)r–   r—   rH   z&invalid-character-after-attribute-namezexpected-end-of-tag-but-got-eof)r   r8   r   rg   r™   r   rc   r
   r   r9   r˜   r|   r'   r   r   r   )r   r&   r"   r"   r#   rš   Ÿ  s:    










z%HTMLTokenizer.afterAttributeNameStatec             C   sh  | j  ¡ }|tkr$| j  td¡ �n@|dkr8| j| _�n,|dkrX| j| _| j  |¡ �n|dkrj| j| _nú|dkr’| j	 
td ddœ¡ |  ¡  nÒ|d	krÔ| j	 
td d
dœ¡ | jd d d  d7  < | j| _n�|dk�r| j	 
td ddœ¡ | jd d d  |7  < | j| _nL|tk�rB| j	 
td ddœ¡ | j| _n"| jd d d  |7  < | j| _dS )NTr—   rG   r–   ru   r$   z.expected-attribute-value-but-got-right-bracket)r%   r&   rd   zinvalid-codepointr&   rJ   r   u   ï¿½)rM   rH   ú`z"equals-in-unquoted-attribute-valuez$expected-attribute-value-but-got-eof)r   r8   r   rg   ÚattributeValueDoubleQuotedStater   ÚattributeValueUnQuotedStater?   ÚattributeValueSingleQuotedStater'   r9   r   rc   r   r   r   )r   r&   r"   r"   r#   r™   ¿  s>    











z'HTMLTokenizer.beforeAttributeValueStatec             C   s¼   | j  ¡ }|dkr| j| _nœ|dkr0|  d¡ nˆ|dkrj| j td ddœ¡ | jd d d	  d
7  < nN|t	kr’| j td ddœ¡ | j
| _n&| jd d d	  || j  d¡ 7  < dS )Nr—   rG   rd   r$   zinvalid-codepoint)r%   r&   r&   rJ   r   u   ï¿½z#eof-in-attribute-value-double-quote)r—   rG   rd   T)r   r8   ÚafterAttributeValueStater   r]   r'   r9   r   r   r   r   rg   )r   r&   r"   r"   r#   rŸ   á  s     




z-HTMLTokenizer.attributeValueDoubleQuotedStatec             C   s¼   | j  ¡ }|dkr| j| _nœ|dkr0|  d¡ nˆ|dkrj| j td ddœ¡ | jd d d	  d
7  < nN|t	kr’| j td ddœ¡ | j
| _n&| jd d d	  || j  d¡ 7  < dS )Nr–   rG   rd   r$   zinvalid-codepoint)r%   r&   r&   rJ   r   u   ï¿½z#eof-in-attribute-value-single-quote)r–   rG   rd   T)r   r8   r¢   r   r]   r'   r9   r   r   r   r   rg   )r   r&   r"   r"   r#   r¡   ô  s     




z-HTMLTokenizer.attributeValueSingleQuotedStatec             C   s  | j  ¡ }|tkr| j| _nð|dkr0|  d¡ nÜ|dkrB|  ¡  nÊ|dkr|| j t	d ddœ¡ | j
d d d	  |7  < n�|d
kr¶| j t	d ddœ¡ | j
d d d	  d7  < nV|tkrÞ| j t	d ddœ¡ | j| _n.| j
d d d	  || j  tdƒtB ¡ 7  < dS )NrG   ru   )r—   r–   rM   rH   rž   r$   z0unexpected-character-in-unquoted-attribute-value)r%   r&   r&   rJ   r   rd   zinvalid-codepointu   ï¿½z eof-in-attribute-value-no-quotes)rG   ru   r—   r–   rM   rH   rž   rd   T)r   r8   r   r{   r   r]   rc   r'   r9   r   r   r   r   rg   r<   )r   r&   r"   r"   r#   r      s,    






z)HTMLTokenizer.attributeValueUnQuotedStatec             C   s¢   | j  ¡ }|tkr| j| _n‚|dkr.|  ¡  np|dkr@| j| _n^|tkrt| j 	t
d ddœ¡ | j  |¡ | j| _n*| j 	t
d ddœ¡ | j  |¡ | j| _dS )Nru   rr   r$   z$unexpected-EOF-after-attribute-value)r%   r&   z*unexpected-character-after-attribute-valueT)r   r8   r   r{   r   rc   r|   r   r'   r9   r   r?   r   )r   r&   r"   r"   r#   r¢      s"    






z&HTMLTokenizer.afterAttributeValueStatec             C   sˆ   | j  ¡ }|dkr&d| jd< |  ¡  n^|tkrZ| j td ddœ¡ | j  |¡ | j	| _
n*| j td ddœ¡ | j  |¡ | j| _
dS )Nru   Tr`   r$   z#unexpected-EOF-after-solidus-in-tag)r%   r&   z)unexpected-character-after-solidus-in-tag)r   r8   r   rc   r   r'   r9   r   r?   r   r   r{   )r   r&   r"   r"   r#   r|   4  s    





z&HTMLTokenizer.selfClosingStartTagStatec             C   sD   | j  d¡}| dd¡}| j td |dœ¡ | j  ¡  | j| _dS )Nru   rd   u   ï¿½ÚComment)r%   r&   T)	r   rg   Úreplacer'   r9   r   r8   r   r   )r   r&   r"   r"   r#   rz   F  s    
zHTMLTokenizer.bogusCommentStatec             C   s†  | j  ¡ g}|d dkrR| | j  ¡ ¡ |d dkrPtd ddœ| _| j| _dS nô|d dkrºd}x.dD ]&}| | j  ¡ ¡ |d |krhd	}P qhW |r¸td
 dd d ddœ| _| j| _dS nŒ|d dk�rF| jd k	�rF| jj	j
�rF| jj	j
d j| jj	jk�rFd}x2dD ]*}| | j  ¡ ¡ |d |k�rd	}P �qW |�rF| j| _dS | j td ddœ¡ x|�rx| j  | ¡ ¡ �q^W | j| _dS )NrJ   r‡   r£   r.   )r%   r&   T)ÚdÚD))ÚoÚO)rD   ÚC)ÚtÚT)ÚyÚY)ÚpÚP)ÚeÚEFÚDoctype)r%   r^   ÚpublicIdÚsystemIdÚcorrectú[)r©   r¦   ÚAr«   r·   r¶   r$   zexpected-dashes-or-doctype)r   r8   r9   r   r   ÚcommentStartStater   ÚdoctypeStater   ÚtreeÚopenElementsÚ	namespaceÚdefaultNamespaceÚcdataSectionStater'   r?   r)   rz   )r   rC   ÚmatchedÚexpectedr"   r"   r#   rw   U  sP    



z(HTMLTokenizer.markupDeclarationOpenStatec             C   sØ   | j  ¡ }|dkr| j| _n¸|dkrN| j td ddœ¡ | jd  d7  < n†|dkr„| j td d	dœ¡ | j | j¡ | j| _nP|t	krº| j td d
dœ¡ | j | j¡ | j| _n| jd  |7  < | j
| _dS )Nr‡   rd   r$   zinvalid-codepoint)r%   r&   r&   u   ï¿½ru   zincorrect-commentzeof-in-commentT)r   r8   ÚcommentStartDashStater   r'   r9   r   r   r   r   ÚcommentState)r   r&   r"   r"   r#   r¸   ‚  s(    






zHTMLTokenizer.commentStartStatec             C   sÜ   | j  ¡ }|dkr| j| _n¼|dkrN| j td ddœ¡ | jd  d7  < nŠ|dkr„| j td d	dœ¡ | j | j¡ | j| _nT|t	krº| j td d
dœ¡ | j | j¡ | j| _n| jd  d| 7  < | j
| _dS )Nr‡   rd   r$   zinvalid-codepoint)r%   r&   r&   u   -ï¿½ru   zincorrect-commentzeof-in-commentT)r   r8   ÚcommentEndStater   r'   r9   r   r   r   r   rÂ   )r   r&   r"   r"   r#   rÁ   ™  s(    






z#HTMLTokenizer.commentStartDashStatec             C   s¦   | j  ¡ }|dkr| j| _n†|dkrN| j td ddœ¡ | jd  d7  < nT|tkr„| j td ddœ¡ | j | j¡ | j	| _n| jd  || j  
d	¡ 7  < d
S )Nr‡   rd   r$   zinvalid-codepoint)r%   r&   r&   u   ï¿½zeof-in-comment)r‡   rd   T)r   r8   ÚcommentEndDashStater   r'   r9   r   r   r   r   rg   )r   r&   r"   r"   r#   rÂ   °  s    




zHTMLTokenizer.commentStatec             C   s®   | j  ¡ }|dkr| j| _nŽ|dkrV| j td ddœ¡ | jd  d7  < | j| _nT|t	krŒ| j td ddœ¡ | j | j¡ | j
| _n| jd  d| 7  < | j| _d	S )
Nr‡   rd   r$   zinvalid-codepoint)r%   r&   r&   u   -ï¿½zeof-in-comment-end-dashT)r   r8   rÃ   r   r'   r9   r   r   rÂ   r   r   )r   r&   r"   r"   r#   rÄ   Â  s     





z!HTMLTokenizer.commentEndDashStatec             C   s,  | j  ¡ }|dkr*| j | j¡ | j| _nþ|dkrd| j td ddœ¡ | jd  d7  < | j| _nÄ|dkrŒ| j td d	dœ¡ | j	| _nœ|d
kr¾| j td ddœ¡ | jd  |7  < nj|t
krô| j td ddœ¡ | j | j¡ | j| _n4| j td ddœ¡ | jd  d| 7  < | j| _dS )Nru   rd   r$   zinvalid-codepoint)r%   r&   r&   u   --ï¿½rq   z,unexpected-bang-after-double-dash-in-commentr‡   z,unexpected-dash-after-double-dash-in-commentzeof-in-comment-double-dashzunexpected-char-in-commentz--T)r   r8   r'   r9   r   r   r   r   rÂ   ÚcommentEndBangStater   )r   r&   r"   r"   r#   rÃ   Õ  s6    









zHTMLTokenizer.commentEndStatec             C   sà   | j  ¡ }|dkr*| j | j¡ | j| _n²|dkrN| jd  d7  < | j| _nŽ|dkrˆ| j td ddœ¡ | jd  d	7  < | j	| _nT|t
kr¾| j td d
dœ¡ | j | j¡ | j| _n| jd  d| 7  < | j	| _dS )Nru   r‡   r&   z--!rd   r$   zinvalid-codepoint)r%   r&   u   --!ï¿½zeof-in-comment-end-bang-stateT)r   r8   r'   r9   r   r   r   rÄ   r   rÂ   r   )r   r&   r"   r"   r#   rÅ   ô  s(    






z!HTMLTokenizer.commentEndBangStatec             C   sŠ   | j  ¡ }|tkr| j| _nj|tkr\| j td ddœ¡ d| j	d< | j | j	¡ | j
| _n*| j td ddœ¡ | j  |¡ | j| _dS )Nr$   z!expected-doctype-name-but-got-eof)r%   r&   Frµ   zneed-space-after-doctypeT)r   r8   r   ÚbeforeDoctypeNameStater   r   r'   r9   r   r   r   r?   )r   r&   r"   r"   r#   r¹     s    





zHTMLTokenizer.doctypeStatec             C   sÜ   | j  ¡ }|tkrnÄ|dkrT| j td ddœ¡ d| jd< | j | j¡ | j| _n„|dkr†| j td ddœ¡ d	| jd
< | j	| _nR|t
krÆ| j td ddœ¡ d| jd< | j | j¡ | j| _n|| jd
< | j	| _dS )Nru   r$   z+expected-doctype-name-but-got-right-bracket)r%   r&   Frµ   rd   zinvalid-codepointu   ï¿½r^   z!expected-doctype-name-but-got-eofT)r   r8   r   r'   r9   r   r   r   r   ÚdoctypeNameStater   )r   r&   r"   r"   r#   rÆ     s.    










z$HTMLTokenizer.beforeDoctypeNameStatec             C   s  | j  ¡ }|tkr2| jd  t¡| jd< | j| _nØ|dkrh| jd  t¡| jd< | j 	| j¡ | j
| _n¢|dkr¢| j 	td ddœ¡ | jd  d7  < | j| _nh|tkrø| j 	td ddœ¡ d	| jd
< | jd  t¡| jd< | j 	| j¡ | j
| _n| jd  |7  < dS )Nr^   ru   rd   r$   zinvalid-codepoint)r%   r&   u   ï¿½zeof-in-doctype-nameFrµ   T)r   r8   r   r   ra   r   ÚafterDoctypeNameStater   r'   r9   r   r   rÇ   r   )r   r&   r"   r"   r#   rÇ   6  s,    







zHTMLTokenizer.doctypeNameStatec             C   sL  | j  ¡ }|tkr�n2|dkr8| j | j¡ | j| _�n|tkr„d| jd< | j  	|¡ | j t
d ddœ¡ | j | j¡ | j| _nÄ|dkrÈd}x$d	D ]}| j  ¡ }||kr–d}P q–W |rÆ| j| _dS nF|d
k�rd}x$dD ]}| j  ¡ }||krÜd}P qÜW |�r| j| _dS | j  	|¡ | j t
d dd|idœ¡ d| jd< | j| _dS )Nru   Frµ   r$   zeof-in-doctype)r%   r&   )r®   r¯   T))ÚuÚU)ÚbÚB)ÚlÚL)ÚiÚI)rD   r©   )ÚsÚS))r¬   r­   )rÑ   rÒ   )rª   r«   )r°   r±   )ÚmÚMz*expected-space-or-right-bracket-in-doctyper&   )r%   r&   r0   )r   r8   r   r'   r9   r   r   r   r   r?   r   ÚafterDoctypePublicKeywordStateÚafterDoctypeSystemKeywordStateÚbogusDoctypeState)r   r&   r¿   rÀ   r"   r"   r#   rÈ   O  sP    









z#HTMLTokenizer.afterDoctypeNameStatec             C   s¨   | j  ¡ }|tkr| j| _nˆ|dkrP| j td ddœ¡ | j  |¡ | j| _nT|t	kr�| j td ddœ¡ d| j
d< | j | j
¡ | j| _n| j  |¡ | j| _dS )	N)r–   r—   r$   zunexpected-char-in-doctype)r%   r&   zeof-in-doctypeFrµ   T)r   r8   r   Ú"beforeDoctypePublicIdentifierStater   r'   r9   r   r?   r   r   r   )r   r&   r"   r"   r#   rÕ   ‚  s"    






z,HTMLTokenizer.afterDoctypePublicKeywordStatec             C   sø   | j  ¡ }|tkrnà|dkr0d| jd< | j| _nÄ|dkrLd| jd< | j| _n¨|dkrŒ| j t	d ddœ¡ d	| jd
< | j | j¡ | j
| _nh|tkrÌ| j t	d ddœ¡ d	| jd
< | j | j¡ | j
| _n(| j t	d ddœ¡ d	| jd
< | j| _dS )Nr—   r.   r³   r–   ru   r$   zunexpected-end-of-doctype)r%   r&   Frµ   zeof-in-doctypezunexpected-char-in-doctypeT)r   r8   r   r   Ú(doctypePublicIdentifierDoubleQuotedStater   Ú(doctypePublicIdentifierSingleQuotedStater'   r9   r   r   r   r×   )r   r&   r"   r"   r#   rØ   –  s4    












z0HTMLTokenizer.beforeDoctypePublicIdentifierStatec             C   sä   | j  ¡ }|dkr| j| _nÄ|dkrN| j td ddœ¡ | jd  d7  < n’|dkrŽ| j td d	dœ¡ d
| jd< | j | j¡ | j| _nR|t	krÎ| j td ddœ¡ d
| jd< | j | j¡ | j| _n| jd  |7  < dS )Nr—   rd   r$   zinvalid-codepoint)r%   r&   r³   u   ï¿½ru   zunexpected-end-of-doctypeFrµ   zeof-in-doctypeT)
r   r8   Ú!afterDoctypePublicIdentifierStater   r'   r9   r   r   r   r   )r   r&   r"   r"   r#   rÙ   ³  s*    








z6HTMLTokenizer.doctypePublicIdentifierDoubleQuotedStatec             C   sä   | j  ¡ }|dkr| j| _nÄ|dkrN| j td ddœ¡ | jd  d7  < n’|dkrŽ| j td d	dœ¡ d
| jd< | j | j¡ | j| _nR|t	krÎ| j td ddœ¡ d
| jd< | j | j¡ | j| _n| jd  |7  < dS )Nr–   rd   r$   zinvalid-codepoint)r%   r&   r³   u   ï¿½ru   zunexpected-end-of-doctypeFrµ   zeof-in-doctypeT)
r   r8   rÛ   r   r'   r9   r   r   r   r   )r   r&   r"   r"   r#   rÚ   Ë  s*    








z6HTMLTokenizer.doctypePublicIdentifierSingleQuotedStatec             C   s  | j  ¡ }|tkr| j| _nì|dkr<| j | j¡ | j| _nÌ|dkrn| j t	d ddœ¡ d| jd< | j
| _nš|dkr | j t	d ddœ¡ d| jd< | j| _nh|tkrà| j t	d d	dœ¡ d
| jd< | j | j¡ | j| _n(| j t	d ddœ¡ d
| jd< | j| _dS )Nru   r—   r$   zunexpected-char-in-doctype)r%   r&   r.   r´   r–   zeof-in-doctypeFrµ   T)r   r8   r   Ú-betweenDoctypePublicAndSystemIdentifiersStater   r'   r9   r   r   r   Ú(doctypeSystemIdentifierDoubleQuotedStateÚ(doctypeSystemIdentifierSingleQuotedStater   r×   )r   r&   r"   r"   r#   rÛ   ã  s6    













z/HTMLTokenizer.afterDoctypePublicIdentifierStatec             C   sØ   | j  ¡ }|tkrnÀ|dkr4| j | j¡ | j| _n |dkrPd| jd< | j| _n„|dkrld| jd< | j	| _nh|t
kr¬| j td ddœ¡ d	| jd
< | j | j¡ | j| _n(| j td ddœ¡ d	| jd
< | j| _dS )Nru   r—   r.   r´   r–   r$   zeof-in-doctype)r%   r&   Frµ   zunexpected-char-in-doctypeT)r   r8   r   r'   r9   r   r   r   rÝ   rÞ   r   r   r×   )r   r&   r"   r"   r#   rÜ     s.    










z;HTMLTokenizer.betweenDoctypePublicAndSystemIdentifiersStatec             C   s¨   | j  ¡ }|tkr| j| _nˆ|dkrP| j td ddœ¡ | j  |¡ | j| _nT|t	kr�| j td ddœ¡ d| j
d< | j | j
¡ | j| _n| j  |¡ | j| _dS )	N)r–   r—   r$   zunexpected-char-in-doctype)r%   r&   zeof-in-doctypeFrµ   T)r   r8   r   Ú"beforeDoctypeSystemIdentifierStater   r'   r9   r   r?   r   r   r   )r   r&   r"   r"   r#   rÖ     s"    






z,HTMLTokenizer.afterDoctypeSystemKeywordStatec             C   sø   | j  ¡ }|tkrnà|dkr0d| jd< | j| _nÄ|dkrLd| jd< | j| _n¨|dkrŒ| j t	d ddœ¡ d	| jd
< | j | j¡ | j
| _nh|tkrÌ| j t	d ddœ¡ d	| jd
< | j | j¡ | j
| _n(| j t	d ddœ¡ d	| jd
< | j| _dS )Nr—   r.   r´   r–   ru   r$   zunexpected-char-in-doctype)r%   r&   Frµ   zeof-in-doctypeT)r   r8   r   r   rÝ   r   rÞ   r'   r9   r   r   r   r×   )r   r&   r"   r"   r#   rß   /  s4    












z0HTMLTokenizer.beforeDoctypeSystemIdentifierStatec             C   sä   | j  ¡ }|dkr| j| _nÄ|dkrN| j td ddœ¡ | jd  d7  < n’|dkrŽ| j td d	dœ¡ d
| jd< | j | j¡ | j| _nR|t	krÎ| j td ddœ¡ d
| jd< | j | j¡ | j| _n| jd  |7  < dS )Nr—   rd   r$   zinvalid-codepoint)r%   r&   r´   u   ï¿½ru   zunexpected-end-of-doctypeFrµ   zeof-in-doctypeT)
r   r8   Ú!afterDoctypeSystemIdentifierStater   r'   r9   r   r   r   r   )r   r&   r"   r"   r#   rÝ   L  s*    








z6HTMLTokenizer.doctypeSystemIdentifierDoubleQuotedStatec             C   sä   | j  ¡ }|dkr| j| _nÄ|dkrN| j td ddœ¡ | jd  d7  < n’|dkrŽ| j td d	dœ¡ d
| jd< | j | j¡ | j| _nR|t	krÎ| j td ddœ¡ d
| jd< | j | j¡ | j| _n| jd  |7  < dS )Nr–   rd   r$   zinvalid-codepoint)r%   r&   r´   u   ï¿½ru   zunexpected-end-of-doctypeFrµ   zeof-in-doctypeT)
r   r8   rà   r   r'   r9   r   r   r   r   )r   r&   r"   r"   r#   rÞ   d  s*    








z6HTMLTokenizer.doctypeSystemIdentifierSingleQuotedStatec             C   s–   | j  ¡ }|tkrn~|dkr4| j | j¡ | j| _n^|tkrt| j t	d ddœ¡ d| jd< | j | j¡ | j| _n| j t	d ddœ¡ | j
| _dS )	Nru   r$   zeof-in-doctype)r%   r&   Frµ   zunexpected-char-in-doctypeT)r   r8   r   r'   r9   r   r   r   r   r   r×   )r   r&   r"   r"   r#   rà   |  s     





z/HTMLTokenizer.afterDoctypeSystemIdentifierStatec             C   sZ   | j  ¡ }|dkr*| j | j¡ | j| _n,|tkrV| j  |¡ | j | j¡ | j| _n dS )Nru   T)	r   r8   r'   r9   r   r   r   r   r?   )r   r&   r"   r"   r#   r×   �  s    


zHTMLTokenizer.bogusDoctypeStatec             C   sü   g }x€|  | j d¡¡ |  | j d¡¡ | j ¡ }|tkr@P q|dksLt‚|d dd … dkrx|d d d… |d< P q|  |¡ qW d |¡}| d¡}|dkrÖx&t|ƒD ]}| j	  t
d	 d
dœ¡ q¬W | dd¡}|rð| j	  t
d |dœ¡ | j| _dS )Nú]ru   rJ   éþÿÿÿz]]r.   rd   r   r$   zinvalid-codepoint)r%   r&   u   ï¿½rO   T)r9   r   rg   r8   r   ÚAssertionErrorr;   ÚcountÚranger'   r   r¤   r   r   )r   r&   r8   Ú	nullCountr�   r"   r"   r#   r¾   �  s0    



zHTMLTokenizer.cdataSectionState)N)NF)NÚ__name__Ú
__module__Ú__qualname__Ú__doc__r   r+   rF   r\   r]   rc   r   re   rk   ri   rm   ro   rp   rf   rx   ry   rj   r~   r   rl   r‚   rƒ   rn   r„   r†   r…   rˆ   rŒ   rŠ   r‰   r‹   r�   r�   rŽ   r‘   r’   r”   r“   r•   r{   r˜   rš   r™   rŸ   r¡   r    r¢   r|   rz   rw   r¸   rÁ   rÂ   rÄ   rÃ   rÅ   r¹   rÆ   rÇ   rÈ   rÕ   rØ   rÙ   rÚ   rÛ   rÜ   rÖ   rß   rÝ   rÞ   rà   r×   r¾   Ú__classcell__r"   r"   )r!   r#   r      s”   H
P#

6 "-3r   N)Ú
__future__r   r   r   Úpip._vendor.sixr   r=   Úcollectionsr   Ú	constantsr   r	   r
   r   r   r   r   r   r   r   Ú_inputstreamr   Ú_trier   rP   Úobjectr   r"   r"   r"   r#   Ú<module>   s   