diff --git a/html5ever/src/tokenizer/mod.rs b/html5ever/src/tokenizer/mod.rs
index dcd372da..b388fe22 100644
--- a/html5ever/src/tokenizer/mod.rs
+++ b/html5ever/src/tokenizer/mod.rs
@@ -724,8 +724,9 @@ impl Tokenizer {
trace!("processing in state {:?}", self.state);
match self.state.get() {
- //§ data-state
+ // https://html.spec.whatwg.org/#data-state
states::Data => loop {
+ // Step 1. Consume the next input character:
let set = small_char_set!('\r' '\0' '&' '<' '\n');
#[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"))]
@@ -776,21 +777,33 @@ impl Tokenizer {
return ProcessResult::Suspend;
};
match set_result {
+ // ↪ U+0026 AMPERSAND (&)
+ FromSet('&') => {
+ // Set the return state to the data state. Switch to the character reference state.
+ go!(self: consume_char_ref)
+ },
+ // ↪ U+003C LESS-THAN SIGN (<)
+ FromSet('<') => {
+ // Switch to the tag open state.
+ go!(self: to State::TagOpen)
+ },
+ // ↪ U+0000 NULL
FromSet('\0') => {
+ // This is an unexpected-null-character parse error.
+ // Emit the current input character as a character token.
self.bad_char_error();
self.emit_char('\0');
},
- FromSet('&') => go!(self: consume_char_ref),
- FromSet('<') => go!(self: to State::TagOpen),
- FromSet(c) => {
- self.emit_char(c);
- },
- NotFromSet(b) => self.emit_chars(b),
+ // ↪ Anything else
+ // Emit the current input character as a character token.
+ FromSet(character) => self.emit_char(character),
+ NotFromSet(characters) => self.emit_chars(characters),
}
},
- //§ rcdata-state
+ // https://html.spec.whatwg.org/#rcdata-state
states::RawData(Rcdata) => loop {
+ // Consume the next input character:
let Some(set_result) =
self.pop_except_from(input, small_char_set!('\r' '\0' '&' '<' '\n'))
else {
@@ -798,19 +811,30 @@ impl Tokenizer {
};
match set_result {
+ // ↪ U+0026 AMPERSAND (&)
+ FromSet('&') => {
+ go!(self: consume_char_ref)
+ },
+ // ↪ U+003C LESS-THAN SIGN (<)
+ FromSet('<') => {
+ // Switch to the RCDATA less-than sign state.
+ go!(self: to State::RawLessThanSign(Rcdata))
+ },
+ // ↪ U+0000 NULL
FromSet('\0') => {
self.bad_char_error();
self.emit_char('\u{fffd}');
},
- FromSet('&') => go!(self: consume_char_ref),
- FromSet('<') => go!(self: to State::RawLessThanSign(Rcdata)),
- FromSet(c) => self.emit_char(c),
- NotFromSet(b) => self.emit_chars(b),
+ // ↪ Anything else
+ // Emit the current input character as a character token.
+ FromSet(character) => self.emit_char(character),
+ NotFromSet(characters) => self.emit_chars(characters),
}
},
- //§ rawtext-state
+ // https://html.spec.whatwg.org/#rawtext-state
states::RawData(Rawtext) => loop {
+ // Consume the next input character:
let Some(set_result) =
self.pop_except_from(input, small_char_set!('\r' '\0' '<' '\n'))
else {
@@ -818,18 +842,28 @@ impl Tokenizer {
};
match set_result {
+ // ↪ U+003C LESS-THAN SIGN (<)
+ FromSet('<') => {
+ // Switch to the RAWTEXT less-than sign state.
+ go!(self: to State::RawLessThanSign(Rawtext));
+ },
+ // ↪ U+0000 NULL
FromSet('\0') => {
+ // This is an unexpected-null-character parse error.
+ // Emit a U+FFFD REPLACEMENT CHARACTER character token.
self.bad_char_error();
self.emit_char('\u{fffd}');
},
- FromSet('<') => go!(self: to State::RawLessThanSign(Rawtext)),
- FromSet(c) => self.emit_char(c),
- NotFromSet(b) => self.emit_chars(b),
+ // ↪ Anything else
+ // Emit the current input character as a character token.
+ FromSet(character) => self.emit_char(character),
+ NotFromSet(characters) => self.emit_chars(characters),
}
},
- //§ script-data-state
+ // https://html.spec.whatwg.org/#script-data-state
states::RawData(ScriptData) => loop {
+ // Consume the next input character:
let Some(set_result) =
self.pop_except_from(input, small_char_set!('\r' '\0' '<' '\n'))
else {
@@ -837,148 +871,185 @@ impl Tokenizer {
};
match set_result {
- FromSet('\0') => {
- self.bad_char_error();
- self.emit_char('\u{fffd}');
- },
- FromSet('<') => go!(self: to State::RawLessThanSign(ScriptData)),
- FromSet(c) => self.emit_char(c),
- NotFromSet(b) => self.emit_chars(b),
- }
- },
-
- //§ script-data-escaped-state
- states::RawData(ScriptDataEscaped(Escaped)) => loop {
- let Some(set_result) =
- self.pop_except_from(input, small_char_set!('\r' '\0' '-' '<' '\n'))
- else {
- return ProcessResult::Suspend;
- };
-
- match set_result {
- FromSet('\0') => {
- self.bad_char_error();
- self.emit_char('\u{fffd}');
- },
- FromSet('-') => {
- self.emit_char('-');
- go!(self: to State::ScriptDataEscapedDash(Escaped));
- },
+ // ↪ U+003C LESS-THAN SIGN (<)
FromSet('<') => {
- go!(self: to State::RawLessThanSign(ScriptDataEscaped(Escaped)))
+ // Switch to the script data less-than sign state.
+ go!(self: to State::RawLessThanSign(ScriptData));
},
- FromSet(c) => self.emit_char(c),
- NotFromSet(b) => self.emit_chars(b),
- }
- },
-
- //§ script-data-double-escaped-state
- states::RawData(ScriptDataEscaped(DoubleEscaped)) => loop {
- let Some(set_result) =
- self.pop_except_from(input, small_char_set!('\r' '\0' '-' '<' '\n'))
- else {
- return ProcessResult::Suspend;
- };
-
- match set_result {
+ // ↪ U+0000 NULL
FromSet('\0') => {
+ // This is an unexpected-null-character parse error.
+ // Emit a U+FFFD REPLACEMENT CHARACTER character token.
self.bad_char_error();
self.emit_char('\u{fffd}');
},
- FromSet('-') => {
- self.emit_char('-');
- go!(self: to State::ScriptDataEscapedDash(DoubleEscaped));
- },
- FromSet('<') => {
- self.emit_char('<');
- go!(self: to State::RawLessThanSign(ScriptDataEscaped(DoubleEscaped)))
- },
- FromSet(c) => self.emit_char(c),
- NotFromSet(b) => self.emit_chars(b),
+ // ↪ Anything else
+ // Emit the current input character as a character token.
+ FromSet(character) => self.emit_char(character),
+ NotFromSet(characters) => self.emit_chars(characters),
}
},
- //§ plaintext-state
+ // https://html.spec.whatwg.org/#plaintext-state
states::Plaintext => loop {
+ // Consume the next input character:
let Some(set_result) = self.pop_except_from(input, small_char_set!('\r' '\0' '\n'))
else {
return ProcessResult::Suspend;
};
match set_result {
+ // ↪ U+0000 NULL
FromSet('\0') => {
+ // This is an unexpected-null-character parse error.
+ // Emit a U+FFFD REPLACEMENT CHARACTER character token.
self.bad_char_error();
self.emit_char('\u{fffd}');
},
- FromSet(c) => self.emit_char(c),
- NotFromSet(b) => self.emit_chars(b),
+ // ↪ Anything else
+ // Emit the current input character as a character token.
+ FromSet(character) => self.emit_char(character),
+ NotFromSet(characters) => self.emit_chars(characters),
}
},
- //§ tag-open-state
+ // https://html.spec.whatwg.org/#tag-open-state
states::TagOpen => loop {
+ // Consume the next input character:
match get_char!(self, input) {
- '!' => go!(self: to State::MarkupDeclarationOpen),
- '/' => go!(self: to State::EndTagOpen),
+ // ↪ U+0021 EXCLAMATION MARK (!)
+ '!' => {
+ // Switch to the markup declaration open state.
+ go!(self: to State::MarkupDeclarationOpen)
+ },
+ // ↪ U+002F SOLIDUS (/)
+ '/' => {
+ // Switch to the end tag open state.
+ go!(self: to State::EndTagOpen)
+ },
+ // ↪ ASCII alpha
+ character if character.is_ascii_alphabetic() => {
+ // Create a new start tag token, set its tag name to the empty string.
+ // Reconsume in the tag name state.
+ // NOTE: We don't reconsume the character but instead immediately append it in lowercase
+ // to the new tag (as that is what the "tag name" state would do).
+ let character = character.to_ascii_lowercase();
+ go!(self: create_tag StartTag character; to State::TagName)
+ },
+ // ↪ U+003F QUESTION MARK (?)
'?' => {
+ // Set the temporary buffer to the empty string.
+ // Switch to the processing instruction open state.
self.bad_char_error();
go!(self: clear_comment; reconsume BogusComment)
},
- c => match lower_ascii_letter(c) {
- Some(cl) => go!(self: create_tag StartTag cl; to State::TagName),
- None => {
- self.bad_char_error();
- self.emit_char('<');
- go!(self: reconsume Data)
- },
+ // ↪ Anything else
+ _ => {
+ // This is an invalid-first-character-of-tag-name parse error.
+ // Emit a U+003C LESS-THAN SIGN character token.
+ // Reconsume in the data state.
+ self.bad_char_error();
+ self.emit_char('<');
+ go!(self: reconsume Data)
},
}
},
- //§ end-tag-open-state
+ // https://html.spec.whatwg.org/#end-tag-open-state
states::EndTagOpen => loop {
+ // Consume the next input character:
match get_char!(self, input) {
+ // ↪ ASCII alpha
+ character if character.is_ascii_alphabetic() => {
+ // Create a new end tag token, set its tag name to the empty string.
+ // Reconsume in the tag name state.
+ // NOTE: We don't reconsume the character but instead immediately append in lowercase
+ // to the new tag (as that is what the "tag name" state would do).
+ let character = character.to_ascii_lowercase();
+ go!(self: create_tag EndTag character; to State::TagName)
+ },
+ // ↪ U+003E GREATER-THAN SIGN (>)
'>' => {
+ // This is a missing-end-tag-name parse error.
+ // Switch to the data state.
self.bad_char_error();
go!(self: to State::Data)
},
- c => match lower_ascii_letter(c) {
- Some(cl) => go!(self: create_tag EndTag cl; to State::TagName),
- None => {
- self.bad_char_error();
- go!(self: clear_comment; reconsume BogusComment)
- },
+ // ↪ Anything else
+ _ => {
+ // This is an invalid-first-character-of-tag-name parse error.
+ // Create a comment token whose data is the empty string.
+ // Reconsume in the bogus comment state.
+ self.bad_char_error();
+ go!(self: clear_comment; reconsume BogusComment)
},
}
},
- //§ tag-name-state
+ // https://html.spec.whatwg.org/#tag-name-state
states::TagName => loop {
+ // Consume the next input character:
match get_char!(self, input) {
- '\t' | '\n' | '\x0C' | ' ' => go!(self: to State::BeforeAttributeName),
- '/' => go!(self: to State::SelfClosingStartTag),
- '>' => go!(self: emit_tag Data),
+ // ↪ U+0009 CHARACTER TABULATION (tab)
+ // ↪ U+000A LINE FEED (LF)
+ // ↪ U+000C FORM FEED (FF)
+ // ↪ U+0020 SPACE
+ '\t' | '\n' | '\x0C' | ' ' => {
+ // Switch to the before attribute name state.
+ go!(self: to State::BeforeAttributeName)
+ },
+ // ↪ U+002F SOLIDUS (/)
+ '/' => {
+ // Switch to the self-closing start tag state.
+ go!(self: to State::SelfClosingStartTag)
+ },
+ // ↪ U+003E GREATER-THAN SIGN (>)
+ '>' => {
+ // Switch to the data state.
+ // Emit the current tag token.
+ go!(self: emit_tag Data)
+ },
+ // ↪ ASCII upper alpha
+ character if character.is_ascii_uppercase() => {
+ // Append the lowercase version of the current input character (add 0x0020 to the
+ // character's code point) to the current tag token's tag name.
+ go!(self: push_tag (character.to_ascii_lowercase()))
+ },
+ // ↪ U+0000 NULL
'\0' => {
self.bad_char_error();
go!(self: push_tag '\u{fffd}')
},
- c => go!(self: push_tag (c.to_ascii_lowercase())),
+ // ↪ Anything else
+ character => {
+ // Append the current input character to the current tag token's tag name.
+ go!(self: push_tag (character))
+ },
}
},
- //§ script-data-escaped-less-than-sign-state
+ // https://html.spec.whatwg.org/#script-data-escaped-less-than-sign-state
states::RawLessThanSign(ScriptDataEscaped(Escaped)) => loop {
+ // Consume the next input character:
match get_char!(self, input) {
+ // ↪ U+002F SOLIDUS (/)
'/' => {
go!(self: clear_temp; to State::RawEndTagOpen(ScriptDataEscaped(Escaped)))
},
- c => match lower_ascii_letter(c) {
- Some(cl) => {
- go!(self: clear_temp; push_temp cl);
+ character => match lower_ascii_letter(character) {
+ // ↪ ASCII alpha
+ Some(character_lowercase) => {
+ // Set the temporary buffer to the empty string.
+ // Emit a U+003C LESS-THAN SIGN character token.
+ // Reconsume in the script data double escape start state.
+ // NOTE: We don't reconsume, and instead emit the lowercased character immediately,
+ // as that is what the "script data double escape start" state would do.
+ go!(self: clear_temp; push_temp character_lowercase);
self.emit_char('<');
- self.emit_char(c);
+ self.emit_char(character);
go!(self: to State::ScriptDataEscapeStart(DoubleEscaped));
},
+ // ↪ Anything else
None => {
self.emit_char('<');
go!(self: reconsume RawData(ScriptDataEscaped(Escaped)));
@@ -987,67 +1058,133 @@ impl Tokenizer {
}
},
- //§ script-data-double-escaped-less-than-sign-state
+ // https://html.spec.whatwg.org/#script-data-double-escaped-less-than-sign-state
states::RawLessThanSign(ScriptDataEscaped(DoubleEscaped)) => loop {
+ // Consume the next input character:
match get_char!(self, input) {
+ // ↪ U+002F SOLIDUS (/)
'/' => {
+ // Set the temporary buffer to the empty string.
+ // Switch to the script data double escape end state.
+ // Emit a U+002F SOLIDUS character token.
go!(self: clear_temp);
self.emit_char('/');
go!(self: to State::ScriptDataDoubleEscapeEnd);
},
- _ => go!(self: reconsume RawData(ScriptDataEscaped(DoubleEscaped))),
+ // ↪ Anything else
+ _ => {
+ // Reconsume in the script data double escaped state.
+ go!(self: reconsume RawData(ScriptDataEscaped(DoubleEscaped)))
+ },
}
},
- //§ rcdata-less-than-sign-state rawtext-less-than-sign-state script-data-less-than-sign-state
- // otherwise
+ // https://html.spec.whatwg.org/#rcdata-less-than-sign-state
+ // https://html.spec.whatwg.org/#script-data-less-than-sign-state
+ // https://html.spec.whatwg.org/#rawtext-less-than-sign-state
states::RawLessThanSign(kind) => loop {
+ // Consume the next input character:
match get_char!(self, input) {
- '/' => go!(self: clear_temp; to State::RawEndTagOpen(kind)),
+ // ↪ U+002F SOLIDUS (/)
+ '/' => {
+ // Set the temporary buffer to the empty string.
+ // Switch to the RCDATA end tag open state.
+ go!(self: clear_temp; to State::RawEndTagOpen(kind))
+ },
+ // ↪ U+0021 EXCLAMATION MARK (!)
'!' if kind == ScriptData => {
+ // Switch to the script data escape start state.
+ // Emit a U+003C LESS-THAN SIGN character token and a U+0021 EXCLAMATION MARK character token.
self.emit_char('<');
self.emit_char('!');
go!(self: to State::ScriptDataEscapeStart(Escaped));
},
+ // ↪ Anything else
_ => {
+ // Emit a U+003C LESS-THAN SIGN character token.
+ // Reconsume in the RCDATA/script data/RAWTEXT state.
self.emit_char('<');
go!(self: reconsume RawData(kind));
},
}
},
- //§ rcdata-end-tag-open-state rawtext-end-tag-open-state script-data-end-tag-open-state script-data-escaped-end-tag-open-state
+ // https://html.spec.whatwg.org/#rcdata-end-tag-open-state
+ // https://html.spec.whatwg.org/#rawtext-end-tag-open-state
+ // https://html.spec.whatwg.org/#script-data-end-tag-open-state
+ // https://html.spec.whatwg.org/#script-data-escaped-end-tag-open-state
states::RawEndTagOpen(kind) => loop {
- let c = get_char!(self, input);
- match lower_ascii_letter(c) {
- Some(cl) => {
- go!(self: create_tag EndTag cl; push_temp c; to State::RawEndTagName(kind))
- },
- None => {
- self.emit_char('<');
- self.emit_char('/');
- go!(self: reconsume RawData(kind));
- },
+ // Consume the next input character:
+ let character = get_char!(self, input);
+
+ // ↪ ASCII alpha
+ if character.is_ascii_alphabetic() {
+ // Create a new end tag token, set its tag name to the empty string.
+ // Reconsume in the RCDATA end tag name state.
+ // NOTE: We don't reconsume the character but instead immediately append its lowercase version
+ // to the end tag (as that is what the new state would do).
+ let character_lowercase = character.to_ascii_lowercase();
+ go!(self: create_tag EndTag character_lowercase; push_temp character; to State::RawEndTagName(kind))
+ }
+ // ↪ Anything else
+ else {
+ // Emit a U+003C LESS-THAN SIGN character token and a U+002F SOLIDUS character token.
+ // Reconsume in the RCDATA/RAWTEXT/script data/script data escaped state.
+ self.emit_char('<');
+ self.emit_char('/');
+ go!(self: reconsume RawData(kind));
}
},
- //§ rcdata-end-tag-name-state rawtext-end-tag-name-state script-data-end-tag-name-state script-data-escaped-end-tag-name-state
+ // https://html.spec.whatwg.org/#rcdata-end-tag-name-state
+ // https://html.spec.whatwg.org/#rawtext-end-tag-name-state
+ // https://html.spec.whatwg.org/#script-data-end-tag-name-state
+ // https://html.spec.whatwg.org/#script-data-escaped-end-tag-name-state
states::RawEndTagName(kind) => loop {
- let c = get_char!(self, input);
+ let character = get_char!(self, input);
+
+ // NOTE: The first three match arms in the specification are treated as "anything else" if the current
+ // end tag token is NOT an appropriate end tag, so we move them into their own match block.
if self.have_appropriate_end_tag() {
- match c {
+ match character {
+ // ↪ U+0009 CHARACTER TABULATION (tab)
+ // ↪ U+000A LINE FEED (LF)
+ // ↪ U+000C FORM FEED (FF)
+ // ↪ U+0020 SPACE
'\t' | '\n' | '\x0C' | ' ' => {
+ // Switch to the before attribute name state
go!(self: clear_temp; to State::BeforeAttributeName)
},
- '/' => go!(self: clear_temp; to State::SelfClosingStartTag),
- '>' => go!(self: clear_temp; emit_tag Data),
- _ => (),
+ // ↪ U+002F SOLIDUS (/)
+ '/' => {
+ // Switch to the self-closing start tag state
+ go!(self: clear_temp; to State::SelfClosingStartTag)
+ },
+ // ↪ U+003E GREATER-THAN SIGN (>)
+ '>' => {
+ // Switch to the data state and emit the current tag token
+ go!(self: clear_temp; emit_tag Data)
+ },
+ _ => {},
}
}
- match lower_ascii_letter(c) {
- Some(cl) => go!(self: push_tag cl; push_temp c),
+ match lower_ascii_letter(character) {
+ // ↪ ASCII upper alpha
+ // NOTE: This is the same as for lower alpha, but the character is lowercased first.
+ // This is handled by lower_ascii_letter.
+ // ↪ ASCII lower alpha
+ Some(character_lowercase) => {
+ // Append the current input character to the current tag token's tag name.
+ // Append the current input character to the temporary buffer.
+ go!(self: push_tag character_lowercase; push_temp character)
+ },
+ // ↪ Anything else
None => {
+ // Emit a U+003C LESS-THAN SIGN character token, a U+002F SOLIDUS character token,
+ // and a character token for each of the characters in the temporary buffer
+ // (in the order they were added to the buffer).
+ // Reconsume in the RCDATA/RAWTEXT/script data/script data escaped state.
go!(self: discard_tag);
self.emit_char('<');
self.emit_char('/');
@@ -1057,216 +1194,486 @@ impl Tokenizer {
}
},
- //§ script-data-double-escape-start-state
+ // https://html.spec.whatwg.org/#script-data-double-escape-start-state
states::ScriptDataEscapeStart(DoubleEscaped) => loop {
- let c = get_char!(self, input);
- match c {
- '\t' | '\n' | '\x0C' | ' ' | '/' | '>' => {
- let esc = if &**self.temp_buf.borrow() == "script" {
+ // Consume the next input character:
+ match get_char!(self, input) {
+ // ↪ U+0009 CHARACTER TABULATION (tab)
+ // ↪ U+000A LINE FEED (LF)
+ // ↪ U+000C FORM FEED (FF)
+ // ↪ U+0020 SPACE
+ // ↪ U+002F SOLIDUS (/)
+ // ↪ U+003E GREATER-THAN SIGN (>)
+ character @ ('\t' | '\n' | '\x0C' | ' ' | '/' | '>') => {
+ // If the temporary buffer is "script", then switch to the script data double escaped state.
+ // Otherwise, switch to the script data escaped state.
+ // Emit the current input character as a character token.
+ let escaped_kind = if &**self.temp_buf.borrow() == "script" {
DoubleEscaped
} else {
Escaped
};
- self.emit_char(c);
- go!(self: to State::RawData(ScriptDataEscaped(esc)));
- },
- _ => match lower_ascii_letter(c) {
- Some(cl) => {
- go!(self: push_temp cl);
- self.emit_char(c);
+ self.emit_char(character);
+ go!(self: to State::RawData(ScriptDataEscaped(escaped_kind)));
+ },
+ // ↪ ASCII upper alpha
+ // NOTE: This is the same as the "ASCII lower alpha" branch, except we lowercase the character first
+ // ↪ ASCII lower alpha
+ character => match lower_ascii_letter(character) {
+ Some(character_lowercase) => {
+ // Append the current input character to the temporary buffer.
+ // Emit the current input character as a character token.
+ go!(self: push_temp character_lowercase);
+ self.emit_char(character);
+ },
+ // ↪ Anything else
+ None => {
+ // Reconsume in the script data escaped state.
+ go!(self: reconsume RawData(ScriptDataEscaped(Escaped)))
},
- None => go!(self: reconsume RawData(ScriptDataEscaped(Escaped))),
},
}
},
- //§ script-data-escape-start-state
+ // https://html.spec.whatwg.org/#script-data-double-escaped-state
+ states::RawData(ScriptDataEscaped(DoubleEscaped)) => loop {
+ // Consume the next input character:
+ let Some(set_result) =
+ self.pop_except_from(input, small_char_set!('\r' '\0' '-' '<' '\n'))
+ else {
+ return ProcessResult::Suspend;
+ };
+
+ match set_result {
+ // ↪ U+002D HYPHEN-MINUS (-)
+ FromSet('-') => {
+ // Switch to the script data double escaped dash state.
+ // Emit a U+002D HYPHEN-MINUS character token.
+ self.emit_char('-');
+ go!(self: to State::ScriptDataEscapedDash(DoubleEscaped));
+ },
+ // ↪ U+003C LESS-THAN SIGN (<)
+ FromSet('<') => {
+ // Switch to the script data double escaped less-than sign state.
+ // Emit a U+003C LESS-THAN SIGN character token.
+ self.emit_char('<');
+ go!(self: to State::RawLessThanSign(ScriptDataEscaped(DoubleEscaped)))
+ },
+ // ↪ U+0000 NULL
+ FromSet('\0') => {
+ // This is an unexpected-null-character parse error.
+ // Emit a U+FFFD REPLACEMENT CHARACTER character token.
+ self.bad_char_error();
+ self.emit_char('\u{fffd}');
+ },
+ // ↪ Anything else
+ // Emit the current input character as a character token.
+ FromSet(character) => self.emit_char(character),
+ NotFromSet(characters) => self.emit_chars(characters),
+ }
+ },
+
+ // https://html.spec.whatwg.org/#script-data-escape-start-state
states::ScriptDataEscapeStart(Escaped) => loop {
+ // Consume the next input character:
match get_char!(self, input) {
+ // ↪ U+002D HYPHEN-MINUS (-)
'-' => {
+ // Switch to the script data escape start dash state.
+ // Emit a U+002D HYPHEN-MINUS character token.
self.emit_char('-');
go!(self: to State::ScriptDataEscapeStartDash);
},
- _ => go!(self: reconsume RawData(ScriptData)),
+ // ↪ Anything else
+ _ => {
+ // Reconsume in the script data state.
+ go!(self: reconsume RawData(ScriptData))
+ },
}
},
- //§ script-data-escape-start-dash-state
+ // https://html.spec.whatwg.org/#script-data-escape-start-dash-state
states::ScriptDataEscapeStartDash => loop {
+ // Consume the next input character:
match get_char!(self, input) {
+ // ↪ U+002D HYPHEN-MINUS (-)
'-' => {
+ // Switch to the script data escaped dash dash state.
+ // Emit a U+002D HYPHEN-MINUS character token.
self.emit_char('-');
go!(self: to State::ScriptDataEscapedDashDash(Escaped));
},
- _ => go!(self: reconsume RawData(ScriptData)),
+ // ↪ Anything else
+ _ => {
+ // Reconsume in the script data state.
+ go!(self: reconsume RawData(ScriptData))
+ },
+ }
+ },
+
+ // https://html.spec.whatwg.org/#script-data-escaped-state
+ states::RawData(ScriptDataEscaped(Escaped)) => loop {
+ // Consume the next input character:
+ let Some(set_result) =
+ self.pop_except_from(input, small_char_set!('\r' '\0' '-' '<' '\n'))
+ else {
+ return ProcessResult::Suspend;
+ };
+
+ match set_result {
+ // ↪ U+002D HYPHEN-MINUS (-)
+ FromSet('-') => {
+ // Switch to the script data escaped dash state.
+ // Emit a U+002D HYPHEN-MINUS character token.
+ self.emit_char('-');
+ go!(self: to State::ScriptDataEscapedDash(Escaped));
+ },
+ // ↪ U+003C LESS-THAN SIGN (<)
+ FromSet('<') => {
+ // Switch to the script data escaped less-than sign state.
+ go!(self: to State::RawLessThanSign(ScriptDataEscaped(Escaped)))
+ },
+ // ↪ U+0000 NULL
+ FromSet('\0') => {
+ // This is an unexpected-null-character parse error.
+ // Emit a U+FFFD REPLACEMENT CHARACTER character token.
+ self.bad_char_error();
+ self.emit_char('\u{fffd}');
+ },
+ // ↪ Anything else
+ // Emit the current input character as a character token.
+ FromSet(character) => self.emit_char(character),
+ NotFromSet(characters) => self.emit_chars(characters),
}
},
- //§ script-data-escaped-dash-state script-data-double-escaped-dash-state
+ // https://html.spec.whatwg.org/#script-data-escaped-dash-state
+ // https://html.spec.whatwg.org/#script-data-double-escaped-dash-state
states::ScriptDataEscapedDash(kind) => loop {
+ // Consume the next input character:
match get_char!(self, input) {
+ // ↪ U+002D HYPHEN-MINUS (-)
'-' => {
+ // Switch to the script data escaped dash dash/script data double escaped dash dash state.
+ // Emit a U+002D HYPHEN-MINUS character token.
self.emit_char('-');
go!(self: to State::ScriptDataEscapedDashDash(kind));
},
+ // ↪ U+003C LESS-THAN SIGN (<)
'<' => {
+ // Switch to the script data escaped less-than sign/script data double escaped less-than sign state.
if kind == DoubleEscaped {
self.emit_char('<');
}
go!(self: to State::RawLessThanSign(ScriptDataEscaped(kind)));
},
+ // ↪ U+0000 NULL
'\0' => {
+ // This is an unexpected-null-character parse error.
+ // Switch to the script data escaped/script data double escaped state.
+ // Emit a U+FFFD REPLACEMENT CHARACTER character token.
self.bad_char_error();
self.emit_char('\u{fffd}');
go!(self: to State::RawData(ScriptDataEscaped(kind)));
},
+ // ↪ Anything else
c => {
+ // Switch to the script data escaped/script data double escaped state.
+ // Emit the current input character as a character token.
self.emit_char(c);
go!(self: to State::RawData(ScriptDataEscaped(kind)));
},
}
},
- //§ script-data-escaped-dash-dash-state script-data-double-escaped-dash-dash-state
+ // https://html.spec.whatwg.org/#script-data-escaped-dash-dash-state
+ // https://html.spec.whatwg.org/#script-data-double-escaped-dash-dash-state
states::ScriptDataEscapedDashDash(kind) => loop {
+ // Consume the next input character:
match get_char!(self, input) {
+ // ↪ U+002D HYPHEN-MINUS (-)
'-' => {
+ // Emit a U+002D HYPHEN-MINUS character token.
self.emit_char('-');
},
+ // ↪ U+003C LESS-THAN SIGN (<)
'<' => {
+ // Switch to the script data escaped less-than sign/script data double escaped less-than sign state.
if kind == DoubleEscaped {
self.emit_char('<');
}
go!(self: to State::RawLessThanSign(ScriptDataEscaped(kind)));
},
+ // ↪ U+003E GREATER-THAN SIGN (>)
'>' => {
+ // Switch to the script data state.
+ // Emit a U+003E GREATER-THAN SIGN character token.
self.emit_char('>');
go!(self: to State::RawData(ScriptData));
},
+ // ↪ U+0000 NULL
'\0' => {
+ // This is an unexpected-null-character parse error.
+ // Switch to the script data escaped/script data double escaped state.
+ // Emit a U+FFFD REPLACEMENT CHARACTER character token.
self.bad_char_error();
self.emit_char('\u{fffd}');
go!(self: to State::RawData(ScriptDataEscaped(kind)))
},
- c => {
- self.emit_char(c);
+ // ↪ Anything else
+ character => {
+ // Switch to the script data escaped/script data double escaped state.
+ // Emit the current input character as a character token.
+ self.emit_char(character);
go!(self: to State::RawData(ScriptDataEscaped(kind)));
},
}
},
- //§ script-data-double-escape-end-state
+ // https://html.spec.whatwg.org/#script-data-double-escape-end-state
states::ScriptDataDoubleEscapeEnd => loop {
- let c = get_char!(self, input);
- match c {
- '\t' | '\n' | '\x0C' | ' ' | '/' | '>' => {
- let esc = if &**self.temp_buf.borrow() == "script" {
+ // Consume the next input character:
+ match get_char!(self, input) {
+ // ↪ U+0009 CHARACTER TABULATION (tab)
+ // ↪ U+000A LINE FEED (LF)
+ // ↪ U+000C FORM FEED (FF)
+ // ↪ U+0020 SPACE
+ // ↪ U+002F SOLIDUS (/)
+ // ↪ U+003E GREATER-THAN SIGN (>)
+ character @ ('\t' | '\n' | '\x0C' | ' ' | '/' | '>') => {
+ // If the temporary buffer is "script", then switch to the script data escaped state.
+ // Otherwise, switch to the script data double escaped state.
+ // Emit the current input character as a character token.
+ let escaped_kind = if &**self.temp_buf.borrow() == "script" {
Escaped
} else {
DoubleEscaped
};
- self.emit_char(c);
- go!(self: to State::RawData(ScriptDataEscaped(esc)));
- },
- _ => match lower_ascii_letter(c) {
- Some(cl) => {
- go!(self: push_temp cl);
- self.emit_char(c);
+ self.emit_char(character);
+ go!(self: to State::RawData(ScriptDataEscaped(escaped_kind)));
+ },
+
+ character => match lower_ascii_letter(character) {
+ // ↪ ASCII upper alpha
+ // NOTE: This is the same as "ASCII lower alpha", except we lowercase the character first.
+ // That is handled by lower_ascii_letter.
+ // ↪ ASCII lower alpha
+ Some(character_lowercase) => {
+ // Append the current input character to the temporary buffer.
+ // Emit the current input character as a character token.
+ go!(self: push_temp character_lowercase);
+ self.emit_char(character);
+ },
+ // ↪ Anything else
+ None => {
+ // Reconsume in the script data double escaped state.
+ go!(self: reconsume RawData(ScriptDataEscaped(DoubleEscaped)))
},
- None => go!(self: reconsume RawData(ScriptDataEscaped(DoubleEscaped))),
},
}
},
- //§ before-attribute-name-state
+ // https://html.spec.whatwg.org/#before-attribute-name-state
states::BeforeAttributeName => loop {
match get_char!(self, input) {
- '\t' | '\n' | '\x0C' | ' ' => (),
- '/' => go!(self: to State::SelfClosingStartTag),
- '>' => go!(self: emit_tag Data),
+ // ↪ U+0009 CHARACTER TABULATION (tab)
+ // ↪ U+000A LINE FEED (LF)
+ // ↪ U+000C FORM FEED (FF)
+ // ↪ U+0020 SPACE
+ '\t' | '\n' | '\x0C' | ' ' => {
+ // Ignore the character.
+ },
+ // U+002F SOLIDUS (/)
+ '/' => {
+ // Reconsume in the after attribute name state.
+ // NOTE: Instead we move to the self closing start tag,
+ // as that is what the "after attribute name" state would do.
+ go!(self: to State::SelfClosingStartTag)
+ },
+ // U+003E GREATER-THAN SIGN (>)
+ '>' => {
+ // Reconsume in the after attribute name state.
+ // NOTE: Instead we emit the current tag and move to the data state,
+ // as that is what the "after attribute name" state would do.
+ go!(self: emit_tag Data)
+ },
+ // NOTE: In the "anything else" case we should reconsume in the attribute name state,
+ // but instead of reconsuming we inline what that state *would* do here.
'\0' => {
self.bad_char_error();
go!(self: create_attr '\u{fffd}'; to State::AttributeName)
},
- c => match lower_ascii_letter(c) {
- Some(cl) => go!(self: create_attr cl; to State::AttributeName),
+ character => match lower_ascii_letter(character) {
+ Some(character) => {
+ go!(self: create_attr character; to State::AttributeName)
+ },
None => {
- if matches!(c, '"' | '\'' | '<' | '=') {
+ if matches!(character, '"' | '\'' | '<' | '=') {
self.bad_char_error();
}
- go!(self: create_attr c; to State::AttributeName);
+ go!(self: create_attr character; to State::AttributeName);
},
},
}
},
- //§ attribute-name-state
+ // https://html.spec.whatwg.org/#attribute-name-state
states::AttributeName => loop {
+ // Consume the next input character:
match get_char!(self, input) {
- '\t' | '\n' | '\x0C' | ' ' => go!(self: to State::AfterAttributeName),
- '/' => go!(self: to State::SelfClosingStartTag),
- '=' => go!(self: to State::BeforeAttributeValue),
- '>' => go!(self: emit_tag Data),
+ // ↪ U+0009 CHARACTER TABULATION (tab)
+ // ↪ U+000A LINE FEED (LF)
+ // ↪ U+000C FORM FEED (FF)
+ // ↪ U+0020 SPACE
+ '\t' | '\n' | '\x0C' | ' ' => {
+ // Reconsume in the after attribute name state.
+ // NOTE: Instead we move to the after attribute name state and ignore
+ // the character, as that state would ignore it as well.
+ go!(self: to State::AfterAttributeName)
+ },
+ // ↪ U+002F SOLIDUS (/)
+ '/' => {
+ // Reconsume in the after attribute name state.
+ // NOTE: Instead we move to the self closing start tag state, as that is
+ // what the after attribute name state would do when it encounters a '/'.
+ go!(self: to State::SelfClosingStartTag)
+ },
+ // ↪ U+003E GREATER-THAN SIGN (>)
+ '>' => {
+ // Reconsume in the after attribute name state.
+ // NOTE: Instead we move to the data state, as that is
+ // what the after attribute name state would do when it encounters a '>'.
+ go!(self: emit_tag Data)
+ },
+ // ↪ U+003D EQUALS SIGN (=)
+ '=' => {
+ // Switch to the before attribute value state.
+ go!(self: to State::BeforeAttributeValue)
+ },
+ // ↪ U+0000 NULL
'\0' => {
+ // This is an unexpected-null-character parse error.
+ // Append a U+FFFD REPLACEMENT CHARACTER character to the current attribute's name.
self.bad_char_error();
go!(self: push_name '\u{fffd}')
},
- c => match lower_ascii_letter(c) {
- Some(cl) => go!(self: push_name cl),
+ character => match lower_ascii_letter(character) {
+ // ↪ ASCII upper alpha
+ Some(character_lowercase) => {
+ // Append the lowercase version of the current input character
+ // (add 0x0020 to the character's code point) to the current attribute's name.
+ go!(self: push_name character_lowercase)
+ },
None => {
- if matches!(c, '"' | '\'' | '<') {
+ // ↪ U+0022 QUOTATION MARK (")
+ // ↪ U+0027 APOSTROPHE (')
+ // ↪ U+003C LESS-THAN SIGN (<)
+ if matches!(character, '"' | '\'' | '<') {
+ // This is an unexpected-character-in-attribute-name parse error.
+ // Treat it as per the "anything else" entry below.
self.bad_char_error();
}
- go!(self: push_name c);
+ // ↪ Anything else
+ // Append the current input character to the current attribute's name.
+ go!(self: push_name character);
},
},
}
},
- //§ after-attribute-name-state
+ // https://html.spec.whatwg.org/#after-attribute-name-state
states::AfterAttributeName => loop {
+ // Consume the next input character:
match get_char!(self, input) {
- '\t' | '\n' | '\x0C' | ' ' => (),
- '/' => go!(self: to State::SelfClosingStartTag),
- '=' => go!(self: to State::BeforeAttributeValue),
- '>' => go!(self: emit_tag Data),
+ // ↪ U+0009 CHARACTER TABULATION (tab)
+ // ↪ U+000A LINE FEED (LF)
+ // ↪ U+000C FORM FEED (FF)
+ // ↪ U+0020 SPACE
+ '\t' | '\n' | '\x0C' | ' ' => {
+ // Ignore the character.
+ },
+ // ↪ U+002F SOLIDUS (/)
+ '/' => {
+ // Switch to the self-closing start tag state.
+ go!(self: to State::SelfClosingStartTag)
+ },
+ // ↪ U+003D EQUALS SIGN (=)
+ '=' => {
+ // Switch to the before attribute value state.
+ go!(self: to State::BeforeAttributeValue)
+ },
+ // ↪ U+003E GREATER-THAN SIGN (>)
+ '>' => {
+ // Switch to the data state. Emit the current tag token.
+ go!(self: emit_tag Data)
+ },
+ // NOTE: The "anything else" match arm in the specification reconsumes
+ // the input in the attribute name state. Instead of reconsuming we inline
+ // what the attribute name state *would* do, and the move to it.
'\0' => {
self.bad_char_error();
go!(self: create_attr '\u{fffd}'; to State::AttributeName)
},
- c => match lower_ascii_letter(c) {
- Some(cl) => go!(self: create_attr cl; to State::AttributeName),
+ character => match lower_ascii_letter(character) {
+ Some(character_lowercase) => {
+ go!(self: create_attr character_lowercase; to State::AttributeName)
+ },
None => {
- if matches!(c, '"' | '\'' | '<') {
+ if matches!(character, '"' | '\'' | '<') {
self.bad_char_error();
}
- go!(self: create_attr c; to State::AttributeName);
+ go!(self: create_attr character; to State::AttributeName);
},
},
}
},
- //§ before-attribute-value-state
+ // https://html.spec.whatwg.org/#before-attribute-value-state
// Use peek so we can handle the first attr character along with the rest,
// hopefully in the same zero-copy buffer.
states::BeforeAttributeValue => loop {
+ // Consume the next input character:
match peek!(self, input) {
- '\t' | '\n' | '\r' | '\x0C' | ' ' => go!(self: discard_char input),
- '"' => go!(self: discard_char input; to State::AttributeValue(DoubleQuoted)),
- '\'' => go!(self: discard_char input; to State::AttributeValue(SingleQuoted)),
+ // ↪ U+0009 CHARACTER TABULATION (tab)
+ // ↪ U+000A LINE FEED (LF)
+ // ↪ U+000C FORM FEED (FF)
+ // ↪ U+0020 SPACE
+ '\t' | '\n' | '\r' | '\x0C' | ' ' => {
+ // Ignore the character.
+ go!(self: discard_char input)
+ },
+ // ↪ U+0022 QUOTATION MARK (")
+ '"' => {
+ // Switch to the attribute value (double-quoted) state.
+ go!(self: discard_char input; to State::AttributeValue(DoubleQuoted))
+ },
+ // ↪ U+0027 APOSTROPHE (')
+ '\'' => {
+ // Switch to the attribute value (single-quoted) state.
+ go!(self: discard_char input; to State::AttributeValue(SingleQuoted))
+ },
+ // ↪ U+003E GREATER-THAN SIGN (>)
'>' => {
+ // This is a missing-attribute-value parse error.
+ // Switch to the data state.
+ // Emit the current tag token.
go!(self: discard_char input);
self.bad_char_error();
go!(self: emit_tag Data)
},
- _ => go!(self: to State::AttributeValue(Unquoted)),
+ // ↪ Anything else
+ _ => {
+ // Reconsume in the attribute value (unquoted) state.
+ go!(self: to State::AttributeValue(Unquoted))
+ },
}
},
- //§ attribute-value-(double-quoted)-state
+ // https://html.spec.whatwg.org/#attribute-value-(double-quoted)-state
states::AttributeValue(DoubleQuoted) => loop {
+ // Consume the next input character:
let Some(set_result) =
self.pop_except_from(input, small_char_set!('\r' '"' '&' '\0' '\n'))
else {
@@ -1274,19 +1681,34 @@ impl Tokenizer {
};
match set_result {
- FromSet('"') => go!(self: to State::AfterAttributeValueQuoted),
- FromSet('&') => go!(self: consume_char_ref),
+ // ↪ U+0022 QUOTATION MARK (")
+ FromSet('"') => {
+ // Switch to the after attribute value (quoted) state.
+ go!(self: to State::AfterAttributeValueQuoted)
+ },
+ // ↪ U+0026 AMPERSAND (&)
+ FromSet('&') => {
+ // Set the return state to the attribute value (double-quoted) state.
+ // Switch to the character reference state.
+ go!(self: consume_char_ref)
+ },
+ // ↪ U+0000 NULL
FromSet('\0') => {
+ // This is an unexpected-null-character parse error.
+ // Append a U+FFFD REPLACEMENT CHARACTER character to the current attribute's value.
self.bad_char_error();
go!(self: push_value '\u{fffd}')
},
- FromSet(c) => go!(self: push_value c),
- NotFromSet(ref b) => go!(self: append_value b),
+ // ↪ Anything else
+ // Append the current input character to the current attribute's value.
+ FromSet(character) => go!(self: push_value character),
+ NotFromSet(ref characters) => go!(self: append_value characters),
}
},
- //§ attribute-value-(single-quoted)-state
+ // https://html.spec.whatwg.org/#attribute-value-(single-quoted)-state
states::AttributeValue(SingleQuoted) => loop {
+ // Consume the next input character:
let Some(set_result) =
self.pop_except_from(input, small_char_set!('\r' '\'' '&' '\0' '\n'))
else {
@@ -1294,19 +1716,34 @@ impl Tokenizer {
};
match set_result {
- FromSet('\'') => go!(self: to State::AfterAttributeValueQuoted),
- FromSet('&') => go!(self: consume_char_ref),
+ // ↪ U+0027 APOSTROPHE (')
+ FromSet('\'') => {
+ // Switch to the after attribute value (quoted) state.
+ go!(self: to State::AfterAttributeValueQuoted)
+ },
+ // ↪ U+0026 AMPERSAND (&)
+ FromSet('&') => {
+ // Set the return state to the attribute value (single-quoted) state.
+ // Switch to the character reference state.
+ go!(self: consume_char_ref)
+ },
+ // ↪ U+0000 NULL
FromSet('\0') => {
+ // This is an unexpected-null-character parse error.
+ // Append a U+FFFD REPLACEMENT CHARACTER character to the current attribute's value.
self.bad_char_error();
go!(self: push_value '\u{fffd}')
},
- FromSet(c) => go!(self: push_value c),
- NotFromSet(ref b) => go!(self: append_value b),
+ // ↪ Anything else
+ // Append the current input character to the current attribute's value.
+ FromSet(character) => go!(self: push_value character),
+ NotFromSet(ref characters) => go!(self: append_value characters),
}
},
- //§ attribute-value-(unquoted)-state
+ // https://html.spec.whatwg.org/#attribute-value-(unquoted)-state
states::AttributeValue(Unquoted) => loop {
+ // Consume the next input character:
let Some(set_result) = self.pop_except_from(
input,
small_char_set!('\r' '\t' '\n' '\x0C' ' ' '&' '>' '\0'),
@@ -1315,224 +1752,543 @@ impl Tokenizer {
};
match set_result {
+ // ↪ U+0009 CHARACTER TABULATION (tab)
+ // ↪ U+000A LINE FEED (LF)
+ // ↪ U+000C FORM FEED (FF)
+ // ↪ U+0020 SPACE
FromSet('\t') | FromSet('\n') | FromSet('\x0C') | FromSet(' ') => {
+ // Switch to the before attribute name state.
go!(self: to State::BeforeAttributeName)
},
- FromSet('&') => go!(self: consume_char_ref),
- FromSet('>') => go!(self: emit_tag Data),
+ // ↪ U+0026 AMPERSAND (&)
+ FromSet('&') => {
+ // Set the return state to the attribute value (unquoted) state.
+ // Switch to the character reference state.
+ go!(self: consume_char_ref)
+ },
+ // ↪ U+003E GREATER-THAN SIGN (>)
+ FromSet('>') => {
+ // Switch to the data state.
+ // Emit the current tag token.
+ go!(self: emit_tag Data)
+ },
+ // ↪ U+0000 NULL
FromSet('\0') => {
+ // This is an unexpected-null-character parse error.
+ // Append a U+FFFD REPLACEMENT CHARACTER character to the current attribute's value.
self.bad_char_error();
go!(self: push_value '\u{fffd}')
},
FromSet(c) => {
+ // ↪ U+0022 QUOTATION MARK (")
+ // ↪ U+0027 APOSTROPHE (')
+ // ↪ U+003C LESS-THAN SIGN (<)
+ // ↪ U+003D EQUALS SIGN (=)
+ // ↪ U+0060 GRAVE ACCENT (`)
if matches!(c, '"' | '\'' | '<' | '=' | '`') {
+ // This is an unexpected-character-in-unquoted-attribute-value parse error.
+ // Treat it as per the "anything else" entry below.
self.bad_char_error();
}
+ // ↪ Anything else
+ // Append the current input character to the current attribute's value.
go!(self: push_value c);
},
- NotFromSet(ref b) => go!(self: append_value b),
+ // ↪ Anything else
+ NotFromSet(ref characters) => {
+ // Append the current input character to the current attribute's value.
+ go!(self: append_value characters)
+ },
}
},
- //§ after-attribute-value-(quoted)-state
+ // https://html.spec.whatwg.org/#after-attribute-value-(quoted)-state
states::AfterAttributeValueQuoted => loop {
+ // Consume the next input character:
match get_char!(self, input) {
- '\t' | '\n' | '\x0C' | ' ' => go!(self: to State::BeforeAttributeName),
- '/' => go!(self: to State::SelfClosingStartTag),
- '>' => go!(self: emit_tag Data),
+ // ↪ U+0009 CHARACTER TABULATION (tab)
+ // ↪ U+000A LINE FEED (LF)
+ // ↪ U+000C FORM FEED (FF)
+ // ↪ U+0020 SPACE
+ '\t' | '\n' | '\x0C' | ' ' => {
+ // Switch to the before attribute name state.
+ go!(self: to State::BeforeAttributeName)
+ },
+ // ↪ U+002F SOLIDUS (/)
+ '/' => {
+ // Switch to the self-closing start tag state.
+ go!(self: to State::SelfClosingStartTag)
+ },
+ // ↪ U+003E GREATER-THAN SIGN (>)
+ '>' => {
+ // Switch to the data state. Emit the current tag token.
+ go!(self: emit_tag Data)
+ },
+ // ↪ Anything else
_ => {
+ // This is a missing-whitespace-between-attributes parse error.
+ // Reconsume in the before attribute name state.
self.bad_char_error();
go!(self: reconsume BeforeAttributeName)
},
}
},
- //§ self-closing-start-tag-state
+ // https://html.spec.whatwg.org/#self-closing-start-tag-state
states::SelfClosingStartTag => loop {
+ // Consume the next input character:
match get_char!(self, input) {
+ // ↪ U+003E GREATER-THAN SIGN (>)
'>' => {
+ // Set the self-closing flag of the current tag token.
+ // Switch to the data state. Emit the current tag token.
self.current_tag_self_closing.set(true);
go!(self: emit_tag Data);
},
+ // ↪ Anything else
_ => {
+ // This is an unexpected-solidus-in-tag parse error.
+ // Reconsume in the before attribute name state.
self.bad_char_error();
go!(self: reconsume BeforeAttributeName)
},
}
},
- //§ comment-start-state
- states::CommentStart => loop {
+ //§ bogus-comment-state
+ states::BogusComment => loop {
+ // Consume the next input character:
match get_char!(self, input) {
- '-' => go!(self: to State::CommentStartDash),
+ // ↪ U+003E GREATER-THAN SIGN (>)
+ '>' => {
+ // Switch to the data state. Emit the current comment token.
+ go!(self: emit_comment; to State::Data)
+ },
+ // ↪ U+0000 NULL
'\0' => {
+ // This is an unexpected-null-character parse error.
+ // Append a U+FFFD REPLACEMENT CHARACTER character to the comment token's data.
self.bad_char_error();
- go!(self: push_comment '\u{fffd}'; to State::Comment)
+ go!(self: push_comment '\u{fffd}')
+ },
+ // ↪ Anything else
+ character => {
+ // Append the current input character to the comment token's data.
+ go!(self: push_comment character)
+ },
+ }
+ },
+
+ // https://html.spec.whatwg.org/#markup-declaration-open-state
+ states::MarkupDeclarationOpen => loop {
+ // If the next few characters are:
+ // ↪ Two U+002D HYPHEN-MINUS characters (-)
+ if eat_exact!(self, input, "--") {
+ go!(self: clear_comment; to State::CommentStart);
+ }
+ // ↪ ASCII case-insensitive match for "DOCTYPE"
+ else if eat!(self, input, "doctype") {
+ go!(self: to State::Doctype);
+ } else {
+ // ↪ "[CDATA["
+ if self
+ .sink
+ .adjusted_current_node_present_but_not_in_html_namespace()
+ && eat_exact!(self, input, "[CDATA[")
+ {
+ // Consume those characters.
+ // If there is an adjusted current node and it is not an element in the HTML namespace,
+ // then switch to the CDATA section state. Otherwise, this is a cdata-in-html-content parse error.
+ // Create a comment token whose data is "[CDATA[". Switch to the bogus comment state.
+ // FIXME: Create that comment token.
+ go!(self: clear_temp; to State::CdataSection);
+ }
+
+ // ↪ Anything else
+ // This is an incorrectly-opened-comment parse error.
+ // Create a comment token whose data is the empty string.
+ // Switch to the bogus comment state (don't consume anything in the current state).
+ // FIXME: Create that comment token.
+ self.bad_char_error();
+ go!(self: clear_comment; to State::BogusComment);
+ }
+ },
+
+ // https://html.spec.whatwg.org/#comment-start-state
+ states::CommentStart => loop {
+ // Consume the next input character:
+ match get_char!(self, input) {
+ // ↪ U+002D HYPHEN-MINUS (-)
+ '-' => {
+ // Switch to the comment start dash state.
+ go!(self: to State::CommentStartDash)
},
+ // ↪ U+003E GREATER-THAN SIGN (>)
'>' => {
+ // This is an abrupt-closing-of-empty-comment parse error.
+ // Switch to the data state.
+ // Emit the current comment token.
self.bad_char_error();
go!(self: emit_comment; to State::Data)
},
- c => go!(self: push_comment c; to State::Comment),
+ // NOTE: The "anything else" case in the specification reconsumes the character in the
+ // comment state, instead we inline what the comment state *would* do if it encountered
+ // that character.
+ '\0' => {
+ self.bad_char_error();
+ go!(self: push_comment '\u{fffd}'; to State::Comment)
+ },
+ character => go!(self: push_comment character; to State::Comment),
}
},
- //§ comment-start-dash-state
+ // https://html.spec.whatwg.org/#comment-start-dash-state
states::CommentStartDash => loop {
+ // Consume the next input character:
match get_char!(self, input) {
- '-' => go!(self: to State::CommentEnd),
- '\0' => {
- self.bad_char_error();
- go!(self: append_comment "-\u{fffd}"; to State::Comment)
+ // ↪ U+002D HYPHEN-MINUS (-)
+ '-' => {
+ // Switch to the comment end state.
+ go!(self: to State::CommentEnd)
},
+ // ↪ U+003E GREATER-THAN SIGN (>)
'>' => {
+ // This is an abrupt-closing-of-empty-comment parse error.
+ // Switch to the data state.
+ // Emit the current comment token.
self.bad_char_error();
go!(self: emit_comment; to State::Data)
},
- c => go!(self: push_comment '-'; push_comment c; to State::Comment),
+ // NOTE: The "anything else" case in the specification reconsumes the character in the
+ // comment state, instead we inline what the comment state *would* do if it encountered
+ // that character.
+ '\0' => {
+ self.bad_char_error();
+ go!(self: append_comment "-\u{fffd}"; to State::Comment)
+ },
+ character => {
+ go!(self: push_comment '-'; push_comment character; to State::Comment)
+ },
}
},
- //§ comment-state
+ // https://html.spec.whatwg.org/#comment-state
states::Comment => loop {
+ // Consume the next input character:
match get_char!(self, input) {
- c @ '<' => go!(self: push_comment c; to State::CommentLessThanSign),
- '-' => go!(self: to State::CommentEndDash),
+ // ↪ U+003C LESS-THAN SIGN (<)
+ c @ '<' => {
+ // Append the current input character to the comment token's data.
+ // Switch to the comment less-than sign state.
+ go!(self: push_comment c; to State::CommentLessThanSign)
+ },
+ // ↪ U+002D HYPHEN-MINUS (-)
+ '-' => {
+ // Switch to the comment end dash state.
+ go!(self: to State::CommentEndDash)
+ },
+ // ↪ U+0000 NULL
'\0' => {
+ // This is an unexpected-null-character parse error.
+ // Append a U+FFFD REPLACEMENT CHARACTER character to the comment token's data.
self.bad_char_error();
go!(self: push_comment '\u{fffd}')
},
- c => go!(self: push_comment c),
+ // ↪ Anything else
+ character => {
+ // Append the current input character to the comment token's data.
+ go!(self: push_comment character)
+ },
}
},
- //§ comment-less-than-sign-state
+ // https://html.spec.whatwg.org/#comment-less-than-sign-state
states::CommentLessThanSign => loop {
+ // Consume the next input character:
match get_char!(self, input) {
- c @ '!' => go!(self: push_comment c; to State::CommentLessThanSignBang),
- c @ '<' => go!(self: push_comment c),
- _ => go!(self: reconsume Comment),
+ // ↪ U+0021 EXCLAMATION MARK (!)
+ c @ '!' => {
+ // Append the current input character to the comment token's data.
+ // Switch to the comment less-than sign bang state.
+ go!(self: push_comment c; to State::CommentLessThanSignBang)
+ },
+ // ↪ U+003C LESS-THAN SIGN (<)
+ c @ '<' => {
+ // Append the current input character to the comment token's data.
+ go!(self: push_comment c)
+ },
+ // ↪ Anything else
+ _ => {
+ // Reconsume in the comment state.
+ go!(self: reconsume Comment)
+ },
}
},
- //§ comment-less-than-sign-bang
+ // https://html.spec.whatwg.org/#comment-less-than-sign-bang-state
states::CommentLessThanSignBang => loop {
+ // Consume the next input character:
match get_char!(self, input) {
- '-' => go!(self: to State::CommentLessThanSignBangDash),
- _ => go!(self: reconsume Comment),
+ // ↪ U+002D HYPHEN-MINUS (-)
+ '-' => {
+ // Switch to the comment less-than sign bang dash state.
+ go!(self: to State::CommentLessThanSignBangDash)
+ },
+ // ↪ Anything else
+ _ => {
+ // Reconsume in the comment state.
+ go!(self: reconsume Comment)
+ },
}
},
- //§ comment-less-than-sign-bang-dash
+ // https://html.spec.whatwg.org/#comment-less-than-sign-bang-dash-state
states::CommentLessThanSignBangDash => loop {
+ // Consume the next input character:
match get_char!(self, input) {
- '-' => go!(self: to State::CommentLessThanSignBangDashDash),
- _ => go!(self: reconsume CommentEndDash),
+ // ↪ U+002D HYPHEN-MINUS (-)
+ '-' => {
+ // Switch to the comment less-than sign bang dash dash state.
+ go!(self: to State::CommentLessThanSignBangDashDash)
+ },
+ // ↪ Anything else
+ _ => {
+ // Reconsume in the comment end dash state.
+ go!(self: reconsume CommentEndDash)
+ },
}
},
- //§ comment-less-than-sign-bang-dash-dash
+ // https://html.spec.whatwg.org/#comment-less-than-sign-bang-dash-dash-state
states::CommentLessThanSignBangDashDash => loop {
match get_char!(self, input) {
- '>' => go!(self: reconsume CommentEnd),
+ // ↪ U+003E GREATER-THAN SIGN (>)
+ '>' => {
+ // Reconsume in the comment end state.
+ go!(self: reconsume CommentEnd)
+ },
+ // ↪ Anything else
_ => {
+ // This is a nested-comment parse error.
+ // Reconsume in the comment end state.
self.bad_char_error();
go!(self: reconsume CommentEnd)
},
}
},
- //§ comment-end-dash-state
+ // https://html.spec.whatwg.org/#comment-end-dash-state
states::CommentEndDash => loop {
+ // Consume the next input character:
match get_char!(self, input) {
- '-' => go!(self: to State::CommentEnd),
+ // ↪ U+002D HYPHEN-MINUS (-)
+ '-' => {
+ // Switch to the comment end state.
+ go!(self: to State::CommentEnd)
+ },
+ // NOTE: The "anything else" case in the specification reconsumes the character in the
+ // comment state. Instead we inline what the comment state *would* do here.
'\0' => {
self.bad_char_error();
go!(self: append_comment "-\u{fffd}"; to State::Comment)
},
- c => go!(self: push_comment '-'; push_comment c; to State::Comment),
+ character => {
+ go!(self: push_comment '-'; push_comment character; to State::Comment)
+ },
}
},
- //§ comment-end-state
+ // https://html.spec.whatwg.org/#comment-end-state
states::CommentEnd => loop {
+ // Consume the next input character:
match get_char!(self, input) {
- '>' => go!(self: emit_comment; to State::Data),
- '!' => go!(self: to State::CommentEndBang),
- '-' => go!(self: push_comment '-'),
- _ => go!(self: append_comment "--"; reconsume Comment),
+ // ↪ U+003E GREATER-THAN SIGN (>)
+ '>' => {
+ // Switch to the data state.
+ // Emit the current comment token.
+ go!(self: emit_comment; to State::Data)
+ },
+ // ↪ U+0021 EXCLAMATION MARK (!)
+ '!' => {
+ // Switch to the comment end bang state.
+ go!(self: to State::CommentEndBang)
+ },
+ // ↪ U+002D HYPHEN-MINUS (-)
+ '-' => {
+ // Append a U+002D HYPHEN-MINUS character (-) to the comment token's data.
+ go!(self: push_comment '-')
+ },
+ // ↪ Anything else
+ _ => {
+ // Append two U+002D HYPHEN-MINUS characters (-) to the comment token's data.
+ // Reconsume in the comment state.
+ go!(self: append_comment "--"; reconsume Comment)
+ },
}
},
- //§ comment-end-bang-state
+ // https://html.spec.whatwg.org/#comment-end-bang-state
states::CommentEndBang => loop {
+ // Consume the next input character:
match get_char!(self, input) {
+ // ↪ U+002D HYPHEN-MINUS (-)
'-' => go!(self: append_comment "--!"; to State::CommentEndDash),
+ // ↪ U+003E GREATER-THAN SIGN (>)
'>' => {
self.bad_char_error();
go!(self: emit_comment; to State::Data)
},
+ // NOTE: The "anything else" case in the specification reconsumes the character in the
+ // comment state. Instead we inline what the comment state *would* do here.
'\0' => {
self.bad_char_error();
go!(self: append_comment "--!\u{fffd}"; to State::Comment)
},
- c => go!(self: append_comment "--!"; push_comment c; to State::Comment),
+ character => {
+ go!(self: append_comment "--!"; push_comment character; to State::Comment)
+ },
}
},
- //§ doctype-state
+ // https://html.spec.whatwg.org/#doctype-state
states::Doctype => loop {
+ // Consume the next input character:
match get_char!(self, input) {
- '\t' | '\n' | '\x0C' | ' ' => go!(self: to State::BeforeDoctypeName),
- '>' => go!(self: reconsume BeforeDoctypeName),
+ // ↪ U+0009 CHARACTER TABULATION (tab)
+ // ↪ U+000A LINE FEED (LF)
+ // ↪ U+000C FORM FEED (FF)
+ // ↪ U+0020 SPACE
+ '\t' | '\n' | '\x0C' | ' ' => {
+ // Switch to the before DOCTYPE name state.
+ go!(self: to State::BeforeDoctypeName)
+ },
+ // ↪ U+003E GREATER-THAN SIGN (>)
+ '>' => {
+ // Reconsume in the before DOCTYPE name state.
+ go!(self: reconsume BeforeDoctypeName)
+ },
+ // ↪ Anything else
_ => {
+ // This is a missing-whitespace-before-doctype-name parse error.
+ // Reconsume in the before DOCTYPE name state.
self.bad_char_error();
go!(self: reconsume BeforeDoctypeName)
},
}
},
- //§ before-doctype-name-state
+ // https://html.spec.whatwg.org/#before-doctype-name-state
states::BeforeDoctypeName => loop {
+ // Consume the next input character:
match get_char!(self, input) {
- '\t' | '\n' | '\x0C' | ' ' => (),
+ // ↪ U+0009 CHARACTER TABULATION (tab)
+ // ↪ U+000A LINE FEED (LF)
+ // ↪ U+000C FORM FEED (FF)
+ // ↪ U+0020 SPACE
+ '\t' | '\n' | '\x0C' | ' ' => {
+ // Ignore the character.
+ },
+ // ↪ U+0000 NULL
'\0' => {
+ // This is an unexpected-null-character parse error.
+ // Create a new DOCTYPE token.
+ // Set the token's name to a U+FFFD REPLACEMENT CHARACTER character.
+ // Switch to the DOCTYPE name state.
self.bad_char_error();
go!(self: create_doctype; push_doctype_name '\u{fffd}'; to State::DoctypeName)
},
+ // ↪ U+003E GREATER-THAN SIGN (>)
'>' => {
+ // This is a missing-doctype-name parse error.
+ // Create a new DOCTYPE token.
+ // Set its force-quirks flag to on.
+ // Switch to the data state.
+ // Emit the current token.
self.bad_char_error();
go!(self: create_doctype; force_quirks; emit_doctype; to State::Data)
},
- c => go!(self: create_doctype; push_doctype_name (c.to_ascii_lowercase());
- to State::DoctypeName),
+ // ↪ ASCII upper alpha
+ // NOTE: This is the same as "anythign else", except we use the lowercase version of the character.
+ // ↪ Anything else
+ character => {
+ // Create a new DOCTYPE token.
+ // Set the token's name to the current input character.
+ // Switch to the DOCTYPE name state.
+ go!(self: create_doctype; push_doctype_name (character.to_ascii_lowercase());
+ to State::DoctypeName)
+ },
}
},
- //§ doctype-name-state
+ // https://html.spec.whatwg.org/#doctype-name-state
states::DoctypeName => loop {
+ // Consume the next input character:
match get_char!(self, input) {
- '\t' | '\n' | '\x0C' | ' ' => go!(self: clear_temp; to State::AfterDoctypeName),
- '>' => go!(self: emit_doctype; to State::Data),
+ // ↪ U+0009 CHARACTER TABULATION (tab)
+ // ↪ U+000A LINE FEED (LF)
+ // ↪ U+000C FORM FEED (FF)
+ // ↪ U+0020 SPACE
+ '\t' | '\n' | '\x0C' | ' ' => {
+ // Switch to the after DOCTYPE name state.
+ go!(self: clear_temp; to State::AfterDoctypeName)
+ },
+ // ↪ U+003E GREATER-THAN SIGN (>)
+ '>' => {
+ // Switch to the data state. Emit the current DOCTYPE token.
+ go!(self: emit_doctype; to State::Data)
+ },
+ // ↪ U+0000 NULL
'\0' => {
+ // This is an unexpected-null-character parse error.
+ // Append a U+FFFD REPLACEMENT CHARACTER character to the current DOCTYPE token's name.
self.bad_char_error();
go!(self: push_doctype_name '\u{fffd}')
},
- c => go!(self: push_doctype_name (c.to_ascii_lowercase())),
+ // ↪ ASCII upper alpha
+ // NOTE: This is the same as "anythign else", except we use the lowercase version of the character.
+ // ↪ Anything else
+ character => {
+ // Append the current input character to the current DOCTYPE token's name.
+ go!(self: push_doctype_name (character.to_ascii_lowercase()))
+ },
}
},
- //§ after-doctype-name-state
+ // https://html.spec.whatwg.org/#after-doctype-name-state
states::AfterDoctypeName => loop {
+ // NOTE: We move some steps out of the "anything else" case to the front for convenience.
+
+ // If the six characters starting from the current input character are an
+ // ASCII case-insensitive match for "PUBLIC", then consume those characters
+ // and switch to the after DOCTYPE public keyword state.
if eat!(self, input, "public") {
go!(self: to State::AfterDoctypeKeyword(Public));
- } else if eat!(self, input, "system") {
+ }
+ // Otherwise, if the six characters starting from the current input character
+ // are an ASCII case-insensitive match for "SYSTEM", then consume those characters
+ // and switch to the after DOCTYPE system keyword state.
+ else if eat!(self, input, "system") {
go!(self: to State::AfterDoctypeKeyword(System));
} else {
+ // Consume the next input character:
match get_char!(self, input) {
- '\t' | '\n' | '\x0C' | ' ' => (),
- '>' => go!(self: emit_doctype; to State::Data),
+ // ↪ U+0009 CHARACTER TABULATION (tab)
+ // ↪ U+000A LINE FEED (LF)
+ // ↪ U+000C FORM FEED (FF)
+ // ↪ U+0020 SPACE
+ '\t' | '\n' | '\x0C' | ' ' => {
+ // Ignore the character.
+ },
+ // ↪ U+003E GREATER-THAN SIGN (>)
+ '>' => {
+ // Switch to the data state. Emit the current DOCTYPE token.
+ go!(self: emit_doctype; to State::Data)
+ },
+ // ↪ Anything else
_ => {
+ // Otherwise, this is an invalid-character-sequence-after-doctype-name parse error.
+ // Set the current DOCTYPE token's force-quirks flag to on.
+ // Reconsume in the bogus DOCTYPE state.
self.bad_char_error();
go!(self: force_quirks; reconsume BogusDoctype)
},
@@ -1540,207 +2296,365 @@ impl Tokenizer {
}
},
- //§ after-doctype-public-keyword-state after-doctype-system-keyword-state
+ // https://html.spec.whatwg.org/#after-doctype-public-keyword-state
+ // https://html.spec.whatwg.org/#after-doctype-system-keyword-state
states::AfterDoctypeKeyword(kind) => loop {
match get_char!(self, input) {
+ // ↪ U+0009 CHARACTER TABULATION (tab)
+ // ↪ U+000A LINE FEED (LF)
+ // ↪ U+000C FORM FEED (FF)
+ // ↪ U+0020 SPACE
'\t' | '\n' | '\x0C' | ' ' => {
+ // Switch to the before DOCTYPE public/system identifier state.
go!(self: to State::BeforeDoctypeIdentifier(kind))
},
+ // ↪ U+0022 QUOTATION MARK (")
'"' => {
+ // This is a missing-whitespace-after-doctype-public/system-keyword parse error.
+ // Set the current DOCTYPE token's public/system identifier to the empty string
+ // (not missing), then switch to the DOCTYPE public/system identifier (double-quoted)
+ // state.
self.bad_char_error();
go!(self: clear_doctype_id kind; to State::DoctypeIdentifierDoubleQuoted(kind))
},
+ // ↪ U+0027 APOSTROPHE (')
'\'' => {
+ // This is a missing-whitespace-after-doctype-public-keyword parse error.
+ // Set the current DOCTYPE token's public identifier to the empty string
+ // (not missing), then switch to the DOCTYPE public/system identifier (single-quoted)
+ // state.
self.bad_char_error();
go!(self: clear_doctype_id kind; to State::DoctypeIdentifierSingleQuoted(kind))
},
+ // ↪ U+003E GREATER-THAN SIGN (>)
'>' => {
+ // This is a missing-doctype-public-identifier parse error.
+ // Set the current DOCTYPE token's force-quirks flag to on.
+ // Switch to the data state.
+ // Emit the current DOCTYPE token.
self.bad_char_error();
go!(self: force_quirks; emit_doctype; to State::Data)
},
+ // ↪ Anything else
_ => {
+ // This is a missing-quote-before-doctype-public-identifier parse error.
+ // Set the current DOCTYPE token's force-quirks flag to on.
+ // Reconsume in the bogus DOCTYPE state.
self.bad_char_error();
go!(self: force_quirks; reconsume BogusDoctype)
},
}
},
- //§ before-doctype-public-identifier-state before-doctype-system-identifier-state
+ // https://html.spec.whatwg.org/#before-doctype-public-identifier-state
+ // https://html.spec.whatwg.org/#before-doctype-system-identifier-state
states::BeforeDoctypeIdentifier(kind) => loop {
+ // Consume the next input character:
match get_char!(self, input) {
- '\t' | '\n' | '\x0C' | ' ' => (),
+ // ↪ U+0009 CHARACTER TABULATION (tab)
+ // ↪ U+000A LINE FEED (LF)
+ // ↪ U+000C FORM FEED (FF)
+ // ↪ U+0020 SPACE
+ '\t' | '\n' | '\x0C' | ' ' => {
+ // Ignore the character.
+ },
+ // ↪ U+0022 QUOTATION MARK (")
'"' => {
+ // Set the current DOCTYPE token's public/system identifier to the empty string
+ // (not missing), then switch to the DOCTYPE public/system identifier (double-quoted)
+ // state.
go!(self: clear_doctype_id kind; to State::DoctypeIdentifierDoubleQuoted(kind))
},
+ // ↪ U+0027 APOSTROPHE (')
'\'' => {
+ // Set the current DOCTYPE token's public/systen identifier to the empty string
+ // (not missing), then switch to the DOCTYPE public/system identifier (single-quoted)
+ // state.
go!(self: clear_doctype_id kind; to State::DoctypeIdentifierSingleQuoted(kind))
},
+ // ↪ U+003E GREATER-THAN SIGN (>)
'>' => {
+ // This is a missing-doctype-public/system-identifier parse error.
+ // Set the current DOCTYPE token's force-quirks flag to on.
+ // Switch to the data state.
+ // Emit the current DOCTYPE token.
self.bad_char_error();
go!(self: force_quirks; emit_doctype; to State::Data)
},
+ // ↪ Anything else
_ => {
+ // This is a missing-quote-before-doctype-public/system-identifier parse error.
+ // Set the current DOCTYPE token's force-quirks flag to on.
+ // Reconsume in the bogus DOCTYPE state.
self.bad_char_error();
go!(self: force_quirks; reconsume BogusDoctype)
},
}
},
- //§ doctype-public-identifier-(double-quoted)-state doctype-system-identifier-(double-quoted)-state
+ // https://html.spec.whatwg.org/#doctype-public-identifier-(double-quoted)-state
+ // https://html.spec.whatwg.org/#doctype-system-identifier-(double-quoted)-state
states::DoctypeIdentifierDoubleQuoted(kind) => loop {
+ // Consume the next input character:
match get_char!(self, input) {
- '"' => go!(self: to State::AfterDoctypeIdentifier(kind)),
+ // ↪ U+0022 QUOTATION MARK (")
+ '"' => {
+ // Switch to the after DOCTYPE public/system identifier state.
+ go!(self: to State::AfterDoctypeIdentifier(kind))
+ },
+ // ↪ U+0000 NULL
'\0' => {
+ // This is an unexpected-null-character parse error.
+ // Append a U+FFFD REPLACEMENT CHARACTER character to the current
+ // DOCTYPE token's public identifier.
self.bad_char_error();
go!(self: push_doctype_id kind '\u{fffd}')
},
+ // ↪ U+003E GREATER-THAN SIGN (>)
'>' => {
+ // This is an abrupt-doctype-public-identifier parse error.
+ // Set the current DOCTYPE token's force-quirks flag to on.
+ // Switch to the data state.
+ // Emit the current DOCTYPE token.
self.bad_char_error();
go!(self: force_quirks; emit_doctype; to State::Data)
},
- c => go!(self: push_doctype_id kind c),
+ // ↪ Anything else
+ character => go!(self: push_doctype_id kind character),
}
},
- //§ doctype-public-identifier-(single-quoted)-state doctype-system-identifier-(single-quoted)-state
+ // https://html.spec.whatwg.org/#doctype-public-identifier-(single-quoted)-state
+ // https://html.spec.whatwg.org/#doctype-system-identifier-(single-quoted)-state
states::DoctypeIdentifierSingleQuoted(kind) => loop {
+ // Consume the next input character:
match get_char!(self, input) {
- '\'' => go!(self: to State::AfterDoctypeIdentifier(kind)),
+ // ↪ U+0027 APOSTROPHE (')
+ '\'' => {
+ // Switch to the after DOCTYPE public/system identifier state.
+ go!(self: to State::AfterDoctypeIdentifier(kind))
+ },
+ // ↪ U+0000 NULL
'\0' => {
+ // This is an unexpected-null-character parse error.
+ // Append a U+FFFD REPLACEMENT CHARACTER character
+ // to the current DOCTYPE token's public/system identifier.
self.bad_char_error();
go!(self: push_doctype_id kind '\u{fffd}')
},
+ // ↪ U+003E GREATER-THAN SIGN (>)
'>' => {
+ // This is an abrupt-doctype-public/system-identifier parse error.
+ // Set the current DOCTYPE token's force-quirks flag to on.
+ // Switch to the data state.
+ // Emit the current DOCTYPE token.
self.bad_char_error();
go!(self: force_quirks; emit_doctype; to State::Data)
},
- c => go!(self: push_doctype_id kind c),
+ // ↪ Anything else
+ character => {
+ // Append the current input character to the current DOCTYPE token's
+ // public/system identifier.
+ go!(self: push_doctype_id kind character)
+ },
}
},
- //§ after-doctype-public-identifier-state
+ // https://html.spec.whatwg.org/#after-doctype-public-identifier-state
states::AfterDoctypeIdentifier(Public) => loop {
+ // Consume the next input character:
match get_char!(self, input) {
+ // ↪ U+0009 CHARACTER TABULATION (tab)
+ // ↪ U+000A LINE FEED (LF)
+ // ↪ U+000C FORM FEED (FF)
+ // ↪ U+0020 SPACE
'\t' | '\n' | '\x0C' | ' ' => {
+ // Switch to the between DOCTYPE public and system identifiers state.
go!(self: to State::BetweenDoctypePublicAndSystemIdentifiers)
},
- '>' => go!(self: emit_doctype; to State::Data),
+ // ↪ U+003E GREATER-THAN SIGN (>)
+ '>' => {
+ // Switch to the data state. Emit the current DOCTYPE token.
+ go!(self: emit_doctype; to State::Data)
+ },
+ // ↪ U+0022 QUOTATION MARK (")
'"' => {
+ // This is a missing-whitespace-between-doctype-public-and-system-identifiers
+ // parse error. Set the current DOCTYPE token's system identifier to the empty string
+ // (not missing), then switch to the DOCTYPE system identifier (double-quoted) state.
self.bad_char_error();
go!(self: clear_doctype_id System; to State::DoctypeIdentifierDoubleQuoted(System))
},
+ // ↪ U+0027 APOSTROPHE (')
'\'' => {
+ // This is a missing-whitespace-between-doctype-public-and-system-identifiers parse error.
+ // Set the current DOCTYPE token's system identifier to the empty string (not missing),
+ // then switch to the DOCTYPE system identifier (single-quoted) state.
self.bad_char_error();
go!(self: clear_doctype_id System; to State::DoctypeIdentifierSingleQuoted(System))
},
+ // ↪ Anything else
_ => {
+ // This is a missing-quote-before-doctype-system-identifier parse error.
+ // Set the current DOCTYPE token's force-quirks flag to on.
+ // Reconsume in the bogus DOCTYPE state.
self.bad_char_error();
go!(self: force_quirks; reconsume BogusDoctype)
},
}
},
- //§ after-doctype-system-identifier-state
- states::AfterDoctypeIdentifier(System) => loop {
- match get_char!(self, input) {
- '\t' | '\n' | '\x0C' | ' ' => (),
- '>' => go!(self: emit_doctype; to State::Data),
- _ => {
- self.bad_char_error();
- go!(self: reconsume BogusDoctype)
- },
- }
- },
-
- //§ between-doctype-public-and-system-identifiers-state
+ // https://html.spec.whatwg.org/#between-doctype-public-and-system-identifiers-state
states::BetweenDoctypePublicAndSystemIdentifiers => loop {
+ // Consume the next input character:
match get_char!(self, input) {
- '\t' | '\n' | '\x0C' | ' ' => (),
- '>' => go!(self: emit_doctype; to State::Data),
+ // ↪ U+0009 CHARACTER TABULATION (tab)
+ // ↪ U+000A LINE FEED (LF)
+ // ↪ U+000C FORM FEED (FF)
+ // ↪ U+0020 SPACE
+ '\t' | '\n' | '\x0C' | ' ' => {
+ // Ignore the character.
+ },
+ // ↪ U+003E GREATER-THAN SIGN (>)
+ '>' => {
+ // Switch to the data state. Emit the current DOCTYPE token.
+ go!(self: emit_doctype; to State::Data)
+ },
+ // ↪ U+0022 QUOTATION MARK (")
'"' => {
+ // Set the current DOCTYPE token's system identifier to the empty string (not missing),
+ // then switch to the DOCTYPE system identifier (double-quoted) state.
go!(self: clear_doctype_id System; to State::DoctypeIdentifierDoubleQuoted(System))
},
+ // ↪ U+0027 APOSTROPHE (')
'\'' => {
+ // Set the current DOCTYPE token's system identifier to the empty string (not missing),
+ // then switch to the DOCTYPE system identifier (single-quoted) state.
go!(self: clear_doctype_id System; to State::DoctypeIdentifierSingleQuoted(System))
},
+ // ↪ Anything else
_ => {
+ // This is a missing-quote-before-doctype-system-identifier parse error.
+ // Set the current DOCTYPE token's force-quirks flag to on.
+ // Reconsume in the bogus DOCTYPE state.
self.bad_char_error();
go!(self: force_quirks; reconsume BogusDoctype)
},
}
},
- //§ bogus-doctype-state
- states::BogusDoctype => loop {
+ // https://html.spec.whatwg.org/#after-doctype-system-identifier-state
+ states::AfterDoctypeIdentifier(System) => loop {
+ // Consume the next input character:
match get_char!(self, input) {
- '>' => go!(self: emit_doctype; to State::Data),
- '\0' => {
+ // ↪ U+0009 CHARACTER TABULATION (tab)
+ // ↪ U+000A LINE FEED (LF)
+ // ↪ U+000C FORM FEED (FF)
+ // ↪ U+0020 SPACE
+ '\t' | '\n' | '\x0C' | ' ' => {
+ // Ignore the character.
+ },
+ // ↪ U+003E GREATER-THAN SIGN (>)
+ '>' => {
+ // Switch to the data state. Emit the current DOCTYPE token.
+ go!(self: emit_doctype; to State::Data)
+ },
+ // ↪ Anything else
+ _ => {
+ // This is an unexpected-character-after-doctype-system-identifier parse error.
+ // Reconsume in the bogus DOCTYPE state.
self.bad_char_error();
+ go!(self: reconsume BogusDoctype)
},
- _ => (),
}
},
- //§ bogus-comment-state
- states::BogusComment => loop {
+ // https://html.spec.whatwg.org/#bogus-doctype-state
+ states::BogusDoctype => loop {
+ // Consume the next input character:
match get_char!(self, input) {
- '>' => go!(self: emit_comment; to State::Data),
+ // ↪ U+003E GREATER-THAN SIGN (>)
+ '>' => {
+ // Switch to the data state.
+ // Emit the current DOCTYPE token.
+ go!(self: emit_doctype; to State::Data)
+ },
+ // ↪ U+0000 NULL
'\0' => {
+ // This is an unexpected-null-character parse error.
+ // Ignore the character.
self.bad_char_error();
- go!(self: push_comment '\u{fffd}')
},
- c => go!(self: push_comment c),
- }
- },
-
- //§ markup-declaration-open-state
- states::MarkupDeclarationOpen => loop {
- if eat_exact!(self, input, "--") {
- go!(self: clear_comment; to State::CommentStart);
- } else if eat!(self, input, "doctype") {
- go!(self: to State::Doctype);
- } else {
- if self
- .sink
- .adjusted_current_node_present_but_not_in_html_namespace()
- && eat_exact!(self, input, "[CDATA[")
- {
- go!(self: clear_temp; to State::CdataSection);
- }
- self.bad_char_error();
- go!(self: clear_comment; to State::BogusComment);
+ // ↪ Anything else
+ _ => {
+ // Ignore the character.
+ },
}
},
- //§ cdata-section-state
+ // https://html.spec.whatwg.org/#cdata-section-state
states::CdataSection => loop {
+ // Consume the next input character:
match get_char!(self, input) {
- ']' => go!(self: to State::CdataSectionBracket),
+ // ↪ U+005D RIGHT SQUARE BRACKET (])
+ ']' => {
+ // Switch to the CDATA section bracket state.
+ go!(self: to State::CdataSectionBracket)
+ },
+ // FIXME: This is not in the specification.
'\0' => {
self.emit_temp_buf();
self.emit_char('\0');
},
- c => go!(self: push_temp c),
+ // ↪ Anything else
+ character => {
+ // Emit the current input character as a character token.
+ go!(self: push_temp character)
+ },
}
},
- //§ cdata-section-bracket
- states::CdataSectionBracket => match get_char!(self, input) {
- ']' => go!(self: to State::CdataSectionEnd),
- _ => go!(self: push_temp ']'; reconsume CdataSection),
+ // https://html.spec.whatwg.org/#cdata-section-bracket-state
+ states::CdataSectionBracket => {
+ // Consume the next input character:
+ match get_char!(self, input) {
+ // ↪ U+005D RIGHT SQUARE BRACKET (])
+ ']' => {
+ // Switch to the CDATA section end state.
+ go!(self: to State::CdataSectionEnd)
+ },
+ // ↪ Anything else
+ _ => {
+ // Emit a U+005D RIGHT SQUARE BRACKET character token. Reconsume in the CDATA section state.
+ go!(self: push_temp ']'; reconsume CdataSection)
+ },
+ }
},
- //§ cdata-section-end
+ // https://html.spec.whatwg.org/#cdata-section-end-state
states::CdataSectionEnd => loop {
+ // Consume the next input character:
match get_char!(self, input) {
- ']' => go!(self: push_temp ']'),
+ // U+005D RIGHT SQUARE BRACKET (])
+ ']' => {
+ // Emit a U+005D RIGHT SQUARE BRACKET character token.
+ go!(self: push_temp ']')
+ },
+ // U+003E GREATER-THAN SIGN (>)
'>' => {
+ // Switch to the data state.
self.emit_temp_buf();
go!(self: to State::Data);
},
- _ => go!(self: push_temp ']'; push_temp ']'; reconsume CdataSection),
+ // Anything else
+ _ => {
+ // Emit two U+005D RIGHT SQUARE BRACKET character tokens.
+ // Reconsume in the CDATA section state.
+ go!(self: push_temp ']'; push_temp ']'; reconsume CdataSection)
+ },
}
},
+ // TODO: What about the processing-instruction related states?
//§ END
}
}