Skip to content

Fix HTML literal recovery at EOF #1062

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
fglock wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Fix HTML literal recovery at EOF #1062

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
110 changes: 90 additions & 20 deletions src/main/java/org/perlonjava/runtime/perlmodule/HTMLParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,8 @@ public static RuntimeList _alloc_pstate(RuntimeArray args, int ctx) {
pstate.put("_eof", scalarFalse);
pstate.put("_started", scalarFalse);
pstate.put("_buf", new RuntimeScalar(""));
pstate.put("_literal_mode", new RuntimeScalar(""));
pstate.put("_pending_end_tag", new RuntimeScalar(""));
pstate.put("_bool_attr_val", scalarUndef);
pstate.put("_in_cdata", scalarFalse);

Expand Down Expand Up @@ -227,12 +229,7 @@ public static RuntimeList parserEof(RuntimeArray args, int ctx) {
pstate.put("_started", scalarTrue);
fireEvent(self, selfHash, pstate, "start_document");
}
// Flush any remaining buffered text
String remaining = pstate.get("_buf").toString();
if (!remaining.isEmpty()) {
pstate.put("_buf", new RuntimeScalar(""));
parseHtml(self, selfHash, pstate, remaining);
}
flushBufferedAtEof(self, selfHash, pstate);
// Fire end_document event
fireEvent(self, selfHash, pstate, "end_document");
pstate.put("_started", scalarFalse);
Expand Down Expand Up @@ -448,6 +445,16 @@ private static RuntimeHash getPstate(RuntimeHash selfHash) {
* @param eventArgs the event-specific arguments
*/
private static void fireEvent(RuntimeScalar self, RuntimeHash selfHash, RuntimeHash pstate, String eventName, RuntimeScalar... eventArgs) {
RuntimeScalar pendingEndTag = pstate.get("_pending_end_tag");
if (pendingEndTag != null && pendingEndTag.getDefinedBoolean()
&& !pendingEndTag.toString().isEmpty()
&& !eventName.equals("text") && !eventName.equals("comment")) {
String tagName = pendingEndTag.toString();
pstate.put("_pending_end_tag", new RuntimeScalar(""));
fireEvent(self, selfHash, pstate, "end",
new RuntimeScalar(tagName), new RuntimeScalar(""));
}

RuntimeHash handlers = pstate.get("_handlers").hashDeref();
RuntimeScalar cb = handlers.get(eventName + "_cb");
String handlerName = eventName;
Expand Down Expand Up @@ -838,6 +845,33 @@ private static void parseHtml(RuntimeScalar self, RuntimeHash selfHash, RuntimeH
int i = 0;
int textStart = 0;

String literalMode = pstate.get("_literal_mode").toString();
if (!literalMode.isEmpty()) {
int endIdx = findLiteralEnd(pstate, html, literalMode, 0);
if (endIdx < 0) {
pstate.put("_buf", new RuntimeScalar(html));
return;
}

int endTagEnd = html.indexOf('>', endIdx);
if (endTagEnd < 0) {
pstate.put("_buf", new RuntimeScalar(html));
return;
}

if (endIdx > 0) {
fireEvent(self, selfHash, pstate, "text",
new RuntimeScalar(html.substring(0, endIdx)));
}
endTagEnd++;
fireEvent(self, selfHash, pstate, "end",
new RuntimeScalar(literalMode),
new RuntimeScalar(html.substring(endIdx, endTagEnd)));
pstate.put("_literal_mode", new RuntimeScalar(""));
i = endTagEnd;
textStart = i;
}

while (i < len) {
if (html.charAt(i) == '<') {
// Flush pending text
Expand Down Expand Up @@ -1093,17 +1127,7 @@ private static void parseHtml(RuntimeScalar self, RuntimeHash selfHash, RuntimeH
|| tagName.equals("xmp") || tagName.equals("listing")
|| tagName.equals("plaintext") || tagName.equals("textarea")
|| tagName.equals("title"))) {
String endTag = "</" + tagName;
// For script content, only marked_sections (not xml_mode) enables CDATA-skipping.
// xml_mode alone doesn't change how script closing tags interact with CDATA sections.
boolean msEnabled = pstate.get("marked_sections").getBoolean();
int endIdx;
if (msEnabled) {
// With marked_sections, skip over <![CDATA[...]]> when looking for end tag
endIdx = findEndTagSkippingCdata(html, endTag, i);
} else {
endIdx = findCaseInsensitive(html, endTag, i);
}
int endIdx = findLiteralEnd(pstate, html, tagName, i);
if (endIdx >= 0) {
// Emit raw content as text
if (endIdx > i) {
Expand All @@ -1124,9 +1148,10 @@ private static void parseHtml(RuntimeScalar self, RuntimeHash selfHash, RuntimeH
return;
}
} else {
// No closing tag found - buffer everything for next parse()
pstate.put("_buf", new RuntimeScalar(html.substring(tagStart)));
// Re-emit the start tag on next parse when we have the full content
// The start event has already fired. Preserve literal mode and only
// buffer its content so a later chunk cannot emit the start twice.
pstate.put("_literal_mode", new RuntimeScalar(tagName));
pstate.put("_buf", new RuntimeScalar(html.substring(i)));
return;
}
}
Expand All @@ -1145,6 +1170,51 @@ private static void parseHtml(RuntimeScalar self, RuntimeHash selfHash, RuntimeH
}
}

private static int findLiteralEnd(RuntimeHash pstate, String html, String tagName, int fromIndex) {
String endTag = "</" + tagName;
if (pstate.get("marked_sections").getBoolean()) {
return findEndTagSkippingCdata(html, endTag, fromIndex);
}
return findCaseInsensitive(html, endTag, fromIndex);
}

private static void flushBufferedAtEof(RuntimeScalar self, RuntimeHash selfHash, RuntimeHash pstate) {
while (true) {
String remaining = pstate.get("_buf").toString();
String literalMode = pstate.get("_literal_mode").toString();
pstate.put("_buf", new RuntimeScalar(""));
pstate.put("_literal_mode", new RuntimeScalar(""));

if (literalMode.isEmpty()) {
if (remaining.isEmpty()) {
return;
}
parseHtml(self, selfHash, pstate, remaining);
} else if (literalMode.equals("script") || literalMode.equals("style")) {
fireEvent(self, selfHash, pstate, "end",
new RuntimeScalar(literalMode), new RuntimeScalar(""));
if (!remaining.isEmpty()) {
parseHtml(self, selfHash, pstate, remaining);
}
} else if (literalMode.equals("title")) {
pstate.put("_pending_end_tag", new RuntimeScalar(literalMode));
if (!remaining.isEmpty()) {
parseHtml(self, selfHash, pstate, remaining);
}
} else if (!remaining.isEmpty()) {
// HTML::Parser treats unterminated textarea/xmp/iframe/plaintext
// literal content as text at EOF. Keep the existing listing mode
// on that same non-markup path.
fireEvent(self, selfHash, pstate, "text", new RuntimeScalar(remaining));
}

if (pstate.get("_buf").toString().isEmpty()
&& pstate.get("_literal_mode").toString().isEmpty()) {
return;
}
}
}

/**
* Case-insensitive search for a substring in a string, starting at fromIndex.
* Used to find closing tags like </script> regardless of case.
Expand Down
47 changes: 47 additions & 0 deletions src/test/resources/unit/html_parser_literal_eof.t
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
use strict;
use warnings;

use HTML::Form;
use HTML::TokeParser;
use Test::More;

my $html = <<'HTML';
<html>
<head><title>Fixed document</head>
<form>
<table>
<tr><select name="foo">
<option value="bar">Bar</option></select></td></tr>
</form>
</html>
HTML

my $parser = HTML::TokeParser->new(\$html);
my @events;
while (my $token = $parser->get_token) {
my $type = $token->[0];
if ($type eq 'S' || $type eq 'E') {
push @events, "$type:$token->[1]";
}
elsif ($type eq 'T' && $token->[1] =~ /\S/) {
my $text = $token->[1];
$text =~ s/^\s+|\s+$//g;
push @events, "T:$text";
}
}

is_deeply(
\@events,
[
'S:html', 'S:head', 'S:title', 'T:Fixed document', 'E:title',
'E:head', 'S:form', 'S:table', 'S:tr', 'S:select', 'S:option',
'T:Bar', 'E:option', 'E:select', 'E:td', 'E:tr', 'E:form',
'E:html',
],
'unterminated title closes at EOF and the remaining markup is tokenized',
);

my @forms = HTML::Form->parse($html, base => 'http://localhost/');
is(scalar @forms, 1, 'form after an unterminated title is discovered');

done_testing;
Loading