handlerRuby
2
foo
" + "X" + ); + + GumboNode* body; + GetAndAssertBody(root_, &body); + ASSERT_EQ(2, GetChildCount(body)); + + GumboNode* p1 = GetChild(body, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, p1->type); + EXPECT_EQ(GUMBO_TAG_P, p1->v.element.tag); + ASSERT_EQ(1, GetChildCount(p1)); + + GumboNode* size1 = GetChild(p1, 0); + GumboNode* red1 = GetChild(size1, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, red1->type); + EXPECT_EQ(GUMBO_TAG_FONT, red1->v.element.tag); + ASSERT_EQ(1, GetAttributeCount(red1)); + GumboAttribute* red1_attr = GetAttribute(red1, 0); + EXPECT_STREQ("color", red1_attr->name); + EXPECT_STREQ("red", red1_attr->value); + ASSERT_EQ(1, GetChildCount(red1)); + + GumboNode* p2 = GetChild(body, 1); + ASSERT_EQ(GUMBO_NODE_ELEMENT, p2->type); + EXPECT_EQ(GUMBO_TAG_P, p2->v.element.tag); + ASSERT_EQ(1, GetChildCount(p2)); + + GumboNode* red2 = GetChild(p2, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, red2->type); + EXPECT_EQ(GUMBO_TAG_FONT, red2->v.element.tag); + ASSERT_EQ(1, GetAttributeCount(red2)); + GumboAttribute* red2_attr = GetAttribute(red2, 0); + EXPECT_STREQ("color", red2_attr->name); + EXPECT_STREQ("red", red2_attr->value); + ASSERT_EQ(1, GetChildCount(red2)); +} + +TEST_F(GumboParserTest, AdoptionAgency1) { + // https://html.spec.whatwg.org/multipage/parsing.html#misnested-tags:-b-i-/b-/i + Parse("12345"); + ASSERT_EQ(1, GetChildCount(root_)); + + GumboNode* html = GetChild(root_, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, html->type); + EXPECT_EQ(GUMBO_INSERTION_BY_PARSER | GUMBO_INSERTION_IMPLICIT_END_TAG | + GUMBO_INSERTION_IMPLIED, + html->parse_flags); + EXPECT_EQ(GUMBO_TAG_HTML, html->v.element.tag); + ASSERT_EQ(2, GetChildCount(html)); + + GumboNode* body = GetChild(html, 1); + ASSERT_EQ(GUMBO_NODE_ELEMENT, body->type); + EXPECT_EQ(GUMBO_INSERTION_BY_PARSER | GUMBO_INSERTION_IMPLICIT_END_TAG | + GUMBO_INSERTION_IMPLIED, + body->parse_flags); + EXPECT_EQ(GUMBO_TAG_BODY, body->v.element.tag); + ASSERT_EQ(1, GetChildCount(body)); + + GumboNode* p = GetChild(body, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, p->type); + EXPECT_EQ(GUMBO_INSERTION_NORMAL, p->parse_flags); + EXPECT_EQ(GUMBO_TAG_P, p->v.element.tag); + ASSERT_EQ(4, GetChildCount(p)); + + GumboNode* text1 = GetChild(p, 0); + ASSERT_EQ(GUMBO_NODE_TEXT, text1->type); + EXPECT_EQ(GUMBO_INSERTION_NORMAL, text1->parse_flags); + EXPECT_STREQ("1", text1->v.text.text); + + GumboNode* b = GetChild(p, 1); + ASSERT_EQ(GUMBO_NODE_ELEMENT, b->type); + EXPECT_EQ(GUMBO_INSERTION_NORMAL, b->parse_flags); + EXPECT_EQ(GUMBO_TAG_B, b->v.element.tag); + ASSERT_EQ(2, GetChildCount(b)); + + GumboNode* text2 = GetChild(b, 0); + ASSERT_EQ(GUMBO_NODE_TEXT, text2->type); + EXPECT_EQ(GUMBO_INSERTION_NORMAL, text2->parse_flags); + EXPECT_STREQ("2", text2->v.text.text); + + GumboNode* i = GetChild(b, 1); + ASSERT_EQ(GUMBO_NODE_ELEMENT, i->type); + EXPECT_EQ(GUMBO_INSERTION_IMPLICIT_END_TAG, i->parse_flags); + EXPECT_EQ(GUMBO_TAG_I, i->v.element.tag); + ASSERT_EQ(1, GetChildCount(i)); + + GumboNode* text3 = GetChild(i, 0); + ASSERT_EQ(GUMBO_NODE_TEXT, text3->type); + EXPECT_EQ(GUMBO_INSERTION_NORMAL, text2->parse_flags); + EXPECT_STREQ("3", text3->v.text.text); + + GumboNode* i2 = GetChild(p, 2); + ASSERT_EQ(GUMBO_NODE_ELEMENT, i2->type); + EXPECT_EQ(GUMBO_INSERTION_BY_PARSER | + GUMBO_INSERTION_RECONSTRUCTED_FORMATTING_ELEMENT, + i2->parse_flags); + EXPECT_EQ(GUMBO_TAG_I, i2->v.element.tag); + ASSERT_EQ(1, GetChildCount(i2)); + + GumboNode* text4 = GetChild(i2, 0); + ASSERT_EQ(GUMBO_NODE_TEXT, text4->type); + EXPECT_EQ(GUMBO_INSERTION_NORMAL, text2->parse_flags); + EXPECT_STREQ("4", text4->v.text.text); + + GumboNode* text5 = GetChild(p, 3); + ASSERT_EQ(GUMBO_NODE_TEXT, text5->type); + EXPECT_EQ(GUMBO_INSERTION_NORMAL, text2->parse_flags); + EXPECT_STREQ("5", text5->v.text.text); +} + +TEST_F(GumboParserTest, AdoptionAgency2) { + // https://html.spec.whatwg.org/multipage/parsing.html#misnested-tags:-b-p-/b-/p + Parse("123"); + ASSERT_EQ(1, GetChildCount(root_)); + + GumboNode* html = GetChild(root_, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, html->type); + EXPECT_EQ(GUMBO_INSERTION_BY_PARSER | GUMBO_INSERTION_IMPLICIT_END_TAG | + GUMBO_INSERTION_IMPLIED, + html->parse_flags); + EXPECT_EQ(GUMBO_TAG_HTML, html->v.element.tag); + ASSERT_EQ(2, GetChildCount(html)); + + GumboNode* body = GetChild(html, 1); + ASSERT_EQ(GUMBO_NODE_ELEMENT, body->type); + EXPECT_EQ(GUMBO_INSERTION_BY_PARSER | GUMBO_INSERTION_IMPLICIT_END_TAG | + GUMBO_INSERTION_IMPLIED, + body->parse_flags); + EXPECT_EQ(GUMBO_TAG_BODY, body->v.element.tag); + ASSERT_EQ(2, GetChildCount(body)); + + GumboNode* b = GetChild(body, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, b->type); + EXPECT_EQ(GUMBO_INSERTION_IMPLICIT_END_TAG, b->parse_flags); + EXPECT_EQ(GUMBO_TAG_B, b->v.element.tag); + ASSERT_EQ(1, GetChildCount(b)); + + GumboNode* text1 = GetChild(b, 0); + ASSERT_EQ(GUMBO_NODE_TEXT, text1->type); + EXPECT_EQ(GUMBO_INSERTION_NORMAL, text1->parse_flags); + EXPECT_STREQ("1", text1->v.text.text); + + GumboNode* p = GetChild(body, 1); + ASSERT_EQ(GUMBO_NODE_ELEMENT, p->type); + EXPECT_EQ(GUMBO_INSERTION_ADOPTION_AGENCY_MOVED, p->parse_flags); + EXPECT_EQ(GUMBO_TAG_P, p->v.element.tag); + ASSERT_EQ(2, GetChildCount(p)); + + GumboNode* b2 = GetChild(p, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, b2->type); + EXPECT_EQ(GUMBO_INSERTION_ADOPTION_AGENCY_CLONED | GUMBO_INSERTION_BY_PARSER, + b2->parse_flags); + EXPECT_EQ(GUMBO_TAG_B, b2->v.element.tag); + ASSERT_EQ(1, GetChildCount(b2)); + + GumboNode* text2 = GetChild(b2, 0); + ASSERT_EQ(GUMBO_NODE_TEXT, text2->type); + EXPECT_EQ(GUMBO_INSERTION_NORMAL, text2->parse_flags); + EXPECT_STREQ("2", text2->v.text.text); + + GumboNode* text3 = GetChild(p, 1); + ASSERT_EQ(GUMBO_NODE_TEXT, text3->type); + EXPECT_EQ(GUMBO_INSERTION_NORMAL, text2->parse_flags); + EXPECT_STREQ("3", text3->v.text.text); +} + +TEST_F(GumboParserTest, AdoptionAgency3) { + Parse(""); +} + +TEST_F(GumboParserTest, ImplicitlyCloseLists) { + Parse( + "\n" + " First\n" + " Second\n" + ""); + + GumboNode* body; + GetAndAssertBody(root_, &body); + ASSERT_EQ(1, GetChildCount(body)); + + GumboNode* ul = GetChild(body, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, ul->type); + EXPECT_EQ(GUMBO_TAG_UL, GetTag(ul)); + ASSERT_EQ(3, GetChildCount(ul)); + + GumboNode* text = GetChild(ul, 0); + ASSERT_EQ(GUMBO_NODE_WHITESPACE, text->type); + EXPECT_EQ(GUMBO_INSERTION_NORMAL, text->parse_flags); + EXPECT_STREQ("\n ", text->v.text.text); + + GumboNode* li1 = GetChild(ul, 1); + ASSERT_EQ(GUMBO_NODE_ELEMENT, li1->type); + EXPECT_EQ(GUMBO_TAG_LI, GetTag(li1)); + ASSERT_EQ(1, GetChildCount(li1)); + + GumboNode* li2 = GetChild(ul, 2); + ASSERT_EQ(GUMBO_NODE_ELEMENT, li2->type); + EXPECT_EQ(GUMBO_TAG_LI, GetTag(li2)); + ASSERT_EQ(1, GetChildCount(li2)); +} + +TEST_F(GumboParserTest, CData) { + Parse(""); + + GumboNode* body; + GetAndAssertBody(root_, &body); + ASSERT_EQ(1, GetChildCount(body)); + + GumboNode* svg = GetChild(body, 0); + ASSERT_EQ(1, GetChildCount(svg)); + + GumboNode* cdata = GetChild(svg, 0); + ASSERT_EQ(GUMBO_NODE_CDATA, cdata->type); + EXPECT_STREQ("this is text", cdata->v.text.text); +} + +TEST_F(GumboParserTest, CDataUnsafe) { + // Can't use Parse() because of the strlen + output_ = + gumbo_parse_with_options(&options_, "", + sizeof("") - 1); + root_ = output_->document; + + GumboNode* body; + GetAndAssertBody(root_, &body); + ASSERT_EQ(1, GetChildCount(body)); + + GumboNode* svg = GetChild(body, 0); + ASSERT_EQ(1, GetChildCount(svg)); + + GumboNode* cdata = GetChild(svg, 0); + ASSERT_EQ(GUMBO_NODE_CDATA, cdata->type); + // \xEF\xBF\xBD = unicode replacement char + EXPECT_STREQ( + "\xEF\xBF\xBD" + "filler\xEF\xBF\xBD" + "text\xEF\xBF\xBD", + cdata->v.text.text); +} + +TEST_F(GumboParserTest, CDataInBody) { + Parse(""); + + GumboNode* body; + GetAndAssertBody(root_, &body); + ASSERT_EQ(1, GetChildCount(body)); + + GumboNode* div = GetChild(body, 0); + ASSERT_EQ(1, GetChildCount(div)); + + GumboNode* cdata = GetChild(div, 0); + ASSERT_EQ(GUMBO_NODE_COMMENT, cdata->type); + EXPECT_STREQ("[CDATA[this is text]]", cdata->v.text.text); +} + +TEST_F(GumboParserTest, CDataInHTMLFragment) { + ParseFragment("", "div", GUMBO_NAMESPACE_HTML); + EXPECT_EQ(1, GetChildCount(root_)); + GumboNode* html = GetChild(root_, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, html->type); + EXPECT_EQ(GUMBO_TAG_HTML, html->v.element.tag); + EXPECT_EQ(1, GetChildCount(html)); + + ASSERT_EQ(1, GetChildCount(html)); + GumboNode* cdata = GetChild(html, 0); + ASSERT_EQ(GUMBO_NODE_COMMENT, cdata->type); + EXPECT_STREQ("[CDATA[this is text]]", cdata->v.text.text); +} + +TEST_F(GumboParserTest, CDataInSVGFragment) { + ParseFragment("", "svg", GUMBO_NAMESPACE_SVG); + EXPECT_EQ(1, GetChildCount(root_)); + GumboNode* html = GetChild(root_, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, html->type); + EXPECT_EQ(GUMBO_TAG_HTML, html->v.element.tag); + EXPECT_EQ(1, GetChildCount(html)); + + ASSERT_EQ(1, GetChildCount(html)); + GumboNode* cdata = GetChild(html, 0); + ASSERT_EQ(GUMBO_NODE_CDATA, cdata->type); + EXPECT_STREQ("this is text", cdata->v.text.text); +} + +TEST_F(GumboParserTest, FormattingTagsInHeading) { + Parse("This is oldtext"); + + GumboNode* body; + GetAndAssertBody(root_, &body); + ASSERT_EQ(2, GetChildCount(body)); + + GumboNode* h2 = GetChild(body, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, h2->type); + EXPECT_EQ(GUMBO_TAG_H2, GetTag(h2)); + ASSERT_EQ(2, GetChildCount(h2)); + + GumboNode* text1 = GetChild(h2, 0); + ASSERT_EQ(GUMBO_NODE_TEXT, text1->type); + EXPECT_EQ(GUMBO_INSERTION_NORMAL, text1->parse_flags); + EXPECT_STREQ("This is ", text1->v.text.text); + + GumboNode* b = GetChild(h2, 1); + ASSERT_EQ(GUMBO_NODE_ELEMENT, b->type); + EXPECT_EQ(GUMBO_TAG_B, GetTag(b)); + EXPECT_EQ(GUMBO_INSERTION_IMPLICIT_END_TAG, b->parse_flags); + ASSERT_EQ(1, GetChildCount(b)); + + GumboNode* text2 = GetChild(b, 0); + ASSERT_EQ(GUMBO_NODE_TEXT, text2->type); + EXPECT_EQ(GUMBO_INSERTION_NORMAL, text2->parse_flags); + EXPECT_STREQ("old", text2->v.text.text); + + GumboNode* b2 = GetChild(body, 1); + ASSERT_EQ(GUMBO_NODE_ELEMENT, b2->type); + EXPECT_EQ(GUMBO_TAG_B, GetTag(b2)); + EXPECT_EQ(GUMBO_INSERTION_IMPLICIT_END_TAG | GUMBO_INSERTION_BY_PARSER | + GUMBO_INSERTION_RECONSTRUCTED_FORMATTING_ELEMENT, + b2->parse_flags); + ASSERT_EQ(1, GetChildCount(b2)); + + GumboNode* text3 = GetChild(b2, 0); + ASSERT_EQ(GUMBO_NODE_TEXT, text3->type); + EXPECT_EQ(GUMBO_INSERTION_NORMAL, text3->parse_flags); + EXPECT_STREQ("text", text3->v.text.text); +} + +TEST_F(GumboParserTest, ExtraReconstruction) { + Parse(""); + + GumboNode* body; + GetAndAssertBody(root_, &body); + ASSERT_EQ(2, GetChildCount(body)); + + EXPECT_EQ(GUMBO_TAG_SPAN, GetTag(GetChild(body, 0))); + EXPECT_EQ(GUMBO_TAG_P, GetTag(GetChild(body, 1))); +} + +TEST_F(GumboParserTest, LinkifiedHeading) { + Parse("TextSummary"); + + GumboNode* body; + GetAndAssertBody(root_, &body); + ASSERT_EQ(1, GetChildCount(body)); + + GumboNode* li = GetChild(body, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, li->type); + EXPECT_EQ(GUMBO_TAG_LI, GetTag(li)); + ASSERT_EQ(2, GetChildCount(li)); + + GumboNode* h3 = GetChild(li, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, h3->type); + EXPECT_EQ(GUMBO_TAG_H3, GetTag(h3)); + ASSERT_EQ(1, GetChildCount(h3)); + + GumboNode* anchor = GetChild(h3, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, anchor->type); + EXPECT_EQ(GUMBO_TAG_A, GetTag(anchor)); + ASSERT_EQ(1, GetChildCount(anchor)); + + GumboNode* div = GetChild(li, 1); + ASSERT_EQ(GUMBO_NODE_ELEMENT, div->type); + EXPECT_EQ(GUMBO_TAG_DIV, GetTag(div)); + ASSERT_EQ(1, GetChildCount(div)); +} + +TEST_F(GumboParserTest, MisnestedHeading) { + Parse( + "" + " " + " " + " List" + " " + " " + " Heading1" + "Heading3" + "After text"); + // The parse of this is pretty weird: according to the spec, it should be: + // + // + // + // + // + // List + // + // Heading1 + // + // Heading3 + // After text + // + // + // Explanation: + // , , and tags are implied. The opening and , the + // does *not* close the existing , and then we enter a definition list. + // The closing , even though it's misnested, causes the to be closed + // implicitly, then also closes the and as a parse error. is + // still open, and so "Heading1" goes into it. Because the current node is a + // heading tag, closes it (as a parse error) and reopens a new node, + // which is closed by the tag. The remaining text goes straight into + // the ; since no heading is open, the tag is ignored and the + // second run is condensed into the first. + // TODO(jdtang): Make sure that parse_flags are set appropriately for this. + GumboNode* body; + GetAndAssertBody(root_, &body); + ASSERT_EQ(3, GetChildCount(body)); + + GumboNode* h1 = GetChild(body, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, h1->type); + EXPECT_EQ(GUMBO_TAG_H1, GetTag(h1)); + ASSERT_EQ(3, GetChildCount(h1)); + // Child 1 is whitespace, as it is for many of these nodes. + + GumboNode* section = GetChild(h1, 1); + ASSERT_EQ(GUMBO_NODE_ELEMENT, section->type); + EXPECT_EQ(GUMBO_TAG_SECTION, GetTag(section)); + ASSERT_EQ(3, GetChildCount(section)); + + GumboNode* h2 = GetChild(section, 1); + ASSERT_EQ(GUMBO_NODE_ELEMENT, h2->type); + EXPECT_EQ(GUMBO_TAG_H2, GetTag(h2)); + ASSERT_EQ(2, GetChildCount(h2)); + + GumboNode* dl = GetChild(h2, 1); + ASSERT_EQ(GUMBO_NODE_ELEMENT, dl->type); + EXPECT_EQ(GUMBO_TAG_DL, GetTag(dl)); + ASSERT_EQ(1, GetChildCount(dl)); + + GumboNode* dt = GetChild(dl, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, dt->type); + EXPECT_EQ(GUMBO_TAG_DT, GetTag(dt)); + ASSERT_EQ(1, GetChildCount(dt)); + + GumboNode* text1 = GetChild(dt, 0); + ASSERT_EQ(GUMBO_NODE_TEXT, text1->type); + EXPECT_EQ(GUMBO_INSERTION_NORMAL, text1->parse_flags); + EXPECT_STREQ("List ", text1->v.text.text); + + GumboNode* text2 = GetChild(h1, 2); + ASSERT_EQ(GUMBO_NODE_TEXT, text2->type); + EXPECT_EQ(GUMBO_INSERTION_NORMAL, text2->parse_flags); + EXPECT_STREQ(" Heading1", text2->v.text.text); + + GumboNode* h3 = GetChild(body, 1); + ASSERT_EQ(GUMBO_NODE_ELEMENT, h3->type); + EXPECT_EQ(GUMBO_TAG_H3, GetTag(h3)); + EXPECT_EQ(1, GetChildCount(h3)); + + GumboNode* text3 = GetChild(h3, 0); + ASSERT_EQ(GUMBO_NODE_TEXT, text3->type); + EXPECT_EQ(GUMBO_INSERTION_NORMAL, text3->parse_flags); + EXPECT_STREQ("Heading3", text3->v.text.text); + + GumboNode* text4 = GetChild(body, 2); + ASSERT_EQ(GUMBO_NODE_TEXT, text4->type); + EXPECT_EQ(GUMBO_INSERTION_NORMAL, text4->parse_flags); + EXPECT_STREQ("After text", text4->v.text.text); +} + +TEST_F(GumboParserTest, DoubleBody) { + Parse("Text"); + + GumboNode* body; + GetAndAssertBody(root_, &body); + ASSERT_EQ(1, GetChildCount(body)); + ASSERT_EQ(2, GetAttributeCount(body)); + + GumboAttribute* clas = GetAttribute(body, 0); + EXPECT_STREQ("class", clas->name); + EXPECT_STREQ("first", clas->value); + + GumboAttribute* id = GetAttribute(body, 1); + EXPECT_STREQ("id", id->name); + EXPECT_STREQ("merged", id->value); + + GumboNode* text = GetChild(body, 0); + ASSERT_EQ(GUMBO_NODE_TEXT, text->type); + EXPECT_EQ(GUMBO_INSERTION_NORMAL, text->parse_flags); + EXPECT_STREQ("Text", text->v.text.text); +} + +TEST_F(GumboParserTest, ThInMathMl) { + Parse("
X" + ); + + GumboNode* body; + GetAndAssertBody(root_, &body); + ASSERT_EQ(2, GetChildCount(body)); + + GumboNode* p1 = GetChild(body, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, p1->type); + EXPECT_EQ(GUMBO_TAG_P, p1->v.element.tag); + ASSERT_EQ(1, GetChildCount(p1)); + + GumboNode* size1 = GetChild(p1, 0); + GumboNode* red1 = GetChild(size1, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, red1->type); + EXPECT_EQ(GUMBO_TAG_FONT, red1->v.element.tag); + ASSERT_EQ(1, GetAttributeCount(red1)); + GumboAttribute* red1_attr = GetAttribute(red1, 0); + EXPECT_STREQ("color", red1_attr->name); + EXPECT_STREQ("red", red1_attr->value); + ASSERT_EQ(1, GetChildCount(red1)); + + GumboNode* p2 = GetChild(body, 1); + ASSERT_EQ(GUMBO_NODE_ELEMENT, p2->type); + EXPECT_EQ(GUMBO_TAG_P, p2->v.element.tag); + ASSERT_EQ(1, GetChildCount(p2)); + + GumboNode* red2 = GetChild(p2, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, red2->type); + EXPECT_EQ(GUMBO_TAG_FONT, red2->v.element.tag); + ASSERT_EQ(1, GetAttributeCount(red2)); + GumboAttribute* red2_attr = GetAttribute(red2, 0); + EXPECT_STREQ("color", red2_attr->name); + EXPECT_STREQ("red", red2_attr->value); + ASSERT_EQ(1, GetChildCount(red2)); +} + +TEST_F(GumboParserTest, AdoptionAgency1) { + // https://html.spec.whatwg.org/multipage/parsing.html#misnested-tags:-b-i-/b-/i + Parse("
12345
23
"); +} + +TEST_F(GumboParserTest, ImplicitlyCloseLists) { + Parse( + "\n" + " First\n" + " Second\n" + ""); + + GumboNode* body; + GetAndAssertBody(root_, &body); + ASSERT_EQ(1, GetChildCount(body)); + + GumboNode* ul = GetChild(body, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, ul->type); + EXPECT_EQ(GUMBO_TAG_UL, GetTag(ul)); + ASSERT_EQ(3, GetChildCount(ul)); + + GumboNode* text = GetChild(ul, 0); + ASSERT_EQ(GUMBO_NODE_WHITESPACE, text->type); + EXPECT_EQ(GUMBO_INSERTION_NORMAL, text->parse_flags); + EXPECT_STREQ("\n ", text->v.text.text); + + GumboNode* li1 = GetChild(ul, 1); + ASSERT_EQ(GUMBO_NODE_ELEMENT, li1->type); + EXPECT_EQ(GUMBO_TAG_LI, GetTag(li1)); + ASSERT_EQ(1, GetChildCount(li1)); + + GumboNode* li2 = GetChild(ul, 2); + ASSERT_EQ(GUMBO_NODE_ELEMENT, li2->type); + EXPECT_EQ(GUMBO_TAG_LI, GetTag(li2)); + ASSERT_EQ(1, GetChildCount(li2)); +} + +TEST_F(GumboParserTest, CData) { + Parse(""); + + GumboNode* body; + GetAndAssertBody(root_, &body); + ASSERT_EQ(1, GetChildCount(body)); + + GumboNode* svg = GetChild(body, 0); + ASSERT_EQ(1, GetChildCount(svg)); + + GumboNode* cdata = GetChild(svg, 0); + ASSERT_EQ(GUMBO_NODE_CDATA, cdata->type); + EXPECT_STREQ("this is text", cdata->v.text.text); +} + +TEST_F(GumboParserTest, CDataUnsafe) { + // Can't use Parse() because of the strlen + output_ = + gumbo_parse_with_options(&options_, "", + sizeof("") - 1); + root_ = output_->document; + + GumboNode* body; + GetAndAssertBody(root_, &body); + ASSERT_EQ(1, GetChildCount(body)); + + GumboNode* svg = GetChild(body, 0); + ASSERT_EQ(1, GetChildCount(svg)); + + GumboNode* cdata = GetChild(svg, 0); + ASSERT_EQ(GUMBO_NODE_CDATA, cdata->type); + // \xEF\xBF\xBD = unicode replacement char + EXPECT_STREQ( + "\xEF\xBF\xBD" + "filler\xEF\xBF\xBD" + "text\xEF\xBF\xBD", + cdata->v.text.text); +} + +TEST_F(GumboParserTest, CDataInBody) { + Parse(""); + + GumboNode* body; + GetAndAssertBody(root_, &body); + ASSERT_EQ(1, GetChildCount(body)); + + GumboNode* div = GetChild(body, 0); + ASSERT_EQ(1, GetChildCount(div)); + + GumboNode* cdata = GetChild(div, 0); + ASSERT_EQ(GUMBO_NODE_COMMENT, cdata->type); + EXPECT_STREQ("[CDATA[this is text]]", cdata->v.text.text); +} + +TEST_F(GumboParserTest, CDataInHTMLFragment) { + ParseFragment("", "div", GUMBO_NAMESPACE_HTML); + EXPECT_EQ(1, GetChildCount(root_)); + GumboNode* html = GetChild(root_, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, html->type); + EXPECT_EQ(GUMBO_TAG_HTML, html->v.element.tag); + EXPECT_EQ(1, GetChildCount(html)); + + ASSERT_EQ(1, GetChildCount(html)); + GumboNode* cdata = GetChild(html, 0); + ASSERT_EQ(GUMBO_NODE_COMMENT, cdata->type); + EXPECT_STREQ("[CDATA[this is text]]", cdata->v.text.text); +} + +TEST_F(GumboParserTest, CDataInSVGFragment) { + ParseFragment("", "svg", GUMBO_NAMESPACE_SVG); + EXPECT_EQ(1, GetChildCount(root_)); + GumboNode* html = GetChild(root_, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, html->type); + EXPECT_EQ(GUMBO_TAG_HTML, html->v.element.tag); + EXPECT_EQ(1, GetChildCount(html)); + + ASSERT_EQ(1, GetChildCount(html)); + GumboNode* cdata = GetChild(html, 0); + ASSERT_EQ(GUMBO_NODE_CDATA, cdata->type); + EXPECT_STREQ("this is text", cdata->v.text.text); +} + +TEST_F(GumboParserTest, FormattingTagsInHeading) { + Parse("This is oldtext"); + + GumboNode* body; + GetAndAssertBody(root_, &body); + ASSERT_EQ(2, GetChildCount(body)); + + GumboNode* h2 = GetChild(body, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, h2->type); + EXPECT_EQ(GUMBO_TAG_H2, GetTag(h2)); + ASSERT_EQ(2, GetChildCount(h2)); + + GumboNode* text1 = GetChild(h2, 0); + ASSERT_EQ(GUMBO_NODE_TEXT, text1->type); + EXPECT_EQ(GUMBO_INSERTION_NORMAL, text1->parse_flags); + EXPECT_STREQ("This is ", text1->v.text.text); + + GumboNode* b = GetChild(h2, 1); + ASSERT_EQ(GUMBO_NODE_ELEMENT, b->type); + EXPECT_EQ(GUMBO_TAG_B, GetTag(b)); + EXPECT_EQ(GUMBO_INSERTION_IMPLICIT_END_TAG, b->parse_flags); + ASSERT_EQ(1, GetChildCount(b)); + + GumboNode* text2 = GetChild(b, 0); + ASSERT_EQ(GUMBO_NODE_TEXT, text2->type); + EXPECT_EQ(GUMBO_INSERTION_NORMAL, text2->parse_flags); + EXPECT_STREQ("old", text2->v.text.text); + + GumboNode* b2 = GetChild(body, 1); + ASSERT_EQ(GUMBO_NODE_ELEMENT, b2->type); + EXPECT_EQ(GUMBO_TAG_B, GetTag(b2)); + EXPECT_EQ(GUMBO_INSERTION_IMPLICIT_END_TAG | GUMBO_INSERTION_BY_PARSER | + GUMBO_INSERTION_RECONSTRUCTED_FORMATTING_ELEMENT, + b2->parse_flags); + ASSERT_EQ(1, GetChildCount(b2)); + + GumboNode* text3 = GetChild(b2, 0); + ASSERT_EQ(GUMBO_NODE_TEXT, text3->type); + EXPECT_EQ(GUMBO_INSERTION_NORMAL, text3->parse_flags); + EXPECT_STREQ("text", text3->v.text.text); +} + +TEST_F(GumboParserTest, ExtraReconstruction) { + Parse(""); + + GumboNode* body; + GetAndAssertBody(root_, &body); + ASSERT_EQ(2, GetChildCount(body)); + + EXPECT_EQ(GUMBO_TAG_SPAN, GetTag(GetChild(body, 0))); + EXPECT_EQ(GUMBO_TAG_P, GetTag(GetChild(body, 1))); +} + +TEST_F(GumboParserTest, LinkifiedHeading) { + Parse("TextSummary"); + + GumboNode* body; + GetAndAssertBody(root_, &body); + ASSERT_EQ(1, GetChildCount(body)); + + GumboNode* li = GetChild(body, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, li->type); + EXPECT_EQ(GUMBO_TAG_LI, GetTag(li)); + ASSERT_EQ(2, GetChildCount(li)); + + GumboNode* h3 = GetChild(li, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, h3->type); + EXPECT_EQ(GUMBO_TAG_H3, GetTag(h3)); + ASSERT_EQ(1, GetChildCount(h3)); + + GumboNode* anchor = GetChild(h3, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, anchor->type); + EXPECT_EQ(GUMBO_TAG_A, GetTag(anchor)); + ASSERT_EQ(1, GetChildCount(anchor)); + + GumboNode* div = GetChild(li, 1); + ASSERT_EQ(GUMBO_NODE_ELEMENT, div->type); + EXPECT_EQ(GUMBO_TAG_DIV, GetTag(div)); + ASSERT_EQ(1, GetChildCount(div)); +} + +TEST_F(GumboParserTest, MisnestedHeading) { + Parse( + "" + " " + " " + " List" + " " + " " + " Heading1" + "Heading3" + "After text"); + // The parse of this is pretty weird: according to the spec, it should be: + // + // + // + // + // + // List + // + // Heading1 + // + // Heading3 + // After text + // + // + // Explanation: + // , , and tags are implied. The opening and , the + // does *not* close the existing , and then we enter a definition list. + // The closing , even though it's misnested, causes the to be closed + // implicitly, then also closes the and as a parse error. is + // still open, and so "Heading1" goes into it. Because the current node is a + // heading tag, closes it (as a parse error) and reopens a new node, + // which is closed by the tag. The remaining text goes straight into + // the ; since no heading is open, the tag is ignored and the + // second run is condensed into the first. + // TODO(jdtang): Make sure that parse_flags are set appropriately for this. + GumboNode* body; + GetAndAssertBody(root_, &body); + ASSERT_EQ(3, GetChildCount(body)); + + GumboNode* h1 = GetChild(body, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, h1->type); + EXPECT_EQ(GUMBO_TAG_H1, GetTag(h1)); + ASSERT_EQ(3, GetChildCount(h1)); + // Child 1 is whitespace, as it is for many of these nodes. + + GumboNode* section = GetChild(h1, 1); + ASSERT_EQ(GUMBO_NODE_ELEMENT, section->type); + EXPECT_EQ(GUMBO_TAG_SECTION, GetTag(section)); + ASSERT_EQ(3, GetChildCount(section)); + + GumboNode* h2 = GetChild(section, 1); + ASSERT_EQ(GUMBO_NODE_ELEMENT, h2->type); + EXPECT_EQ(GUMBO_TAG_H2, GetTag(h2)); + ASSERT_EQ(2, GetChildCount(h2)); + + GumboNode* dl = GetChild(h2, 1); + ASSERT_EQ(GUMBO_NODE_ELEMENT, dl->type); + EXPECT_EQ(GUMBO_TAG_DL, GetTag(dl)); + ASSERT_EQ(1, GetChildCount(dl)); + + GumboNode* dt = GetChild(dl, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, dt->type); + EXPECT_EQ(GUMBO_TAG_DT, GetTag(dt)); + ASSERT_EQ(1, GetChildCount(dt)); + + GumboNode* text1 = GetChild(dt, 0); + ASSERT_EQ(GUMBO_NODE_TEXT, text1->type); + EXPECT_EQ(GUMBO_INSERTION_NORMAL, text1->parse_flags); + EXPECT_STREQ("List ", text1->v.text.text); + + GumboNode* text2 = GetChild(h1, 2); + ASSERT_EQ(GUMBO_NODE_TEXT, text2->type); + EXPECT_EQ(GUMBO_INSERTION_NORMAL, text2->parse_flags); + EXPECT_STREQ(" Heading1", text2->v.text.text); + + GumboNode* h3 = GetChild(body, 1); + ASSERT_EQ(GUMBO_NODE_ELEMENT, h3->type); + EXPECT_EQ(GUMBO_TAG_H3, GetTag(h3)); + EXPECT_EQ(1, GetChildCount(h3)); + + GumboNode* text3 = GetChild(h3, 0); + ASSERT_EQ(GUMBO_NODE_TEXT, text3->type); + EXPECT_EQ(GUMBO_INSERTION_NORMAL, text3->parse_flags); + EXPECT_STREQ("Heading3", text3->v.text.text); + + GumboNode* text4 = GetChild(body, 2); + ASSERT_EQ(GUMBO_NODE_TEXT, text4->type); + EXPECT_EQ(GUMBO_INSERTION_NORMAL, text4->parse_flags); + EXPECT_STREQ("After text", text4->v.text.text); +} + +TEST_F(GumboParserTest, DoubleBody) { + Parse("Text"); + + GumboNode* body; + GetAndAssertBody(root_, &body); + ASSERT_EQ(1, GetChildCount(body)); + ASSERT_EQ(2, GetAttributeCount(body)); + + GumboAttribute* clas = GetAttribute(body, 0); + EXPECT_STREQ("class", clas->name); + EXPECT_STREQ("first", clas->value); + + GumboAttribute* id = GetAttribute(body, 1); + EXPECT_STREQ("id", id->name); + EXPECT_STREQ("merged", id->value); + + GumboNode* text = GetChild(body, 0); + ASSERT_EQ(GUMBO_NODE_TEXT, text->type); + EXPECT_EQ(GUMBO_INSERTION_NORMAL, text->parse_flags); + EXPECT_STREQ("Text", text->v.text.text); +} + +TEST_F(GumboParserTest, ThInMathMl) { + Parse("
"); + + GumboNode* body; + GetAndAssertBody(root_, &body); + ASSERT_EQ(2, GetChildCount(body)); + + GumboNode* foo = GetChild(body, 0); + // If foo becomes a tag at some point, change the example. + ASSERT_EQ(GUMBO_TAG_UNKNOWN, foo->v.element.tag); + EXPECT_EQ(std::string("foo"), foo->v.element.name); + ASSERT_EQ(1, GetChildCount(foo)); + + GumboNode* bar = GetChild(foo, 0); + ASSERT_EQ(GUMBO_TAG_UNKNOWN, bar->v.element.tag); + EXPECT_EQ(std::string("bar"), bar->v.element.name); + + GumboNode* p = GetChild(body, 1); + EXPECT_EQ(std::string("p"), p->v.element.name); +} + +TEST_F(GumboParserTest, ForeignElementMismatch) { + Parse(""); + + GumboNode* body; + GetAndAssertBody(root_, &body); + ASSERT_EQ(1, GetChildCount(body)); + + GumboNode* math = GetChild(body, 0); + ASSERT_EQ(std::string("math"), math->v.element.name); + ASSERT_EQ(2, GetChildCount(math)); + + GumboNode* foo = GetChild(math, 0); + // If foo becomes a tag at some point, change the example. + ASSERT_EQ(GUMBO_TAG_UNKNOWN, foo->v.element.tag); + EXPECT_EQ(std::string("foo"), foo->v.element.name); + ASSERT_EQ(1, GetChildCount(foo)); + + GumboNode* bar = GetChild(foo, 0); + ASSERT_EQ(GUMBO_TAG_UNKNOWN, bar->v.element.tag); + EXPECT_EQ(std::string("bar"), bar->v.element.name); + + GumboNode* extra = GetChild(math, 1); + EXPECT_EQ(std::string("extra"), extra->v.element.name); +} + +TEST_F(GumboParserTest, FragmentAtAnnotationXML_text_html) { + ParseFragment("", "annotation-xml", GUMBO_NAMESPACE_MATHML, "text/html"); + + ASSERT_EQ(1, GetChildCount(root_)); + GumboNode* html = GetChild(root_, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, html->type); + EXPECT_EQ(GUMBO_TAG_HTML, html->v.element.tag); + ASSERT_EQ(1, GetChildCount(html)); + + GumboNode* a = GetChild(html, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, a->type); + EXPECT_EQ(GUMBO_TAG_A, a->v.element.tag); + EXPECT_EQ(GUMBO_NAMESPACE_HTML, a->v.element.tag_namespace); + EXPECT_EQ(0, GetChildCount(a)); +} + +TEST_F(GumboParserTest, FragmentAtAnnotationXML_application_xhtml_xml) { + ParseFragment("", "annotation-xml", GUMBO_NAMESPACE_MATHML, "application/xhtml+xml"); + + ASSERT_EQ(1, GetChildCount(root_)); + GumboNode* html = GetChild(root_, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, html->type); + EXPECT_EQ(GUMBO_TAG_HTML, html->v.element.tag); + ASSERT_EQ(1, GetChildCount(html)); + + GumboNode* a = GetChild(html, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, a->type); + EXPECT_EQ(GUMBO_TAG_A, a->v.element.tag); + EXPECT_EQ(GUMBO_NAMESPACE_HTML, a->v.element.tag_namespace); + EXPECT_EQ(0, GetChildCount(a)); +} + +TEST_F(GumboParserTest, FragmentAtAnnotationXML_other) { + ParseFragment("", "annotation-xml", GUMBO_NAMESPACE_MATHML, "not one of those"); + + ASSERT_EQ(1, GetChildCount(root_)); + GumboNode* html = GetChild(root_, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, html->type); + EXPECT_EQ(GUMBO_TAG_HTML, html->v.element.tag); + ASSERT_EQ(1, GetChildCount(html)); + + GumboNode* a = GetChild(html, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, a->type); + EXPECT_EQ(GUMBO_TAG_A, a->v.element.tag); + EXPECT_EQ(GUMBO_NAMESPACE_MATHML, a->v.element.tag_namespace); + EXPECT_EQ(0, GetChildCount(a)); +} + +TEST_F(GumboParserTest, DoctypePrefixPublicId) { + Parse(""); + EXPECT_EQ(std::string("+//Silmaril//dtd html Pro v0r11 19970101//extra"), + output_->document->v.document.public_identifier); + EXPECT_EQ(GUMBO_DOCTYPE_QUIRKS, output_->document->v.document.doc_type_quirks_mode); +} + +TEST_F(GumboParserTest, DoctypePublicIdHTML) { + Parse(""); + EXPECT_EQ(std::string("HTML"), + output_->document->v.document.public_identifier); + EXPECT_EQ(GUMBO_DOCTYPE_QUIRKS, output_->document->v.document.doc_type_quirks_mode); +} + +TEST_F(GumboParserTest, DoctypePublicIdHTMLblah) { + Parse(""); + EXPECT_EQ(std::string("HTMLblah"), + output_->document->v.document.public_identifier); + EXPECT_EQ(GUMBO_DOCTYPE_NO_QUIRKS, output_->document->v.document.doc_type_quirks_mode); +} + +TEST_F(GumboParserTest, DoctypeLimitedQuirksPrefix) { + Parse(""); + EXPECT_EQ(std::string("-//W3C//DTD XHTML 1.0 Frameset//asdf"), + output_->document->v.document.public_identifier); + EXPECT_EQ(GUMBO_DOCTYPE_LIMITED_QUIRKS, output_->document->v.document.doc_type_quirks_mode); +} + +TEST_F(GumboParserTest, DoctypeLimitedQuirks) { + Parse(""); + EXPECT_EQ(std::string("-//W3C//DTD XHTML 1.0 Frameset//"), + output_->document->v.document.public_identifier); + EXPECT_EQ(GUMBO_DOCTYPE_LIMITED_QUIRKS, output_->document->v.document.doc_type_quirks_mode); +} + +TEST_F(GumboParserTest, FragmentWithForm) { + ParseFragment("
baz
quux"); + EXPECT_EQ(1, GetChildCount(root_)); + GumboNode* html = GetChild(root_, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, html->type); + EXPECT_EQ(GUMBO_TAG_HTML, html->v.element.tag); + EXPECT_EQ(2, GetChildCount(html)); + + GumboNode* body = GetChild(html, 1); + ASSERT_EQ(GUMBO_NODE_ELEMENT, body->type); + EXPECT_EQ(GUMBO_TAG_BODY, body->v.element.tag); + EXPECT_EQ(4, GetChildCount(body)); + + GumboNode* svg = GetChild(body, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, svg->type); + EXPECT_EQ(GUMBO_TAG_SVG, svg->v.element.tag); + EXPECT_EQ(GUMBO_NAMESPACE_SVG, svg->v.element.tag_namespace); + EXPECT_EQ(2, GetChildCount(svg)); + + GumboNode* g = GetChild(svg, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, g->type); + EXPECT_EQ(std::string("g"), g->v.element.name); + EXPECT_EQ(GUMBO_NAMESPACE_SVG, g->v.element.tag_namespace); + EXPECT_EQ(1, GetChildCount(g)); + + GumboNode* text = GetChild(g, 0); + ASSERT_EQ(GUMBO_NODE_TEXT, text->type); + EXPECT_EQ(std::string("foo"), text->v.text.text); + + g = GetChild(svg, 1); + ASSERT_EQ(GUMBO_NODE_ELEMENT, g->type); + EXPECT_EQ(std::string("g"), g->v.element.name); + EXPECT_EQ(GUMBO_NAMESPACE_SVG, g->v.element.tag_namespace); + EXPECT_EQ(1, GetChildCount(g)); + + text = GetChild(g, 0); + ASSERT_EQ(GUMBO_NODE_TEXT, text->type); + EXPECT_EQ(std::string("bar"), text->v.text.text); + + GumboNode* p = GetChild(body, 1); + ASSERT_EQ(GUMBO_NODE_ELEMENT, p->type); + EXPECT_EQ(GUMBO_TAG_P, p->v.element.tag); + EXPECT_EQ(GUMBO_NAMESPACE_HTML, p->v.element.tag_namespace); + EXPECT_EQ(1, GetChildCount(p)); + + text = GetChild(p, 0); + ASSERT_EQ(GUMBO_NODE_TEXT, text->type); + EXPECT_EQ(std::string("baz"), text->v.text.text); + + GumboNode* table = GetChild(body, 2); + ASSERT_EQ(GUMBO_NODE_ELEMENT, table->type); + EXPECT_EQ(GUMBO_TAG_TABLE, table->v.element.tag); + EXPECT_EQ(GUMBO_NAMESPACE_HTML, table->v.element.tag_namespace); + EXPECT_EQ(1, GetChildCount(table)); + + GumboNode* colgroup = GetChild(table, 0); + ASSERT_EQ(GUMBO_NODE_ELEMENT, colgroup->type); + EXPECT_EQ(GUMBO_TAG_COLGROUP, colgroup->v.element.tag); + EXPECT_EQ(GUMBO_NAMESPACE_HTML, colgroup->v.element.tag_namespace); + EXPECT_EQ(0, GetChildCount(colgroup)); + + p = GetChild(body, 3); + ASSERT_EQ(GUMBO_NODE_ELEMENT, p->type); + EXPECT_EQ(GUMBO_TAG_P, p->v.element.tag); + EXPECT_EQ(GUMBO_NAMESPACE_HTML, p->v.element.tag_namespace); + EXPECT_EQ(1, GetChildCount(p)); + + text = GetChild(p, 0); + ASSERT_EQ(GUMBO_NODE_TEXT, text->type); + EXPECT_EQ(std::string("quux"), text->v.text.text); +} + +} // namespace diff --git a/gumbo-parser/test/string_buffer.cc b/gumbo-parser/test/string_buffer.cc new file mode 100644 index 0000000000..01102985d1 --- /dev/null +++ b/gumbo-parser/test/string_buffer.cc @@ -0,0 +1,101 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: jdtang@google.com (Jonathan Tang) + +#include "string_buffer.h" + +#include +#include + +#include "gtest/gtest.h" +#include "test_utils.h" +#include "util.h" + +namespace { + +#define INIT_GUMBO_STRING(varname, literal) \ + GumboStringPiece varname = {literal, sizeof(literal) - 1} + +class GumboStringBufferTest : public GumboTest { + protected: + GumboStringBufferTest() { gumbo_string_buffer_init(&buffer_); } + + ~GumboStringBufferTest() { gumbo_string_buffer_destroy(&buffer_); } + + void NullTerminateBuffer() { buffer_.data[buffer_.length++] = 0; } + + GumboStringBuffer buffer_; +}; + +TEST_F(GumboStringBufferTest, Reserve) { + gumbo_string_buffer_reserve(21, &buffer_); + EXPECT_EQ(40, buffer_.capacity); + strcpy(buffer_.data, "01234567890123456789"); + buffer_.length = 20; + NullTerminateBuffer(); + EXPECT_EQ(21, buffer_.length); + EXPECT_STREQ("01234567890123456789", buffer_.data); +} + +TEST_F(GumboStringBufferTest, AppendString) { + INIT_GUMBO_STRING(str, "01234567"); + gumbo_string_buffer_append_string(&str, &buffer_); + NullTerminateBuffer(); + EXPECT_STREQ("01234567", buffer_.data); +} + +TEST_F(GumboStringBufferTest, AppendStringWithResize) { + INIT_GUMBO_STRING(str, "01234567"); + gumbo_string_buffer_append_string(&str, &buffer_); + gumbo_string_buffer_append_string(&str, &buffer_); + NullTerminateBuffer(); + EXPECT_STREQ("0123456701234567", buffer_.data); +} + +TEST_F(GumboStringBufferTest, AppendCodepoint_1Byte) { + gumbo_string_buffer_append_codepoint('a', &buffer_); + NullTerminateBuffer(); + EXPECT_STREQ("a", buffer_.data); +} + +TEST_F(GumboStringBufferTest, AppendCodepoint_2Bytes) { + gumbo_string_buffer_append_codepoint(0xE5, &buffer_); + NullTerminateBuffer(); + EXPECT_STREQ("\xC3\xA5", buffer_.data); +} + +TEST_F(GumboStringBufferTest, AppendCodepoint_3Bytes) { + gumbo_string_buffer_append_codepoint(0x39E7, &buffer_); + NullTerminateBuffer(); + EXPECT_STREQ("\xE3\xA7\xA7", buffer_.data); +} + +TEST_F(GumboStringBufferTest, AppendCodepoint_4Bytes) { + gumbo_string_buffer_append_codepoint(0x679E7, &buffer_); + NullTerminateBuffer(); + EXPECT_STREQ("\xF1\xA7\xA7\xA7", buffer_.data); +} + +TEST_F(GumboStringBufferTest, ToString) { + gumbo_string_buffer_reserve(8, &buffer_); + strcpy(buffer_.data, "012345"); + buffer_.length = 7; + + char* dest = gumbo_string_buffer_to_string(&buffer_); + EXPECT_STREQ("012345", dest); + gumbo_free(dest); +} + +} // namespace diff --git a/gumbo-parser/test/string_piece.cc b/gumbo-parser/test/string_piece.cc new file mode 100644 index 0000000000..db2d5f3cd2 --- /dev/null +++ b/gumbo-parser/test/string_piece.cc @@ -0,0 +1,67 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: jdtang@google.com (Jonathan Tang) + +#include "test_utils.h" + +namespace { + +typedef GumboTest GumboStringPieceTest; + +#define STRING(s) {"" s, sizeof(s) - 1} + +TEST_F(GumboStringPieceTest, Equal) { + const GumboStringPiece str1 = STRING("foo"); + const GumboStringPiece str2 = STRING("foo"); + EXPECT_TRUE(gumbo_string_equals(&str1, &str2)); +} + +TEST_F(GumboStringPieceTest, NotEqual_DifferingCase) { + const GumboStringPiece str1 = STRING("foo"); + const GumboStringPiece str2 = STRING("Foo"); + EXPECT_FALSE(gumbo_string_equals(&str1, &str2)); +} + +TEST_F(GumboStringPieceTest, NotEqual_Str1Shorter) { + const GumboStringPiece str1 = STRING("foo"); + const GumboStringPiece str2 = STRING("foobar"); + EXPECT_FALSE(gumbo_string_equals(&str1, &str2)); +} + +TEST_F(GumboStringPieceTest, NotEqual_Str2Shorter) { + const GumboStringPiece str1 = STRING("foobar"); + const GumboStringPiece str2 = STRING("foo"); + EXPECT_FALSE(gumbo_string_equals(&str1, &str2)); +} + +TEST_F(GumboStringPieceTest, NotEqual_DifferentText) { + const GumboStringPiece str1 = STRING("bar"); + const GumboStringPiece str2 = STRING("foo"); + EXPECT_FALSE(gumbo_string_equals(&str1, &str2)); +} + +TEST_F(GumboStringPieceTest, CaseEqual) { + const GumboStringPiece str1 = STRING("foo"); + const GumboStringPiece str2 = STRING("fOO"); + EXPECT_TRUE(gumbo_string_equals_ignore_case(&str1, &str2)); +} + +TEST_F(GumboStringPieceTest, CaseNotEqual_Str2Shorter) { + const GumboStringPiece str1 = STRING("foobar"); + const GumboStringPiece str2 = STRING("foo"); + EXPECT_FALSE(gumbo_string_equals_ignore_case(&str1, &str2)); +} + +} // namespace diff --git a/gumbo-parser/test/test_utils.cc b/gumbo-parser/test/test_utils.cc new file mode 100644 index 0000000000..b39bac7cc1 --- /dev/null +++ b/gumbo-parser/test/test_utils.cc @@ -0,0 +1,166 @@ +// Copyright 2013 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: jdtang@google.com (Jonathan Tang) + +#include "test_utils.h" + +#include "error.h" +#include "util.h" + +int GetChildCount(GumboNode* node) { + if (node->type == GUMBO_NODE_DOCUMENT) { + return node->v.document.children.length; + } else { + return node->v.element.children.length; + } +} + +GumboTag GetTag(GumboNode* node) { return node->v.element.tag; } + +GumboNode* GetChild(GumboNode* parent, int index) { + if (parent->type == GUMBO_NODE_DOCUMENT) { + return static_cast(parent->v.document.children.data[index]); + } else { + return static_cast(parent->v.element.children.data[index]); + } +} + +int GetAttributeCount(GumboNode* node) { + return node->v.element.attributes.length; +} + +GumboAttribute* GetAttribute(GumboNode* node, int index) { + return static_cast(node->v.element.attributes.data[index]); +} + +// Convenience function to do some basic assertions on the structure of the +// document (nodes are elements, nodes have the right tags) and then return +// the body node. +void GetAndAssertBody(GumboNode* root, GumboNode** body) { + GumboNode* html = NULL; + for (int i = 0; i < GetChildCount(root); ++i) { + GumboNode* child = GetChild(root, i); + if (child->type != GUMBO_NODE_ELEMENT) { + ASSERT_EQ(GUMBO_NODE_COMMENT, child->type); + continue; + } + ASSERT_TRUE(html == NULL); + html = child; + } + ASSERT_TRUE(html != NULL); + ASSERT_EQ(GUMBO_NODE_ELEMENT, html->type); + EXPECT_EQ(GUMBO_TAG_HTML, GetTag(html)); + + // There may be comment/whitespace nodes; this walks through the children of + // and assigns head/body based on them, or assert-fails if there are + // fewer/more than 2 such nodes. + GumboNode* head = NULL; + *body = NULL; + for (int i = 0; i < GetChildCount(html); ++i) { + GumboNode* child = GetChild(html, i); + if (child->type != GUMBO_NODE_ELEMENT) { + continue; + } + + if (!head) { + head = child; + EXPECT_EQ(GUMBO_TAG_HEAD, GetTag(head)); + } else if (!(*body)) { + *body = child; + EXPECT_EQ(GUMBO_TAG_BODY, GetTag(*body)); + } else { + ASSERT_TRUE("More than two elements found inside " != NULL); + } + } + EXPECT_TRUE(head != NULL); + ASSERT_TRUE(*body != NULL); +} + +void SanityCheckPointers( + const char* input, size_t input_length, const GumboNode* node, int depth) { + ASSERT_GE(input_length, (size_t) 0); + ASSERT_TRUE(node != NULL); + // There are some truly pathological HTML documents out there - the + // integration tests for this include one where the DOM "tree" is actually a + // linked list 27,500 nodes deep - and so we need a limit on the recursion + // depth here to avoid blowing the stack. Alternatively, we could externalize + // the stack and use an iterative algorithm, but that gets us very little for + // the additional programming complexity. + if (node->type == GUMBO_NODE_DOCUMENT || depth > 400) { + // Don't sanity-check the document as well...we start with the root. + return; + } + if (node->type == GUMBO_NODE_ELEMENT) { + const GumboElement* element = &node->v.element; + // Sanity checks on original* pointers, making sure they fall within the + // original input. + if (element->original_tag.data && element->original_tag.length) { + EXPECT_GE(element->original_tag.data, input); + EXPECT_LT(element->original_tag.data, input + input_length); + EXPECT_LE(element->original_tag.length, input_length); + } + if (element->original_end_tag.data && element->original_tag.length) { + EXPECT_GE(element->original_end_tag.data, input); + EXPECT_LT(element->original_end_tag.data, input + input_length); + EXPECT_LE(element->original_end_tag.length, input_length); + } + EXPECT_GE(element->start_pos.offset, 0); + EXPECT_LE(element->start_pos.offset, input_length); + EXPECT_GE(element->end_pos.offset, 0); + EXPECT_LE(element->end_pos.offset, input_length); + + const GumboVector* children = &element->children; + for (unsigned int i = 0; i < children->length; ++i) { + const GumboNode* child = static_cast(children->data[i]); + // Checks on parent/child links. + ASSERT_TRUE(child != NULL); + EXPECT_EQ(node, child->parent); + EXPECT_EQ(i, child->index_within_parent); + SanityCheckPointers(input, input_length, child, depth + 1); + } + } else { + const GumboText* text = &node->v.text; + EXPECT_GE(text->original_text.data, input); + EXPECT_LT(text->original_text.data, input + input_length); + EXPECT_LE(text->original_text.length, input_length); + EXPECT_GE(text->start_pos.offset, 0); + EXPECT_LT(text->start_pos.offset, input_length); + } +} + +GumboTest::GumboTest() + : options_(kGumboDefaultOptions), errors_are_expected_(false), text_("") { + options_.max_errors = 100; + parser_._options = &options_; + parser_._output = static_cast(gumbo_alloc(sizeof(GumboOutput))); + gumbo_init_errors(&parser_); +} + +GumboTest::~GumboTest() { + if (!errors_are_expected_) { + // TODO(jdtang): A googlemock matcher may be a more appropriate solution for + // this; we only want to pretty-print errors that are not an expected + // output of the test. + for (unsigned int i = 0; i < parser_._output->errors.length && i < 1; ++i) { + gumbo_print_caret_diagnostic ( + static_cast(parser_._output->errors.data[i]), + text_, + strlen(text_) + ); + } + } + gumbo_destroy_errors(&parser_); + gumbo_free(parser_._output); +} diff --git a/gumbo-parser/test/test_utils.h b/gumbo-parser/test/test_utils.h new file mode 100644 index 0000000000..620ba99c6b --- /dev/null +++ b/gumbo-parser/test/test_utils.h @@ -0,0 +1,65 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: jdtang@google.com (Jonathan Tang) + +#ifndef GUMBO_TEST_UTILS_H_ +#define GUMBO_TEST_UTILS_H_ + +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "gumbo.h" +#include "parser.h" + +inline std::string ToString(const GumboStringPiece& str) { + return std::string(str.data, str.length); +} + +int GetChildCount(GumboNode* node); +GumboTag GetTag(GumboNode* node); +GumboNode* GetChild(GumboNode* parent, int index); +int GetAttributeCount(GumboNode* node); +GumboAttribute* GetAttribute(GumboNode* node, int index); + +// Convenience function to do some basic assertions on the structure of the +// document (nodes are elements, nodes have the right tags) and then return +// the body node. +void GetAndAssertBody(GumboNode* root, GumboNode** body); + +void SanityCheckPointers ( + const char* input, + size_t input_length, + const GumboNode* node, + int depth +); + +// Base class for Gumbo tests. This provides an GumboParser object that's +// been initialized to sane values, as normally happens in the beginning of +// gumbo_parse, and then a destructor that cleans up after it. +class GumboTest : public ::testing::Test { + protected: + GumboTest(); + virtual ~GumboTest(); + + GumboOptions options_; + GumboParser parser_; + bool errors_are_expected_; + const char* text_; +}; + +#endif // GUMBO_TEST_UTILS_H_ diff --git a/gumbo-parser/test/token_buffer.cc b/gumbo-parser/test/token_buffer.cc new file mode 100644 index 0000000000..35a9bb2c8f --- /dev/null +++ b/gumbo-parser/test/token_buffer.cc @@ -0,0 +1,200 @@ +// Copyright 2018 Stephen Checkoway +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "token_buffer.h" +#include "tokenizer.h" + +#include "gtest/gtest.h" +#include "test_utils.h" +#include "util.h" + +namespace { + +class GumboCharacterTokenBufferTest : public GumboTest { + protected: + GumboCharacterTokenBufferTest() { + gumbo_character_token_buffer_init(&buffer_); + } + virtual ~GumboCharacterTokenBufferTest() { + gumbo_character_token_buffer_destroy(&buffer_); + } + + void Fill(const char* input, size_t size = -1) { + gumbo_character_token_buffer_clear(&buffer_); + if (size == static_cast(-1)) + size = strlen(input); + text_ = input; + gumbo_tokenizer_state_init(&parser_, input, size); + + while (true) { + GumboToken token; + gumbo_lex(&parser_, &token); + if (token.type == GUMBO_TOKEN_EOF) { + gumbo_token_destroy(&token); + break; + } + ASSERT_TRUE(token.type == GUMBO_TOKEN_CHARACTER + || token.type == GUMBO_TOKEN_WHITESPACE); + gumbo_character_token_buffer_append(&token, &buffer_); + gumbo_token_destroy(&token); + } + } + + GumboToken Get(size_t index) { + GumboToken ret; + gumbo_character_token_buffer_get(&buffer_, index, &ret); + return ret; + } + + GumboCharacterTokenBuffer buffer_; +}; + +TEST_F(GumboCharacterTokenBufferTest, AsciiCharacters) { + Fill("abcXYZ!@#$%^&*()-=_+[]{}()"); + for (size_t i = 0; text_[i] != 0; ++i) { + GumboToken t = Get(i); + EXPECT_EQ(GUMBO_TOKEN_CHARACTER, t.type); + EXPECT_EQ(text_[i], t.v.character); + EXPECT_EQ(1, t.position.line); + EXPECT_EQ(i+1, t.position.column); + EXPECT_EQ(i, t.position.offset); + EXPECT_EQ(&text_[i], t.original_text.data); + EXPECT_EQ(1, t.original_text.length); + } +} + +TEST_F(GumboCharacterTokenBufferTest, Whitespace) { + Fill("\n\f\t \r\r\n"); + GumboToken t = Get(0); + EXPECT_EQ(GUMBO_TOKEN_WHITESPACE, t.type); + EXPECT_EQ('\n', t.v.character); + EXPECT_EQ(1, t.position.line); + EXPECT_EQ(1, t.position.column); + EXPECT_EQ(0, t.position.offset); + EXPECT_EQ(&text_[0], t.original_text.data); + EXPECT_EQ(1, t.original_text.length); + + t = Get(1); + EXPECT_EQ(GUMBO_TOKEN_WHITESPACE, t.type); + EXPECT_EQ('\f', t.v.character); + EXPECT_EQ(2, t.position.line); + EXPECT_EQ(1, t.position.column); + EXPECT_EQ(1, t.position.offset); + EXPECT_EQ(&text_[1], t.original_text.data); + EXPECT_EQ(1, t.original_text.length); + + t = Get(2); + EXPECT_EQ(GUMBO_TOKEN_WHITESPACE, t.type); + EXPECT_EQ('\t', t.v.character); + EXPECT_EQ(2, t.position.line); + EXPECT_EQ(2, t.position.column); + EXPECT_EQ(2, t.position.offset); + EXPECT_EQ(&text_[2], t.original_text.data); + EXPECT_EQ(1, t.original_text.length); + + t = Get(3); + EXPECT_EQ(GUMBO_TOKEN_WHITESPACE, t.type); + EXPECT_EQ(' ', t.v.character); + EXPECT_EQ(2, t.position.line); + EXPECT_EQ(8, t.position.column); + EXPECT_EQ(3, t.position.offset); + EXPECT_EQ(&text_[3], t.original_text.data); + EXPECT_EQ(1, t.original_text.length); + + t = Get(4); + EXPECT_EQ(GUMBO_TOKEN_WHITESPACE, t.type); + EXPECT_EQ('\n', t.v.character); + EXPECT_EQ(2, t.position.line); + EXPECT_EQ(9, t.position.column); + EXPECT_EQ(4, t.position.offset); + EXPECT_EQ(&text_[4], t.original_text.data); + EXPECT_EQ(1, t.original_text.length); + + t = Get(5); + EXPECT_EQ(GUMBO_TOKEN_WHITESPACE, t.type); + EXPECT_EQ('\n', t.v.character); + EXPECT_EQ(3, t.position.line); + EXPECT_EQ(1, t.position.column); + EXPECT_EQ(6, t.position.offset); + EXPECT_EQ(&text_[6], t.original_text.data); + EXPECT_EQ(1, t.original_text.length); +} + +TEST_F(GumboCharacterTokenBufferTest, Alternate) { + Fill("a b c"); + for (size_t i = 0; text_[i] != 0; ++i) { + GumboToken t = Get(i); + if (i & 1) + EXPECT_EQ(GUMBO_TOKEN_WHITESPACE, t.type); + else + EXPECT_EQ(GUMBO_TOKEN_CHARACTER, t.type); + EXPECT_EQ(text_[i], t.v.character); + EXPECT_EQ(1, t.position.line); + EXPECT_EQ(i+1, t.position.column); + EXPECT_EQ(i, t.position.offset); + EXPECT_EQ(&text_[i], t.original_text.data); + EXPECT_EQ(1, t.original_text.length); + } +} + +TEST_F(GumboCharacterTokenBufferTest, Entities) { + Fill("⊂⃒Ü"); + GumboToken t = Get(0); + EXPECT_EQ(GUMBO_TOKEN_CHARACTER, t.type); + EXPECT_EQ(0x2282, t.v.character); + EXPECT_EQ(1, t.position.line); + EXPECT_EQ(1, t.position.column); + EXPECT_EQ(0, t.position.offset); + EXPECT_EQ(&text_[0], t.original_text.data); + EXPECT_EQ(7, t.original_text.length); + + t = Get(1); + EXPECT_EQ(GUMBO_TOKEN_CHARACTER, t.type); + EXPECT_EQ(0x20D2, t.v.character); + // XXX: These should probably report the same location in the source as the + // first token from the entity. + // EXPECT_EQ(1, t.position.line); + // EXPECT_EQ(1, t.position.column); + // EXPECT_EQ(0, t.position.offset); + // EXPECT_EQ(&text_[0], t.original_text.data); + // EXPECT_EQ(7, t.original_text.length); + + t = Get(2); + EXPECT_EQ(GUMBO_TOKEN_CHARACTER, t.type); + EXPECT_EQ(0x00DC, t.v.character); + EXPECT_EQ(1, t.position.line); + EXPECT_EQ(8, t.position.column); + EXPECT_EQ(7, t.position.offset); + EXPECT_EQ(&text_[7], t.original_text.data); + EXPECT_EQ(6, t.original_text.length); +} + +TEST_F(GumboCharacterTokenBufferTest, CarriageReturn) { + Fill(" "); + GumboToken t = Get(0); + EXPECT_EQ(GUMBO_TOKEN_WHITESPACE, t.type); + EXPECT_EQ(0x000D, t.v.character); + EXPECT_EQ(1, t.position.line); + EXPECT_EQ(1, t.position.column); + EXPECT_EQ(0, t.position.offset); + EXPECT_EQ(&text_[0], t.original_text.data); + EXPECT_EQ(6, t.original_text.length); + errors_are_expected_ = true; +} + +} // namespace + +// vim: set sw=2 sts=2 ts=8 et: diff --git a/gumbo-parser/test/tokenizer.cc b/gumbo-parser/test/tokenizer.cc new file mode 100644 index 0000000000..ede4e9be47 --- /dev/null +++ b/gumbo-parser/test/tokenizer.cc @@ -0,0 +1,4790 @@ +// Copyright 2011 Google Inc. +// Copyright 2018 Stephen Checkoway +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: jdtang@google.com (Jonathan Tang) + +#include + +#include "gtest/gtest.h" +#include "test_utils.h" + +#include "tokenizer.h" +#include "error.h" + +extern const char* kGumboTagNames[]; + +namespace { + +// Tests for tokenizer.c +class GumboTokenizerTest : public GumboTest { + protected: + GumboTokenizerTest() : at_start_(true), error_index_(0) { + gumbo_tokenizer_state_init(&parser_, "", 0); + } + + virtual ~GumboTokenizerTest() { + EXPECT_EQ(error_index_, parser_._output->errors.length); + gumbo_tokenizer_state_destroy(&parser_); + gumbo_token_destroy(&token_); + } + + void SetInput(const char* input, size_t size = -1) { + if (!at_start_) + gumbo_token_destroy(&token_); + text_ = input; + gumbo_tokenizer_state_destroy(&parser_); + if (size == static_cast(-1)) + size = strlen(input); + gumbo_tokenizer_state_init(&parser_, input, size); + at_start_ = true; + error_index_ = 0; + } + + void SetState(GumboTokenizerEnum state) { + gumbo_tokenizer_set_state(&parser_, state); + } + + void Foreign() { + gumbo_tokenizer_set_is_adjusted_current_node_foreign(&parser_, true); + } + + void Advance(int num_tokens) { + for (int i = 0; i < num_tokens; ++i) { + Next(); + ASSERT_NE(GUMBO_TOKEN_EOF, token_.type); + } + } + + void Next(bool errors_are_expected = false) { + if (!at_start_) + gumbo_token_destroy(&token_); + at_start_ = false; + // Reset the document errors. + parser_._output->document_error = false; + gumbo_lex(&parser_, &token_); + EXPECT_EQ(errors_are_expected, parser_._output->document_error); + errors_are_expected_ = errors_are_expected; + } + + void NextChar(int c, bool errors_are_expected = false) { + Next(errors_are_expected); + ASSERT_EQ(GUMBO_TOKEN_CHARACTER, token_.type); + EXPECT_EQ(c, token_.v.character); + } + + void NextCdata(int c, bool errors_are_expected = false) { + Next(errors_are_expected); + ASSERT_EQ(GUMBO_TOKEN_CDATA, token_.type); + EXPECT_EQ(c, token_.v.character); + } + + void NextSpace(int c, bool errors_are_expected = false) { + Next(errors_are_expected); + ASSERT_EQ(GUMBO_TOKEN_WHITESPACE, token_.type); + EXPECT_EQ(c, token_.v.character); + } + + void NextStartTag(GumboTag tag, bool errors_are_expected = false) { + Next(errors_are_expected); + ASSERT_EQ(GUMBO_TOKEN_START_TAG, token_.type); + EXPECT_EQ(tag, token_.v.start_tag.tag); + } + + void NextEndTag(GumboTag tag, bool errors_are_expected = false) { + Next(errors_are_expected); + ASSERT_EQ(GUMBO_TOKEN_END_TAG, token_.type); + EXPECT_EQ(tag, token_.v.end_tag.tag); + } + + void NextComment(const char *text, bool errors_are_expected = false) { + Next(errors_are_expected); + ASSERT_EQ(GUMBO_TOKEN_COMMENT, token_.type); + EXPECT_STREQ(text, token_.v.text); + } + + void NextDoctype(const char *name, const char *pub, const char *sys, bool errors_are_expected = false) { + Next(errors_are_expected); + ASSERT_EQ(GUMBO_TOKEN_DOCTYPE, token_.type); + // XXX: There's a difference between missing and empty. + if (!name) + name = ""; + EXPECT_STREQ(name, token_.v.doc_type.name); + if (!pub) { + EXPECT_FALSE(token_.v.doc_type.has_public_identifier); + } else { + EXPECT_TRUE(token_.v.doc_type.has_public_identifier); + EXPECT_STREQ(pub, token_.v.doc_type.public_identifier); + } + if (!sys) { + EXPECT_FALSE(token_.v.doc_type.has_system_identifier); + } else { + EXPECT_TRUE(token_.v.doc_type.has_system_identifier); + EXPECT_STREQ(sys, token_.v.doc_type.system_identifier); + } + } + + void AtEnd(bool errors_are_expected = false) { + Next(errors_are_expected); + ASSERT_EQ(GUMBO_TOKEN_EOF, token_.type); + EXPECT_EQ(-1, token_.v.character); + } + + void Error(GumboErrorType err) { + ASSERT_LT(error_index_, parser_._output->errors.length); + GumboError* error = + static_cast(parser_._output->errors.data[error_index_++]); + EXPECT_EQ(err, error->type); + } + + bool at_start_; + size_t error_index_; + GumboToken token_; +}; + +TEST(GumboTagEnumTest, TagEnumIncludesAllTags) { + EXPECT_EQ(0, GUMBO_TAG_HTML); + for (unsigned int i = 0; i < (unsigned int) GUMBO_TAG_UNKNOWN; i++) { + const char* tagname = gumbo_normalized_tagname((GumboTag)i); + EXPECT_FALSE(tagname == NULL); + EXPECT_FALSE(tagname[0] == '\0'); + EXPECT_TRUE(strlen(tagname) < 15); + } + EXPECT_STREQ("", gumbo_normalized_tagname(GUMBO_TAG_UNKNOWN)); + EXPECT_STREQ("html", gumbo_normalized_tagname(GUMBO_TAG_HTML)); + EXPECT_STREQ("a", gumbo_normalized_tagname(GUMBO_TAG_A)); + EXPECT_STREQ("dialog", gumbo_normalized_tagname(GUMBO_TAG_DIALOG)); + EXPECT_STREQ("template", gumbo_normalized_tagname(GUMBO_TAG_TEMPLATE)); +} + +TEST(GumboTagEnumTest, TagLookupCaseSensitivity) { + EXPECT_EQ(GUMBO_TAG_HTML, gumbo_tagn_enum("HTML", 4)); + EXPECT_EQ(GUMBO_TAG_BODY, gumbo_tagn_enum("boDy", 4)); + EXPECT_EQ(GUMBO_TAG_A, gumbo_tagn_enum("A", 1)); + EXPECT_EQ(GUMBO_TAG_A, gumbo_tagn_enum("a", 1)); + EXPECT_EQ(GUMBO_TAG_TEMPLATE, gumbo_tagn_enum("Template", 8)); + EXPECT_EQ(GUMBO_TAG_DIALOG, gumbo_tagn_enum("diAloG", 6)); + EXPECT_EQ(GUMBO_TAG_ANNOTATION_XML, gumbo_tagn_enum("annotation-xml", 14)); + EXPECT_EQ(GUMBO_TAG_ANNOTATION_XML, gumbo_tagn_enum("ANNOTATION-XML", 14)); + EXPECT_EQ(GUMBO_TAG_UNKNOWN, gumbo_tagn_enum("ANNOTATION-XML-", 15)); + EXPECT_EQ(GUMBO_TAG_UNKNOWN, gumbo_tagn_enum("ANNOTATION-XM", 13)); + EXPECT_EQ(GUMBO_TAG_UNKNOWN, gumbo_tagn_enum("", 0)); + EXPECT_EQ(GUMBO_TAG_B, gumbo_tagn_enum("b", 1)); + EXPECT_EQ(GUMBO_TAG_I, gumbo_tagn_enum("i", 1)); + EXPECT_EQ(GUMBO_TAG_U, gumbo_tagn_enum("u", 1)); + EXPECT_EQ(GUMBO_TAG_UNKNOWN, gumbo_tagn_enum("x", 1)); + EXPECT_EQ(GUMBO_TAG_UNKNOWN, gumbo_tagn_enum("c", 1)); +} + +TEST_F(GumboTokenizerTest, LexCharToken) { + SetInput("a"); + NextChar('a'); + EXPECT_EQ(1, token_.position.column); + EXPECT_EQ(1, token_.position.line); + EXPECT_EQ(0, token_.position.offset); + EXPECT_EQ('a', *token_.original_text.data); + EXPECT_EQ(1, token_.original_text.length); + EXPECT_EQ('a', token_.v.character); + + AtEnd(); + EXPECT_EQ(1, token_.position.offset); +} + +TEST_F(GumboTokenizerTest, AttrWithCRNL) { + SetInput(""); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast(token_.v.start_tag.attributes.data[0]); + EXPECT_EQ(3, attr->original_name.length); + EXPECT_EQ('O', attr->original_name.data[attr->original_name.length-1]); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, LexCharRef) { + SetInput(" Text"); + NextChar(0xa0); + EXPECT_EQ(1, token_.position.column); + EXPECT_EQ(1, token_.position.line); + EXPECT_EQ(0, token_.position.offset); + EXPECT_EQ('&', *token_.original_text.data); + EXPECT_EQ(6, token_.original_text.length); + + NextSpace(' '); + EXPECT_EQ(' ', *token_.original_text.data); + + NextChar('T'); + NextChar('e'); + NextChar('x'); + NextChar('t'); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, LexCharRef_NotCharRef) { + SetInput("&xyz"); + NextChar('&'); + EXPECT_EQ(0, token_.position.offset); + + NextChar('x'); + EXPECT_EQ(1, token_.position.offset); + + NextChar('y'); + EXPECT_EQ(2, token_.position.offset); + + NextChar('z'); + EXPECT_EQ(3, token_.position.offset); + + AtEnd(); +} + +TEST_F(GumboTokenizerTest, LeadingWhitespace) { + SetInput( + "\n" + " "); + Advance(4); + + NextStartTag(GUMBO_TAG_SPAN); + EXPECT_EQ(2, token_.position.line); + EXPECT_EQ(3, token_.position.column); + GumboTokenStartTag* start_tag = &token_.v.start_tag; + ASSERT_EQ(1, start_tag->attributes.length); + + GumboAttribute* clas = + static_cast(start_tag->attributes.data[0]); + EXPECT_STREQ("class", clas->name); + EXPECT_EQ("class", ToString(clas->original_name)); + EXPECT_EQ(2, clas->name_start.line); + EXPECT_EQ(9, clas->name_start.column); + EXPECT_EQ(14, clas->name_end.column); + EXPECT_STREQ("foo", clas->value); + EXPECT_EQ("foo", ToString(clas->original_value)); + EXPECT_EQ(15, clas->value_start.column); + EXPECT_EQ(18, clas->value_end.column); +} + +TEST_F(GumboTokenizerTest, Doctype) { + SetInput(""); + Next(); + ASSERT_EQ(GUMBO_TOKEN_DOCTYPE, token_.type); + EXPECT_EQ(0, token_.position.offset); + + GumboTokenDocType* doc_type = &token_.v.doc_type; + EXPECT_FALSE(doc_type->force_quirks); + EXPECT_FALSE(doc_type->has_public_identifier); + EXPECT_FALSE(doc_type->has_system_identifier); + EXPECT_STREQ("html", doc_type->name); + EXPECT_STREQ("", doc_type->public_identifier); + EXPECT_STREQ("", doc_type->system_identifier); +} + +TEST_F(GumboTokenizerTest, DoctypePublic) { + SetInput( + ""); + Next(); + ASSERT_EQ(GUMBO_TOKEN_DOCTYPE, token_.type); + EXPECT_EQ(0, token_.position.offset); + + GumboTokenDocType* doc_type = &token_.v.doc_type; + EXPECT_FALSE(doc_type->force_quirks); + EXPECT_TRUE(doc_type->has_public_identifier); + EXPECT_TRUE(doc_type->has_system_identifier); + EXPECT_STREQ("html", doc_type->name); + EXPECT_STREQ( + "-//W3C//DTD XHTML 1.0 Transitional//EN", doc_type->public_identifier); + EXPECT_STREQ("http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd", + doc_type->system_identifier); +} + +TEST_F(GumboTokenizerTest, DoctypeSystem) { + SetInput(""); + Next(); + ASSERT_EQ(GUMBO_TOKEN_DOCTYPE, token_.type); + EXPECT_EQ(0, token_.position.offset); + + GumboTokenDocType* doc_type = &token_.v.doc_type; + EXPECT_FALSE(doc_type->force_quirks); + EXPECT_FALSE(doc_type->has_public_identifier); + EXPECT_TRUE(doc_type->has_system_identifier); + EXPECT_STREQ("root_element", doc_type->name); + EXPECT_STREQ("DTD_location", doc_type->system_identifier); +} + +TEST_F(GumboTokenizerTest, RawtextEnd) { + SetInput("x ignores <tag>"); + NextStartTag(GUMBO_TAG_TITLE); + gumbo_tokenizer_set_state(&parser_, GUMBO_LEX_RAWTEXT); + NextChar('x'); + Advance(9); + NextChar('<'); + NextChar('t'); + Advance(3); + NextEndTag(GUMBO_TAG_TITLE); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, RCDataEnd) { + SetInput("x"); + NextStartTag(GUMBO_TAG_TITLE); + gumbo_tokenizer_set_state(&parser_, GUMBO_LEX_RCDATA); + NextChar('x'); + NextEndTag(GUMBO_TAG_TITLE); +} + +TEST_F(GumboTokenizerTest, ScriptEnd) { + SetInput(""); + NextStartTag(GUMBO_TAG_SCRIPT); + + gumbo_tokenizer_set_state(&parser_, GUMBO_LEX_SCRIPT_DATA); + NextChar('x'); + + Advance(6); + NextChar('<'); + NextChar('/'); + NextChar('\''); + NextChar(';'); + NextEndTag(GUMBO_TAG_SCRIPT); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, ScriptEscapedEnd) { + SetInput("x"); + NextStartTag(GUMBO_TAG_TITLE); + gumbo_tokenizer_set_state(&parser_, GUMBO_LEX_SCRIPT_DATA_ESCAPED); + NextChar('x'); + NextEndTag(GUMBO_TAG_TITLE); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, ScriptCommentEscaped) { + SetInput( + ""); + NextStartTag(GUMBO_TAG_SCRIPT); + gumbo_tokenizer_set_state(&parser_, GUMBO_LEX_SCRIPT_DATA); + Advance(15); + NextChar('x'); + NextSpace(' '); + NextChar('<'); + NextSpace(' '); + NextChar('7'); + Advance(4); + NextChar('<'); + NextChar('/'); + NextChar('d'); + Advance(27); + NextChar('-'); + NextChar('-'); + NextChar('>'); + NextSpace('\n'); + NextEndTag(GUMBO_TAG_SCRIPT); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, ScriptEscapedEmbeddedLessThan) { + SetInput(""); + NextStartTag(GUMBO_TAG_SCRIPT); + gumbo_tokenizer_set_state(&parser_, GUMBO_LEX_SCRIPT_DATA); + Advance(14); + NextChar('x'); + NextChar('<'); + NextChar('7'); + Advance(8); + NextEndTag(GUMBO_TAG_SCRIPT); +} + +TEST_F(GumboTokenizerTest, ScriptHasTagEmbedded) { + SetInput(""); + NextStartTag(GUMBO_TAG_SCRIPT); + gumbo_tokenizer_set_state(&parser_, GUMBO_LEX_SCRIPT_DATA); + Advance(11); + NextChar('<'); + NextChar('/'); + NextChar('d'); + NextChar('i'); + Advance(4); + NextEndTag(GUMBO_TAG_SCRIPT); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, ScriptDoubleEscaped) { + SetInput( + ""); + NextStartTag(GUMBO_TAG_SCRIPT); + gumbo_tokenizer_set_state(&parser_, GUMBO_LEX_SCRIPT_DATA); + Advance(34); + + NextChar('<'); + NextChar('s'); + NextChar('C'); + Advance(20); + NextChar('-'); + NextChar('-'); + NextChar('>'); + NextChar(';'); + NextEndTag(GUMBO_TAG_SCRIPT); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, CDataNulls) { + SetInput("", sizeof("")-1); + Foreign(); + + Next(); + EXPECT_EQ(GUMBO_TOKEN_NULL, token_.type); + EXPECT_EQ(0, token_.v.character); + Next(); + EXPECT_EQ(GUMBO_TOKEN_CDATA, token_.type); + EXPECT_EQ('f', token_.v.character); +} + +TEST_F(GumboTokenizerTest, StyleHasTagEmbedded) { + SetInput(""); + NextStartTag(GUMBO_TAG_STYLE); + gumbo_tokenizer_set_state(&parser_, GUMBO_LEX_RCDATA); + Advance(7); + NextChar('<'); + NextChar('h'); + NextChar('e'); +} + +TEST_F(GumboTokenizerTest, PreWithNewlines) { + SetInput("\r\na"); + Next(); + ASSERT_EQ(GUMBO_TOKEN_DOCTYPE, token_.type); + EXPECT_EQ(0, token_.position.offset); + + NextStartTag(GUMBO_TAG_PRE); + EXPECT_EQ("", ToString(token_.original_text)); + EXPECT_EQ(15, token_.position.offset); +} + +TEST_F(GumboTokenizerTest, SelfClosingStartTag) { + SetInput(""); + NextStartTag(GUMBO_TAG_BR); + EXPECT_EQ(0, token_.position.offset); + EXPECT_EQ("", ToString(token_.original_text)); + + GumboTokenStartTag* start_tag = &token_.v.start_tag; + EXPECT_EQ(0, start_tag->attributes.length); + EXPECT_TRUE(start_tag->is_self_closing); +} + +TEST_F(GumboTokenizerTest, SelfClosingEndTag) { + SetInput(""); + NextEndTag(GUMBO_TAG_P, true); + Error(GUMBO_ERR_END_TAG_WITH_TRAILING_SOLIDUS); + EXPECT_EQ(0, token_.position.offset); + EXPECT_EQ("", ToString(token_.original_text)); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, OpenTagWithAttributes) { + SetInput(""); + NextStartTag(GUMBO_TAG_A); + + GumboTokenStartTag* start_tag = &token_.v.start_tag; + EXPECT_FALSE(start_tag->is_self_closing); + ASSERT_EQ(2, start_tag->attributes.length); + + GumboAttribute* href = + static_cast(start_tag->attributes.data[0]); + EXPECT_STREQ("href", href->name); + EXPECT_EQ("href", ToString(href->original_name)); + EXPECT_STREQ("/search?q=foo&hl=en", href->value); + EXPECT_EQ("'/search?q=foo&hl=en'", ToString(href->original_value)); + + GumboAttribute* id = + static_cast(start_tag->attributes.data[1]); + EXPECT_STREQ("id", id->name); + EXPECT_EQ("id", ToString(id->original_name)); + EXPECT_STREQ("link", id->value); + EXPECT_EQ("link", ToString(id->original_value)); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, BogusComment1) { + SetInput("Text"); + NextComment("?xml is bogus-comment", true); + Error(GUMBO_ERR_UNEXPECTED_QUESTION_MARK_INSTEAD_OF_TAG_NAME); + NextChar('T'); + NextChar('e'); + NextChar('x'); + NextChar('t'); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, BogusComment2) { + SetInput(""); + NextStartTag(GUMBO_TAG_UNKNOWN); + + GumboTokenStartTag* start_tag = &token_.v.start_tag; + EXPECT_TRUE(start_tag->is_self_closing); + ASSERT_EQ(1, start_tag->attributes.length); + + GumboAttribute* long_attr = + static_cast(start_tag->attributes.data[0]); + EXPECT_STREQ("long_attr", long_attr->name); + EXPECT_EQ("long_attr", ToString(long_attr->original_name)); + EXPECT_STREQ( + "SomeCode;\n" + " calls_a_big_long_function();\n" + " return true;", + long_attr->value); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, DoubleAmpersand) { + SetInput(""); + NextStartTag(GUMBO_TAG_SPAN); + GumboTokenStartTag* start_tag = &token_.v.start_tag; + EXPECT_FALSE(start_tag->is_self_closing); + ASSERT_EQ(1, start_tag->attributes.length); + + GumboAttribute* jsif = + static_cast(start_tag->attributes.data[0]); + EXPECT_STREQ("jsif", jsif->name); + EXPECT_EQ("jsif", ToString(jsif->original_name)); + EXPECT_STREQ("foo && bar", jsif->value); + EXPECT_EQ("\"foo && bar\"", ToString(jsif->original_value)); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, MatchedTagPair) { + SetInput("a"); + NextStartTag(GUMBO_TAG_DIV, true); + Error(GUMBO_ERR_UNEXPECTED_CHARACTER_IN_UNQUOTED_ATTRIBUTE_VALUE); + EXPECT_EQ(0, token_.position.offset); + + GumboTokenStartTag* start_tag = &token_.v.start_tag; + EXPECT_FALSE(start_tag->is_self_closing); + ASSERT_EQ(2, start_tag->attributes.length); + + GumboAttribute* id = + static_cast(start_tag->attributes.data[0]); + EXPECT_STREQ("id", id->name); + EXPECT_EQ("id", ToString(id->original_name)); + EXPECT_EQ(1, id->name_start.line); + EXPECT_EQ(5, id->name_start.offset); + EXPECT_EQ(6, id->name_start.column); + EXPECT_EQ(8, id->name_end.column); + EXPECT_STREQ("dash<-Dash", id->value); + EXPECT_EQ("dash<-Dash", ToString(id->original_value)); + EXPECT_EQ(9, id->value_start.column); + EXPECT_EQ(19, id->value_end.column); + + GumboAttribute* data_attr = + static_cast(start_tag->attributes.data[1]); + EXPECT_STREQ("data-test", data_attr->name); + EXPECT_EQ("data-test", ToString(data_attr->original_name)); + EXPECT_EQ(20, data_attr->name_start.column); + EXPECT_EQ(29, data_attr->name_end.column); + EXPECT_STREQ("bar", data_attr->value); + EXPECT_EQ("\"bar\"", ToString(data_attr->original_value)); + EXPECT_EQ(30, data_attr->value_start.column); + EXPECT_EQ(35, data_attr->value_end.column); + + NextChar('a'); + EXPECT_EQ(35, token_.position.offset); + NextEndTag(GUMBO_TAG_DIV); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, BogusEndTag) { + // According to the spec, the correct parse of this is an end tag token for + // "" (notice the ending bracket) with the attribute "th=th" (ignored + // because end tags don't take attributes). + SetInput(""); + NextEndTag(GUMBO_TAG_UNKNOWN, true); + Error(GUMBO_ERR_UNEXPECTED_SOLIDUS_IN_TAG); + Error(GUMBO_ERR_END_TAG_WITH_ATTRIBUTES); + EXPECT_STREQ("div<", token_.v.end_tag.name); + EXPECT_EQ(0, token_.position.offset); + EXPECT_EQ("", ToString(token_.original_text)); +} + +TEST_F(GumboTokenizerTest, Whitespace) { + SetInput(" "); + NextChar(0xa0); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, NumericHex) { + SetInput("ካ"); + NextChar(0x12ab); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, NumericDecimal) { + SetInput("Ӓ"); + NextChar(1234); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, NumericInvalidDigit) { + SetInput("&#google"); + NextChar('&', true); + Error(GUMBO_ERR_ABSENCE_OF_DIGITS_IN_NUMERIC_CHARACTER_REFERENCE); + NextChar('#'); + NextChar('g'); + NextChar('o'); + NextChar('o'); + NextChar('g'); + NextChar('l'); + NextChar('e'); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, NumericNoSemicolon) { + SetInput("Ӓgoogle"); + NextChar(1234, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + NextChar('g'); + NextChar('o'); + NextChar('o'); + NextChar('g'); + NextChar('l'); + NextChar('e'); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, NamedReplacement) { + SetInput("<"); + NextChar('<'); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, NamedReplacementNoSemicolon) { + SetInput(">"); + NextChar('>', true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, NamedReplacementWithInvalidUtf8) { + SetInput("&\xc3\xa5"); + NextChar('&'); + NextChar(229); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, NamedReplacementInvalid) { + SetInput("&google;"); + // This is an error on the semicolon since that's where the error occurs in + // the spec. Anything other than a semicolon would cause no error. + NextChar('&'); + NextChar('g'); + NextChar('o'); + NextChar('o'); + NextChar('g'); + NextChar('l'); + NextChar('e'); + NextChar(';', true); + Error(GUMBO_ERR_UNKNOWN_NAMED_CHARACTER_REFERENCE); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, NamedReplacementInvalidNoSemicolon) { + SetInput("&google"); + NextChar('&'); + NextChar('g'); + NextChar('o'); + NextChar('o'); + NextChar('g'); + NextChar('l'); + NextChar('e'); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, InAttribute) { + SetInput(""); + NextStartTag(GUMBO_TAG_SPAN); + GumboTokenStartTag* start_tag = &token_.v.start_tag; + ASSERT_EQ(1, start_tag->attributes.length); + GumboAttribute* attr = static_cast(start_tag->attributes.data[0]); + EXPECT_STREQ("¬ed", attr->value); + NextEndTag(GUMBO_TAG_SPAN); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, MultiChars) { + SetInput("⋵̸"); + NextChar(0x22F5); + NextChar(0x0338); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, CharAfter) { + SetInput("<x"); + NextChar('<'); + NextChar('x'); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, MaxNumericCharRef) { + SetInput("FOOZOO"); + NextChar('F'); + NextChar('O'); + NextChar('O'); + NextChar(0x10FFFF, true); + Error(GUMBO_ERR_NONCHARACTER_CHARACTER_REFERENCE); + NextChar('Z'); + NextChar('O'); + NextChar('O'); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, InvalidRef) { + SetInput("ZZ&prod_id=23"); + NextChar('Z'); + NextChar('Z'); + NextChar('&'); + NextChar('p'); + NextChar('r'); + NextChar('o'); + NextChar('d'); + NextChar('_'); + NextChar('i'); + NextChar('d'); + NextChar('='); + NextChar('2'); + NextChar('3'); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, InvalidRefInAttr) { + SetInput(""); + NextStartTag(GUMBO_TAG_SPAN); + GumboTokenStartTag* start_tag = &token_.v.start_tag; + ASSERT_EQ(1, start_tag->attributes.length); + GumboAttribute* attr = static_cast(start_tag->attributes.data[0]); + EXPECT_STREQ("ZZ&prod_id=23", attr->value); + NextEndTag(GUMBO_TAG_SPAN); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, NumericTooLarge) { + SetInput("�"); + NextChar(0xFFFD, true); + Error(GUMBO_ERR_CHARACTER_REFERENCE_OUTSIDE_UNICODE_RANGE); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, CdataWithBrackets) { + SetInput("y"); + Foreign(); + Next(); + EXPECT_EQ(GUMBO_TOKEN_CDATA, token_.type); + EXPECT_EQ('x', token_.v.character); + EXPECT_EQ(10, token_.position.column); + EXPECT_EQ(1, token_.position.line); + EXPECT_EQ(9, token_.position.offset); + + Next(); + EXPECT_EQ(GUMBO_TOKEN_CDATA, token_.type); + EXPECT_EQ(']', token_.v.character); + EXPECT_EQ(11, token_.position.column); + EXPECT_EQ(1, token_.position.line); + EXPECT_EQ(10, token_.position.offset); + + Next(); + EXPECT_EQ(GUMBO_TOKEN_CDATA, token_.type); + EXPECT_EQ(']', token_.v.character); + EXPECT_EQ(12, token_.position.column); + EXPECT_EQ(1, token_.position.line); + EXPECT_EQ(11, token_.position.offset); + + Next(); + EXPECT_EQ(GUMBO_TOKEN_CDATA, token_.type); + EXPECT_EQ(']', token_.v.character); + EXPECT_EQ(13, token_.position.column); + EXPECT_EQ(1, token_.position.line); + EXPECT_EQ(12, token_.position.offset); + + NextChar('y'); + EXPECT_EQ(17, token_.position.column); + EXPECT_EQ(1, token_.position.line); + EXPECT_EQ(16, token_.position.offset); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, EscapedScriptStates) { + SetInput(""); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('/'); + NextChar('>'); + NextChar('<'); + NextChar('/'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextEndTag(GUMBO_TAG_SCRIPT); +} + +TEST_F(GumboTokenizerTest, EscapedScriptStates2) { + SetInput(""); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextChar('-'); + NextChar('-'); + NextChar('>'); + NextEndTag(GUMBO_TAG_SCRIPT); +} + +TEST_F(GumboTokenizerTest, ReadOutOfBounds) { + SetInput("⋵̸", 6); + NextChar(0x00AC, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + NextChar('i'); + NextChar('n'); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, ControlCharRefs) { + SetInput("€‚ƒ„…†‡ˆ‰" + "Š‹ŒŽ‘’“”•" + "–—˜™š›œžŸ"); + NextChar(0x20AC, true); + NextChar(0x201A, true); + NextChar(0x0192, true); + NextChar(0x201E, true); + NextChar(0x2026, true); + NextChar(0x2020, true); + NextChar(0x2021, true); + NextChar(0x02C6, true); + NextChar(0x2030, true); + NextChar(0x0160, true); + NextChar(0x2039, true); + NextChar(0x0152, true); + NextChar(0x017D, true); + NextChar(0x2018, true); + NextChar(0x2019, true); + NextChar(0x201C, true); + NextChar(0x201D, true); + NextChar(0x2022, true); + NextChar(0x2013, true); + NextChar(0x2014, true); + NextChar(0x02DC, true); + NextChar(0x2122, true); + NextChar(0x0161, true); + NextChar(0x203A, true); + NextChar(0x0153, true); + NextChar(0x017E, true); + NextChar(0x0178, true); + for (int i = 0; i < 27; ++i) + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + AtEnd(); +} + +// The tests below here are designed to test all of the tokenizers +// transitions. The name of each test has a specific form (for the most part): +// StartState_Rule1_Rule2_..._Rulen +// The parser sets the tokenizer in just a handful of states: plaintext, +// rawtext, rcdata, script data, and data. The tests are divided into those 5 +// categories and should cover every possible transition from each state. +// (Some transitions are covered more than once, and I'm sure some transitions +// are missing and should be added when found.) +// +// When a character is reconsumed in a new state, the test name _should_ +// include which rule gets matched for the new state, but not all of them do. +// That's just an oversight and should be fixed for consistency. +// +// Many states require additional characters in order to emit a token (or do +// so without an error). The rules used to match those tokens are omitted from +// the state name. + +// Starting in the PLAINTEXT state +TEST_F(GumboTokenizerTest, Plaintext_NULL) { + SetInput("", 1); + SetState(GUMBO_LEX_PLAINTEXT); + NextChar(0xFFFD, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); +} + +TEST_F(GumboTokenizerTest, Plaintext_EOF) { + SetInput(""); + SetState(GUMBO_LEX_PLAINTEXT); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Plaintext_AnythingElse) { + SetInput("a\xce\xb2\xd7\x92</plaintext>¬"); + NextStartTag(GUMBO_TAG_PLAINTEXT); + SetState(GUMBO_LEX_PLAINTEXT); + NextChar(0x61); + NextChar(0x03B2); + NextChar(0x05D2); + NextChar('<'); + NextChar('/'); + NextChar('p'); + NextChar('l'); + NextChar('a'); + NextChar('i'); + NextChar('n'); + NextChar('t'); + NextChar('e'); + NextChar('x'); + NextChar('t'); + NextChar('>'); + NextChar('&'); + NextChar('n'); + NextChar('o'); + NextChar('t'); + NextChar(';'); +} + +// Starting in the RAWTEXT state +TEST_F(GumboTokenizerTest, Rawtext_LT_Slash_Alpha_InappropriateTab) { + SetInput("<iframe></i\tx"); + NextStartTag(GUMBO_TAG_IFRAME); + SetState(GUMBO_LEX_RAWTEXT); + NextChar('<'); + NextChar('/'); + NextChar('i'); + NextSpace('\t'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rawtext_LT_Slash_Alpha_InappropriateLF) { + SetInput("<noframes></n\nx"); + NextStartTag(GUMBO_TAG_NOFRAMES); + SetState(GUMBO_LEX_RAWTEXT); + NextChar('<'); + NextChar('/'); + NextChar('n'); + NextSpace('\n'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rawtext_LT_Slash_Alpha_InappropriateFF) { + SetInput("<iframe></i\fx"); + NextStartTag(GUMBO_TAG_IFRAME); + SetState(GUMBO_LEX_RAWTEXT); + NextChar('<'); + NextChar('/'); + NextChar('i'); + NextSpace('\f'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rawtext_LT_Slash_Alpha_InappropriateSpace) { + SetInput("<noframes></n x"); + NextStartTag(GUMBO_TAG_NOFRAMES); + SetState(GUMBO_LEX_RAWTEXT); + NextChar('<'); + NextChar('/'); + NextChar('n'); + NextSpace(' '); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rawtext_LT_Slash_Alpha_InappropriateSlash) { + SetInput("<iframe></i/x"); + NextStartTag(GUMBO_TAG_IFRAME); + SetState(GUMBO_LEX_RAWTEXT); + NextChar('<'); + NextChar('/'); + NextChar('i'); + NextChar('/'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rawtext_LT_Slash_Alpha_InappropriateGT) { + SetInput("<noframes></n>x"); + NextStartTag(GUMBO_TAG_NOFRAMES); + SetState(GUMBO_LEX_RAWTEXT); + NextChar('<'); + NextChar('/'); + NextChar('n'); + NextChar('>'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rawtext_LT_Slash_Alpha_AppropriateLF) { + SetInput("<noframes></nofRAmes\n>x"); + NextStartTag(GUMBO_TAG_NOFRAMES); + SetState(GUMBO_LEX_RAWTEXT); + NextEndTag(GUMBO_TAG_NOFRAMES); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rawtext_LT_Slash_Alpha_AppropriateFF) { + SetInput("<iframe></ifrAMe\f foo=bar>x"); + NextStartTag(GUMBO_TAG_IFRAME); + SetState(GUMBO_LEX_RAWTEXT); + NextEndTag(GUMBO_TAG_IFRAME, true); + Error(GUMBO_ERR_END_TAG_WITH_ATTRIBUTES); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rawtext_LT_Slash_Alpha_AppropriateSpace) { + SetInput("<noframes></Noframes >x"); + NextStartTag(GUMBO_TAG_NOFRAMES); + SetState(GUMBO_LEX_RAWTEXT); + NextEndTag(GUMBO_TAG_NOFRAMES); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rawtext_LT_Slash_Alpha_AppropriateSlash_GT) { + SetInput("<iframe></IFRAME/>x"); + NextStartTag(GUMBO_TAG_IFRAME); + SetState(GUMBO_LEX_RAWTEXT); + NextEndTag(GUMBO_TAG_IFRAME, true); + Error(GUMBO_ERR_END_TAG_WITH_TRAILING_SOLIDUS); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rawtext_LT_Slash_Alpha_AppropriateSlash_EOF) { + SetInput("<iframe></iframe/"); + NextStartTag(GUMBO_TAG_IFRAME); + SetState(GUMBO_LEX_RAWTEXT); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_TAG); +} + +TEST_F(GumboTokenizerTest, Rawtext_LT_Slash_Alpha_AppropriateSlash_AnythingElse) { + SetInput("<iframe></iframe/ >x"); + NextStartTag(GUMBO_TAG_IFRAME); + SetState(GUMBO_LEX_RAWTEXT); + NextEndTag(GUMBO_TAG_IFRAME, true); + Error(GUMBO_ERR_UNEXPECTED_SOLIDUS_IN_TAG); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rawtext_LT_Slash_Alpha_AppropriateGT) { + SetInput("<noframes></noframes>x"); + NextStartTag(GUMBO_TAG_NOFRAMES); + SetState(GUMBO_LEX_RAWTEXT); + NextEndTag(GUMBO_TAG_NOFRAMES); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rawtext_LT_Slash_Alpha_AnythingElse) { + SetInput("<iframe></if\xce\xb2"); + NextStartTag(GUMBO_TAG_IFRAME); + SetState(GUMBO_LEX_RAWTEXT); + NextChar('<'); + NextChar('/'); + NextChar('i'); + NextChar('f'); + NextChar(0x03B2); +} + +TEST_F(GumboTokenizerTest, Rawtext_LT_Slash_AnythingElse) { + SetInput("</0"); + SetState(GUMBO_LEX_RAWTEXT); + NextChar('<'); + NextChar('/'); + NextChar('0'); +} + +TEST_F(GumboTokenizerTest, Rawtext_LT_AnythingElse) { + SetInput("<p>"); + SetState(GUMBO_LEX_RAWTEXT); + NextChar('<'); + NextChar('p'); + NextChar('>'); +} + +TEST_F(GumboTokenizerTest, Rawtext_NULL) { + SetInput("", 1); + SetState(GUMBO_LEX_RAWTEXT); + NextChar(0xFFFD, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); +} + +TEST_F(GumboTokenizerTest, Rawtext_EOF) { + SetInput(""); + SetState(GUMBO_LEX_RAWTEXT); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Rawtext_AnythingElse) { + SetInput("a\xce\xb2\xd7\x92>¬"); + SetState(GUMBO_LEX_RAWTEXT); + NextChar(0x61); + NextChar(0x03B2); + NextChar(0x05D2); + NextChar('>'); + NextChar('&'); + NextChar('n'); + NextChar('o'); + NextChar('t'); + NextChar(';'); +} + +// Starting in the RCDATA state +TEST_F(GumboTokenizerTest, Rcdata_Amp_Alnum_Match_Semicolon) { + SetInput("fjx"); + SetState(GUMBO_LEX_RCDATA); + NextChar(0x0066); + NextChar(0x006A); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_Alnum_Match) { + SetInput("áx"); + SetState(GUMBO_LEX_RCDATA); + NextChar(0x00E1, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_Alnum_Otherwise_Alnum_Semicolon) { + SetInput("&1f;x"); + SetState(GUMBO_LEX_RCDATA); + NextChar('&'); + NextChar('1'); + NextChar('f'); + NextChar(';', true); + Error(GUMBO_ERR_UNKNOWN_NAMED_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_Alnum_Otherwise_Alnum_AnythingElse) { + SetInput("&1f.x"); + SetState(GUMBO_LEX_RCDATA); + NextChar('&'); + NextChar('1'); + NextChar('f'); + NextChar('.'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_NumberSign_X_Hex_Semicolon_Zero) { + SetInput("�x"); + SetState(GUMBO_LEX_RCDATA); + NextChar(0xFFFD, true); + Error(GUMBO_ERR_NULL_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_NumberSign_X_Hex_Semicolon_TooLarge) { + SetInput("�x"); + SetState(GUMBO_LEX_RCDATA); + NextChar(0xFFFD, true); + Error(GUMBO_ERR_CHARACTER_REFERENCE_OUTSIDE_UNICODE_RANGE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_NumberSign_X_Hex_Semicolon_Surrogate) { + SetInput("�x"); + SetState(GUMBO_LEX_RCDATA); + NextChar(0xFFFD, true); + Error(GUMBO_ERR_SURROGATE_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_NumberSign_x_Hex_Semicolon_Noncharacter) { + SetInput("x"); + SetState(GUMBO_LEX_RCDATA); + NextChar(0xFDD8, true); + Error(GUMBO_ERR_NONCHARACTER_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_NumberSign_x_Hex_Semicolon_Noncharacter2) { + SetInput("x"); + SetState(GUMBO_LEX_RCDATA); + NextChar(0x7FFFE, true); + Error(GUMBO_ERR_NONCHARACTER_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_NumberSign_x_Hex_Semicolon_C0Control) { + SetInput("x"); + SetState(GUMBO_LEX_RCDATA); + NextChar(3, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_NumberSign_x_Hex_Semicolon_NL) { + SetInput("
x"); + SetState(GUMBO_LEX_RCDATA); + NextSpace('\n'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_NumberSign_x_Hex_Semicolon_FF) { + SetInput("x"); + SetState(GUMBO_LEX_RCDATA); + NextSpace('\f'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_NumberSign_x_Hex_Semicolon_CR) { + SetInput("
x"); + SetState(GUMBO_LEX_RCDATA); + NextSpace('\r', true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_NumberSign_X_Hex_Semicolon_Control) { + SetInput("x"); + SetState(GUMBO_LEX_RCDATA); + NextChar(0x81, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_NumberSign_x_Hex_Semicolon_Control2) { + SetInput("‚x"); + SetState(GUMBO_LEX_RCDATA); + NextChar(0x201A, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_NumberSign_x_Hex_AnythingElse) { + SetInput("⼚x"); + SetState(GUMBO_LEX_RCDATA); + NextChar(0x2F1A, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_NumberSign_x_Hex_AnythingElse2) { + SetInput("⼚-"); + SetState(GUMBO_LEX_RCDATA); + NextChar(0x2F1A, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + NextChar('-'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_NumberSign_x_AnythingElse) { + SetInput("&#xG"); + SetState(GUMBO_LEX_RCDATA); + NextChar('&', true); + Error(GUMBO_ERR_ABSENCE_OF_DIGITS_IN_NUMERIC_CHARACTER_REFERENCE); + NextChar('#'); + NextChar('x'); + NextChar('G'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_NumberSign_AnythingElse_Digit_Semicolon) { + SetInput("Ӓx"); + SetState(GUMBO_LEX_RCDATA); + NextChar(1234); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_NumberSign_AnythingElse_Digit_Semicolon_Control) { + SetInput("€x"); + SetState(GUMBO_LEX_RCDATA); + NextChar(0x20AC, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_NumberSign_AnythingElse_Digit_AnythingElse) { + SetInput("Ӓa"); + SetState(GUMBO_LEX_RCDATA); + NextChar(1234, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + NextChar('a'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_NumberSign_AnythingElse_Digit_AnythingElse_Control) { + SetInput("€a"); + SetState(GUMBO_LEX_RCDATA); + NextChar(0x20AC, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + NextChar('a'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_AnythingElse) { + SetInput("&#x"); + SetState(GUMBO_LEX_RCDATA); + NextChar('&', true); + Error(GUMBO_ERR_ABSENCE_OF_DIGITS_IN_NUMERIC_CHARACTER_REFERENCE); + NextChar('#'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_LT_Slash_Alpha_InappropriateTab) { + SetInput("<title></t\tx"); + NextStartTag(GUMBO_TAG_TITLE); + SetState(GUMBO_LEX_RCDATA); + NextChar('<'); + NextChar('/'); + NextChar('t'); + NextSpace('\t'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_LT_Slash_Alpha_InappropriateLF) { + SetInput("<textarea></t\nx"); + NextStartTag(GUMBO_TAG_TEXTAREA); + SetState(GUMBO_LEX_RCDATA); + NextChar('<'); + NextChar('/'); + NextChar('t'); + NextSpace('\n'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_LT_Slash_Alpha_InappropriateFF) { + SetInput("<title></t\fx"); + NextStartTag(GUMBO_TAG_TITLE); + SetState(GUMBO_LEX_RCDATA); + NextChar('<'); + NextChar('/'); + NextChar('t'); + NextSpace('\f'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_LT_Slash_Alpha_InappropriateSpace) { + SetInput("<textarea></t x"); + NextStartTag(GUMBO_TAG_TEXTAREA); + SetState(GUMBO_LEX_RCDATA); + NextChar('<'); + NextChar('/'); + NextChar('t'); + NextSpace(' '); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_LT_Slash_Alpha_InappropriateSlash) { + SetInput("<title></t/x"); + NextStartTag(GUMBO_TAG_TITLE); + SetState(GUMBO_LEX_RCDATA); + NextChar('<'); + NextChar('/'); + NextChar('t'); + NextChar('/'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_LT_Slash_Alpha_InappropriateGT) { + SetInput("<textarea></t>x"); + NextStartTag(GUMBO_TAG_TEXTAREA); + SetState(GUMBO_LEX_RCDATA); + NextChar('<'); + NextChar('/'); + NextChar('t'); + NextChar('>'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_LT_Slash_Alpha_AppropriateLF) { + SetInput("<textarea></teXTarea\n>x"); + NextStartTag(GUMBO_TAG_TEXTAREA); + SetState(GUMBO_LEX_RCDATA); + NextEndTag(GUMBO_TAG_TEXTAREA); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_LT_Slash_Alpha_AppropriateFF) { + SetInput("<title></titlE\f foo=bar>x"); + NextStartTag(GUMBO_TAG_TITLE); + SetState(GUMBO_LEX_RCDATA); + NextEndTag(GUMBO_TAG_TITLE, true); + Error(GUMBO_ERR_END_TAG_WITH_ATTRIBUTES); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_LT_Slash_Alpha_AppropriateSpace) { + SetInput("<textarea></Textarea >x"); + NextStartTag(GUMBO_TAG_TEXTAREA); + SetState(GUMBO_LEX_RCDATA); + NextEndTag(GUMBO_TAG_TEXTAREA); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_LT_Slash_Alpha_AppropriateSlash_GT) { + SetInput("<title></TITLE/>x"); + NextStartTag(GUMBO_TAG_TITLE); + SetState(GUMBO_LEX_RCDATA); + NextEndTag(GUMBO_TAG_TITLE, true); + Error(GUMBO_ERR_END_TAG_WITH_TRAILING_SOLIDUS); + NextChar('x'); +} +TEST_F(GumboTokenizerTest, Rcdata_LT_Slash_Alpha_AppropriateSlash_EOF) { + SetInput("<title></tITle/"); + NextStartTag(GUMBO_TAG_TITLE); + SetState(GUMBO_LEX_RCDATA); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_TAG); +} + +TEST_F(GumboTokenizerTest, Rcdata_LT_Slash_Alpha_AppropriateSlash_AnythingElse) { + SetInput("<title></title/ >x"); + NextStartTag(GUMBO_TAG_TITLE); + SetState(GUMBO_LEX_RCDATA); + NextEndTag(GUMBO_TAG_TITLE, true); + Error(GUMBO_ERR_UNEXPECTED_SOLIDUS_IN_TAG); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_LT_Slash_Alpha_AppropriateGT) { + SetInput("<textarea></textarea>x"); + NextStartTag(GUMBO_TAG_TEXTAREA); + SetState(GUMBO_LEX_RCDATA); + NextEndTag(GUMBO_TAG_TEXTAREA); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_LT_Slash_Alpha_AnythingElse) { + SetInput("<title></ti\xce\xb2"); + NextStartTag(GUMBO_TAG_TITLE); + SetState(GUMBO_LEX_RCDATA); + NextChar('<'); + NextChar('/'); + NextChar('t'); + NextChar('i'); + NextChar(0x03B2); +} + +TEST_F(GumboTokenizerTest, Rcdata_LT_Slash_AnythingElse) { + SetInput("</0"); + SetState(GUMBO_LEX_RCDATA); + NextChar('<'); + NextChar('/'); + NextChar('0'); +} + +TEST_F(GumboTokenizerTest, Rcdata_LT_AnythingElse) { + SetInput("<p>"); + SetState(GUMBO_LEX_RCDATA); + NextChar('<'); + NextChar('p'); + NextChar('>'); +} + +TEST_F(GumboTokenizerTest, Rcdata_NULL) { + SetInput("", 1); + SetState(GUMBO_LEX_RCDATA); + NextChar(0xFFFD, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); +} + +TEST_F(GumboTokenizerTest, Rcdata_EOF) { + SetInput(""); + SetState(GUMBO_LEX_RCDATA); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Rcdata_AnythingElse) { + SetInput("a\xce\xb2\xd7\x92"); + SetState(GUMBO_LEX_RCDATA); + NextChar(0x61); + NextChar(0x03B2); + NextChar(0x05D2); +} + +// Starting in the script data state +TEST_F(GumboTokenizerTest, ScriptData_LT_Slash_Alpha_InappropriateTab) { + SetInput("<script></s\tx"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('/'); + NextChar('s'); + NextSpace('\t'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Slash_Alpha_InappropriateLF) { + SetInput("<script></s\nx"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('/'); + NextChar('s'); + NextSpace('\n'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Slash_Alpha_InappropriateFF) { + SetInput("<script></s\fx"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('/'); + NextChar('s'); + NextSpace('\f'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Slash_Alpha_InappropriateSpace) { + SetInput("<script></s x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('/'); + NextChar('s'); + NextSpace(' '); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Slash_Alpha_InappropriateSlash) { + SetInput("<script></s/x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('/'); + NextChar('s'); + NextChar('/'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Slash_Alpha_InappropriateGT) { + SetInput("<script></s>x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('/'); + NextChar('s'); + NextChar('>'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Slash_Alpha_AppropriateLF) { + SetInput("<script></scRIpt\n>x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextEndTag(GUMBO_TAG_SCRIPT); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Slash_Alpha_AppropriateFF) { + SetInput("<script></scripT\f foo=bar>x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextEndTag(GUMBO_TAG_SCRIPT, true); + Error(GUMBO_ERR_END_TAG_WITH_ATTRIBUTES); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Slash_Alpha_AppropriateSpace) { + SetInput("<script></Script >x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextEndTag(GUMBO_TAG_SCRIPT); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Slash_Alpha_AppropriateSlash_GT) { + SetInput("<script></SCRIPT/>x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextEndTag(GUMBO_TAG_SCRIPT, true); + Error(GUMBO_ERR_END_TAG_WITH_TRAILING_SOLIDUS); + NextChar('x'); +} +TEST_F(GumboTokenizerTest, ScriptData_LT_Slash_Alpha_AppropriateSlash_EOF) { + SetInput("<script></script/"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_TAG); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Slash_Alpha_AppropriateSlash_AnythingElse) { + SetInput("<script></script/ >x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextEndTag(GUMBO_TAG_SCRIPT, true); + Error(GUMBO_ERR_UNEXPECTED_SOLIDUS_IN_TAG); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Slash_Alpha_AppropriateGT) { + SetInput("<script></script>x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextEndTag(GUMBO_TAG_SCRIPT); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Slash_Alpha_AnythingElse) { + SetInput("<script></sc</script>x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('/'); + NextChar('s'); + NextChar('c'); + NextEndTag(GUMBO_TAG_SCRIPT); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Slash_AnythingElse) { + SetInput("</0"); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('/'); + NextChar('0'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_Dash_LT_Slash_Alpha_InappropriateTab_EOF) { + SetInput("<script><!---</s\t"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('/'); + NextChar('s'); + NextSpace('\t'); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_SCRIPT_HTML_COMMENT_LIKE_TEXT); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Slash_Alpha_InappropriateLF_NULL) { + SetInput("<script><!--</s\n\x00", sizeof("<script><!--</s\n\x00")-1); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('/'); + NextChar('s'); + NextSpace('\n'); + NextChar(0xFFFD, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Slash_Alpha_InappropriateFF_AnythingElse) { + SetInput("<script><!--</s\fx"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('/'); + NextChar('s'); + NextSpace('\f'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Slash_Alpha_InappropriateSpace) { + SetInput("<script><!--</s x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('/'); + NextChar('s'); + NextSpace(' '); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Slash_Alpha_InappropriateSlash) { + SetInput("<script><!--</s/x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('/'); + NextChar('s'); + NextChar('/'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Slash_Alpha_InappropriateGT) { + SetInput("<script><!--</s>x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('/'); + NextChar('s'); + NextChar('>'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Slash_Alpha_AppropriateTab) { + SetInput("<script><!--</sCrIpT\t>x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextEndTag(GUMBO_TAG_SCRIPT); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Slash_Alpha_AppropriateLF) { + SetInput("<script><!--</Script\n>x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextEndTag(GUMBO_TAG_SCRIPT); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Slash_Alpha_AppropriateFF) { + SetInput("<script><!--</SCRIPT\f>x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextEndTag(GUMBO_TAG_SCRIPT); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Slash_Alpha_AppropriateSpace) { + SetInput("<script><!--</script >x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextEndTag(GUMBO_TAG_SCRIPT); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Slash_Alpha_AppropriateSlash_GT) { + SetInput("<script><!--</script/>x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextEndTag(GUMBO_TAG_SCRIPT, true); + Error(GUMBO_ERR_END_TAG_WITH_TRAILING_SOLIDUS); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Slash_Alpha_AppropriateSlash_EOF) { + SetInput("<script><!--</script/"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_TAG); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Slash_Alpha_AppropriateSlash_AnythingElse) { + SetInput("<script><!--</script/ >x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextEndTag(GUMBO_TAG_SCRIPT, true); + Error(GUMBO_ERR_UNEXPECTED_SOLIDUS_IN_TAG); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Slash_Alpha_AppropriateGT) { + SetInput("<script><!--</scrIPT>x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextEndTag(GUMBO_TAG_SCRIPT); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Slash_Alpha_AnythingElse) { + SetInput("<script><!--</s.x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('/'); + NextChar('s'); + NextChar('.'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Slash_AnythingElse) { + SetInput("<!--</>"); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('/'); + NextChar('>'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptTab_Dash_Dash_Dash_NULL_EOF) { + SetInput("<!--<scRIpt\t---\x00", sizeof("<!--<scRIpt\t---\x00")-1); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('R'); + NextChar('I'); + NextChar('p'); + NextChar('t'); + NextSpace('\t'); + NextChar('-'); + NextChar('-'); + NextChar('-'); + NextChar(0xFFFD, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_SCRIPT_HTML_COMMENT_LIKE_TEXT); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptGT_Dash_Dash_LT) { + SetInput("<script><!--<script>--</script></script>"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('/'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextEndTag(GUMBO_TAG_SCRIPT); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptGT_Dash_Dash_EOF) { + SetInput("<script><!--<script>--"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextChar('-'); + NextChar('-'); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_SCRIPT_HTML_COMMENT_LIKE_TEXT); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptGT_Dash_Dash_AnythingElse) { + SetInput("<script><!--<script>--X--></script>"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextChar('-'); + NextChar('-'); + NextChar('X'); + NextChar('-'); + NextChar('-'); + NextChar('>'); + NextEndTag(GUMBO_TAG_SCRIPT); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptTab_Dash_NULL) { + SetInput("<!--<scRIpt\t-\x00", sizeof("<!--<scRIpt\t---\x00")-1); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('R'); + NextChar('I'); + NextChar('p'); + NextChar('t'); + NextSpace('\t'); + NextChar('-'); + NextChar(0xFFFD, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptGT_Dash_LT) { + SetInput("<script><!--<script>-</script></script>"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextChar('-'); + NextChar('<'); + NextChar('/'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextEndTag(GUMBO_TAG_SCRIPT); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptGT_Dash_EOF) { + SetInput("<script><!--<script>-"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextChar('-'); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_SCRIPT_HTML_COMMENT_LIKE_TEXT); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptGT_Dash_AnythingElse) { + SetInput("<script><!--<script>-X--></script>"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextChar('-'); + NextChar('X'); + NextChar('-'); + NextChar('-'); + NextChar('>'); + NextEndTag(GUMBO_TAG_SCRIPT); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptNL_LT_Slash_ScriptTab) { + SetInput("<script><!--<script\n</Script\tx"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextSpace('\n'); + NextChar('<'); + NextChar('/'); + NextChar('S'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextSpace('\t'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptFF_LT_Slash_ScriptNL) { + SetInput("<script><!--<script\f</Script\nx"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextSpace('\f'); + NextChar('<'); + NextChar('/'); + NextChar('S'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextSpace('\n'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptSpace_LT_Slash_ScriptFF) { + SetInput("<script><!--<script </Script\fx"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextSpace(' '); + NextChar('<'); + NextChar('/'); + NextChar('S'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextSpace('\f'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptSlash_LT_Slash_ScriptSpace) { + SetInput("<script><!--<script/</Script x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('/'); + NextChar('<'); + NextChar('/'); + NextChar('S'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextSpace(' '); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptGT_LT_Slash_ScriptSlash) { + SetInput("<script><!--<script></Script/x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextChar('<'); + NextChar('/'); + NextChar('S'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('/'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptGT_LT_Slash_ScriptGT) { + SetInput("<script><!--<script></Script>x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextChar('<'); + NextChar('/'); + NextChar('S'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptGT_LT_Slash_Tab) { + SetInput("<script><!--<script></\t>x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextChar('<'); + NextChar('/'); + NextSpace('\t'); + NextChar('>'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptGT_LT_Slash_LF) { + SetInput("<script><!--<script></\n>x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextChar('<'); + NextChar('/'); + NextSpace('\n'); + NextChar('>'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptGT_LT_Slash_FF) { + SetInput("<script><!--<script></\f>x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextChar('<'); + NextChar('/'); + NextSpace('\f'); + NextChar('>'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptGT_LT_Slash_Slash) { + SetInput("<script><!--<script><//>x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextChar('<'); + NextChar('/'); + NextChar('/'); + NextChar('>'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptGT_LT_Slash_AnythingElse) { + SetInput("<script><!--<script></.>x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextChar('<'); + NextChar('/'); + NextChar('.'); + NextChar('>'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptGT_AnythingElse) { + SetInput("<script><!--<script>x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptGT_NULL) { + SetInput("<script><!--<script>\x00", sizeof("<script><!--<script>\x00")-1); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextChar(0xFFFD, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_Tab_LT_Slash_Alpha_AppropriateTab) { + SetInput("<script><!--<s\t</SCRIPT\t>"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextSpace('\t'); + NextEndTag(GUMBO_TAG_SCRIPT); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_LF_LT_Slash_Alpha_AppropriateLF) { + SetInput("<script><!--<s\n</SCRIPT\n>"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextSpace('\n'); + NextEndTag(GUMBO_TAG_SCRIPT); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_FF_LT_Slash_Alpha_AppropriateFF) { + SetInput("<script><!--<s\f</SCRIPT\f>"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextSpace('\f'); + NextEndTag(GUMBO_TAG_SCRIPT); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_Space_LT_Slash_Alpha_AppropriateSpace) { + SetInput("<script><!--<s </SCRIPT >"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextSpace(' '); + NextEndTag(GUMBO_TAG_SCRIPT); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_Slash_LT_Slash_Alpha_AppropriateSlash) { + SetInput("<script><!--<s/</SCRIPT/>"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('/'); + NextEndTag(GUMBO_TAG_SCRIPT, true); + Error(GUMBO_ERR_END_TAG_WITH_TRAILING_SOLIDUS); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_GT_LT_Slash_Alpha_AppropriateGT) { + SetInput("<script><!--<s></SCRIPT>"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('>'); + NextEndTag(GUMBO_TAG_SCRIPT); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_AnythingElse_LT_Slash_Alpha_AnythingElse) { + SetInput("<script><!--<s.</S."); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('.'); + NextChar('<'); + NextChar('/'); + NextChar('S'); + NextChar('.'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_AnythingElse_NULL_AnythingElse_EOF) { + SetInput("<script><!--<\x00.", sizeof("<script><!--<\x00.")-1); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar(0xFFFD, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + NextChar('.'); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_SCRIPT_HTML_COMMENT_LIKE_TEXT); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_AnythingElse) { + SetInput("<script><!-x</script>"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('x'); + NextEndTag(GUMBO_TAG_SCRIPT); +} + +TEST_F(GumboTokenizerTest, ScriptData_Bang_Dash_Dash_LT_AnythingElse_Dash_Dash_NULL_AnythingElse_EOF) { + SetInput("<script><!--<.--\x00.", sizeof("<script><!--<.--\x00.")-1); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('.'); + NextChar('-'); + NextChar('-'); + NextChar(0xFFFD, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + NextChar('.'); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_SCRIPT_HTML_COMMENT_LIKE_TEXT); +} + +TEST_F(GumboTokenizerTest, ScriptData_Bang_Dash_Dash_EOF) { + SetInput("<script><!--"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_SCRIPT_HTML_COMMENT_LIKE_TEXT); +} + +TEST_F(GumboTokenizerTest, ScriptData_Bang_Dash_Dash_GT_EOF) { + SetInput("<script><!-->"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('>'); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, ScriptData_Bang_Dash_Dash_AnythingElse_Dash_LT) { + SetInput("<script><!--X-<</script>"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('X'); + NextChar('-'); + NextChar('<'); + NextEndTag(GUMBO_TAG_SCRIPT); +} + +TEST_F(GumboTokenizerTest, ScriptData_Bang_Dash_Dash_AnythingElse_Dash_NULL) { + SetInput("<script><!--X-\x00</script>", sizeof("<script><!--X-\x00</script>")-1); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('X'); + NextChar('-'); + NextChar(0xFFFD, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + NextEndTag(GUMBO_TAG_SCRIPT); +} + +TEST_F(GumboTokenizerTest, ScriptData_Bang_Dash_Dash_AnythingElse_Dash_EOF) { + SetInput("<script><!--X-"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('X'); + NextChar('-'); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_SCRIPT_HTML_COMMENT_LIKE_TEXT); +} + +TEST_F(GumboTokenizerTest, ScriptData_Bang_Dash_Dash_AnythingElse_Dash_AnythingElse) { + SetInput("<script><!--X-X</script>"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('X'); + NextChar('-'); + NextChar('X'); + NextEndTag(GUMBO_TAG_SCRIPT); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_AnythingElse) { + SetInput("<p>"); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('p'); + NextChar('>'); +} + +TEST_F(GumboTokenizerTest, ScriptData_NULL) { + SetInput("", 1); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar(0xFFFD, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); +} + +TEST_F(GumboTokenizerTest, Data_Amp_Alnum_Otherwise_Alnum_Semicolon) { + SetInput("&1f;x"); + NextChar('&'); + NextChar('1'); + NextChar('f'); + NextChar(';', true); + Error(GUMBO_ERR_UNKNOWN_NAMED_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_Alnum_Otherwise_Alnum_AnythingElse) { + SetInput("&1f.x"); + NextChar('&'); + NextChar('1'); + NextChar('f'); + NextChar('.'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_NumberSign_X_Hex_Semicolon_Zero) { + SetInput("�x"); + NextChar(0xFFFD, true); + Error(GUMBO_ERR_NULL_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_NumberSign_X_Hex_Semicolon_TooLarge) { + SetInput("�x"); + NextChar(0xFFFD, true); + Error(GUMBO_ERR_CHARACTER_REFERENCE_OUTSIDE_UNICODE_RANGE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_NumberSign_X_Hex_Semicolon_Surrogate) { + SetInput("�x"); + NextChar(0xFFFD, true); + Error(GUMBO_ERR_SURROGATE_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_NumberSign_x_Hex_Semicolon_Noncharacter) { + SetInput("x"); + NextChar(0xFDD8, true); + Error(GUMBO_ERR_NONCHARACTER_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_NumberSign_x_Hex_Semicolon_Noncharacter2) { + SetInput("x"); + NextChar(0x7FFFE, true); + Error(GUMBO_ERR_NONCHARACTER_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_NumberSign_x_Hex_Semicolon_C0Control) { + SetInput("x"); + NextChar(3, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_NumberSign_x_Hex_Semicolon_NL) { + SetInput("
x"); + NextSpace('\n'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_NumberSign_x_Hex_Semicolon_FF) { + SetInput("x"); + NextSpace('\f'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_NumberSign_x_Hex_Semicolon_CR) { + SetInput("
x"); + NextSpace('\r', true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_NumberSign_X_Hex_Semicolon_Control) { + SetInput("x"); + NextChar(0x81, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_NumberSign_x_Hex_Semicolon_Control2) { + SetInput("‚x"); + NextChar(0x201A, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_NumberSign_x_Hex_AnythingElse) { + SetInput("⼚x"); + NextChar(0x2F1A, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_NumberSign_x_Hex_AnythingElse2) { + SetInput("⼚-"); + NextChar(0x2F1A, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + NextChar('-'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_NumberSign_x_AnythingElse) { + SetInput("&#xG"); + NextChar('&', true); + Error(GUMBO_ERR_ABSENCE_OF_DIGITS_IN_NUMERIC_CHARACTER_REFERENCE); + NextChar('#'); + NextChar('x'); + NextChar('G'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_NumberSign_AnythingElse_Digit_Semicolon) { + SetInput("Ӓx"); + NextChar(1234); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_NumberSign_AnythingElse_Digit_Semicolon_TooLarge) { + SetInput("�x"); + NextChar(0xFFFD, true); + Error(GUMBO_ERR_CHARACTER_REFERENCE_OUTSIDE_UNICODE_RANGE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_NumberSign_AnythingElse_Digit_Semicolon_Control) { + SetInput("€x"); + NextChar(0x20AC, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_NumberSign_AnythingElse_Digit_AnythingElse) { + SetInput("Ӓa"); + NextChar(1234, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + NextChar('a'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_NumberSign_AnythingElse_Digit_AnythingElse_Control) { + SetInput("€a"); + NextChar(0x20AC, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + NextChar('a'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_AnythingElse) { + SetInput("&#x"); + NextChar('&', true); + Error(GUMBO_ERR_ABSENCE_OF_DIGITS_IN_NUMERIC_CHARACTER_REFERENCE); + NextChar('#'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_DashDash_Dash_Dash_GT) { + SetInput("<!---->"); + NextComment(""); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_DashDash_Dash_Dash_Bang_Dash_EOF) { + SetInput("<!----!-"); + NextComment("--!", true); + Error(GUMBO_ERR_EOF_IN_COMMENT); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_DashDash_Dash_Dash_Bang_GT) { + SetInput("<!----!>"); + NextComment("", true); + Error(GUMBO_ERR_INCORRECTLY_CLOSED_COMMENT); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_DashDash_Dash_Dash_Bang_EOF) { + SetInput("<!----!"); + NextComment("", true); + Error(GUMBO_ERR_EOF_IN_COMMENT); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_DashDash_Dash_Dash_Bang_AnythingElse_AnythingElse_Dash_EOF) { + SetInput("<!----!x-"); + NextComment("--!x", true); + Error(GUMBO_ERR_EOF_IN_COMMENT); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_DashDash_Dash_Dash_Dash_EOF) { + SetInput("<!-----"); + NextComment("-", true); + Error(GUMBO_ERR_EOF_IN_COMMENT); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_DashDash_Dash_Dash_AnythingElse_AnythingElse_EOF) { + SetInput("<!----x"); + NextComment("--x", true); + Error(GUMBO_ERR_EOF_IN_COMMENT); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_DashDash_Dash_GT) { + SetInput("<!--->"); + NextComment("", true); + Error(GUMBO_ERR_ABRUPT_CLOSING_OF_EMPTY_COMMENT); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_DashDash_Dash_EOF) { + SetInput("<!---"); + NextComment("", true); + Error(GUMBO_ERR_EOF_IN_COMMENT); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_DashDash_Dash_AnythingElse) { + SetInput("<!---x-->"); + NextComment("-x"); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_DashDash_GT) { + SetInput("<!-->"); + NextComment("", true); + Error(GUMBO_ERR_ABRUPT_CLOSING_OF_EMPTY_COMMENT); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_DashDash_AnythingElse_LT_Bang_Dash_Dash_GT_GT) { + SetInput("<!--<!-->"); + NextComment("<!"); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_DashDash_AnythingElse_LT_Bang_Dash_Dash_EOF) { + SetInput("<!--<!--"); + NextComment("<!", true); + Error(GUMBO_ERR_EOF_IN_COMMENT); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_DashDash_AnythingElse_LT_Bang_Dash_Dash_AnythingElse) { + SetInput("<!--<!--x-->"); + NextComment("<!--x", true); + Error(GUMBO_ERR_NESTED_COMMENT); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_DashDash_AnythingElse_LT_Bang_Dash_AnythingElse) { + SetInput("<!--<!-x-->"); + NextComment("<!-x"); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_DashDash_AnythingElse_LT_Bang_AnythingElse) { + SetInput("<!--<!x-->"); + NextComment("<!x"); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_DashDash_AnythingElse_LT_LT_AnythingElse) { + SetInput("<!--<<-->"); + NextComment("<<"); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_DashDash_Dash_AnythingElse_AnythingElse_NULL) { + SetInput("<!---x\x00-->", sizeof("<!---x\x00-->") - 1); + NextComment("-x\xEF\xBF\xBD", true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_DashDash_AnythingElse_AnythingElse_NULL_EOF) { + SetInput("<!--0\x00", sizeof("<!--0\x00")-1); + NextComment("0\xEF\xBF\xBD", true); // Two errors. + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + Error(GUMBO_ERR_EOF_IN_COMMENT); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Tab_Tab_LF_FF_Space_Alpha_Tab_Tab_LF_FF_Space_GT) { + SetInput("<!DoCtyPE\t\t\n\f H\t\t\n\f >"); + NextDoctype("h", NULL, NULL); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_LF_Tab_LF_FF_Space_Alpha_LF_Tab_LF_FF_Space_GT) { + SetInput("<!DoCtyPE\n\t\n\f H\n\t\n\f >"); + NextDoctype("h", NULL, NULL); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_FF_Tab_LF_FF_Space_Alpha_FF_Tab_LF_FF_Space_GT) { + SetInput("<!DoCtyPE\f\t\n\f H\f\t\n\f >"); + NextDoctype("h", NULL, NULL); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Tab_LF_FF_Space_Alpha_Space_Tab_LF_FF_Space_GT) { + SetInput("<!DoCtyPE \t\n\f H \t\n\f >"); + NextDoctype("h", NULL, NULL); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_NULL_GT) { + SetInput("<!DOCTYPE \x00>", sizeof("<!DOCTYPE \x00>")-1); + NextDoctype("\xEF\xBF\xBD", NULL, NULL, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_GT) { + SetInput("<!DOCTYPE >"); + NextDoctype(NULL, NULL, NULL, true); + Error(GUMBO_ERR_MISSING_DOCTYPE_NAME); + EXPECT_TRUE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_EOF) { + SetInput("<!DOCTYPE "); + NextDoctype(NULL, NULL, NULL, true); + Error(GUMBO_ERR_EOF_IN_DOCTYPE); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_AnythingElse_NULL) { + SetInput("<!doctype ht\x00ml>", sizeof("<!doctype ht\x00ml>")-1); + NextDoctype("ht\xEF\xBF\xBDml", NULL, NULL, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_AnythingElse_EOF) { + SetInput("<!doctype html"); + NextDoctype("html", NULL, NULL, true); + Error(GUMBO_ERR_EOF_IN_DOCTYPE); + EXPECT_TRUE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_EOF) { + SetInput("<!DOCtype HTML "); + NextDoctype("html", NULL, NULL, true); + Error(GUMBO_ERR_EOF_IN_DOCTYPE); + EXPECT_TRUE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Tab_Tab_LF_FF_Space_Quote_Quote_Tab_Tab_LF_FF_Space_GT) { + SetInput("<!docTYPE HTML PUBLIC\t\t\n\f \"\"\t\t\n\f >"); + NextDoctype("html", "", NULL); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_LF_Tab_LF_FF_Space_Quote_Quote_LF_Tab_LF_FF_Space_GT) { + SetInput("<!docTYPE HTML PUBLIC\n\t\n\f \"\"\n\t\n\f >"); + NextDoctype("html", "", NULL); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_FF_Tab_LF_FF_Space_Quote_Quote_FF_Tab_LF_FF_Space_GT) { + SetInput("<!docTYPE HTML PUBLIC\f\t\n\f \"\"\f\t\n\f >"); + NextDoctype("html", "", NULL); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Tab_LF_FF_Space_Quote_Quote_Space_Tab_LF_FF_Space_GT) { + SetInput("<!docTYPE HTML PUBLIC \t\n\f \"\" \t\n\f >"); + NextDoctype("html", "", NULL); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_Quote_Space_Quote_Quote_Tab_LF_FF_Space_GT) { + SetInput("<!DOCTYPE html pubLIC \"\" \"\"\t\n\f >"); + NextDoctype("html", "", ""); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_Quote_Space_Quote_Quote_EOF) { + SetInput("<!DOCTYPE html pubLIC \"\" \"\""); + NextDoctype("html", "", "", true); + Error(GUMBO_ERR_EOF_IN_DOCTYPE); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_Quote_Space_Quote_Quote_AnythingElse_GT) { + SetInput("<!DOCTYPE html public \"\" \"\"x>"); + NextDoctype("html", "", "", true); + Error(GUMBO_ERR_UNEXPECTED_CHARACTER_AFTER_DOCTYPE_SYSTEM_IDENTIFIER); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_Quote_Space_Quote_Quote_AnythingElse_NULL_GT) { + SetInput("<!DOCTYPE html public \"\" \"\"\x00>", sizeof("<!DOCTYPE html public \"\" \"\"\x00>")-1); + NextDoctype("html", "", "", true); // Two errors! + Error(GUMBO_ERR_UNEXPECTED_CHARACTER_AFTER_DOCTYPE_SYSTEM_IDENTIFIER); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_Quote_Space_Quote_Quote_AnythingElse_EOF) { + SetInput("<!DOCTYPE html public \"\" \"\"x"); + NextDoctype("html", "", "", true); // Only one error. + Error(GUMBO_ERR_UNEXPECTED_CHARACTER_AFTER_DOCTYPE_SYSTEM_IDENTIFIER); + EXPECT_FALSE(token_.v.doc_type.force_quirks); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_Quote_Space_Quote_NULL) { + SetInput("<!DOCTYPE html public \"\" \"\x00\">", sizeof("<!DOCTYPE html public \"\" \"\x00\">")-1); + NextDoctype("html", "", "\xEF\xBF\xBD", true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_Quote_Space_Quote_GT) { + SetInput("<!DOCTYPE html public \"\" \">"); + NextDoctype("html", "", "", true); + Error(GUMBO_ERR_ABRUPT_DOCTYPE_SYSTEM_IDENTIFIER); + EXPECT_TRUE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_Quote_Space_Quote_EOF) { + SetInput("<!DOCTYPE html public \"\" \""); + NextDoctype("html", "", "", true); + Error(GUMBO_ERR_EOF_IN_DOCTYPE); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_Quote_Space_Quote_AnythingElse) { + SetInput("<!DOCTYPE html public \"\" \"SyS\">"); + NextDoctype("html", "", "SyS"); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_Quote_Space_Apos_AnythingElse) { + SetInput("<!DOCTYPE html public \"\" 'sys'>"); + NextDoctype("html", "", "sys"); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_Quote_Space_EOF) { + SetInput("<!DOCTYPE html public \"\" "); + NextDoctype("html", "", NULL, true); + Error(GUMBO_ERR_EOF_IN_DOCTYPE); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_Quote_Space_AnythingElse) { + SetInput("<!DOCTYPE html public \"\" x>z"); + NextDoctype("html", "", NULL, true); + Error(GUMBO_ERR_MISSING_QUOTE_BEFORE_DOCTYPE_SYSTEM_IDENTIFIER); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_Quote_GT) { + SetInput("<!DOCTYPE html public \"\">"); + NextDoctype("html", "", NULL); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_Quote_Quote_Quote) { + SetInput("<!DOCTYPE html public \"\"\"\">"); + NextDoctype("html", "", "", true); + Error(GUMBO_ERR_MISSING_WHITESPACE_BETWEEN_DOCTYPE_PUBLIC_AND_SYSTEM_IDENTIFIERS); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_Quote_Apos) { + SetInput("<!DOCTYPE html public \"\"''>"); + NextDoctype("html", "", "", true); + Error(GUMBO_ERR_MISSING_WHITESPACE_BETWEEN_DOCTYPE_PUBLIC_AND_SYSTEM_IDENTIFIERS); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_Quote_EOF) { + SetInput("<!DOCTYPE html public \"\""); + NextDoctype("html", "", NULL, true); + Error(GUMBO_ERR_EOF_IN_DOCTYPE); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_Quote_AnythingElse) { + SetInput("<!DOCTYPE html public \"\"x>"); + NextDoctype("html", "", NULL, true); + Error(GUMBO_ERR_MISSING_QUOTE_BEFORE_DOCTYPE_SYSTEM_IDENTIFIER); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_NULL) { + SetInput("<!DOCTYPE html public \"\x00\">", sizeof("<!DOCTYPE html public \"\x00\">")-1); + NextDoctype("html", "\xEF\xBF\xBD", NULL, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_GT) { + SetInput("<!DOCTYPE html public \">"); + NextDoctype("html", "", NULL, true); + Error(GUMBO_ERR_ABRUPT_DOCTYPE_PUBLIC_IDENTIFIER); + EXPECT_TRUE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_EOF) { + SetInput("<!DOCTYPE html public \"foo"); + NextDoctype("html", "foo", NULL, true); + Error(GUMBO_ERR_EOF_IN_DOCTYPE); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Apos_Apos) { + SetInput("<!DOCTYPE html public ''>"); + NextDoctype("html", "", NULL); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Apos_NULL) { + SetInput("<!DOCTYPE html public '\x00'>", sizeof("<!DOCTYPE html public '\x00'>")-1); + NextDoctype("html", "\xEF\xBF\xBD", NULL, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Apos_GT) { + SetInput("<!DOCTYPE html public 'foo>"); + NextDoctype("html", "foo", NULL, true); + Error(GUMBO_ERR_ABRUPT_DOCTYPE_PUBLIC_IDENTIFIER); + EXPECT_TRUE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Apos_EOF) { + SetInput("<!DOCTYPE html public 'foo"); + NextDoctype("html", "foo", NULL, true); + Error(GUMBO_ERR_EOF_IN_DOCTYPE); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_GT) { + SetInput("<!DOCTYPE HTML PUblic >"); + NextDoctype("html", NULL, NULL, true); + Error(GUMBO_ERR_MISSING_DOCTYPE_PUBLIC_IDENTIFIER); + EXPECT_TRUE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_EOF) { + SetInput("<!DOCTYPE HTML PUblic "); + NextDoctype("html", NULL, NULL, true); + Error(GUMBO_ERR_EOF_IN_DOCTYPE); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_AnythingElse) { + SetInput("<!DOCTYPE HTML PUblic x>"); + NextDoctype("html", NULL, NULL, true); + Error(GUMBO_ERR_MISSING_QUOTE_BEFORE_DOCTYPE_PUBLIC_IDENTIFIER); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Quote) { + SetInput("<!DOCTYPE html public\"foo\">"); + NextDoctype("html", "foo", NULL, true); + Error(GUMBO_ERR_MISSING_WHITESPACE_AFTER_DOCTYPE_PUBLIC_KEYWORD); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Apos) { + SetInput("<!DOCTYPE html public''>"); + NextDoctype("html", "", NULL, true); + Error(GUMBO_ERR_MISSING_WHITESPACE_AFTER_DOCTYPE_PUBLIC_KEYWORD); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_GT) { + SetInput("<!DOCTYPE html public>"); + NextDoctype("html", NULL, NULL, true); + Error(GUMBO_ERR_MISSING_DOCTYPE_PUBLIC_IDENTIFIER); + EXPECT_TRUE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_EOF) { + SetInput("<!DOCTYPE html public"); + NextDoctype("html", NULL, NULL, true); + Error(GUMBO_ERR_EOF_IN_DOCTYPE); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_AnythingElse) { + SetInput("<!DOCTYPE html publicX>"); + NextDoctype("html", NULL, NULL, true); + Error(GUMBO_ERR_MISSING_QUOTE_BEFORE_DOCTYPE_PUBLIC_IDENTIFIER); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_System_Tab_Tab_LF_FF_Space_Quote_Quote_GT) { + SetInput("<!DOCTYPE HtMl SySTem\t\t\n\f \"\">"); + NextDoctype("html", NULL, ""); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_System_LF_Apos_Apos_GT) { + SetInput("<!docTYPE hTmL sYstEM\n''>"); + NextDoctype("html", NULL, ""); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_System_FF_Apos_Apos_GT) { + SetInput("<!docTYPE hTmL sYstEM\f''>"); + NextDoctype("html", NULL, ""); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_System_Space_Apos_NULL) { + SetInput("<!DOCTYPE html SYSTEM '\x00'>", sizeof("<!DOCTYPE html SYSTEM '\x00'>")-1); + NextDoctype("html", NULL, "\xEF\xBF\xBD", true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_System_Space_Apos_GT) { + SetInput("<!DOCTYPE html SYSTEM '>"); + NextDoctype("html", NULL, "", true); + Error(GUMBO_ERR_ABRUPT_DOCTYPE_SYSTEM_IDENTIFIER); + EXPECT_TRUE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_System_Space_Apos_EOF) { + SetInput("<!DOCTYPE html SYSTEM '"); + NextDoctype("html", NULL, "", true); + Error(GUMBO_ERR_EOF_IN_DOCTYPE); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_System_Space_GT) { + SetInput("<!docTYPE hTmL sYstEM >"); + NextDoctype("html", NULL, NULL, true); + Error(GUMBO_ERR_MISSING_DOCTYPE_SYSTEM_IDENTIFIER); + EXPECT_TRUE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_System_Space_EOF) { + SetInput("<!docTYPE hTmL sYstEM "); + NextDoctype("html", NULL, NULL, true); + Error(GUMBO_ERR_EOF_IN_DOCTYPE); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_System_Space_AnythingElse) { + SetInput("<!docTYPE hTmL sYstEM x>z"); + NextDoctype("html", NULL, NULL, true); + Error(GUMBO_ERR_MISSING_QUOTE_BEFORE_DOCTYPE_SYSTEM_IDENTIFIER); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_System_Quote) { + SetInput("<!DOCTYPE foo system\"bar\">z"); + NextDoctype("foo", NULL, "bar", true); + Error(GUMBO_ERR_MISSING_WHITESPACE_AFTER_DOCTYPE_SYSTEM_KEYWORD); + EXPECT_FALSE(token_.v.doc_type.force_quirks); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_System_Apos) { + SetInput("<!DOCTYPE foo system'bar'>z"); + NextDoctype("foo", NULL, "bar", true); + Error(GUMBO_ERR_MISSING_WHITESPACE_AFTER_DOCTYPE_SYSTEM_KEYWORD); + EXPECT_FALSE(token_.v.doc_type.force_quirks); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_System_GT) { + SetInput("<!DOCTYPE foo system>z"); + NextDoctype("foo", NULL, NULL, true); + Error(GUMBO_ERR_MISSING_DOCTYPE_SYSTEM_IDENTIFIER); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_System_EOF) { + SetInput("<!DOCTYPE foo system"); + NextDoctype("foo", NULL, NULL, true); + Error(GUMBO_ERR_EOF_IN_DOCTYPE); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_System_AnythingElse) { + SetInput("<!DOCTYPE foo systemX>z"); + NextDoctype("foo", NULL, NULL, true); + Error(GUMBO_ERR_MISSING_QUOTE_BEFORE_DOCTYPE_SYSTEM_IDENTIFIER); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Bogus) { + SetInput("<!DOCTYPE html foobar>z"); + NextDoctype("html", NULL, NULL, true); + Error(GUMBO_ERR_INVALID_CHARACTER_SEQUENCE_AFTER_DOCTYPE_NAME); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_GT) { + SetInput("<!DOCTYPE html>z"); + NextDoctype("html", NULL, NULL); + EXPECT_FALSE(token_.v.doc_type.force_quirks); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_NULL) { + SetInput("<!DOCTYPE html\x00>z", sizeof("<!DOCTYPE html\x00>z")-1); + NextDoctype("html\xEF\xBF\xBD", NULL, NULL, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + EXPECT_FALSE(token_.v.doc_type.force_quirks); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_EOF) { + SetInput("<!DOCTYPE html"); + NextDoctype("html", NULL, NULL, true); + Error(GUMBO_ERR_EOF_IN_DOCTYPE); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_NULL) { + SetInput("<!DOCTYPE \x00>z", sizeof("<!DOCTYPE \x00>z")-1); + NextDoctype("\xEF\xBF\xBD", NULL, NULL, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + EXPECT_FALSE(token_.v.doc_type.force_quirks); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_GT) { + SetInput("<!DOCtype>z"); + NextDoctype(NULL, NULL, NULL, true); + Error(GUMBO_ERR_MISSING_DOCTYPE_NAME); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_EOF) { + SetInput("<!DOCTYPE"); + NextDoctype(NULL, NULL, NULL, true); + Error(GUMBO_ERR_EOF_IN_DOCTYPE); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_AnythingElse) { + SetInput("<!DOCTYPEhtml>z"); + NextDoctype("html", NULL, NULL, true); + Error(GUMBO_ERR_MISSING_WHITESPACE_BEFORE_DOCTYPE_NAME); + EXPECT_FALSE(token_.v.doc_type.force_quirks); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Cdata_Bracket_Bracket_Bracket_GT) { + SetInput("<![CDATA[]]]>x"); + Foreign(); + NextCdata(']'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Cdata_Bracket_Bracket_AnythingElse) { + SetInput("<![CDATA[]]x]]>z"); + Foreign(); + NextCdata(']'); + NextCdata(']'); + NextCdata('x'); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Cdata_Bracket_AnythingElse) { + SetInput("<![CDATA[]>x]]>z"); + Foreign(); + NextCdata(']'); + NextCdata('>'); + NextCdata('x'); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Cdata_AnythingElse_EOF) { + SetInput("<![CDATA[x"); + Foreign(); + NextCdata('x'); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_CDATA); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_CdataHtml_GT) { + SetInput("<![CDATA[bogus>z"); + NextComment("[CDATA[bogus", true); + Error(GUMBO_ERR_CDATA_IN_HTML_CONTENT); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_CdataHtml_EOF) { + SetInput("<![CDATA[bogus"); + NextComment("[CDATA[bogus", true); + Error(GUMBO_ERR_CDATA_IN_HTML_CONTENT); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Cdata_Html_NULL_AnythingElse) { + SetInput("<![CDATA[bogus\x00""comment>z", sizeof("<![CDATA[bogus\x00""comment>z")-1); + NextComment("[CDATA[bogus\xEF\xBF\xBD""comment", true); // Two errors. + Error(GUMBO_ERR_CDATA_IN_HTML_CONTENT); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_AnythingElse) { + SetInput("<!4asdf>z"); + NextComment("4asdf", true); + Error(GUMBO_ERR_INCORRECTLY_OPENED_COMMENT); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Slash_Alpha_Tab_Tab_Slash_Slash_GT) { + SetInput("</p\t\t/>z"); + NextEndTag(GUMBO_TAG_P, true); + Error(GUMBO_ERR_END_TAG_WITH_TRAILING_SOLIDUS); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Slash_Alpha_LF_LF_Slash_Slash_EOF) { + SetInput("</p\n\n/"); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_TAG); +} + +TEST_F(GumboTokenizerTest, Data_LT_Slash_Alpha_FF_FF_GT) { + SetInput("</p\f\f>z"); + NextEndTag(GUMBO_TAG_P); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Slash_Alpha_Space_Space_GT) { + SetInput("</p >z"); + NextEndTag(GUMBO_TAG_P); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Slash_Alpha_Space_Alpha_EQ_Alpha_GT) { + SetInput("</span foo=bar>z"); + NextEndTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_END_TAG_WITH_ATTRIBUTES); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Slash_GT) { + SetInput("</>z"); + NextChar('z', true); + Error(GUMBO_ERR_MISSING_END_TAG_NAME); +} + +TEST_F(GumboTokenizerTest, Data_LT_Slash_EOF) { + SetInput("</"); + NextChar('<', true); + Error(GUMBO_ERR_EOF_BEFORE_TAG_NAME); + NextChar('/'); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Slash_AnythingElse) { + SetInput("</?>z"); + NextComment("?", true); + Error(GUMBO_ERR_INVALID_FIRST_CHARACTER_OF_TAG_NAME); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Tab_Tab_Slash_Slash_GT) { + SetInput("<hr\t\t/>z"); + NextStartTag(GUMBO_TAG_HR); + EXPECT_TRUE(token_.v.start_tag.is_self_closing); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_LF_LF_GT) { + SetInput("<hr\n\n>z"); + NextStartTag(GUMBO_TAG_HR); + EXPECT_FALSE(token_.v.start_tag.is_self_closing); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_FF_FF_Slash_Slash_GT) { + SetInput("<hr\f\f/>z"); + NextStartTag(GUMBO_TAG_HR); + EXPECT_TRUE(token_.v.start_tag.is_self_closing); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_Space_GT) { + SetInput("<hr >z"); + NextStartTag(GUMBO_TAG_HR); + EXPECT_FALSE(token_.v.start_tag.is_self_closing); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_EOF) { + SetInput("<span "); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_TAG); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_EQ_GT) { + SetInput("<span =>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_UNEXPECTED_EQUALS_SIGN_BEFORE_ATTRIBUTE_NAME); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("=", attr->name); + EXPECT_STREQ("", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_Tab_Tab_GT) { + SetInput("<body X\t>z"); + NextStartTag(GUMBO_TAG_BODY); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("x", attr->name); + EXPECT_STREQ("", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_AnythingElse_LF_LF_GT) { + SetInput("<body ?\n>z"); + NextStartTag(GUMBO_TAG_BODY); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("?", attr->name); + EXPECT_STREQ("", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_AnythingElse_FF_FF_GT) { + SetInput("<body &$\f>z"); + NextStartTag(GUMBO_TAG_BODY); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("&$", attr->name); + EXPECT_STREQ("", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_Space_Space_GT) { + SetInput("<body xml:lang >z"); + NextStartTag(GUMBO_TAG_BODY); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("xml:lang", attr->name); + EXPECT_STREQ("", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_Slash_GT) { + SetInput("<br foo/>z"); + NextStartTag(GUMBO_TAG_BR); + EXPECT_TRUE(token_.v.start_tag.is_self_closing); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_GT) { + SetInput("<br foo>z"); + NextStartTag(GUMBO_TAG_BR); + EXPECT_FALSE(token_.v.start_tag.is_self_closing); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EOF) { + SetInput("<br foo"); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_TAG); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Tab_LF_FF_Space_Quote_Quote_Slash) { + SetInput("<span foo=\t\n\f \"\"/>z"); + NextStartTag(GUMBO_TAG_SPAN); + EXPECT_TRUE(token_.v.start_tag.is_self_closing); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Tab_LF_FF_Space_Quote_Quote_GT) { + SetInput("<span foo=\t\n\f \"\">z"); + NextStartTag(GUMBO_TAG_SPAN); + EXPECT_FALSE(token_.v.start_tag.is_self_closing); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Space_Quote_Quote_EOF) { + SetInput("<span foo= \"\""); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_TAG); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Space_Quote_Quote_AnythingElse) { + SetInput("<span foo= \"\"bar=''>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_MISSING_WHITESPACE_BETWEEN_ATTRIBUTES); + EXPECT_FALSE(token_.v.start_tag.is_self_closing); + ASSERT_EQ(2, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("", attr->value); + + attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[1]); + EXPECT_STREQ("bar", attr->name); + EXPECT_STREQ("", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_Alnum_Match_Semicolon) { + SetInput("<span foo=\"fjx\">z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\x66\x6Ax", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_Alnum_Match_Alpha) { + SetInput("<span foo=\"áx\">z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("áx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_Alnum_Match_Num) { + SetInput("<span foo=\"á0\">z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("á0", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_Alnum_Match_EQ) { + SetInput("<span foo=\"á=\">z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("á=", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_Alnum_Match_Otherwise) { + SetInput("<span foo=\"á\">z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xC3\xA1", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_Alnum_Otherwise_Alnum_Semicolon) { + SetInput("<span foo=\"&1f;x\">z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_UNKNOWN_NAMED_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("&1f;x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_Alnum_Otherwise_Alnum_AnythingElse) { + SetInput("<span foo=\"&1f.x\">z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("&1f.x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_NumberSign_X_Hex_Semicolon_Zero) { + SetInput("<span foo=\"�x\">z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_NULL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xEF\xBF\xBDx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_NumberSign_X_Hex_Semicolon_TooLarge) { + SetInput("<span foo=\"�x\">z"); + NextStartTag(GUMBO_TAG_SPAN, true); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + Error(GUMBO_ERR_CHARACTER_REFERENCE_OUTSIDE_UNICODE_RANGE); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xEF\xBF\xBDx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_NumberSign_X_Hex_Semicolon_Surrogate) { + SetInput("<span foo=\"�x\">z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_SURROGATE_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xEF\xBF\xBDx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_NumberSign_x_Hex_Semicolon_Noncharacter) { + SetInput("<span foo=\"x\">z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_NONCHARACTER_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xEF\xB7\x98x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_NumberSign_x_Hex_Semicolon_Noncharacter2) { + SetInput("<span foo=\"x\">z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_NONCHARACTER_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xF1\xBF\xBF\xBEx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_NumberSign_x_Hex_Semicolon_C0Control) { + SetInput("<span foo=\"x\">z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\x03x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_NumberSign_x_Hex_Semicolon_NL) { + SetInput("<span foo=\"
x\">z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\nx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_NumberSign_x_Hex_Semicolon_FF) { + SetInput("<span foo=\"x\">z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\fx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_NumberSign_x_Hex_Semicolon_CR) { + SetInput("<span foo=\"
x\">z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\rx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_NumberSign_X_Hex_Semicolon_Control) { + SetInput("<span foo=\"x\">z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xC2\x81x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_NumberSign_x_Hex_Semicolon_Control2) { + SetInput("<span foo=\"‚x\">z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xE2\x80\x9Ax", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_NumberSign_x_Hex_AnythingElse) { + SetInput("<span foo=\"⼚x\">z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xE2\xBC\x9Ax", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_NumberSign_x_Hex_AnythingElse2) { + SetInput("<span foo=\"⼚-\">z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xE2\xBC\x9A-", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_NumberSign_x_AnythingElse) { + SetInput("<span foo=\"&#xG\">z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_ABSENCE_OF_DIGITS_IN_NUMERIC_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("&#xG", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_NumberSign_AnythingElse_Digit_Semicolon) { + SetInput("<span foo=\"Ӓx\">z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xD3\x92x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_NumberSign_AnythingElse_Digit_Semicolon_Control) { + SetInput("<span foo=\"€x\">z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xE2\x82\xACx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_NumberSign_AnythingElse_Digit_AnythingElse) { + SetInput("<span foo=\"Ӓa\">z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xD3\x92""a", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_NumberSign_AnythingElse_Digit_AnythingElse_Control) { + SetInput("<span foo=\"€a\">z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xE2\x82\xAC""a", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_AnythingElse) { + SetInput("<span foo=\"&#x\">z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_ABSENCE_OF_DIGITS_IN_NUMERIC_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("&#x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_NULL) { + SetInput("<span foo=\"\x00\">z", sizeof("<span foo=\"\x00\">z")-1); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xEF\xBF\xBD", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_EOF) { + SetInput("<span foo=\""); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_TAG); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_AnythingElse) { + SetInput("<span foo=\"b@#$'\t\n\f =<>!\">z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("b@#$'\t\n\f =<>!", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_Alnum_Match_Semicolon) { + SetInput("<span foo='fjx'>z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\x66\x6Ax", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_Alnum_Match_Alpha) { + SetInput("<span foo='áx'>z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("áx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_Alnum_Match_Num) { + SetInput("<span foo='á0'>z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("á0", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_Alnum_Match_EQ) { + SetInput("<span foo='á='>z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("á=", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_Alnum_Match_Otherwise) { + SetInput("<span foo='á'>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xC3\xA1", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_Alnum_Otherwise_Alnum_Semicolon) { + SetInput("<span foo='&1f;x'>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_UNKNOWN_NAMED_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("&1f;x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_Alnum_Otherwise_Alnum_AnythingElse) { + SetInput("<span foo='&1f.x'>z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("&1f.x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_NumberSign_X_Hex_Semicolon_Zero) { + SetInput("<span foo='�x'>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_NULL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xEF\xBF\xBDx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_NumberSign_X_Hex_Semicolon_TooLarge) { + SetInput("<span foo='�x'>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_CHARACTER_REFERENCE_OUTSIDE_UNICODE_RANGE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xEF\xBF\xBDx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_NumberSign_X_Hex_Semicolon_Surrogate) { + SetInput("<span foo='�x'>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_SURROGATE_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xEF\xBF\xBDx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_NumberSign_x_Hex_Semicolon_Noncharacter) { + SetInput("<span foo='x'>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_NONCHARACTER_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xEF\xB7\x98x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_NumberSign_x_Hex_Semicolon_Noncharacter2) { + SetInput("<span foo='x'>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_NONCHARACTER_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xF1\xBF\xBF\xBEx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_NumberSign_x_Hex_Semicolon_C0Control) { + SetInput("<span foo='x'>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\x03x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_NumberSign_x_Hex_Semicolon_NL) { + SetInput("<span foo='
x'>z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\nx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_NumberSign_x_Hex_Semicolon_FF) { + SetInput("<span foo='x'>z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\fx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_NumberSign_x_Hex_Semicolon_CR) { + SetInput("<span foo='
x'>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\rx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_NumberSign_X_Hex_Semicolon_Control) { + SetInput("<span foo='x'>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xC2\x81x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_NumberSign_x_Hex_Semicolon_Control2) { + SetInput("<span foo='‚x'>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xE2\x80\x9Ax", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_NumberSign_x_Hex_AnythingElse) { + SetInput("<span foo='⼚x'>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xE2\xBC\x9Ax", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_NumberSign_x_Hex_AnythingElse2) { + SetInput("<span foo='⼚-'>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xE2\xBC\x9A-", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_NumberSign_x_AnythingElse) { + SetInput("<span foo='&#xG'>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_ABSENCE_OF_DIGITS_IN_NUMERIC_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("&#xG", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_NumberSign_AnythingElse_Digit_Semicolon) { + SetInput("<span foo='Ӓx'>z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xD3\x92x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_NumberSign_AnythingElse_Digit_Semicolon_Control) { + SetInput("<span foo='€x'>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xE2\x82\xACx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_NumberSign_AnythingElse_Digit_AnythingElse) { + SetInput("<span foo='Ӓa'>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xD3\x92""a", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_NumberSign_AnythingElse_Digit_AnythingElse_Control) { + SetInput("<span foo='€a'>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xE2\x82\xAC""a", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_AnythingElse) { + SetInput("<span foo='&#x'>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_ABSENCE_OF_DIGITS_IN_NUMERIC_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("&#x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_NULL) { + SetInput("<span foo='\x00'>z", sizeof("<span foo='\x00'>z")-1); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xEF\xBF\xBD", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_EOF) { + SetInput("<span foo='"); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_TAG); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_AnythingElse) { + SetInput("<span foo='b@#$\"\t\n\f =<>!'>z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("b@#$\"\t\n\f =<>!", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_GT) { + SetInput("<span foo=>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_MISSING_ATTRIBUTE_VALUE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_Alnum_Match_Semicolon) { + SetInput("<span foo=fjx>z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\x66\x6Ax", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_Alnum_Match_Alpha) { + SetInput("<span foo=áx>z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("áx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_Alnum_Match_Num) { + SetInput("<span foo=á0>z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("á0", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_Alnum_Match_Otherwise) { + SetInput("<span foo=á>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xC3\xA1", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_Alnum_Otherwise_Alnum_Semicolon) { + SetInput("<span foo=&1f;x>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_UNKNOWN_NAMED_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("&1f;x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_Alnum_Otherwise_Alnum_AnythingElse) { + SetInput("<span foo=&1f.x>z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("&1f.x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_NumberSign_X_Hex_Semicolon_Zero) { + SetInput("<span foo=�x>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_NULL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xEF\xBF\xBDx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_NumberSign_X_Hex_Semicolon_TooLarge) { + SetInput("<span foo=�x>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_CHARACTER_REFERENCE_OUTSIDE_UNICODE_RANGE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xEF\xBF\xBDx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_NumberSign_X_Hex_Semicolon_Surrogate) { + SetInput("<span foo=�x>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_SURROGATE_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xEF\xBF\xBDx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_NumberSign_x_Hex_Semicolon_Noncharacter) { + SetInput("<span foo=x>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_NONCHARACTER_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xEF\xB7\x98x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_NumberSign_x_Hex_Semicolon_Noncharacter2) { + SetInput("<span foo=x>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_NONCHARACTER_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xF1\xBF\xBF\xBEx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_NumberSign_x_Hex_Semicolon_C0Control) { + SetInput("<span foo=x>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\x03x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_NumberSign_x_Hex_Semicolon_NL) { + SetInput("<span foo=
x>z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\nx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_NumberSign_x_Hex_Semicolon_FF) { + SetInput("<span foo=x>z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\fx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_NumberSign_x_Hex_Semicolon_CR) { + SetInput("<span foo=
x>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\rx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_NumberSign_X_Hex_Semicolon_Control) { + SetInput("<span foo=x>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xC2\x81x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_NumberSign_x_Hex_Semicolon_Control2) { + SetInput("<span foo=‚x>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xE2\x80\x9Ax", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_NumberSign_x_Hex_AnythingElse) { + SetInput("<span foo=⼚x>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xE2\xBC\x9Ax", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_NumberSign_x_Hex_AnythingElse2) { + SetInput("<span foo=⼚->z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xE2\xBC\x9A-", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_NumberSign_x_AnythingElse) { + SetInput("<span foo=&#xG>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_ABSENCE_OF_DIGITS_IN_NUMERIC_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("&#xG", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_NumberSign_AnythingElse_Digit_Semicolon) { + SetInput("<span foo=Ӓx>z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xD3\x92x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_NumberSign_AnythingElse_Digit_Semicolon_Control) { + SetInput("<span foo=€x>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xE2\x82\xACx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_NumberSign_AnythingElse_Digit_AnythingElse) { + SetInput("<span foo=Ӓa>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xD3\x92""a", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_NumberSign_AnythingElse_Digit_AnythingElse_Control) { + SetInput("<span foo=€a>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xE2\x82\xAC""a", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_AnythingElse) { + SetInput("<span foo=&#x>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_ABSENCE_OF_DIGITS_IN_NUMERIC_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("&#x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_NULL) { + SetInput("<span foo=\x00>z", sizeof("<span foo=\x00>z")-1); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xEF\xBF\xBD", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_EOF) { + SetInput("<span foo="); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_TAG); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_AnythingElse) { + SetInput("<span foo=b@#$!>z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("b@#$!", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_Space_AnythingElse) { + SetInput("<span foo bar=''>z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(2, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("", attr->value); + + attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[1]); + EXPECT_STREQ("bar", attr->name); + EXPECT_STREQ("", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_NULL) { + SetInput("<span \x00=''>z", sizeof("<span \x00=''>z")-1); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("\xEF\xBF\xBD", attr->name); + EXPECT_STREQ("", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Quote) { + SetInput("<span \"=''>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_UNEXPECTED_CHARACTER_IN_ATTRIBUTE_NAME); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("\"", attr->name); + EXPECT_STREQ("", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Apos) { + SetInput("<span '=''>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_UNEXPECTED_CHARACTER_IN_ATTRIBUTE_NAME); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("'", attr->name); + EXPECT_STREQ("", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_LT) { + SetInput("<span <=''>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_UNEXPECTED_CHARACTER_IN_ATTRIBUTE_NAME); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("<", attr->name); + EXPECT_STREQ("", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_MultipleAttributes) { + SetInput("<span foo=\"123\"\tbar='456'\nbaz=789\f>z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(3, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("123", attr->value); + + attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[1]); + EXPECT_STREQ("bar", attr->name); + EXPECT_STREQ("456", attr->value); + + attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[2]); + EXPECT_STREQ("baz", attr->name); + EXPECT_STREQ("789", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Slash_GT) { + SetInput("<br/>z"); + NextStartTag(GUMBO_TAG_BR); + EXPECT_TRUE(token_.v.start_tag.is_self_closing); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Slash_EOF) { + SetInput("<br/"); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_TAG); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Slash_AnythingElse) { + SetInput("<br/ >z"); + NextStartTag(GUMBO_TAG_BR, true); + Error(GUMBO_ERR_UNEXPECTED_SOLIDUS_IN_TAG); + EXPECT_FALSE(token_.v.start_tag.is_self_closing); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_GT) { + SetInput("<mATh>z"); + NextStartTag(GUMBO_TAG_MATH); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_NULL) { + SetInput("<s\x00pan>z", sizeof("<s\x00pan>z")-1); + Next(true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + ASSERT_EQ(GUMBO_TOKEN_START_TAG, token_.type); + EXPECT_EQ(GUMBO_TAG_UNKNOWN, token_.v.start_tag.tag); + EXPECT_STREQ("s\xEF\xBF\xBDpan", token_.v.start_tag.name); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_EOF) { + SetInput("<sp"); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_TAG); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_AnythingElse) { + SetInput("<h5?-&!]>z"); + Next(); + ASSERT_EQ(GUMBO_TOKEN_START_TAG, token_.type); + EXPECT_EQ(GUMBO_TAG_UNKNOWN, token_.v.start_tag.tag); + EXPECT_STREQ("h5?-&!]", token_.v.start_tag.name); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Question) { + SetInput("<?php ?>z"); + NextComment("?php ?", true); + Error(GUMBO_ERR_UNEXPECTED_QUESTION_MARK_INSTEAD_OF_TAG_NAME); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_EOF) { + SetInput("<"); + NextChar('<', true); + Error(GUMBO_ERR_EOF_BEFORE_TAG_NAME); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_AnythingElse) { + SetInput("<0z"); + NextChar('<', true); + Error(GUMBO_ERR_INVALID_FIRST_CHARACTER_OF_TAG_NAME); + NextChar('0'); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_NULL) { + SetInput("", 1); + Next(true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + ASSERT_EQ(GUMBO_TOKEN_NULL, token_.type); + EXPECT_EQ(0, token_.v.character); +} + +TEST_F(GumboTokenizerTest, Data_EOF) { + SetInput(""); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_AnythingElse) { + SetInput("a\xce\xb2\xd7\x92"); + NextChar(0x61); + NextChar(0x03B2); + NextChar(0x05D2); +} + +TEST_F(GumboTokenizerTest, UTF8_BOM) { + SetInput("\xEF\xBB\xBF<b>"); + Next(); + ASSERT_EQ(GUMBO_TOKEN_START_TAG, token_.type); + EXPECT_EQ(GUMBO_TAG_B, token_.v.start_tag.tag); + AtEnd(); +} + +} // namespace +// vim: set sw=2 sts=2 ts=8 et: diff --git a/gumbo-parser/test/utf8.cc b/gumbo-parser/test/utf8.cc new file mode 100644 index 0000000000..7235342d1f --- /dev/null +++ b/gumbo-parser/test/utf8.cc @@ -0,0 +1,623 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: jdtang@google.com (Jonathan Tang) + +#include "utf8.h" + +#include <string.h> + +#include "gtest/gtest.h" +#include "error.h" +#include "gumbo.h" +#include "test_utils.h" + +namespace { + +// Tests for utf8.c +class Utf8Test : public GumboTest { + protected: + void Advance(int num_chars) { + for (int i = 0; i < num_chars; ++i) { + utf8iterator_next(&input_); + } + } + + void ResetText(const char* text) { + text_ = text; + utf8iterator_init(&parser_, text, strlen(text), &input_); + } + + GumboError* GetFirstError() { + return static_cast<GumboError*>(parser_._output->errors.data[0]); + } + + int GetNumErrors() { return parser_._output->errors.length; } + + Utf8Iterator input_; +}; + +TEST_F(Utf8Test, EmptyString) { + ResetText(""); + EXPECT_EQ(-1, utf8iterator_current(&input_)); +} + +TEST_F(Utf8Test, GetPosition_EmptyString) { + ResetText(""); + GumboSourcePosition pos; + + utf8iterator_get_position(&input_, &pos); + EXPECT_EQ(1, pos.line); + EXPECT_EQ(1, pos.column); + EXPECT_EQ(0, pos.offset); +} + +TEST_F(Utf8Test, Null) { + // Can't use ResetText, as the implicit strlen will choke on the null. + text_ = "\0f"; + utf8iterator_init(&parser_, text_, 2, &input_); + + EXPECT_EQ(0, utf8iterator_current(&input_)); + EXPECT_EQ('\0', *utf8iterator_get_char_pointer(&input_)); + utf8iterator_next(&input_); + EXPECT_EQ('f', utf8iterator_current(&input_)); + EXPECT_EQ('f', *utf8iterator_get_char_pointer(&input_)); +} + +TEST_F(Utf8Test, OneByteChar) { + ResetText("a"); + + EXPECT_EQ(0, GetNumErrors()); + EXPECT_EQ('a', utf8iterator_current(&input_)); + EXPECT_EQ('a', *utf8iterator_get_char_pointer(&input_)); + + GumboSourcePosition pos; + utf8iterator_get_position(&input_, &pos); + EXPECT_EQ(1, pos.line); + EXPECT_EQ(1, pos.column); + EXPECT_EQ(0, pos.offset); + + utf8iterator_next(&input_); + EXPECT_EQ(-1, utf8iterator_current(&input_)); +} + +TEST_F(Utf8Test, ContinuationByte) { + ResetText("\x85"); + + EXPECT_EQ(1, GetNumErrors()); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + EXPECT_EQ('\x85', *utf8iterator_get_char_pointer(&input_)); + + errors_are_expected_ = true; + GumboError* error = GetFirstError(); + EXPECT_EQ(GUMBO_ERR_UTF8_INVALID, error->type); + EXPECT_EQ('\x85', *error->original_text.data); + // This isn't a valid code point so it doesn't matter what value it has. + // EXPECT_EQ(0x85, error->v.tokenizer.codepoint); + + utf8iterator_next(&input_); + EXPECT_EQ(-1, utf8iterator_current(&input_)); +} + +TEST_F(Utf8Test, MultipleContinuationBytes) { + ResetText("a\x85\xA0\xC2x\x9A"); + EXPECT_EQ('a', utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ('x', utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ(-1, utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ(4, GetNumErrors()); + errors_are_expected_ = true; +} + +TEST_F(Utf8Test, OverlongEncoding) { + // \xC0\x75 = 11000000 01110101. + ResetText("\xC0\x75"); + + ASSERT_EQ(1, GetNumErrors()); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + EXPECT_EQ('\xC0', *utf8iterator_get_char_pointer(&input_)); + + errors_are_expected_ = true; + GumboError* error = GetFirstError(); + EXPECT_EQ(GUMBO_ERR_UTF8_INVALID, error->type); + EXPECT_EQ(1, error->position.line); + EXPECT_EQ(1, error->position.column); + EXPECT_EQ(0, error->position.offset); + EXPECT_EQ('\xC0', *error->original_text.data); + // This isn't a valid code point so it doesn't matter what value it has. + // EXPECT_EQ(0xC0, error->v.tokenizer.codepoint); + + utf8iterator_next(&input_); + EXPECT_EQ(0x75, utf8iterator_current(&input_)); + EXPECT_EQ('\x75', *utf8iterator_get_char_pointer(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ(-1, utf8iterator_current(&input_)); +} + +TEST_F(Utf8Test, OverlongEncodingWithContinuationByte) { + // \xC0\x85 = 11000000 10000101. + ResetText("\xC0\x85"); + + ASSERT_EQ(1, GetNumErrors()); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + EXPECT_EQ('\xC0', *utf8iterator_get_char_pointer(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + + errors_are_expected_ = true; + GumboError* error = GetFirstError(); + EXPECT_EQ(GUMBO_ERR_UTF8_INVALID, error->type); + EXPECT_EQ(1, error->position.line); + EXPECT_EQ(1, error->position.column); + EXPECT_EQ(0, error->position.offset); + EXPECT_EQ('\xC0', *error->original_text.data); + // This isn't a valid code point so it doesn't matter what value it has. + // EXPECT_EQ(0xC0, error->v.tokenizer.codepoint); + + utf8iterator_next(&input_); + EXPECT_EQ(-1, utf8iterator_current(&input_)); +} + +TEST_F(Utf8Test, TwoByteChar) { + // \xC3\xA5 = 11000011 10100101. + ResetText("\xC3\xA5o"); + + EXPECT_EQ(0, GetNumErrors()); + // Codepoint = 000 11100101 = 0xE5. + EXPECT_EQ(0xE5, utf8iterator_current(&input_)); + EXPECT_EQ('\xC3', *utf8iterator_get_char_pointer(&input_)); + + GumboSourcePosition pos; + utf8iterator_get_position(&input_, &pos); + EXPECT_EQ(1, pos.line); + EXPECT_EQ(1, pos.column); + EXPECT_EQ(0, pos.offset); + + utf8iterator_next(&input_); + EXPECT_EQ('o', utf8iterator_current(&input_)); + + utf8iterator_get_position(&input_, &pos); + EXPECT_EQ(1, pos.line); + EXPECT_EQ(2, pos.column); + EXPECT_EQ(2, pos.offset); +} + +TEST_F(Utf8Test, TwoByteChar2) { + // \xC2\xA5 = 11000010 10100101. + ResetText("\xC2\xA5"); + + EXPECT_EQ(0, GetNumErrors()); + // Codepoint = 000 10100101 = 0xA5. + EXPECT_EQ(0xA5, utf8iterator_current(&input_)); + EXPECT_EQ('\xC2', *utf8iterator_get_char_pointer(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ(-1, utf8iterator_current(&input_)); +} + +TEST_F(Utf8Test, ThreeByteChar) { + // \xE3\xA7\xA7 = 11100011 10100111 10100111 + ResetText("\xE3\xA7\xA7\xB0"); + + EXPECT_EQ(0, GetNumErrors()); + // Codepoint = 00111001 11100111 = 0x39E7 + EXPECT_EQ(0x39E7, utf8iterator_current(&input_)); + EXPECT_EQ('\xE3', *utf8iterator_get_char_pointer(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ(1, GetNumErrors()); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + EXPECT_EQ('\xB0', *utf8iterator_get_char_pointer(&input_)); + + GumboSourcePosition pos; + utf8iterator_get_position(&input_, &pos); + EXPECT_EQ(1, pos.line); + EXPECT_EQ(2, pos.column); + EXPECT_EQ(3, pos.offset); + + utf8iterator_next(&input_); + EXPECT_EQ(-1, utf8iterator_current(&input_)); + errors_are_expected_ = true; +} + +TEST_F(Utf8Test, FourByteChar) { + // \xC3\x9A = 11000011 10011010 + // \xF1\xA7\xA7\xA7 = 11110001 10100111 10100111 10100111 + ResetText("\xC3\x9A\xF1\xA7\xA7\xA7"); + + // Codepoint = 000 11011010 = 0xDA. + EXPECT_EQ(0xDA, utf8iterator_current(&input_)); + EXPECT_EQ('\xC3', *utf8iterator_get_char_pointer(&input_)); + + utf8iterator_next(&input_); + // Codepoint = 00110 01111001 11100111 = 0x679E7. + EXPECT_EQ(0x679E7, utf8iterator_current(&input_)); + EXPECT_EQ('\xF1', *utf8iterator_get_char_pointer(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ(-1, utf8iterator_current(&input_)); +} + +TEST_F(Utf8Test, FourByteCharWithoutContinuationChars) { + // \xF1\xA7\xA7\xA7 = 11110001 10100111 10100111 10100111 + ResetText("\xF1\xA7\xA7-"); + + EXPECT_EQ(1, GetNumErrors()); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + EXPECT_EQ('\xF1', *utf8iterator_get_char_pointer(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ('-', utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ(-1, utf8iterator_current(&input_)); + errors_are_expected_ = true; +} + +TEST_F(Utf8Test, FiveByteCharIsError) { + ResetText("\xF6\xA7\xA7\xA7\xA7x"); + + EXPECT_EQ(1, GetNumErrors()); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + utf8iterator_next(&input_); + utf8iterator_next(&input_); + utf8iterator_next(&input_); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + utf8iterator_next(&input_); + EXPECT_EQ('x', utf8iterator_current(&input_)); + errors_are_expected_ = true; +} + +TEST_F(Utf8Test, SixByteCharIsError) { + ResetText("\xF8\xA7\xA7\xA7\xA7\xA7x"); + + EXPECT_EQ(1, GetNumErrors()); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + utf8iterator_next(&input_); + utf8iterator_next(&input_); + utf8iterator_next(&input_); + utf8iterator_next(&input_); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + utf8iterator_next(&input_); + EXPECT_EQ('x', utf8iterator_current(&input_)); + errors_are_expected_ = true; +} + +TEST_F(Utf8Test, SevenByteCharIsError) { + ResetText("\xFC\xA7\xA7\xA7\xA7\xA7\xA7x"); + + EXPECT_EQ(1, GetNumErrors()); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + utf8iterator_next(&input_); + utf8iterator_next(&input_); + utf8iterator_next(&input_); + utf8iterator_next(&input_); + utf8iterator_next(&input_); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + utf8iterator_next(&input_); + EXPECT_EQ('x', utf8iterator_current(&input_)); + errors_are_expected_ = true; +} + +TEST_F(Utf8Test, 0xFFIsError) { + ResetText("\xFFx"); + + EXPECT_EQ(1, GetNumErrors()); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ('x', utf8iterator_current(&input_)); + errors_are_expected_ = true; +} + +TEST_F(Utf8Test, InvalidControlCharIsNotReplaced) { + ResetText("\x1Bx"); + + EXPECT_EQ(1, GetNumErrors()); + EXPECT_EQ(0x001b, utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ('x', utf8iterator_current(&input_)); + errors_are_expected_ = true; +} + +TEST_F(Utf8Test, TruncatedInput) { + ResetText("\xF1\xA7"); + + EXPECT_EQ(1, GetNumErrors()); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + + errors_are_expected_ = true; + GumboError* error = GetFirstError(); + EXPECT_EQ(GUMBO_ERR_UTF8_TRUNCATED, error->type); + EXPECT_EQ(1, error->position.line); + EXPECT_EQ(1, error->position.column); + EXPECT_EQ(0, error->position.offset); + EXPECT_EQ('\xF1', *error->original_text.data); + // This isn't a valid code point so it doesn't matter what value it has. + // EXPECT_EQ(0xF1A7, error->v.tokenizer.codepoint); + + utf8iterator_next(&input_); + EXPECT_EQ(-1, utf8iterator_current(&input_)); +} + +TEST_F(Utf8Test, Html5SpecExample) { + // This example has since been removed from the spec, and the spec has been + // changed to reference the Unicode Standard 6.2, 5.22 "Best practices for + // U+FFFD substitution." + errors_are_expected_ = true; + ResetText("\x41\x98\xBA\x42\xE2\x98\x43\xE2\x98\xBA\xE2\x98"); + + EXPECT_EQ('A', utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ('B', utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ('C', utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + // \xE2\x98\xBA = 11100010 10011000 10111010 + // Codepoint = 00100110 00111010 = 0x263A + EXPECT_EQ(0x263A, utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + utf8iterator_next(&input_); +} + +TEST_F(Utf8Test, MultipleEOFReads) { + ResetText("a"); + Advance(2); + EXPECT_EQ(-1, utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ(-1, utf8iterator_current(&input_)); +} + +TEST_F(Utf8Test, AsciiOnly) { + ResetText("hello"); + Advance(4); + + EXPECT_EQ('o', utf8iterator_current(&input_)); + EXPECT_EQ('o', *utf8iterator_get_char_pointer(&input_)); + + GumboSourcePosition pos; + utf8iterator_get_position(&input_, &pos); + EXPECT_EQ(1, pos.line); + EXPECT_EQ(5, pos.column); + EXPECT_EQ(4, pos.offset); + + Advance(1); + EXPECT_EQ(-1, utf8iterator_current(&input_)); +} + +TEST_F(Utf8Test, NewlinePosition) { + ResetText("a\nnewline"); + Advance(1); + + // Newline itself should register as being at the end of a line. + GumboSourcePosition pos; + utf8iterator_get_position(&input_, &pos); + EXPECT_EQ(1, pos.line); + EXPECT_EQ(2, pos.column); + EXPECT_EQ(1, pos.offset); + + // The next character should be at the next line. + Advance(1); + utf8iterator_get_position(&input_, &pos); + EXPECT_EQ(2, pos.line); + EXPECT_EQ(1, pos.column); + EXPECT_EQ(2, pos.offset); +} + +TEST_F(Utf8Test, TabPositionFreshTabstop) { + ResetText("a\n\ttab"); + Advance(sizeof("a\n\t") - 1); + + GumboSourcePosition pos; + utf8iterator_get_position(&input_, &pos); + EXPECT_EQ(2, pos.line); + EXPECT_EQ(8, pos.column); + EXPECT_EQ(3, pos.offset); +} + +TEST_F(Utf8Test, TabPositionMidTabstop) { + ResetText("a tab\tinline"); + Advance(sizeof("a tab\t") - 1); + + GumboSourcePosition pos; + utf8iterator_get_position(&input_, &pos); + EXPECT_EQ(1, pos.line); + EXPECT_EQ(8, pos.column); + EXPECT_EQ(6, pos.offset); +} + +TEST_F(Utf8Test, ConfigurableTabstop) { + options_.tab_stop = 4; + ResetText("a\n\ttab"); + Advance(sizeof("a\n\t") - 1); + + GumboSourcePosition pos; + utf8iterator_get_position(&input_, &pos); + EXPECT_EQ(2, pos.line); + EXPECT_EQ(4, pos.column); + EXPECT_EQ(3, pos.offset); +} + +TEST_F(Utf8Test, CRLF) { + ResetText("Windows\r\nlinefeeds"); + Advance(sizeof("Windows") - 1); + + EXPECT_EQ('\n', utf8iterator_current(&input_)); + EXPECT_EQ('\n', *utf8iterator_get_char_pointer(&input_)); + + GumboSourcePosition pos; + utf8iterator_get_position(&input_, &pos); + EXPECT_EQ(1, pos.line); + // The carriage return should be ignore in column calculations, treating the + // CRLF combination as one character. + EXPECT_EQ(8, pos.column); + // However, it should not be ignored in computing offsets, which are often + // used by other tools to index into the original buffer. We don't expect + // other unicode-aware tools to have the same \r\n handling as HTML5. + EXPECT_EQ(8, pos.offset); +} + +TEST_F(Utf8Test, SkipBom) { + ResetText("\xEF\xBB\xBFX"); + + EXPECT_EQ(0, GetNumErrors()); + EXPECT_EQ('X', utf8iterator_current(&input_)); + + GumboSourcePosition pos; + utf8iterator_get_position(&input_, &pos); + EXPECT_EQ(1, pos.line); + EXPECT_EQ(1, pos.column); + EXPECT_EQ(3, pos.offset); +} + +TEST_F(Utf8Test, CarriageReturn) { + ResetText("Mac\rlinefeeds"); + Advance(sizeof("Mac") - 1); + + EXPECT_EQ('\n', utf8iterator_current(&input_)); + // We don't change the original pointer, which is part of the const input + // buffer. original_text pointers will see a carriage return as original + // written. + EXPECT_EQ('\r', *utf8iterator_get_char_pointer(&input_)); + + GumboSourcePosition pos; + utf8iterator_get_position(&input_, &pos); + EXPECT_EQ(1, pos.line); + EXPECT_EQ(4, pos.column); + EXPECT_EQ(3, pos.offset); + + Advance(1); + EXPECT_EQ('l', utf8iterator_current(&input_)); + EXPECT_EQ('l', *utf8iterator_get_char_pointer(&input_)); + + utf8iterator_get_position(&input_, &pos); + EXPECT_EQ(2, pos.line); + EXPECT_EQ(1, pos.column); + EXPECT_EQ(4, pos.offset); +} + +TEST_F(Utf8Test, Matches) { + ResetText("\xC2\xA5goobar"); + Advance(1); + EXPECT_TRUE(utf8iterator_maybe_consume_match(&input_, "goo", 3, true)); + EXPECT_EQ('b', utf8iterator_current(&input_)); +} + +TEST_F(Utf8Test, MatchesOverflow) { + ResetText("goo"); + EXPECT_FALSE(utf8iterator_maybe_consume_match(&input_, "goobar", 6, true)); + EXPECT_EQ('g', utf8iterator_current(&input_)); +} + +TEST_F(Utf8Test, MatchesEof) { + ResetText("goo"); + EXPECT_TRUE(utf8iterator_maybe_consume_match(&input_, "goo", 3, true)); + EXPECT_EQ(-1, utf8iterator_current(&input_)); +} + +TEST_F(Utf8Test, MatchesCaseSensitivity) { + ResetText("gooBAR"); + EXPECT_FALSE(utf8iterator_maybe_consume_match(&input_, "goobar", 6, true)); + EXPECT_EQ('g', utf8iterator_current(&input_)); +} + +TEST_F(Utf8Test, MatchesCaseInsensitive) { + ResetText("gooBAR"); + EXPECT_TRUE(utf8iterator_maybe_consume_match(&input_, "goobar", 6, false)); + EXPECT_EQ(-1, utf8iterator_current(&input_)); +} + +TEST_F(Utf8Test, MatchFollowedByNullByte) { + // Can't use ResetText, as the implicit strlen will choke on the null. + text_ = "CDATA\0f"; + utf8iterator_init(&parser_, text_, 7, &input_); + + EXPECT_TRUE(utf8iterator_maybe_consume_match( + &input_, "cdata", sizeof("cdata") - 1, false)); + + EXPECT_EQ(0, utf8iterator_current(&input_)); + EXPECT_EQ('\0', *utf8iterator_get_char_pointer(&input_)); + utf8iterator_next(&input_); + EXPECT_EQ('f', utf8iterator_current(&input_)); + EXPECT_EQ('f', *utf8iterator_get_char_pointer(&input_)); +} + +TEST_F(Utf8Test, MarkReset) { + ResetText("this is a test"); + Advance(5); + EXPECT_EQ('i', utf8iterator_current(&input_)); + utf8iterator_mark(&input_); + + Advance(3); + EXPECT_EQ('a', utf8iterator_current(&input_)); + + GumboSourcePosition position = utf8iterator_get_mark_position(&input_); + const char* original_text = utf8iterator_get_mark_pointer(&input_); + EXPECT_EQ('i', *original_text); + EXPECT_EQ(1, position.line); + EXPECT_EQ(6, position.column); + EXPECT_EQ(5, position.offset); + + utf8iterator_reset(&input_); + EXPECT_EQ('i', utf8iterator_current(&input_)); + EXPECT_EQ('i', *utf8iterator_get_char_pointer(&input_)); + + utf8iterator_get_position(&input_, &position); + EXPECT_EQ(1, position.line); + EXPECT_EQ(6, position.column); + EXPECT_EQ(5, position.offset); +} + +} // namespace diff --git a/gumbo-parser/test/vector.cc b/gumbo-parser/test/vector.cc new file mode 100644 index 0000000000..33b15c3621 --- /dev/null +++ b/gumbo-parser/test/vector.cc @@ -0,0 +1,129 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: jdtang@google.com (Jonathan Tang) + +#include <stddef.h> +#include "vector.h" +#include "gtest/gtest.h" +#include "test_utils.h" + +namespace { + +class GumboVectorTest : public GumboTest { + protected: + GumboVectorTest(): one_(1), two_(2), three_(3) { + gumbo_vector_init(2, &vector_); + } + + ~GumboVectorTest() { gumbo_vector_destroy(&vector_); } + + GumboVector vector_; + + // dummy ints that we can use to take addresses of. + int one_; + int two_; + int three_; +}; + +TEST_F(GumboVectorTest, Init) { + EXPECT_EQ(0, vector_.length); + EXPECT_EQ(2, vector_.capacity); +} + +TEST_F(GumboVectorTest, InitZeroCapacity) { + gumbo_vector_destroy(&vector_); + gumbo_vector_init(0, &vector_); + + gumbo_vector_add(&one_, &vector_); + EXPECT_EQ(1, vector_.length); + EXPECT_EQ(1, *(static_cast<int*>(vector_.data[0]))); +} + +TEST_F(GumboVectorTest, Add) { + gumbo_vector_add(&one_, &vector_); + EXPECT_EQ(1, vector_.length); + EXPECT_EQ(1, *(static_cast<int*>(vector_.data[0]))); + EXPECT_EQ(0, gumbo_vector_index_of(&vector_, &one_)); + EXPECT_EQ(-1, gumbo_vector_index_of(&vector_, &two_)); +} + +TEST_F(GumboVectorTest, AddMultiple) { + gumbo_vector_add(&one_, &vector_); + gumbo_vector_add(&two_, &vector_); + EXPECT_EQ(2, vector_.length); + EXPECT_EQ(2, *(static_cast<int*>(vector_.data[1]))); + EXPECT_EQ(1, gumbo_vector_index_of(&vector_, &two_)); +} + +TEST_F(GumboVectorTest, Realloc) { + gumbo_vector_add(&one_, &vector_); + gumbo_vector_add(&two_, &vector_); + gumbo_vector_add(&three_, &vector_); + EXPECT_EQ(3, vector_.length); + EXPECT_EQ(4, vector_.capacity); + EXPECT_EQ(3, *(static_cast<int*>(vector_.data[2]))); +} + +TEST_F(GumboVectorTest, Pop) { + gumbo_vector_add(&one_, &vector_); + int result = *static_cast<int*>(gumbo_vector_pop(&vector_)); + EXPECT_EQ(1, result); + EXPECT_EQ(0, vector_.length); +} + +TEST_F(GumboVectorTest, PopEmpty) { + EXPECT_EQ(NULL, gumbo_vector_pop(&vector_)); +} + +TEST_F(GumboVectorTest, InsertAtFirst) { + gumbo_vector_add(&one_, &vector_); + gumbo_vector_add(&two_, &vector_); + gumbo_vector_insert_at(&three_, 0, &vector_); + EXPECT_EQ(3, vector_.length); + int result = *static_cast<int*>(vector_.data[0]); + EXPECT_EQ(3, result); +} + +TEST_F(GumboVectorTest, InsertAtLast) { + gumbo_vector_add(&one_, &vector_); + gumbo_vector_add(&two_, &vector_); + gumbo_vector_insert_at(&three_, 2, &vector_); + EXPECT_EQ(3, vector_.length); + int result = *static_cast<int*>(vector_.data[2]); + EXPECT_EQ(3, result); +} + +TEST_F(GumboVectorTest, Remove) { + gumbo_vector_add(&one_, &vector_); + gumbo_vector_add(&two_, &vector_); + gumbo_vector_add(&three_, &vector_); + gumbo_vector_remove(&two_, &vector_); + EXPECT_EQ(2, vector_.length); + int three = *static_cast<int*>(vector_.data[1]); + EXPECT_EQ(3, three); +} + +TEST_F(GumboVectorTest, RemoveAt) { + gumbo_vector_add(&one_, &vector_); + gumbo_vector_add(&two_, &vector_); + gumbo_vector_add(&three_, &vector_); + int result = *static_cast<int*>(gumbo_vector_remove_at(1, &vector_)); + EXPECT_EQ(2, result); + EXPECT_EQ(2, vector_.length); + int three = *static_cast<int*>(vector_.data[1]); + EXPECT_EQ(3, three); +} + +} // namespace diff --git a/lib/nokogiri.rb b/lib/nokogiri.rb index bf5663dad4..b396f47e16 100644 --- a/lib/nokogiri.rb +++ b/lib/nokogiri.rb @@ -2,38 +2,29 @@ # frozen_string_literal: true # Modify the PATH on windows so that the external DLLs will get loaded. -require 'rbconfig' +require "rbconfig" if defined?(RUBY_ENGINE) && RUBY_ENGINE == "jruby" - require 'nokogiri/jruby/dependencies' + require_relative "nokogiri/jruby/dependencies" end -require 'nokogiri/extension' - -require 'nokogiri/version' -require 'nokogiri/syntax_error' -require 'nokogiri/xml' -require 'nokogiri/xslt' -require 'nokogiri/html' -require 'nokogiri/decorators/slop' -require 'nokogiri/css' -require 'nokogiri/html/builder' +require_relative "nokogiri/extension" # Nokogiri parses and searches XML/HTML very quickly, and also has # correctly implemented CSS3 selector support as well as XPath 1.0 # support. # # Parsing a document returns either a Nokogiri::XML::Document, or a -# Nokogiri::HTML::Document depending on the kind of document you parse. +# Nokogiri::HTML4::Document depending on the kind of document you parse. # # Here is an example: # # require 'nokogiri' # require 'open-uri' # -# # Get a Nokogiri::HTML:Document for the page we’re interested in... +# # Get a Nokogiri::HTML4::Document for the page we’re interested in... # -# doc = Nokogiri::HTML(URI.open('http://www.google.com/search?q=tenderlove')) +# doc = Nokogiri::HTML4(URI.open('http://www.google.com/search?q=tenderlove')) # # # Do funky things with it using Nokogiri::XML::Node methods... # @@ -49,27 +40,27 @@ module Nokogiri class << self ### # Parse an HTML or XML document. +string+ contains the document. - def parse string, url = nil, encoding = nil, options = nil + def parse(string, url = nil, encoding = nil, options = nil) if string.respond_to?(:read) || /^\s*<(?:!DOCTYPE\s+)?html[\s>]/i === string[0, 512] # Expect an HTML indicator to appear within the first 512 # characters of a document. (<?xml ?> + <?xml-stylesheet ?> # shouldn't be that long) - Nokogiri.HTML(string, url, encoding, + Nokogiri.HTML4(string, url, encoding, options || XML::ParseOptions::DEFAULT_HTML) else Nokogiri.XML(string, url, encoding, options || XML::ParseOptions::DEFAULT_XML) - end.tap { |doc| + end.tap do |doc| yield doc if block_given? - } + end end ### # Create a new Nokogiri::XML::DocumentFragment - def make input = nil, opts = {}, &blk + def make(input = nil, opts = {}, &blk) if input - Nokogiri::HTML.fragment(input).children.first + Nokogiri::HTML4.fragment(input).children.first else Nokogiri(&blk) end @@ -98,10 +89,10 @@ def install_default_aliases # Make sure to support some popular encoding aliases not known by # all iconv implementations. { - 'Windows-31J' => 'CP932', # Windows-31J is the IANA registered name of CP932. - }.each { |alias_name, name| + "Windows-31J" => "CP932", # Windows-31J is the IANA registered name of CP932. + }.each do |alias_name, name| EncodingHandler.alias(name, alias_name) if EncodingHandler[alias_name].nil? - } + end end end @@ -109,15 +100,26 @@ def install_default_aliases end ### -# Parse a document contained in +args+. Nokogiri will try to guess what -# type of document you are attempting to parse. For more information, see -# Nokogiri.parse +# Parse a document contained in +args+. Nokogiri will try to guess what type of document you are +# attempting to parse. For more information, see Nokogiri.parse # -# To specify the type of document, use Nokogiri.XML or Nokogiri.HTML. +# To specify the type of document, use {Nokogiri.XML}, {Nokogiri.HTML4}, or {Nokogiri.HTML5}. def Nokogiri(*args, &block) if block_given? - Nokogiri::HTML::Builder.new(&block).doc.root + Nokogiri::HTML4::Builder.new(&block).doc.root else Nokogiri.parse(*args) end end + +require_relative "nokogiri/version" +require_relative "nokogiri/syntax_error" +require_relative "nokogiri/xml" +require_relative "nokogiri/xslt" +require_relative "nokogiri/html4" +require_relative "nokogiri/html" +require_relative "nokogiri/decorators/slop" +require_relative "nokogiri/css" +require_relative "nokogiri/html4/builder" + +require_relative "nokogiri/html5" if Nokogiri.uses_gumbo? diff --git a/lib/nokogiri/css.rb b/lib/nokogiri/css.rb index 786aa5e239..1b8fbdca24 100644 --- a/lib/nokogiri/css.rb +++ b/lib/nokogiri/css.rb @@ -1,28 +1,28 @@ # frozen_string_literal: true -require 'nokogiri/css/node' -require 'nokogiri/css/xpath_visitor' -x = $-w -$-w = false -require 'nokogiri/css/parser' -$-w = x - -require 'nokogiri/css/tokenizer' -require 'nokogiri/css/syntax_error' - module Nokogiri module CSS class << self ### # Parse this CSS selector in +selector+. Returns an AST. - def parse selector - Parser.new.parse selector + def parse(selector) + Parser.new.parse(selector) end ### # Get the XPath for +selector+. - def xpath_for selector, options={} - Parser.new(options[:ns] || {}).xpath_for selector, options + def xpath_for(selector, options = {}) + Parser.new(options[:ns] || {}).xpath_for(selector, options) end end end end + +require_relative "css/node" +require_relative "css/xpath_visitor" +x = $-w +$-w = false +require_relative "css/parser" +$-w = x + +require_relative "css/tokenizer" +require_relative "css/syntax_error" diff --git a/lib/nokogiri/css/parser.rb b/lib/nokogiri/css/parser.rb index 52b96826c9..29bdf59e79 100644 --- a/lib/nokogiri/css/parser.rb +++ b/lib/nokogiri/css/parser.rb @@ -8,7 +8,7 @@ require 'racc/parser.rb' -require 'nokogiri/css/parser_extras' +require_relative "parser_extras" module Nokogiri module CSS diff --git a/lib/nokogiri/css/parser.y b/lib/nokogiri/css/parser.y index 4ac8f9d69b..9ed97c8559 100644 --- a/lib/nokogiri/css/parser.y +++ b/lib/nokogiri/css/parser.y @@ -253,7 +253,7 @@ end ---- header -require 'nokogiri/css/parser_extras' +require_relative "parser_extras" ---- inner diff --git a/lib/nokogiri/css/syntax_error.rb b/lib/nokogiri/css/syntax_error.rb index db3e2e583e..7b48806468 100644 --- a/lib/nokogiri/css/syntax_error.rb +++ b/lib/nokogiri/css/syntax_error.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true -require 'nokogiri/syntax_error' +require_relative "../syntax_error" module Nokogiri module CSS class SyntaxError < ::Nokogiri::SyntaxError diff --git a/lib/nokogiri/extension.rb b/lib/nokogiri/extension.rb index b7de485bb5..a7e8ca8637 100644 --- a/lib/nokogiri/extension.rb +++ b/lib/nokogiri/extension.rb @@ -2,8 +2,9 @@ # load the C or Java extension begin + # native precompiled gems package shared libraries in <gem_dir>/lib/nokogiri/<ruby_version> ::RUBY_VERSION =~ /(\d+\.\d+)/ - require "nokogiri/#{Regexp.last_match(1)}/nokogiri" + require_relative "#{Regexp.last_match(1)}/nokogiri" rescue LoadError => e if e.message =~ /GLIBC/ warn(<<~EOM) @@ -22,5 +23,9 @@ EOM raise e end - require 'nokogiri/nokogiri' + + # use "require" instead of "require_relative" because non-native gems will place C extension files + # in Gem::BasicSpecification#extension_dir after compilation (during normal installation), which + # is in $LOAD_PATH but not necessarily relative to this file (see #2300) + require "nokogiri/nokogiri" end diff --git a/lib/nokogiri/gumbo.rb b/lib/nokogiri/gumbo.rb new file mode 100644 index 0000000000..d6f7403ec3 --- /dev/null +++ b/lib/nokogiri/gumbo.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true +module Nokogiri + module Gumbo + # The default maximum number of attributes per element. + DEFAULT_MAX_ATTRIBUTES = 400 + + # The default maximum number of errors for parsing a document or a fragment. + DEFAULT_MAX_ERRORS = 0 + + # The default maximum depth of the DOM tree produced by parsing a document + # or fragment. + DEFAULT_MAX_TREE_DEPTH = 400 + end +end diff --git a/lib/nokogiri/html.rb b/lib/nokogiri/html.rb index 21ce61a8dd..4fbed9bccc 100644 --- a/lib/nokogiri/html.rb +++ b/lib/nokogiri/html.rb @@ -1,38 +1,42 @@ # frozen_string_literal: true -require 'nokogiri/html/entity_lookup' -require 'nokogiri/html/document' -require 'nokogiri/html/document_fragment' -require 'nokogiri/html/sax/parser_context' -require 'nokogiri/html/sax/parser' -require 'nokogiri/html/sax/push_parser' -require 'nokogiri/html/element_description' -require 'nokogiri/html/element_description_defaults' +require_relative "html4" module Nokogiri - class << self - ### - # Parse HTML. Convenience method for Nokogiri::HTML::Document.parse - def HTML thing, url = nil, encoding = nil, options = XML::ParseOptions::DEFAULT_HTML, &block - Nokogiri::HTML::Document.parse(thing, url, encoding, options, &block) - end - end + HTML = Nokogiri::HTML4 + + # @!method HTML(input, url = nil, encoding = nil, options = XML::ParseOptions::DEFAULT_HTML, &block) + # Parse HTML. Convenience method for Nokogiri::HTML4::Document.parse + # @!scope class + define_singleton_method(:HTML, Nokogiri.method(:HTML4)) + # @note This module/namespace is an alias for {Nokogiri::HTML4} as of v1.12.0. Before v1.12.0, + # {Nokogiri::HTML4} did not exist, and this was the module/namespace for all HTML-related + # classes. module HTML - class << self - ### - # Parse HTML. Convenience method for Nokogiri::HTML::Document.parse - def parse thing, url = nil, encoding = nil, options = XML::ParseOptions::DEFAULT_HTML, &block - Document.parse(thing, url, encoding, options, &block) + # @note This class is an alias for {Nokogiri::HTML4::Document} as of v1.12.0. + class Document < Nokogiri::XML::Document + end + + # @note This class is an alias for {Nokogiri::HTML4::DocumentFragment} as of v1.12.0. + class DocumentFragment < Nokogiri::XML::DocumentFragment + end + + # @note This class is an alias for {Nokogiri::HTML4::Builder} as of v1.12.0. + class Builder < Nokogiri::XML::Builder + end + + module SAX + # @note This class is an alias for {Nokogiri::HTML4::SAX::Parser} as of v1.12.0. + class Parser < Nokogiri::XML::SAX::Parser end - #### - # Parse a fragment from +string+ in to a NodeSet. - def fragment string, encoding = nil - HTML::DocumentFragment.parse string, encoding + # @note This class is an alias for {Nokogiri::HTML4::SAX::ParserContext} as of v1.12.0. + class ParserContext < Nokogiri::XML::SAX::ParserContext end - end - # Instance of Nokogiri::HTML::EntityLookup - NamedCharacters = EntityLookup.new + # @note This class is an alias for {Nokogiri::HTML4::SAX::PushParser} as of v1.12.0. + class PushParser + end + end end end diff --git a/lib/nokogiri/html/sax/parser_context.rb b/lib/nokogiri/html/sax/parser_context.rb deleted file mode 100644 index 5eb48ca329..0000000000 --- a/lib/nokogiri/html/sax/parser_context.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true -module Nokogiri - module HTML - module SAX - ### - # Context for HTML SAX parsers. This class is usually not instantiated - # by the user. Instead, you should be looking at - # Nokogiri::HTML::SAX::Parser - class ParserContext < Nokogiri::XML::SAX::ParserContext - def self.new thing, encoding = 'UTF-8' - [:read, :close].all? { |x| thing.respond_to?(x) } ? super : - memory(thing, encoding) - end - end - end - end -end diff --git a/lib/nokogiri/html4.rb b/lib/nokogiri/html4.rb new file mode 100644 index 0000000000..b2fe513509 --- /dev/null +++ b/lib/nokogiri/html4.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true +module Nokogiri + class << self + ### + # Parse HTML. Convenience method for Nokogiri::HTML4::Document.parse + def HTML4(input, url = nil, encoding = nil, options = XML::ParseOptions::DEFAULT_HTML, &block) + Nokogiri::HTML4::Document.parse(input, url, encoding, options, &block) + end + end + + # @since v1.12.0 + # @note Before v1.12.0, {Nokogiri::HTML4} did not exist, and {Nokogiri::HTML} was the module/namespace for parsing HTML. + module HTML4 + class << self + ### + # Parse HTML. Convenience method for Nokogiri::HTML4::Document.parse + def parse(input, url = nil, encoding = nil, options = XML::ParseOptions::DEFAULT_HTML, &block) + Document.parse(input, url, encoding, options, &block) + end + + #### + # Parse a fragment from +string+ in to a NodeSet. + def fragment(string, encoding = nil) + HTML4::DocumentFragment.parse(string, encoding) + end + end + + # Instance of Nokogiri::HTML4::EntityLookup + NamedCharacters = EntityLookup.new + end +end + +require_relative "html4/entity_lookup" +require_relative "html4/document" +require_relative "html4/document_fragment" +require_relative "html4/sax/parser_context" +require_relative "html4/sax/parser" +require_relative "html4/sax/push_parser" +require_relative "html4/element_description" +require_relative "html4/element_description_defaults" diff --git a/lib/nokogiri/html/builder.rb b/lib/nokogiri/html4/builder.rb similarity index 93% rename from lib/nokogiri/html/builder.rb rename to lib/nokogiri/html4/builder.rb index 098588ae87..a875e4eedc 100644 --- a/lib/nokogiri/html/builder.rb +++ b/lib/nokogiri/html4/builder.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true module Nokogiri - module HTML + module HTML4 ### # Nokogiri HTML builder is used for building HTML documents. It is very # similar to the Nokogiri::XML::Builder. In fact, you should go read the @@ -12,7 +12,7 @@ module HTML # Create an HTML document with a body that has an onload attribute, and a # span tag with a class of "bold" that has content of "Hello world". # - # builder = Nokogiri::HTML::Builder.new do |doc| + # builder = Nokogiri::HTML4::Builder.new do |doc| # doc.html { # doc.body(:onload => 'some_func();') { # doc.span.bold { diff --git a/lib/nokogiri/html/document.rb b/lib/nokogiri/html4/document.rb similarity index 97% rename from lib/nokogiri/html/document.rb rename to lib/nokogiri/html4/document.rb index c54676c92f..07f4d0968e 100644 --- a/lib/nokogiri/html/document.rb +++ b/lib/nokogiri/html4/document.rb @@ -3,7 +3,7 @@ require 'pathname' module Nokogiri - module HTML + module HTML4 class Document < Nokogiri::XML::Document ### # Get the meta tag encoding for this document. If there is no meta tag, @@ -268,12 +268,12 @@ def self.detect_encoding(chunk) m = chunk.match(/(<meta\s)(.*)(charset\s*=\s*([\w-]+))(.*)/i) and return m[4] catch(:encoding_found) { - Nokogiri::HTML::SAX::Parser.new(JumpSAXHandler.new(:encoding_found)).parse(chunk) + Nokogiri::HTML4::SAX::Parser.new(JumpSAXHandler.new(:encoding_found)).parse(chunk) nil } else handler = SAXHandler.new - parser = Nokogiri::HTML::SAX::PushParser.new(handler) + parser = Nokogiri::HTML4::SAX::PushParser.new(handler) parser << chunk rescue Nokogiri::SyntaxError handler.encoding end @@ -286,7 +286,7 @@ def initialize(io) end # This method is used by the C extension so that - # Nokogiri::HTML::Document#read_io() does not leak memory when + # Nokogiri::HTML4::Document#read_io() does not leak memory when # EncodingFound is raised. attr_reader :encoding_found diff --git a/lib/nokogiri/html/document_fragment.rb b/lib/nokogiri/html4/document_fragment.rb similarity index 90% rename from lib/nokogiri/html/document_fragment.rb rename to lib/nokogiri/html4/document_fragment.rb index ce85cd78e3..09f9cdc8d0 100644 --- a/lib/nokogiri/html/document_fragment.rb +++ b/lib/nokogiri/html4/document_fragment.rb @@ -1,11 +1,11 @@ # frozen_string_literal: true module Nokogiri - module HTML + module HTML4 class DocumentFragment < Nokogiri::XML::DocumentFragment #### # Create a Nokogiri::XML::DocumentFragment from +tags+, using +encoding+ def self.parse(tags, encoding = nil) - doc = HTML::Document.new + doc = HTML4::Document.new encoding ||= if tags.respond_to?(:encoding) encoding = tags.encoding @@ -39,7 +39,7 @@ def initialize(document, tags = nil, ctx = nil) "/html/body/node()" end - temp_doc = HTML::Document.parse("<html><body>#{tags}", nil, document.encoding) + temp_doc = HTML4::Document.parse("<html><body>#{tags}", nil, document.encoding) temp_doc.xpath(path).each { |child| child.parent = self } self.errors = temp_doc.errors end diff --git a/lib/nokogiri/html/element_description.rb b/lib/nokogiri/html4/element_description.rb similarity index 96% rename from lib/nokogiri/html/element_description.rb rename to lib/nokogiri/html4/element_description.rb index 6c415774da..b025a2ea9e 100644 --- a/lib/nokogiri/html/element_description.rb +++ b/lib/nokogiri/html4/element_description.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true module Nokogiri - module HTML + module HTML4 class ElementDescription ### # Is this element a block element? diff --git a/lib/nokogiri/html/element_description_defaults.rb b/lib/nokogiri/html4/element_description_defaults.rb similarity index 99% rename from lib/nokogiri/html/element_description_defaults.rb rename to lib/nokogiri/html4/element_description_defaults.rb index 6763cd496e..f336eec612 100644 --- a/lib/nokogiri/html/element_description_defaults.rb +++ b/lib/nokogiri/html4/element_description_defaults.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true module Nokogiri - module HTML + module HTML4 class ElementDescription # Methods are defined protected by method_defined? because at diff --git a/lib/nokogiri/html/entity_lookup.rb b/lib/nokogiri/html4/entity_lookup.rb similarity index 94% rename from lib/nokogiri/html/entity_lookup.rb rename to lib/nokogiri/html4/entity_lookup.rb index bc66e41afe..a6034e278c 100644 --- a/lib/nokogiri/html/entity_lookup.rb +++ b/lib/nokogiri/html4/entity_lookup.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true module Nokogiri - module HTML + module HTML4 class EntityDescription < Struct.new(:value, :name, :description); end class EntityLookup diff --git a/lib/nokogiri/html/sax/parser.rb b/lib/nokogiri/html4/sax/parser.rb similarity index 75% rename from lib/nokogiri/html/sax/parser.rb rename to lib/nokogiri/html4/sax/parser.rb index 16853e6957..356f37c96c 100644 --- a/lib/nokogiri/html/sax/parser.rb +++ b/lib/nokogiri/html4/sax/parser.rb @@ -1,18 +1,15 @@ # frozen_string_literal: true module Nokogiri - module HTML + module HTML4 ### - # Nokogiri lets you write a SAX parser to process HTML but get HTML - # correction features. + # Nokogiri lets you write a SAX parser to process HTML but get HTML correction features. # - # See Nokogiri::HTML::SAX::Parser for a basic example of using a - # SAX parser with HTML. + # See Nokogiri::HTML4::SAX::Parser for a basic example of using a SAX parser with HTML. # # For more information on SAX parsers, see Nokogiri::XML::SAX module SAX ### - # This class lets you perform SAX style parsing on HTML with HTML - # error correction. + # This class lets you perform SAX style parsing on HTML with HTML error correction. # # Here is a basic usage example: # @@ -22,40 +19,40 @@ module SAX # end # end # - # parser = Nokogiri::HTML::SAX::Parser.new(MyDoc.new) + # parser = Nokogiri::HTML4::SAX::Parser.new(MyDoc.new) # parser.parse(File.read(ARGV[0], mode: 'rb')) # # For more information on SAX parsers, see Nokogiri::XML::SAX class Parser < Nokogiri::XML::SAX::Parser ### # Parse html stored in +data+ using +encoding+ - def parse_memory data, encoding = 'UTF-8' + def parse_memory(data, encoding = "UTF-8") raise ArgumentError unless data return unless data.length > 0 ctx = ParserContext.memory(data, encoding) yield ctx if block_given? - ctx.parse_with self + ctx.parse_with(self) end ### # Parse given +io+ - def parse_io io, encoding = 'UTF-8' + def parse_io(io, encoding = "UTF-8") check_encoding(encoding) @encoding = encoding ctx = ParserContext.io(io, ENCODINGS[encoding]) yield ctx if block_given? - ctx.parse_with self + ctx.parse_with(self) end ### # Parse a file with +filename+ - def parse_file filename, encoding = 'UTF-8' + def parse_file(filename, encoding = "UTF-8") raise ArgumentError unless filename raise Errno::ENOENT unless File.exist?(filename) raise Errno::EISDIR if File.directory?(filename) ctx = ParserContext.file(filename, encoding) yield ctx if block_given? - ctx.parse_with self + ctx.parse_with(self) end end end diff --git a/lib/nokogiri/html4/sax/parser_context.rb b/lib/nokogiri/html4/sax/parser_context.rb new file mode 100644 index 0000000000..6d964a3059 --- /dev/null +++ b/lib/nokogiri/html4/sax/parser_context.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true +module Nokogiri + module HTML4 + module SAX + ### + # Context for HTML SAX parsers. This class is usually not instantiated by the user. Instead, + # you should be looking at Nokogiri::HTML4::SAX::Parser + class ParserContext < Nokogiri::XML::SAX::ParserContext + def self.new(thing, encoding = "UTF-8") + if [:read, :close].all? { |x| thing.respond_to?(x) } + super + else + memory(thing, encoding) + end + end + end + end + end +end diff --git a/lib/nokogiri/html/sax/push_parser.rb b/lib/nokogiri/html4/sax/push_parser.rb similarity index 70% rename from lib/nokogiri/html/sax/push_parser.rb rename to lib/nokogiri/html4/sax/push_parser.rb index 6054e00a27..2794757199 100644 --- a/lib/nokogiri/html/sax/push_parser.rb +++ b/lib/nokogiri/html4/sax/push_parser.rb @@ -1,17 +1,17 @@ # frozen_string_literal: true module Nokogiri - module HTML + module HTML4 module SAX class PushParser - # The Nokogiri::HTML::SAX::Document on which the PushParser will be + # The Nokogiri::HTML4::SAX::Document on which the PushParser will be # operating attr_accessor :document - def initialize(doc = HTML::SAX::Document.new, file_name = nil, encoding = 'UTF-8') + def initialize(doc = HTML4::SAX::Document.new, file_name = nil, encoding = 'UTF-8') @document = doc @encoding = encoding - @sax_parser = HTML::SAX::Parser.new(doc, @encoding) + @sax_parser = HTML4::SAX::Parser.new(doc, @encoding) ## Create our push parser context initialize_native(@sax_parser, file_name, encoding) @@ -27,7 +27,7 @@ def write chunk, last_chunk = false ### # Finish the parsing. This method is only necessary for - # Nokogiri::HTML::SAX::Document#end_document to be called. + # Nokogiri::HTML4::SAX::Document#end_document to be called. def finish write '', true end diff --git a/lib/nokogiri/html5.rb b/lib/nokogiri/html5.rb new file mode 100644 index 0000000000..86b4bba717 --- /dev/null +++ b/lib/nokogiri/html5.rb @@ -0,0 +1,473 @@ +# coding: utf-8 +# frozen_string_literal: true +# +# Copyright 2013-2021 Sam Ruby, Stephen Checkoway +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require_relative 'html5/document' +require_relative 'html5/document_fragment' +require_relative 'html5/node' + +module Nokogiri + # @since v1.12.0 + # @note HTML5 functionality is not available when running JRuby. + # Parse an HTML5 document. Convenience method for {Nokogiri::HTML5::Document.parse} + def self.HTML5(input, url = nil, encoding = nil, **options, &block) + Nokogiri::HTML5::Document.parse(input, url, encoding, **options, &block) + end + + # == Usage + # + # Parse an HTML5 document: + # + # doc = Nokogiri.HTML5(string) + # + # Parse an HTML5 fragment: + # + # fragment = Nokogiri::HTML5.fragment(string) + # + # == Parsing options + # + # The document and fragment parsing methods support options that are different from Nokogiri's. + # + # - <tt>Nokogiri.HTML5(html, url = nil, encoding = nil, options = {})</tt> + # - <tt>Nokogiri::HTML5.parse(html, url = nil, encoding = nil, options = {})</tt> + # - <tt>Nokogiri::HTML5::Document.parse(html, url = nil, encoding = nil, options = {})</tt> + # - <tt>Nokogiri::HTML5.fragment(html, encoding = nil, options = {})</tt> + # - <tt>Nokogiri::HTML5::DocumentFragment.parse(html, encoding = nil, options = {})</tt> + # + # The three currently supported options are +:max_errors+, +:max_tree_depth+ and + # +:max_attributes+, described below. + # + # === Error reporting + # + # Nokogiri contains an experimental HTML5 parse error reporting facility. By default, no parse + # errors are reported but this can be configured by passing the +:max_errors+ option to + # {HTML5.parse} or {HTML5.fragment}. + # + # For example, this script: + # + # doc = Nokogiri::HTML5.parse('<span/>Hi there!</span foo=bar />', max_errors: 10) + # doc.errors.each do |err| + # puts(err) + # end + # + # Emits: + # + # 1:1: ERROR: Expected a doctype token + # <span/>Hi there!</span foo=bar /> + # ^ + # 1:1: ERROR: Start tag of nonvoid HTML element ends with '/>', use '>'. + # <span/>Hi there!</span foo=bar /> + # ^ + # 1:17: ERROR: End tag ends with '/>', use '>'. + # <span/>Hi there!</span foo=bar /> + # ^ + # 1:17: ERROR: End tag contains attributes. + # <span/>Hi there!</span foo=bar /> + # ^ + # + # Using <tt>max_errors: -1</tt> results in an unlimited number of errors being returned. + # + # The errors returned by {HTML5::Document#errors} are instances of {Nokogiri::XML::SyntaxError}. + # + # The {https://html.spec.whatwg.org/multipage/parsing.html#parse-errors HTML standard} defines a + # number of standard parse error codes. These error codes only cover the "tokenization" stage of + # parsing HTML. The parse errors in the "tree construction" stage do not have standardized error + # codes (yet). + # + # As a convenience to Nokogiri users, the defined error codes are available via + # {Nokogiri::XML::SyntaxError#str1} method. + # + # doc = Nokogiri::HTML5.parse('<span/>Hi there!</span foo=bar />', max_errors: 10) + # doc.errors.each do |err| + # puts("#{err.line}:#{err.column}: #{err.str1}") + # end + # # => 1:1: generic-parser + # # 1:1: non-void-html-element-start-tag-with-trailing-solidus + # # 1:17: end-tag-with-trailing-solidus + # # 1:17: end-tag-with-attributes + # + # Note that the first error is +generic-parser+ because it's an error from the tree construction + # stage and doesn't have a standardized error code. + # + # For the purposes of semantic versioning, the error messages, error locations, and error codes + # are not part of Nokogiri's public API. That is, these are subject to change without Nokogiri's + # major version number changing. These may be stabilized in the future. + # + # === Maximum tree depth + # + # The maximum depth of the DOM tree parsed by the various parsing methods is configurable by the + # +:max_tree_depth+ option. If the depth of the tree would exceed this limit, then an + # {::ArgumentError} is thrown. + # + # This limit (which defaults to <tt>Nokogiri::Gumbo::DEFAULT_MAX_TREE_DEPTH = 400</tt>) can be + # removed by giving the option <tt>max_tree_depth: -1</tt>. + # + # html = '<!DOCTYPE html>' + '<div>' * 1000 + # doc = Nokogiri.HTML5(html) + # # raises ArgumentError: Document tree depth limit exceeded + # doc = Nokogiri.HTML5(html, max_tree_depth: -1) + # + # === Attribute limit per element + # + # The maximum number of attributes per DOM element is configurable by the +:max_attributes+ + # option. If a given element would exceed this limit, then an {::ArgumentError} is thrown. + # + # This limit (which defaults to <tt>Nokogiri::Gumbo::DEFAULT_MAX_ATTRIBUTES = 400</tt>) can be + # removed by giving the option <tt>max_attributes: -1</tt>. + # + # html = '<!DOCTYPE html><div ' + (1..1000).map { |x| "attr-#{x}" }.join(' ') + '>' + # # "<!DOCTYPE html><div attr-1 attr-2 attr-3 ... attr-1000>" + # doc = Nokogiri.HTML5(html) + # # raises ArgumentError: Attributes per element limit exceeded + # doc = Nokogiri.HTML5(html, max_attributes: -1) + # + # == HTML Serialization + # + # After parsing HTML, it may be serialized using any of the {Nokogiri::XML::Node} serialization + # methods. In particular, {XML::Node#serialize}, {XML::Node#to_html}, and {XML::Node#to_s} will + # serialize a given node and its children. (This is the equivalent of JavaScript's + # +Element.outerHTML+.) Similarly, {XML::Node#inner_html} will serialize the children of a given + # node. (This is the equivalent of JavaScript's +Element.innerHTML+.) + # + # doc = Nokogiri::HTML5("<!DOCTYPE html><span>Hello world!</span>") + # puts doc.serialize + # # => <!DOCTYPE html><html><head> <style> .commit-tease, .user-profile-mini-avatar, .avatar, .vcard-details, .signup-prompt-bg { display: none !IMPORTANT; } </style> <script> document.addEventListener('DOMContentLoaded', function() { this.querySelectorAll('a').forEach(anchor => { anchor.addEventListener('click', e => { e.preventDefault(); const redact = new URLSearchParams(window.location.search).get('redact'); const hasExistingParams = anchor.href.includes('?'); window.location.href = anchor.href + (hasExistingParams ? `&redact=${redact}` : `?redact=${redact}`); }); }); }); </script> </head><body><span>Hello world!</span></body></html> + # + # Due to quirks in how HTML is parsed and serialized, it's possible for a DOM tree to be + # serialized and then re-parsed, resulting in a different DOM. Mostly, this happens with DOMs + # produced from invalid HTML. Unfortunately, even valid HTML may not survive serialization and + # re-parsing. + # + # In particular, a newline at the start of +pre+, +listing+, and +textarea+ elements is ignored by + # the parser. + # + # doc = Nokogiri::HTML5(<<-EOF) + # <!DOCTYPE html> + # <pre> + # Content</pre> + # EOF + # puts doc.at('/html/body/pre').serialize + # # => <pre>Content</pre> + # + # In this case, the original HTML is semantically equivalent to the serialized version. If the + # +pre+, +listing+, or +textarea+ content starts with two newlines, the first newline will be + # stripped on the first parse and the second newline will be stripped on the second, leading to + # semantically different DOMs. Passing the parameter <tt>preserve_newline: true</tt> will cause + # two or more newlines to be preserved. (A single leading newline will still be removed.) + # + # doc = Nokogiri::HTML5(<<-EOF) + # <!DOCTYPE html> + # <listing> + # + # Content</listing> + # EOF + # puts doc.at('/html/body/listing').serialize(preserve_newline: true) + # # => <listing> + # # + # # Content</listing> + # + # == Encodings + # + # Nokogiri always parses HTML5 using {https://en.wikipedia.org/wiki/UTF-8 UTF-8}; however, the + # encoding of the input can be explicitly selected via the optional +encoding+ parameter. This is + # most useful when the input comes not from a string but from an IO object. + # + # When serializing a document or node, the encoding of the output string can be specified via the + # +:encoding+ options. Characters that cannot be encoded in the selected encoding will be encoded + # as {https://en.wikipedia.org/wiki/List_of_XML_and_HTML_character_entity_references HTML numeric + # entities}. + # + # frag = Nokogiri::HTML5.fragment('<span>아는 길도 물어가라</span>') + # html = frag.serialize(encoding: 'US-ASCII') + # puts html + # # => <span>아는 길도 물어가라</span> + # frag = Nokogiri::HTML5.fragment(html) + # puts frag.serialize + # # => <span>아는 길도 물어가라</span> + # + # (There's a {https://bugs.ruby-lang.org/issues/15033 bug} in all current versions of Ruby that + # can cause the entity encoding to fail. Of the mandated supported encodings for HTML, the only + # encoding I'm aware of that has this bug is <tt>'ISO-2022-JP'</tt>. We recommend avoiding this + # encoding.) + # + # == Notes + # + # * The {Nokogiri::HTML5.fragment} function takes a string and parses it + # as a HTML5 document. The +<html>+, +<head>+, and +<body>+ elements are + # removed from this document, and any children of these elements that remain + # are returned as a {Nokogiri::HTML5::DocumentFragment}. + # + # * The {Nokogiri::HTML5.parse} function takes a string and passes it to the + # <code>gumbo_parse_with_options</code> method, using the default options. + # The resulting Gumbo parse tree is then walked. + # + # * Instead of uppercase element names, lowercase element names are produced. + # + # * Instead of returning +unknown+ as the element name for unknown tags, the + # original tag name is returned verbatim. + # + # @since v1.12.0 + # @note HTML5 functionality is not available when running JRuby. + module HTML5 + # HTML uses the XHTML namespace. + HTML_NAMESPACE = 'http://www.w3.org/1999/xhtml'.freeze + MATHML_NAMESPACE = 'http://www.w3.org/1998/Math/MathML'.freeze + SVG_NAMESPACE = 'http://www.w3.org/2000/svg'.freeze + XLINK_NAMESPACE = 'http://www.w3.org/1999/xlink'.freeze + XML_NAMESPACE = 'http://www.w3.org/XML/1998/namespace'.freeze + XMLNS_NAMESPACE = 'http://www.w3.org/2000/xmlns/'.freeze + + # Parse an HTML 5 document. Convenience method for {Nokogiri::HTML5::Document.parse} + def self.parse(string, url = nil, encoding = nil, **options, &block) + Document.parse(string, url, encoding, **options, &block) + end + + # Parse a fragment from +string+. Convenience method for + # {Nokogiri::HTML5::DocumentFragment.parse}. + def self.fragment(string, encoding = nil, **options) + DocumentFragment.parse(string, encoding, options) + end + + # Fetch and parse a HTML document from the web, following redirects, + # handling https, and determining the character encoding using HTML5 + # rules. +uri+ may be a +String+ or a +URI+. +options+ contains + # http headers and special options. Everything which is not a + # special option is considered a header. Special options include: + # * :follow_limit => number of redirects which are followed + # * :basic_auth => [username, password] + def self.get(uri, options={}) + warn("Nokogiri::HTML5.get is deprecated and will be removed in a future version of Nokogiri.", + uplevel: 1, category: :deprecated) + get_impl(uri, options) + end + + private + + def self.get_impl(uri, options={}) + headers = options.clone + headers = {:follow_limit => headers} if Numeric === headers # deprecated + limit=headers[:follow_limit] ? headers.delete(:follow_limit).to_i : 10 + + require 'net/http' + uri = URI(uri) unless URI === uri + + http = Net::HTTP.new(uri.host, uri.port) + + # TLS / SSL support + http.use_ssl = true if uri.scheme == 'https' + + # Pass through Net::HTTP override values, which currently include: + # :ca_file, :ca_path, :cert, :cert_store, :ciphers, + # :close_on_empty_response, :continue_timeout, :key, :open_timeout, + # :read_timeout, :ssl_timeout, :ssl_version, :use_ssl, + # :verify_callback, :verify_depth, :verify_mode + options.each do |key, value| + http.send "#{key}=", headers.delete(key) if http.respond_to? "#{key}=" + end + + request = Net::HTTP::Get.new(uri.request_uri) + + # basic authentication + auth = headers.delete(:basic_auth) + auth ||= [uri.user, uri.password] if uri.user && uri.password + request.basic_auth auth.first, auth.last if auth + + # remaining options are treated as headers + headers.each {|key, value| request[key.to_s] = value.to_s} + + response = http.request(request) + + case response + when Net::HTTPSuccess + doc = parse(reencode(response.body, response['content-type']), options) + doc.instance_variable_set('@response', response) + doc.class.send(:attr_reader, :response) + doc + when Net::HTTPRedirection + response.value if limit <= 1 + location = URI.join(uri, response['location']) + get_impl(location, options.merge(:follow_limit => limit-1)) + else + response.value + end + end + + def self.read_and_encode(string, encoding) + # Read the string with the given encoding. + if string.respond_to?(:read) + if encoding.nil? + string = string.read + else + string = string.read(encoding: encoding) + end + else + # Otherwise the string has the given encoding. + string = string.to_s + if encoding + string = string.dup + string.force_encoding(encoding) + end + end + + # convert to UTF-8 + if string.encoding != Encoding::UTF_8 + string = reencode(string) + end + string + end + + # Charset sniffing is a complex and controversial topic that understandably isn't done _by + # default_ by the Ruby Net::HTTP library. This being said, it is a very real problem for + # consumers of HTML as the default for HTML is iso-8859-1, most "good" producers use utf-8, and + # the Gumbo parser *only* supports utf-8. + # + # Accordingly, Nokogiri::HTML4::Document.parse provides limited encoding detection. Following + # this lead, Nokogiri::HTML5 attempts to do likewise, while attempting to more closely follow + # the HTML5 standard. + # + # http://bugs.ruby-lang.org/issues/2567 + # http://www.w3.org/TR/html5/syntax.html#determining-the-character-encoding + # + def self.reencode(body, content_type=nil) + if body.encoding == Encoding::ASCII_8BIT + encoding = nil + + # look for a Byte Order Mark (BOM) + initial_bytes = body[0..2].bytes + if initial_bytes[0..2] == [0xEF, 0xBB, 0xBF] + encoding = Encoding::UTF_8 + elsif initial_bytes[0..1] == [0xFE, 0xFF] + encoding = Encoding::UTF_16BE + elsif initial_bytes[0..1] == [0xFF, 0xFE] + encoding = Encoding::UTF_16LE + end + + # look for a charset in a content-encoding header + if content_type + encoding ||= content_type[/charset=["']?(.*?)($|["';\s])/i, 1] + end + + # look for a charset in a meta tag in the first 1024 bytes + if not encoding + data = body[0..1023].gsub(/<!--.*?(-->|\Z)/m, '') + data.scan(/<meta.*?>/m).each do |meta| + encoding ||= meta[/charset=["']?([^>]*?)($|["'\s>])/im, 1] + end + end + + # if all else fails, default to the official default encoding for HTML + encoding ||= Encoding::ISO_8859_1 + + # change the encoding to match the detected or inferred encoding + body = body.dup + begin + body.force_encoding(encoding) + rescue ArgumentError + body.force_encoding(Encoding::ISO_8859_1) + end + end + + body.encode(Encoding::UTF_8) + end + + def self.serialize_node_internal(current_node, io, encoding, options) + case current_node.type + when XML::Node::ELEMENT_NODE + ns = current_node.namespace + ns_uri = ns.nil? ? nil : ns.href + # XXX(sfc): attach namespaces to all nodes, even html? + if ns_uri.nil? || ns_uri == HTML_NAMESPACE || ns_uri == MATHML_NAMESPACE || ns_uri == SVG_NAMESPACE + tagname = current_node.name + else + tagname = "#{ns.prefix}:#{current_node.name}" + end + io << '<' << tagname + current_node.attribute_nodes.each do |attr| + attr_ns = attr.namespace + if attr_ns.nil? + attr_name = attr.name + else + ns_uri = attr_ns.href + if ns_uri == XML_NAMESPACE + attr_name = 'xml:' + attr.name.sub(/^[^:]*:/, '') + elsif ns_uri == XMLNS_NAMESPACE && attr.name.sub(/^[^:]*:/, '') == 'xmlns' + attr_name = 'xmlns' + elsif ns_uri == XMLNS_NAMESPACE + attr_name = 'xmlns:' + attr.name.sub(/^[^:]*:/, '') + elsif ns_uri == XLINK_NAMESPACE + attr_name = 'xlink:' + attr.name.sub(/^[^:]*:/, '') + else + attr_name = "#{attr_ns.prefix}:#{attr.name}" + end + end + io << ' ' << attr_name << '="' << escape_text(attr.content, encoding, true) << '"' + end + io << '>' + if !%w[area base basefont bgsound br col embed frame hr img input keygen + link meta param source track wbr].include?(current_node.name) + io << "\n" if options[:preserve_newline] && prepend_newline?(current_node) + current_node.children.each do |child| + # XXX(sfc): Templates handled specially? + serialize_node_internal(child, io, encoding, options) + end + io << '</' << tagname << '>' + end + when XML::Node::TEXT_NODE + parent = current_node.parent + if parent.element? && %w[style script xmp iframe noembed noframes plaintext noscript].include?(parent.name) + io << current_node.content + else + io << escape_text(current_node.content, encoding, false) + end + when XML::Node::CDATA_SECTION_NODE + io << '<![CDATA[' << current_node.content << ']]>' + when XML::Node::COMMENT_NODE + io << '<!--' << current_node.content << '-->' + when XML::Node::PI_NODE + io << '<?' << current_node.content << '>' + when XML::Node::DOCUMENT_TYPE_NODE, XML::Node::DTD_NODE + io << '<!DOCTYPE ' << current_node.name << '>' + when XML::Node::HTML_DOCUMENT_NODE, XML::Node::DOCUMENT_FRAG_NODE + current_node.children.each do |child| + serialize_node_internal(child, io, encoding, options) + end + else + raise "Unexpected node '#{current_node.name}' of type #{current_node.type}" + end + end + + def self.escape_text(text, encoding, attribute_mode) + if attribute_mode + text = text.gsub(/[&\u00a0"]/, + '&' => '&', "\u00a0" => ' ', '"' => '"') + else + text = text.gsub(/[&\u00a0<>]/, + '&' => '&', "\u00a0" => ' ', '<' => '<', '>' => '>') + end + # Not part of the standard + text.encode(encoding, fallback: lambda { |c| "&\#x#{c.ord.to_s(16)};" }) + end + + def self.prepend_newline?(node) + return false unless %w[pre textarea listing].include?(node.name) && !node.children.empty? + first_child = node.children[0] + first_child.text? && first_child.content.start_with?("\n") + end + end +end + +require_relative 'gumbo' diff --git a/lib/nokogiri/html5/document.rb b/lib/nokogiri/html5/document.rb new file mode 100644 index 0000000000..fe74d99b75 --- /dev/null +++ b/lib/nokogiri/html5/document.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true +# +# Copyright 2013-2021 Sam Ruby, Stephen Checkoway +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require_relative "../html4/document" + +module Nokogiri + module HTML5 + # @since v1.12.0 + # @note HTML5 functionality is not available when running JRuby. + class Document < Nokogiri::HTML4::Document + def self.parse(string_or_io, url = nil, encoding = nil, **options, &block) + yield options if block_given? + string_or_io = '' unless string_or_io + + if string_or_io.respond_to?(:encoding) && string_or_io.encoding.name != 'ASCII-8BIT' + encoding ||= string_or_io.encoding.name + end + + if string_or_io.respond_to?(:read) && string_or_io.respond_to?(:path) + url ||= string_or_io.path + end + unless string_or_io.respond_to?(:read) || string_or_io.respond_to?(:to_str) + raise ArgumentError.new("not a string or IO object") + end + do_parse(string_or_io, url, encoding, options) + end + + def self.read_io(io, url = nil, encoding = nil, **options) + raise ArgumentError.new("io object doesn't respond to :read") unless io.respond_to?(:read) + do_parse(io, url, encoding, options) + end + + def self.read_memory(string, url = nil, encoding = nil, **options) + raise ArgumentError.new("string object doesn't respond to :to_str") unless string.respond_to?(:to_str) + do_parse(string, url, encoding, options) + end + + def fragment(tags = nil) + DocumentFragment.new(self, tags, self.root) + end + + def to_xml(options = {}, &block) + # Bypass XML::Document#to_xml which doesn't add + # XML::Node::SaveOptions::AS_XML like XML::Node#to_xml does. + XML::Node.instance_method(:to_xml).bind(self).call(options, &block) + end + + private + def self.do_parse(string_or_io, url, encoding, options) + string = HTML5.read_and_encode(string_or_io, encoding) + max_attributes = options[:max_attributes] || Nokogiri::Gumbo::DEFAULT_MAX_ATTRIBUTES + max_errors = options[:max_errors] || options[:max_parse_errors] || Nokogiri::Gumbo::DEFAULT_MAX_ERRORS + max_depth = options[:max_tree_depth] || Nokogiri::Gumbo::DEFAULT_MAX_TREE_DEPTH + doc = Nokogiri::Gumbo.parse(string, url, max_attributes, max_errors, max_depth) + doc.encoding = 'UTF-8' + doc + end + end + end +end diff --git a/lib/nokogiri/html5/document_fragment.rb b/lib/nokogiri/html5/document_fragment.rb new file mode 100644 index 0000000000..66c6b413a9 --- /dev/null +++ b/lib/nokogiri/html5/document_fragment.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true +# +# Copyright 2013-2021 Sam Ruby, Stephen Checkoway +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require_relative "../html4/document_fragment" + +module Nokogiri + module HTML5 + # @since v1.12.0 + # @note HTML5 functionality is not available when running JRuby. + class DocumentFragment < Nokogiri::HTML4::DocumentFragment + attr_accessor :document + attr_accessor :errors + + # Create a document fragment. + def initialize(doc, tags = nil, ctx = nil, options = {}) + self.document = doc + self.errors = [] + return self unless tags + + max_attributes = options[:max_attributes] || Nokogiri::Gumbo::DEFAULT_MAX_ATTRIBUTES + max_errors = options[:max_errors] || Nokogiri::Gumbo::DEFAULT_MAX_ERRORS + max_depth = options[:max_tree_depth] || Nokogiri::Gumbo::DEFAULT_MAX_TREE_DEPTH + tags = Nokogiri::HTML5.read_and_encode(tags, nil) + Nokogiri::Gumbo.fragment(self, tags, ctx, max_attributes, max_errors, max_depth) + end + + def serialize(options = {}, &block) + # Bypass XML::Document.serialize which doesn't support options even + # though XML::Node.serialize does! + XML::Node.instance_method(:serialize).bind(self).call(options, &block) + end + + # Parse a document fragment from +tags+, returning a Nodeset. + def self.parse(tags, encoding = nil, options = {}) + doc = HTML5::Document.new + tags = HTML5.read_and_encode(tags, encoding) + doc.encoding = "UTF-8" + new(doc, tags, nil, options) + end + + def extract_params(params) # :nodoc: + handler = params.find do |param| + ![Hash, String, Symbol].include?(param.class) + end + params -= [handler] if handler + + hashes = [] + while Hash === params.last || params.last.nil? + hashes << params.pop + break if params.empty? + end + ns, binds = hashes.reverse + + ns ||= + begin + ns = {} + children.each { |child| ns.merge!(child.namespaces) } + ns + end + + [params, handler, ns, binds] + end + end + end +end +# vim: set shiftwidth=2 softtabstop=2 tabstop=8 expandtab: diff --git a/lib/nokogiri/html5/node.rb b/lib/nokogiri/html5/node.rb new file mode 100644 index 0000000000..92cf1b9470 --- /dev/null +++ b/lib/nokogiri/html5/node.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true +# +# Copyright 2013-2021 Sam Ruby, Stephen Checkoway +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require_relative "../xml/node" + +module Nokogiri + module HTML5 + # @since v1.12.0 + # @note HTML5 functionality is not available when running JRuby. + module Node + def inner_html(options = {}) + return super(options) unless document.is_a?(HTML5::Document) + result = options[:preserve_newline] && HTML5.prepend_newline?(self) ? String.new("\n") : String.new + result << children.map { |child| child.to_html(options) }.join + result + end + + def write_to(io, *options) + return super(io, *options) unless document.is_a?(HTML5::Document) + options = options.first.is_a?(Hash) ? options.shift : {} + encoding = options[:encoding] || options[0] + if Nokogiri.jruby? + save_options = options[:save_with] || options[1] + indent_times = options[:indent] || 0 + else + save_options = options[:save_with] || options[1] || XML::Node::SaveOptions::FORMAT + indent_times = options[:indent] || 2 + end + indent_string = (options[:indent_text] || " ") * indent_times + + config = XML::Node::SaveOptions.new(save_options.to_i) + yield config if block_given? + + config_options = config.options + if config_options & (XML::Node::SaveOptions::AS_XML | XML::Node::SaveOptions::AS_XHTML) != 0 + # Use Nokogiri's serializing code. + native_write_to(io, encoding, indent_string, config_options) + else + # Serialize including the current node. + encoding ||= document.encoding || Encoding::UTF_8 + internal_ops = { + preserve_newline: options[:preserve_newline] || false, + } + HTML5.serialize_node_internal(self, io, encoding, internal_ops) + end + end + + def fragment(tags) + return super(tags) unless document.is_a?(HTML5::Document) + DocumentFragment.new(document, tags, self) + end + + private + + # HTML elements can have attributes that contain colons. + # Nokogiri::XML::Node#[]= treats names with colons as a prefixed QName + # and tries to create an attribute in a namespace. This is especially + # annoying with attribute names like xml:lang since libxml2 will + # actually create the xml namespace if it doesn't exist already. + def add_child_node_and_reparent_attrs(node) + return super(node) unless document.is_a?(HTML5::Document) + # I'm not sure what this method is supposed to do. Reparenting + # namespaces is handled by libxml2, including child namespaces which + # this method wouldn't handle. + # https://github.com/sparklemotion/nokogiri/issues/1790 + add_child_node(node) + # node.attribute_nodes.find_all { |a| a.namespace }.each do |attr| + # attr.remove + # ns = attr.namespace + # a["#{ns.prefix}:#{attr.name}"] = attr.value + # end + end + end + # Monkey patch + XML::Node.prepend(HTML5::Node) + end +end + +# vim: set shiftwidth=2 softtabstop=2 tabstop=8 expandtab: diff --git a/lib/nokogiri/version/constant.rb b/lib/nokogiri/version/constant.rb index bb3e4ee505..c6b1a69c42 100644 --- a/lib/nokogiri/version/constant.rb +++ b/lib/nokogiri/version/constant.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module Nokogiri # The version of Nokogiri you are using - VERSION = "1.11.3" + VERSION = "1.12.5" end diff --git a/lib/nokogiri/version/info.rb b/lib/nokogiri/version/info.rb index 1bfea42908..460408d186 100644 --- a/lib/nokogiri/version/info.rb +++ b/lib/nokogiri/version/info.rb @@ -52,6 +52,10 @@ def libxml2_has_iconv? defined?(Nokogiri::LIBXML_ICONV_ENABLED) && Nokogiri::LIBXML_ICONV_ENABLED end + def libxslt_has_datetime? + defined?(Nokogiri::LIBXSLT_DATETIME_ENABLED) && Nokogiri::LIBXSLT_DATETIME_ENABLED + end + def libxml2_using_packaged? libxml2? && Nokogiri::PACKAGED_LIBRARIES end @@ -137,6 +141,7 @@ def to_hash else libxml["source"] = "system" end + libxml["memory_management"] = Nokogiri::LIBXML_MEMORY_MANAGEMENT libxml["iconv_enabled"] = libxml2_has_iconv? libxml["compiled"] = compiled_libxml_version.to_s libxml["loaded"] = loaded_libxml_version.to_s @@ -150,6 +155,7 @@ def to_hash else libxslt["source"] = "system" end + libxslt["datetime_enabled"] = libxslt_has_datetime? libxslt["compiled"] = compiled_libxslt_version.to_s libxslt["loaded"] = loaded_libxslt_version.to_s end @@ -190,15 +196,19 @@ def self.uses_libxml?(requirement = nil) # :nodoc: Gem::Requirement.new(requirement).satisfied_by?(VersionInfo.instance.loaded_libxml_version) end + def self.uses_gumbo? + uses_libxml? # TODO: replace with Gumbo functionality + end + def self.jruby? # :nodoc: VersionInfo.instance.jruby? end # Ensure constants used in this file are loaded - see #1896 if Nokogiri.jruby? - require "nokogiri/jruby/dependencies" + require_relative "../jruby/dependencies" end - require "nokogiri/extension" + require_relative "../extension" # More complete version information about libxml VERSION_INFO = VersionInfo.instance.to_hash diff --git a/lib/nokogiri/xml.rb b/lib/nokogiri/xml.rb index 93a898b951..e1e21c05ea 100644 --- a/lib/nokogiri/xml.rb +++ b/lib/nokogiri/xml.rb @@ -1,38 +1,9 @@ # frozen_string_literal: true -require 'nokogiri/xml/pp' -require 'nokogiri/xml/parse_options' -require 'nokogiri/xml/sax' -require 'nokogiri/xml/searchable' -require 'nokogiri/xml/node' -require 'nokogiri/xml/attribute_decl' -require 'nokogiri/xml/element_decl' -require 'nokogiri/xml/element_content' -require 'nokogiri/xml/character_data' -require 'nokogiri/xml/namespace' -require 'nokogiri/xml/attr' -require 'nokogiri/xml/dtd' -require 'nokogiri/xml/cdata' -require 'nokogiri/xml/text' -require 'nokogiri/xml/document' -require 'nokogiri/xml/document_fragment' -require 'nokogiri/xml/processing_instruction' -require 'nokogiri/xml/node_set' -require 'nokogiri/xml/syntax_error' -require 'nokogiri/xml/xpath' -require 'nokogiri/xml/xpath_context' -require 'nokogiri/xml/builder' -require 'nokogiri/xml/reader' -require 'nokogiri/xml/notation' -require 'nokogiri/xml/entity_decl' -require 'nokogiri/xml/entity_reference' -require 'nokogiri/xml/schema' -require 'nokogiri/xml/relax_ng' - module Nokogiri class << self ### # Parse XML. Convenience method for Nokogiri::XML::Document.parse - def XML thing, url = nil, encoding = nil, options = XML::ParseOptions::DEFAULT_XML, &block + def XML(thing, url = nil, encoding = nil, options = XML::ParseOptions::DEFAULT_XML, &block) Nokogiri::XML::Document.parse(thing, url, encoding, options, &block) end end @@ -41,20 +12,19 @@ module XML # Original C14N 1.0 spec canonicalization XML_C14N_1_0 = 0 # Exclusive C14N 1.0 spec canonicalization - XML_C14N_EXCLUSIVE_1_0 = 1 + XML_C14N_EXCLUSIVE_1_0 = 1 # C14N 1.1 spec canonicalization XML_C14N_1_1 = 2 class << self ### # Parse an XML document using the Nokogiri::XML::Reader API. See # Nokogiri::XML::Reader for mor information - def Reader string_or_io, url = nil, encoding = nil, options = ParseOptions::STRICT - + def Reader(string_or_io, url = nil, encoding = nil, options = ParseOptions::STRICT) options = Nokogiri::XML::ParseOptions.new(options) if Integer === options # Give the options to the user yield options if block_given? - if string_or_io.respond_to? :read + if string_or_io.respond_to?(:read) return Reader.from_io(string_or_io, url, encoding, options.to_i) end Reader.from_memory(string_or_io, url, encoding, options.to_i) @@ -62,15 +32,44 @@ def Reader string_or_io, url = nil, encoding = nil, options = ParseOptions::STRI ### # Parse XML. Convenience method for Nokogiri::XML::Document.parse - def parse thing, url = nil, encoding = nil, options = ParseOptions::DEFAULT_XML, &block + def parse(thing, url = nil, encoding = nil, options = ParseOptions::DEFAULT_XML, &block) Document.parse(thing, url, encoding, options, &block) end #### # Parse a fragment from +string+ in to a NodeSet. - def fragment string + def fragment(string) XML::DocumentFragment.parse(string) end end end end + +require_relative "xml/pp" +require_relative "xml/parse_options" +require_relative "xml/sax" +require_relative "xml/searchable" +require_relative "xml/node" +require_relative "xml/attribute_decl" +require_relative "xml/element_decl" +require_relative "xml/element_content" +require_relative "xml/character_data" +require_relative "xml/namespace" +require_relative "xml/attr" +require_relative "xml/dtd" +require_relative "xml/cdata" +require_relative "xml/text" +require_relative "xml/document" +require_relative "xml/document_fragment" +require_relative "xml/processing_instruction" +require_relative "xml/node_set" +require_relative "xml/syntax_error" +require_relative "xml/xpath" +require_relative "xml/xpath_context" +require_relative "xml/builder" +require_relative "xml/reader" +require_relative "xml/notation" +require_relative "xml/entity_decl" +require_relative "xml/entity_reference" +require_relative "xml/schema" +require_relative "xml/relax_ng" diff --git a/lib/nokogiri/xml/builder.rb b/lib/nokogiri/xml/builder.rb index 1c7ed087fe..d08ca83f31 100644 --- a/lib/nokogiri/xml/builder.rb +++ b/lib/nokogiri/xml/builder.rb @@ -196,6 +196,41 @@ module XML # # Note the "foo:object" tag. # + # === Namespace inheritance + # + # In the Builder context, children will inherit their parent's namespace. This is the same + # behavior as if the underlying {XML::Document} set +namespace_inheritance+ to +true+: + # + # result = Nokogiri::XML::Builder.new do |xml| + # xml["soapenv"].Envelope("xmlns:soapenv" => "http://schemas.xmlsoap.org/soap/envelope/") do + # xml.Header + # end + # end + # result.doc.to_xml + # # => <?xml version="1.0" encoding="utf-8"?> + # # <soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/"> + # # <soapenv:Header/> + # # </soapenv:Envelope> + # + # Users may turn this behavior off by passing a keyword argument +namespace_inheritance:false+ + # to the initializer: + # + # result = Nokogiri::XML::Builder.new(namespace_inheritance: false) do |xml| + # xml["soapenv"].Envelope("xmlns:soapenv" => "http://schemas.xmlsoap.org/soap/envelope/") do + # xml.Header + # xml["soapenv"].Body # users may explicitly opt into the namespace + # end + # end + # result.doc.to_xml + # # => <?xml version="1.0" encoding="utf-8"?> + # # <soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/"> + # # <Header/> + # # <soapenv:Body/> + # # </soapenv:Envelope> + # + # For more information on namespace inheritance, please see {XML::Document#namespace_inheritance} + # + # # == Document Types # # To create a document type (DTD), access use the Builder#doc method to get @@ -226,6 +261,8 @@ module XML # </root> # class Builder + DEFAULT_DOCUMENT_OPTIONS = {namespace_inheritance: true} + # The current Document object being built attr_accessor :doc @@ -282,6 +319,7 @@ def initialize(options = {}, root = nil, &block) @arity = nil @ns = nil + options = DEFAULT_DOCUMENT_OPTIONS.merge(options) options.each do |k, v| @doc.send(:"#{k}=", v) end diff --git a/lib/nokogiri/xml/document.rb b/lib/nokogiri/xml/document.rb index d09c0e41ad..4a4e5087a1 100644 --- a/lib/nokogiri/xml/document.rb +++ b/lib/nokogiri/xml/document.rb @@ -113,9 +113,55 @@ def self.parse string_or_io, url = nil, encoding = nil, options = ParseOptions:: # A list of Nokogiri::XML::SyntaxError found when parsing a document attr_accessor :errors + # When true, reparented elements without a namespace will inherit their new parent's + # namespace (if one exists). Defaults to +false+. + # + # @example Default behavior of namespace inheritance + # xml = <<~EOF + # <root xmlns:foo="http://nokogiri.org/default_ns/test/foo"> + # <foo:parent> + # </foo:parent> + # </root> + # EOF + # doc = Nokogiri::XML(xml) + # parent = doc.at_xpath("//foo:parent", "foo" => "http://nokogiri.org/default_ns/test/foo") + # parent.add_child("<child></child>") + # doc.to_xml + # # => <?xml version="1.0"?> + # # <root xmlns:foo="http://nokogiri.org/default_ns/test/foo"> + # # <foo:parent> + # # <child/> + # # </foo:parent> + # # </root> + # + # @example Setting namespace inheritance to +true+ + # xml = <<~EOF + # <root xmlns:foo="http://nokogiri.org/default_ns/test/foo"> + # <foo:parent> + # </foo:parent> + # </root> + # EOF + # doc = Nokogiri::XML(xml) + # doc.namespace_inheritance = true + # parent = doc.at_xpath("//foo:parent", "foo" => "http://nokogiri.org/default_ns/test/foo") + # parent.add_child("<child></child>") + # doc.to_xml + # # => <?xml version="1.0"?> + # # <root xmlns:foo="http://nokogiri.org/default_ns/test/foo"> + # # <foo:parent> + # # <foo:child/> + # # </foo:parent> + # # </root> + # + # @return [Boolean] + # + # @since v1.12.4 + attr_accessor :namespace_inheritance + def initialize *args # :nodoc: @errors = [] @decorators = nil + @namespace_inheritance = false end ## diff --git a/lib/nokogiri/xml/node.rb b/lib/nokogiri/xml/node.rb index 44f94ccd49..a06ba5fa56 100644 --- a/lib/nokogiri/xml/node.rb +++ b/lib/nokogiri/xml/node.rb @@ -1,7 +1,6 @@ # encoding: UTF-8 # frozen_string_literal: true require "stringio" -require "nokogiri/xml/node/save_options" module Nokogiri module XML @@ -837,7 +836,7 @@ def parse(string_or_io, options = nil) node_set = in_context(contents, options.to_i) if (node_set.empty? && (document.errors.length > error_count)) if options.recover? - fragment = Nokogiri::HTML::DocumentFragment.parse contents + fragment = Nokogiri::HTML4::DocumentFragment.parse contents node_set = fragment.children else raise document.errors[error_count] @@ -883,7 +882,7 @@ def xml? type == DOCUMENT_NODE end - # Returns true if this is an HTML::Document node + # Returns true if this is an HTML4::Document node def html? type == HTML_DOCUMENT_NODE end @@ -909,11 +908,11 @@ def fragment? end ### - # Fetch the Nokogiri::HTML::ElementDescription for this node. Returns + # Fetch the Nokogiri::HTML4::ElementDescription for this node. Returns # nil on XML documents and on unknown tags. def description return nil if document.xml? - Nokogiri::HTML::ElementDescription[name] + Nokogiri::HTML4::ElementDescription[name] end ### @@ -1235,3 +1234,5 @@ def add_child_node_and_reparent_attrs(node) end end end + +require_relative "node/save_options" diff --git a/lib/nokogiri/xml/node/save_options.rb b/lib/nokogiri/xml/node/save_options.rb index 6da00134bb..8afeef345a 100644 --- a/lib/nokogiri/xml/node/save_options.rb +++ b/lib/nokogiri/xml/node/save_options.rb @@ -34,7 +34,7 @@ class SaveOptions DEFAULT_HTML = FORMAT | NO_DECLARATION | NO_EMPTY_TAGS | AS_HTML end # the default for XHTML document - DEFAULT_XHTML = FORMAT | NO_DECLARATION | NO_EMPTY_TAGS | AS_XHTML + DEFAULT_XHTML = FORMAT | NO_DECLARATION | AS_XHTML # Integer representation of the SaveOptions attr_reader :options diff --git a/lib/nokogiri/xml/parse_options.rb b/lib/nokogiri/xml/parse_options.rb index a266d5ba07..36186ab7b9 100644 --- a/lib/nokogiri/xml/parse_options.rb +++ b/lib/nokogiri/xml/parse_options.rb @@ -71,6 +71,8 @@ class ParseOptions # the default options used for parsing XML documents DEFAULT_XML = RECOVER | NONET + # the default options used for parsing XSLT stylesheets + DEFAULT_XSLT = RECOVER | NONET | NOENT | DTDLOAD | DTDATTR | NOCDATA # the default options used for parsing HTML documents DEFAULT_HTML = RECOVER | NOERROR | NOWARNING | NONET # the default options used for parsing XML schemas diff --git a/lib/nokogiri/xml/pp.rb b/lib/nokogiri/xml/pp.rb index 1b0f4df850..591bc8fa79 100644 --- a/lib/nokogiri/xml/pp.rb +++ b/lib/nokogiri/xml/pp.rb @@ -1,3 +1,3 @@ # frozen_string_literal: true -require 'nokogiri/xml/pp/node' -require 'nokogiri/xml/pp/character_data' +require_relative "pp/node" +require_relative "pp/character_data" diff --git a/lib/nokogiri/xml/sax.rb b/lib/nokogiri/xml/sax.rb index e965f677b0..f4c35b7137 100644 --- a/lib/nokogiri/xml/sax.rb +++ b/lib/nokogiri/xml/sax.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true -require 'nokogiri/xml/sax/document' -require 'nokogiri/xml/sax/parser_context' -require 'nokogiri/xml/sax/parser' -require 'nokogiri/xml/sax/push_parser' +require_relative "sax/document" +require_relative "sax/parser_context" +require_relative "sax/parser" +require_relative "sax/push_parser" diff --git a/lib/nokogiri/xml/sax/document.rb b/lib/nokogiri/xml/sax/document.rb index 0f32e8930a..8336b74789 100644 --- a/lib/nokogiri/xml/sax/document.rb +++ b/lib/nokogiri/xml/sax/document.rb @@ -2,20 +2,19 @@ module Nokogiri module XML ### - # SAX Parsers are event driven parsers. Nokogiri provides two different - # event based parsers when dealing with XML. If you want to do SAX style - # parsing using HTML, check out Nokogiri::HTML::SAX. + # SAX Parsers are event driven parsers. Nokogiri provides two different event based parsers when + # dealing with XML. If you want to do SAX style parsing using HTML, check out + # Nokogiri::HTML4::SAX. # - # The basic way a SAX style parser works is by creating a parser, - # telling the parser about the events we're interested in, then giving - # the parser some XML to process. The parser will notify you when - # it encounters events you said you would like to know about. + # The basic way a SAX style parser works is by creating a parser, telling the parser about the + # events we're interested in, then giving the parser some XML to process. The parser will notify + # you when it encounters events you said you would like to know about. # - # To register for events, you simply subclass Nokogiri::XML::SAX::Document, - # and implement the methods for which you would like notification. + # To register for events, you simply subclass Nokogiri::XML::SAX::Document, and implement the + # methods for which you would like notification. # - # For example, if I want to be notified when a document ends, and when an - # element starts, I would write a class like this: + # For example, if I want to be notified when a document ends, and when an element starts, I + # would write a class like this: # # class MyDocument < Nokogiri::XML::SAX::Document # def end_document @@ -27,8 +26,7 @@ module XML # end # end # - # Then I would instantiate a SAX parser with this document, and feed the - # parser some XML + # Then I would instantiate a SAX parser with this document, and feed the parser some XML # # # Create a new parser # parser = Nokogiri::XML::SAX::Parser.new(MyDocument.new) @@ -36,25 +34,21 @@ module XML # # Feed the parser some XML # parser.parse(File.open(ARGV[0])) # - # Now my document handler will be called when each node starts, and when - # then document ends. To see what kinds of events are available, take - # a look at Nokogiri::XML::SAX::Document. + # Now my document handler will be called when each node starts, and when then document ends. To + # see what kinds of events are available, take a look at Nokogiri::XML::SAX::Document. # - # Two SAX parsers for XML are available, a parser that reads from a string - # or IO object as it feels necessary, and a parser that lets you spoon - # feed it XML. If you want to let Nokogiri deal with reading your XML, - # use the Nokogiri::XML::SAX::Parser. If you want to have fine grain + # Two SAX parsers for XML are available, a parser that reads from a string or IO object as it + # feels necessary, and a parser that lets you spoon feed it XML. If you want to let Nokogiri + # deal with reading your XML, use the Nokogiri::XML::SAX::Parser. If you want to have fine grain # control over the XML input, use the Nokogiri::XML::SAX::PushParser. module SAX ### - # This class is used for registering types of events you are interested - # in handling. All of the methods on this class are available as - # possible events while parsing an XML document. To register for any - # particular event, just subclass this class and implement the methods - # you are interested in knowing about. + # This class is used for registering types of events you are interested in handling. All of + # the methods on this class are available as possible events while parsing an XML document. To + # register for any particular event, just subclass this class and implement the methods you + # are interested in knowing about. # - # To only be notified about start and end element events, write a class - # like this: + # To only be notified about start and end element events, write a class like this: # # class MyDocument < Nokogiri::XML::SAX::Document # def start_element name, attrs = [] @@ -66,8 +60,8 @@ module SAX # end # end # - # You can use this event handler for any SAX style parser included with - # Nokogiri. See Nokogiri::XML::SAX, and Nokogiri::HTML::SAX. + # You can use this event handler for any SAX style parser included with Nokogiri. See + # Nokogiri::XML::SAX, and Nokogiri::HTML4::SAX. class Document ### # Called when an XML declaration is parsed @@ -129,7 +123,7 @@ def end_element_namespace name, prefix = nil, uri = nil end ### - # Characters read between a tag. This method might be called multiple + # Characters read between a tag. This method might be called multiple # times given one contiguous string of characters. # # +string+ contains the character data diff --git a/lib/nokogiri/xml/xpath.rb b/lib/nokogiri/xml/xpath.rb index e2f30fe0e4..4caba97f8d 100644 --- a/lib/nokogiri/xml/xpath.rb +++ b/lib/nokogiri/xml/xpath.rb @@ -1,9 +1,9 @@ # frozen_string_literal: true -require 'nokogiri/xml/xpath/syntax_error' - module Nokogiri module XML module XPath end end end + +require_relative "xpath/syntax_error" diff --git a/lib/nokogiri/xslt.rb b/lib/nokogiri/xslt.rb index 50cdcaf9ae..87e87839c5 100644 --- a/lib/nokogiri/xslt.rb +++ b/lib/nokogiri/xslt.rb @@ -1,6 +1,4 @@ # frozen_string_literal: true -require 'nokogiri/xslt/stylesheet' - module Nokogiri class << self ### @@ -22,32 +20,32 @@ module XSLT class << self ### # Parse the stylesheet in +string+, register any +modules+ - def parse string, modules = {} + def parse(string, modules = {}) modules.each do |url, klass| - XSLT.register url, klass + XSLT.register(url, klass) end + doc = XML::Document.parse(string, nil, nil, XML::ParseOptions::DEFAULT_XSLT) if Nokogiri.jruby? - Stylesheet.parse_stylesheet_doc(XML.parse(string), string) + Stylesheet.parse_stylesheet_doc(doc, string) else - Stylesheet.parse_stylesheet_doc(XML.parse(string)) + Stylesheet.parse_stylesheet_doc(doc) end end ### # Quote parameters in +params+ for stylesheet safety - def quote_params params + def quote_params(params) parray = (params.instance_of?(Hash) ? params.to_a.flatten : params).dup - parray.each_with_index do |v,i| - if i % 2 > 0 - parray[i]= - if v =~ /'/ - "concat('#{ v.gsub(/'/, %q{', "'", '}) }')" - else - "'#{v}'"; - end + parray.each_with_index do |v, i| + parray[i] = if i % 2 > 0 + if v =~ /'/ + "concat('#{v.gsub(/'/, %q{', "'", '})}')" + else + "'#{v}'" + end else - parray[i] = v.to_s + v.to_s end end parray.flatten @@ -55,3 +53,5 @@ def quote_params params end end end + +require_relative "xslt/stylesheet" diff --git a/lib/nokogiri/xslt/stylesheet.rb b/lib/nokogiri/xslt/stylesheet.rb index acdd9cb3f1..4d445d7946 100644 --- a/lib/nokogiri/xslt/stylesheet.rb +++ b/lib/nokogiri/xslt/stylesheet.rb @@ -18,7 +18,7 @@ class Stylesheet # Apply an XSLT stylesheet to an XML::Document. # +params+ is an array of strings used as XSLT parameters. # returns serialized document - def apply_to document, params = [] + def apply_to(document, params = []) serialize(transform(document, params)) end end diff --git a/nokogiri.gemspec b/nokogiri.gemspec index cfbd69c50b..9a36de9f96 100644 --- a/nokogiri.gemspec +++ b/nokogiri.gemspec @@ -27,6 +27,9 @@ Gem::Specification.new do |spec| "Akinori MUSHA", "John Shahid", "Karol Bucek", + "Sam Ruby", + "Craig Barnes", + "Stephen Checkoway", "Lars Kanis", "Sergio Arbeo", "Timothy Elliott", @@ -56,11 +59,11 @@ Gem::Specification.new do |spec| "bin/nokogiri", "dependencies.yml", "ext/java/nokogiri/EncodingHandler.java", - "ext/java/nokogiri/HtmlDocument.java", - "ext/java/nokogiri/HtmlElementDescription.java", - "ext/java/nokogiri/HtmlEntityLookup.java", - "ext/java/nokogiri/HtmlSaxParserContext.java", - "ext/java/nokogiri/HtmlSaxPushParser.java", + "ext/java/nokogiri/Html4Document.java", + "ext/java/nokogiri/Html4ElementDescription.java", + "ext/java/nokogiri/Html4EntityLookup.java", + "ext/java/nokogiri/Html4SaxParserContext.java", + "ext/java/nokogiri/Html4SaxPushParser.java", "ext/java/nokogiri/NokogiriService.java", "ext/java/nokogiri/XmlAttr.java", "ext/java/nokogiri/XmlAttributeDecl.java", @@ -144,11 +147,11 @@ Gem::Specification.new do |spec| "ext/java/nokogiri/internals/dom2dtm/DOM2DTMdefaultNamespaceDeclarationNode.java", "ext/nokogiri/depend", "ext/nokogiri/extconf.rb", - "ext/nokogiri/html_document.c", - "ext/nokogiri/html_element_description.c", - "ext/nokogiri/html_entity_lookup.c", - "ext/nokogiri/html_sax_parser_context.c", - "ext/nokogiri/html_sax_push_parser.c", + "ext/nokogiri/html4_document.c", + "ext/nokogiri/html4_element_description.c", + "ext/nokogiri/html4_entity_lookup.c", + "ext/nokogiri/html4_sax_parser_context.c", + "ext/nokogiri/html4_sax_push_parser.c", "ext/nokogiri/libxml2_backwards_compat.c", "ext/nokogiri/nokogiri.c", "ext/nokogiri/nokogiri.h", @@ -178,6 +181,51 @@ Gem::Specification.new do |spec| "ext/nokogiri/xml_text.c", "ext/nokogiri/xml_xpath_context.c", "ext/nokogiri/xslt_stylesheet.c", + "gumbo-parser/CHANGES.md", + "gumbo-parser/Makefile", + "gumbo-parser/THANKS", + "gumbo-parser/src/Makefile", + "gumbo-parser/src/README.md", + "gumbo-parser/src/ascii.c", + "gumbo-parser/src/ascii.h", + "gumbo-parser/src/attribute.c", + "gumbo-parser/src/attribute.h", + "gumbo-parser/src/char_ref.c", + "gumbo-parser/src/char_ref.h", + "gumbo-parser/src/char_ref.rl", + "gumbo-parser/src/error.c", + "gumbo-parser/src/error.h", + "gumbo-parser/src/foreign_attrs.c", + "gumbo-parser/src/foreign_attrs.gperf", + "gumbo-parser/src/gumbo.h", + "gumbo-parser/src/insertion_mode.h", + "gumbo-parser/src/macros.h", + "gumbo-parser/src/parser.c", + "gumbo-parser/src/parser.h", + "gumbo-parser/src/replacement.h", + "gumbo-parser/src/string_buffer.c", + "gumbo-parser/src/string_buffer.h", + "gumbo-parser/src/string_piece.c", + "gumbo-parser/src/svg_attrs.c", + "gumbo-parser/src/svg_attrs.gperf", + "gumbo-parser/src/svg_tags.c", + "gumbo-parser/src/svg_tags.gperf", + "gumbo-parser/src/tag.c", + "gumbo-parser/src/tag_lookup.c", + "gumbo-parser/src/tag_lookup.gperf", + "gumbo-parser/src/tag_lookup.h", + "gumbo-parser/src/token_buffer.c", + "gumbo-parser/src/token_buffer.h", + "gumbo-parser/src/token_type.h", + "gumbo-parser/src/tokenizer.c", + "gumbo-parser/src/tokenizer.h", + "gumbo-parser/src/tokenizer_states.h", + "gumbo-parser/src/utf8.c", + "gumbo-parser/src/utf8.h", + "gumbo-parser/src/util.c", + "gumbo-parser/src/util.h", + "gumbo-parser/src/vector.c", + "gumbo-parser/src/vector.h", "lib/isorelax.jar", "lib/jing.jar", "lib/nekodtd.jar", @@ -194,16 +242,22 @@ Gem::Specification.new do |spec| "lib/nokogiri/css/xpath_visitor.rb", "lib/nokogiri/decorators/slop.rb", "lib/nokogiri/extension.rb", + "lib/nokogiri/gumbo.rb", "lib/nokogiri/html.rb", - "lib/nokogiri/html/builder.rb", - "lib/nokogiri/html/document.rb", - "lib/nokogiri/html/document_fragment.rb", - "lib/nokogiri/html/element_description.rb", - "lib/nokogiri/html/element_description_defaults.rb", - "lib/nokogiri/html/entity_lookup.rb", - "lib/nokogiri/html/sax/parser.rb", - "lib/nokogiri/html/sax/parser_context.rb", - "lib/nokogiri/html/sax/push_parser.rb", + "lib/nokogiri/html4.rb", + "lib/nokogiri/html4/builder.rb", + "lib/nokogiri/html4/document.rb", + "lib/nokogiri/html4/document_fragment.rb", + "lib/nokogiri/html4/element_description.rb", + "lib/nokogiri/html4/element_description_defaults.rb", + "lib/nokogiri/html4/entity_lookup.rb", + "lib/nokogiri/html4/sax/parser.rb", + "lib/nokogiri/html4/sax/parser_context.rb", + "lib/nokogiri/html4/sax/push_parser.rb", + "lib/nokogiri/html5.rb", + "lib/nokogiri/html5/document.rb", + "lib/nokogiri/html5/document_fragment.rb", + "lib/nokogiri/html5/node.rb", "lib/nokogiri/jruby/dependencies.rb", "lib/nokogiri/syntax_error.rb", "lib/nokogiri/version.rb", @@ -263,10 +317,9 @@ Gem::Specification.new do |spec| spec.rdoc_options = ["--main", "README.md"] spec.add_runtime_dependency("racc", "~> 1.4") - spec.add_runtime_dependency("mini_portile2", "~> 2.5.0") unless java_p # keep version in sync with extconf.rb + spec.add_runtime_dependency("mini_portile2", "~> 2.6.1") unless java_p # keep version in sync with extconf.rb spec.add_development_dependency("bundler", "~> 2.2") - spec.add_development_dependency("concourse", "~> 0.41") spec.add_development_dependency("hoe-markdown", "~> 1.4") spec.add_development_dependency("minitest", "~> 5.8") spec.add_development_dependency("minitest-reporters", "~> 1.4") diff --git a/concourse/images/Dockerfile.alpine.erb b/oci-images/nokogiri-test/Dockerfile.alpine.erb similarity index 100% rename from concourse/images/Dockerfile.alpine.erb rename to oci-images/nokogiri-test/Dockerfile.alpine.erb diff --git a/concourse/images/Dockerfile.alpine32.erb b/oci-images/nokogiri-test/Dockerfile.alpine32.erb similarity index 100% rename from concourse/images/Dockerfile.alpine32.erb rename to oci-images/nokogiri-test/Dockerfile.alpine32.erb diff --git a/concourse/images/Dockerfile.bionic.erb b/oci-images/nokogiri-test/Dockerfile.bionic.erb similarity index 100% rename from concourse/images/Dockerfile.bionic.erb rename to oci-images/nokogiri-test/Dockerfile.bionic.erb diff --git a/concourse/images/Dockerfile.bionic32.erb b/oci-images/nokogiri-test/Dockerfile.bionic32.erb similarity index 100% rename from concourse/images/Dockerfile.bionic32.erb rename to oci-images/nokogiri-test/Dockerfile.bionic32.erb diff --git a/concourse/images/Dockerfile.jruby.erb b/oci-images/nokogiri-test/Dockerfile.jruby.erb similarity index 100% rename from concourse/images/Dockerfile.jruby.erb rename to oci-images/nokogiri-test/Dockerfile.jruby.erb diff --git a/concourse/images/Dockerfile.mri.erb b/oci-images/nokogiri-test/Dockerfile.mri.erb similarity index 100% rename from concourse/images/Dockerfile.mri.erb rename to oci-images/nokogiri-test/Dockerfile.mri.erb diff --git a/concourse/images/Dockerfile.truffle.erb b/oci-images/nokogiri-test/Dockerfile.truffle.erb similarity index 100% rename from concourse/images/Dockerfile.truffle.erb rename to oci-images/nokogiri-test/Dockerfile.truffle.erb diff --git a/concourse/images/bundle-install.step b/oci-images/nokogiri-test/bundle-install.step similarity index 100% rename from concourse/images/bundle-install.step rename to oci-images/nokogiri-test/bundle-install.step diff --git a/concourse/images/debian-libxml-et-al.step b/oci-images/nokogiri-test/debian-libxml-et-al.step similarity index 100% rename from concourse/images/debian-libxml-et-al.step rename to oci-images/nokogiri-test/debian-libxml-et-al.step diff --git a/concourse/images/debian-prelude.step b/oci-images/nokogiri-test/debian-prelude.step similarity index 100% rename from concourse/images/debian-prelude.step rename to oci-images/nokogiri-test/debian-prelude.step diff --git a/concourse/images/debian-ruby.step b/oci-images/nokogiri-test/debian-ruby.step similarity index 100% rename from concourse/images/debian-ruby.step rename to oci-images/nokogiri-test/debian-ruby.step diff --git a/concourse/images/debian-valgrind.step b/oci-images/nokogiri-test/debian-valgrind.step similarity index 100% rename from concourse/images/debian-valgrind.step rename to oci-images/nokogiri-test/debian-valgrind.step diff --git a/concourse/images/java-opts.step b/oci-images/nokogiri-test/java-opts.step similarity index 100% rename from concourse/images/java-opts.step rename to oci-images/nokogiri-test/java-opts.step diff --git a/concourse/images/update-bundler.step b/oci-images/nokogiri-test/update-bundler.step similarity index 100% rename from concourse/images/update-bundler.step rename to oci-images/nokogiri-test/update-bundler.step diff --git a/patches/libxml2/0002-Remove-script-macro-support.patch b/patches/libxml2/0001-Remove-script-macro-support.patch similarity index 100% rename from patches/libxml2/0002-Remove-script-macro-support.patch rename to patches/libxml2/0001-Remove-script-macro-support.patch diff --git a/patches/libxml2/0001-Revert-Do-not-URI-escape-in-server-side-includes.patch b/patches/libxml2/0001-Revert-Do-not-URI-escape-in-server-side-includes.patch deleted file mode 100644 index e3ef2f2a68..0000000000 --- a/patches/libxml2/0001-Revert-Do-not-URI-escape-in-server-side-includes.patch +++ /dev/null @@ -1,78 +0,0 @@ -From c5538465c08a8ea248a370bf55bc39cd3385e4af Mon Sep 17 00:00:00 2001 -From: Mike Dalessio <mike.dalessio@gmail.com> -Date: Thu, 29 Mar 2018 14:09:00 -0400 -Subject: [PATCH] Revert "Do not URI escape in server side includes" - -This reverts commit 960f0e275616cadc29671a218d7fb9b69eb35588. ---- - HTMLtree.c | 49 +++++++++++-------------------------------------- - 1 file changed, 11 insertions(+), 38 deletions(-) - -diff --git a/HTMLtree.c b/HTMLtree.c -index 2fd0c9c..67160c5 100644 ---- a/HTMLtree.c -+++ b/HTMLtree.c -@@ -717,49 +717,22 @@ htmlAttrDumpOutput(xmlOutputBufferPtr buf, xmlDocPtr doc, xmlAttrPtr cur, - (!xmlStrcasecmp(cur->name, BAD_CAST "src")) || - ((!xmlStrcasecmp(cur->name, BAD_CAST "name")) && - (!xmlStrcasecmp(cur->parent->name, BAD_CAST "a"))))) { -+ xmlChar *escaped; - xmlChar *tmp = value; -- /* xmlURIEscapeStr() escapes '"' so it can be safely used. */ -- xmlBufCCat(buf->buffer, "\""); - - while (IS_BLANK_CH(*tmp)) tmp++; - -- /* URI Escape everything, except server side includes. */ -- for ( ; ; ) { -- xmlChar *escaped; -- xmlChar endChar; -- xmlChar *end = NULL; -- xmlChar *start = (xmlChar *)xmlStrstr(tmp, BAD_CAST "<!--"); -- if (start != NULL) { -- end = (xmlChar *)xmlStrstr(tmp, BAD_CAST "-->"); -- if (end != NULL) { -- *start = '\0'; -- } -- } -- -- /* Escape the whole string, or until start (set to '\0'). */ -- escaped = xmlURIEscapeStr(tmp, BAD_CAST"@/:=?;#%&,+"); -- if (escaped != NULL) { -- xmlBufCat(buf->buffer, escaped); -- xmlFree(escaped); -- } else { -- xmlBufCat(buf->buffer, tmp); -- } -- -- if (end == NULL) { /* Everything has been written. */ -- break; -- } -- -- /* Do not escape anything within server side includes. */ -- *start = '<'; /* Restore the first character of "<!--". */ -- end += 3; /* strlen("-->") */ -- endChar = *end; -- *end = '\0'; -- xmlBufCat(buf->buffer, start); -- *end = endChar; -- tmp = end; -+ /* -+ * the < and > have already been escaped at the entity level -+ * And doing so here breaks server side includes -+ */ -+ escaped = xmlURIEscapeStr(tmp, BAD_CAST"@/:=?;#%&,+<>"); -+ if (escaped != NULL) { -+ xmlBufWriteQuotedString(buf->buffer, escaped); -+ xmlFree(escaped); -+ } else { -+ xmlBufWriteQuotedString(buf->buffer, value); - } -- -- xmlBufCCat(buf->buffer, "\""); - } else { - xmlBufWriteQuotedString(buf->buffer, value); - } --- -2.9.5 - diff --git a/patches/libxml2/0003-Update-entities-to-remove-handling-of-ssi.patch b/patches/libxml2/0002-Update-entities-to-remove-handling-of-ssi.patch similarity index 100% rename from patches/libxml2/0003-Update-entities-to-remove-handling-of-ssi.patch rename to patches/libxml2/0002-Update-entities-to-remove-handling-of-ssi.patch diff --git a/patches/libxml2/0004-libxml2.la-is-in-top_builddir.patch b/patches/libxml2/0003-libxml2.la-is-in-top_builddir.patch similarity index 94% rename from patches/libxml2/0004-libxml2.la-is-in-top_builddir.patch rename to patches/libxml2/0003-libxml2.la-is-in-top_builddir.patch index 30c81dc77c..b6add83769 100644 --- a/patches/libxml2/0004-libxml2.la-is-in-top_builddir.patch +++ b/patches/libxml2/0003-libxml2.la-is-in-top_builddir.patch @@ -16,7 +16,7 @@ index cf96d41..1372d8b 100644 } -libxml2.la: $(libxml2_la_OBJECTS) $(libxml2_la_DEPENDENCIES) $(EXTRA_libxml2_la_DEPENDENCIES) -+$(top_builddir)/libxml2.la: $(libxml2_la_OBJECTS) $(libxml2_la_DEPENDENCIES) $(EXTRA_libxml2_la_DEPENDENCIES) ++$(top_builddir)/libxml2.la: $(libxml2_la_OBJECTS) $(libxml2_la_DEPENDENCIES) $(EXTRA_libxml2_la_DEPENDENCIES) $(AM_V_CCLD)$(libxml2_la_LINK) -rpath $(libdir) $(libxml2_la_OBJECTS) $(libxml2_la_LIBADD) $(LIBS) testdso.la: $(testdso_la_OBJECTS) $(testdso_la_DEPENDENCIES) $(EXTRA_testdso_la_DEPENDENCIES) diff --git a/patches/libxml2/0008-use-glibc-strlen.patch b/patches/libxml2/0004-use-glibc-strlen.patch similarity index 100% rename from patches/libxml2/0008-use-glibc-strlen.patch rename to patches/libxml2/0004-use-glibc-strlen.patch diff --git a/patches/libxml2/0005-Fix-infinite-loop-in-xmlStringLenDecodeEntities.patch b/patches/libxml2/0005-Fix-infinite-loop-in-xmlStringLenDecodeEntities.patch deleted file mode 100644 index 56026fb569..0000000000 --- a/patches/libxml2/0005-Fix-infinite-loop-in-xmlStringLenDecodeEntities.patch +++ /dev/null @@ -1,32 +0,0 @@ -From 0e1a49c8907645d2e155f0d89d4d9895ac5112b5 Mon Sep 17 00:00:00 2001 -From: Zhipeng Xie <xiezhipeng1@huawei.com> -Date: Thu, 12 Dec 2019 17:30:55 +0800 -Subject: [PATCH] Fix infinite loop in xmlStringLenDecodeEntities - -When ctxt->instate == XML_PARSER_EOF,xmlParseStringEntityRef -return NULL which cause a infinite loop in xmlStringLenDecodeEntities - -Found with libFuzzer. - -Signed-off-by: Zhipeng Xie <xiezhipeng1@huawei.com> ---- - parser.c | 3 ++- - 1 file changed, 2 insertions(+), 1 deletion(-) - -diff --git a/parser.c b/parser.c -index d1c3196..a34bb6c 100644 ---- a/parser.c -+++ b/parser.c -@@ -2646,7 +2646,8 @@ xmlStringLenDecodeEntities(xmlParserCtxtPtr ctxt, const xmlChar *str, int len, - else - c = 0; - while ((c != 0) && (c != end) && /* non input consuming loop */ -- (c != end2) && (c != end3)) { -+ (c != end2) && (c != end3) && -+ (ctxt->instate != XML_PARSER_EOF)) { - - if (c == 0) break; - if ((c == '&') && (str[1] == '#')) { --- -2.17.1 - diff --git a/patches/libxml2/0009-avoid-isnan-isinf.patch b/patches/libxml2/0005-avoid-isnan-isinf.patch similarity index 94% rename from patches/libxml2/0009-avoid-isnan-isinf.patch rename to patches/libxml2/0005-avoid-isnan-isinf.patch index ffd82cba6c..38a867522b 100644 --- a/patches/libxml2/0009-avoid-isnan-isinf.patch +++ b/patches/libxml2/0005-avoid-isnan-isinf.patch @@ -51,7 +51,7 @@ diff --git a/xpath.c b/xpath.c index 9f64ab9..5b6d999 100644 --- a/xpath.c +++ b/xpath.c -@@ -509,11 +509,7 @@ xmlXPathInit(void) { +@@ -515,11 +515,7 @@ xmlXPathInit(void) { */ int xmlXPathIsNaN(double val) { @@ -63,16 +63,16 @@ index 9f64ab9..5b6d999 100644 } /** -@@ -524,15 +520,11 @@ xmlXPathIsNaN(double val) { +@@ -530,15 +530,11 @@ xmlXPathIsNaN(double val) { */ int xmlXPathIsInf(double val) { -#ifdef isinf - return isinf(val) ? (val > 0 ? 1 : -1) : 0; -#else - if (val >= INFINITY) + if (val >= xmlXPathPINF) return 1; - if (val <= -INFINITY) + if (val <= -xmlXPathPINF) return -1; return 0; -#endif diff --git a/patches/libxml2/0006-htmlParseComment-treat-as-if-it-closed-the-comment.patch b/patches/libxml2/0006-htmlParseComment-treat-as-if-it-closed-the-comment.patch deleted file mode 100644 index 5a687dfba1..0000000000 --- a/patches/libxml2/0006-htmlParseComment-treat-as-if-it-closed-the-comment.patch +++ /dev/null @@ -1,73 +0,0 @@ -From 4f51a6d2b1755ce5b36c524c215aad70d864ac1d Mon Sep 17 00:00:00 2001 -From: Mike Dalessio <mike.dalessio@gmail.com> -Date: Mon, 3 Aug 2020 17:36:05 -0400 -Subject: [PATCH 1/2] htmlParseComment: treat `--!>` as if it closed the - comment - -See guidance provided on incorrectly-closed comments here: - -https://html.spec.whatwg.org/multipage/parsing.html#parse-error-incorrectly-closed-comment ---- - HTMLparser.c | 28 ++++++++++++++++++++-------- - 1 file changed, 20 insertions(+), 8 deletions(-) - -diff --git a/HTMLparser.c b/HTMLparser.c -index 7b6d689..4d43479 100644 ---- a/HTMLparser.c -+++ b/HTMLparser.c -@@ -3300,6 +3300,7 @@ htmlParseComment(htmlParserCtxtPtr ctxt) { - int q, ql; - int r, rl; - int cur, l; -+ int next, nl; - xmlParserInputState state; - - /* -@@ -3332,6 +3333,21 @@ htmlParseComment(htmlParserCtxtPtr ctxt) { - while (IS_CHAR(cur) && - ((cur != '>') || - (r != '-') || (q != '-'))) { -+ NEXTL(l); -+ next = CUR_CHAR(nl); -+ if (next == 0) { -+ SHRINK; -+ GROW; -+ next = CUR_CHAR(nl); -+ } -+ -+ if ((q == '-') && (r == '-') && (cur == '!') && (next == '>')) { -+ htmlParseErr(ctxt, XML_ERR_COMMENT_NOT_FINISHED, -+ "Comment incorrectly closed by '--!>'", NULL, NULL); -+ cur = '>'; -+ break; -+ } -+ - if (len + 5 >= size) { - xmlChar *tmp; - -@@ -3345,18 +3361,14 @@ htmlParseComment(htmlParserCtxtPtr ctxt) { - } - buf = tmp; - } -- COPY_BUF(ql,buf,len,q); -+ COPY_BUF(ql,buf,len,q); -+ - q = r; - ql = rl; - r = cur; - rl = l; -- NEXTL(l); -- cur = CUR_CHAR(l); -- if (cur == 0) { -- SHRINK; -- GROW; -- cur = CUR_CHAR(l); -- } -+ cur = next; -+ l = nl; - } - buf[len] = 0; - if (IS_CHAR(cur)) { --- -2.25.1 - diff --git a/patches/libxml2/0011-update-automake-files-for-arm64.patch b/patches/libxml2/0006-update-automake-files-for-arm64.patch similarity index 100% rename from patches/libxml2/0011-update-automake-files-for-arm64.patch rename to patches/libxml2/0006-update-automake-files-for-arm64.patch diff --git a/patches/libxml2/0007-Fix-XPath-recursion-limit.patch b/patches/libxml2/0007-Fix-XPath-recursion-limit.patch new file mode 100644 index 0000000000..11342b33d8 --- /dev/null +++ b/patches/libxml2/0007-Fix-XPath-recursion-limit.patch @@ -0,0 +1,31 @@ +From 3e1aad4fe584747fd7d17cc7b2863a78e2d21a77 Mon Sep 17 00:00:00 2001 +From: Nick Wellnhofer <wellnhofer@aevum.de> +Date: Wed, 2 Jun 2021 17:31:49 +0200 +Subject: [PATCH] Fix XPath recursion limit + +Fix accounting of recursion depth when parsing XPath expressions. + +This silly bug introduced in commit 804c5297 could lead to spurious +errors when parsing larger expressions or XSLT documents. + +Should fix #264. +--- + xpath.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/xpath.c b/xpath.c +index 7497ba0..1aa2f1a 100644 +--- a/xpath.c ++++ b/xpath.c +@@ -10983,7 +10983,7 @@ xmlXPathCompileExpr(xmlXPathParserContextPtr ctxt, int sort) { + } + + if (xpctxt != NULL) +- xpctxt->depth -= 1; ++ xpctxt->depth -= 10; + } + + /** +-- +2.31.0 + diff --git a/patches/libxml2/0007-use-new-htmlParseLookupCommentEnd-to-find-comment-en.patch b/patches/libxml2/0007-use-new-htmlParseLookupCommentEnd-to-find-comment-en.patch deleted file mode 100644 index 3ba50bff8e..0000000000 --- a/patches/libxml2/0007-use-new-htmlParseLookupCommentEnd-to-find-comment-en.patch +++ /dev/null @@ -1,103 +0,0 @@ -From b20d746fa7cbb74716171bc49d836af99927e41e Mon Sep 17 00:00:00 2001 -From: Mike Dalessio <mike.dalessio@gmail.com> -Date: Sun, 11 Oct 2020 14:15:37 -0400 -Subject: [PATCH 2/2] use new htmlParseLookupCommentEnd to find comment ends - -Note that the caret in error messages generated during comment parsing -may have moved by one byte. - -See guidance provided on incorrectly-closed comments here: - -https://html.spec.whatwg.org/multipage/parsing.html#parse-error-incorrectly-closed-comment ---- - HTMLparser.c | 46 +++++++++++++++++++++++++++++++++++++--------- - 1 file changed, 37 insertions(+), 9 deletions(-) - -diff --git a/HTMLparser.c b/HTMLparser.c -index 4d43479..000dc3d 100644 ---- a/HTMLparser.c -+++ b/HTMLparser.c -@@ -5331,6 +5331,39 @@ htmlParseLookupChars(htmlParserCtxtPtr ctxt, const xmlChar * stop, - return (-1); - } - -+/** -+ * htmlParseLookupCommentEnd: -+ * @ctxt: an HTML parser context -+ * -+ * Try to find a comment end tag in the input stream -+ * The search includes "-->" as well as WHATWG-recommended incorrectly-closed tags. -+ * (See https://html.spec.whatwg.org/multipage/parsing.html#parse-error-incorrectly-closed-comment) -+ * This function has a side effect of (possibly) incrementing ctxt->checkIndex -+ * to avoid rescanning sequences of bytes, it DOES change the state of the -+ * parser, do not use liberally. -+ * This wraps to htmlParseLookupSequence() -+ * -+ * Returns the index to the current parsing point if the full sequence is available, -1 otherwise. -+ */ -+static int -+htmlParseLookupCommentEnd(htmlParserCtxtPtr ctxt) -+{ -+ int mark = 0; -+ int cur = CUR_PTR - BASE_PTR; -+ -+ while (mark >= 0) { -+ mark = htmlParseLookupSequence(ctxt, '-', '-', 0, 1, 1); -+ if ((mark < 0) || -+ (NXT(mark+2) == '>') || -+ ((NXT(mark+2) == '!') && (NXT(mark+3) == '>'))) { -+ return mark; -+ } -+ ctxt->checkIndex = cur + mark + 1; -+ } -+ return mark; -+} -+ -+ - /** - * htmlParseTryOrFinish: - * @ctxt: an HTML parser context -@@ -5507,8 +5540,7 @@ htmlParseTryOrFinish(htmlParserCtxtPtr ctxt, int terminate) { - cur = in->cur[0]; - if ((cur == '<') && (next == '!') && - (in->cur[2] == '-') && (in->cur[3] == '-')) { -- if ((!terminate) && -- (htmlParseLookupSequence(ctxt, '-', '-', '>', 1, 1) < 0)) -+ if ((!terminate) && (htmlParseLookupCommentEnd(ctxt) < 0)) - goto done; - #ifdef DEBUG_PUSH - xmlGenericError(xmlGenericErrorContext, -@@ -5567,8 +5599,7 @@ htmlParseTryOrFinish(htmlParserCtxtPtr ctxt, int terminate) { - next = in->cur[1]; - if ((cur == '<') && (next == '!') && - (in->cur[2] == '-') && (in->cur[3] == '-')) { -- if ((!terminate) && -- (htmlParseLookupSequence(ctxt, '-', '-', '>', 1, 1) < 0)) -+ if ((!terminate) && (htmlParseLookupCommentEnd(ctxt) < 0)) - goto done; - #ifdef DEBUG_PUSH - xmlGenericError(xmlGenericErrorContext, -@@ -5614,8 +5645,7 @@ htmlParseTryOrFinish(htmlParserCtxtPtr ctxt, int terminate) { - next = in->cur[1]; - if ((cur == '<') && (next == '!') && - (in->cur[2] == '-') && (in->cur[3] == '-')) { -- if ((!terminate) && -- (htmlParseLookupSequence(ctxt, '-', '-', '>', 1, 1) < 0)) -+ if ((!terminate) && (htmlParseLookupCommentEnd(ctxt) < 0)) - goto done; - #ifdef DEBUG_PUSH - xmlGenericError(xmlGenericErrorContext, -@@ -5871,9 +5901,7 @@ htmlParseTryOrFinish(htmlParserCtxtPtr ctxt, int terminate) { - htmlParseDocTypeDecl(ctxt); - } else if ((cur == '<') && (next == '!') && - (in->cur[2] == '-') && (in->cur[3] == '-')) { -- if ((!terminate) && -- (htmlParseLookupSequence( -- ctxt, '-', '-', '>', 1, 1) < 0)) -+ if ((!terminate) && (htmlParseLookupCommentEnd(ctxt) < 0)) - goto done; - #ifdef DEBUG_PUSH - xmlGenericError(xmlGenericErrorContext, --- -2.25.1 - diff --git a/patches/libxml2/0010-parser.c-shrink-the-input-buffer-when-appropriate.patch b/patches/libxml2/0010-parser.c-shrink-the-input-buffer-when-appropriate.patch deleted file mode 100644 index d3d9ad46a4..0000000000 --- a/patches/libxml2/0010-parser.c-shrink-the-input-buffer-when-appropriate.patch +++ /dev/null @@ -1,70 +0,0 @@ -From ca565c1edef9a455453fa8564270cc9c5813e1b9 Mon Sep 17 00:00:00 2001 -From: Mike Dalessio <mike.dalessio@gmail.com> -Date: Sun, 31 Jan 2021 09:53:56 -0500 -Subject: [PATCH] parser.c: shrink the input buffer when appropriate - -Fixes GNOME/libxml2#200 - -Also see discussions at: -- GNOME/libxml2#192 -- https://gitlab.gnome.org/nwellnhof/libxml2/-/commit/99bda1e -- https://github.com/sparklemotion/nokogiri/issues/2132 ---- - parser.c | 6 ++++++ - 1 file changed, 6 insertions(+) - -diff --git a/parser.c b/parser.c -index a7bdc7f..efde672 100644 ---- a/parser.c -+++ b/parser.c -@@ -4204,6 +4204,7 @@ xmlParseSystemLiteral(xmlParserCtxtPtr ctxt) { - } - count++; - if (count > 50) { -+ SHRINK; - GROW; - count = 0; - if (ctxt->instate == XML_PARSER_EOF) { -@@ -4291,6 +4292,7 @@ xmlParsePubidLiteral(xmlParserCtxtPtr ctxt) { - buf[len++] = cur; - count++; - if (count > 50) { -+ SHRINK; - GROW; - count = 0; - if (ctxt->instate == XML_PARSER_EOF) { -@@ -4571,6 +4573,7 @@ xmlParseCharDataComplex(xmlParserCtxtPtr ctxt, int cdata) { - } - count++; - if (count > 50) { -+ SHRINK; - GROW; - count = 0; - if (ctxt->instate == XML_PARSER_EOF) -@@ -4776,6 +4779,7 @@ xmlParseCommentComplex(xmlParserCtxtPtr ctxt, xmlChar *buf, - - count++; - if (count > 50) { -+ SHRINK; - GROW; - count = 0; - if (ctxt->instate == XML_PARSER_EOF) { -@@ -5186,6 +5190,7 @@ xmlParsePI(xmlParserCtxtPtr ctxt) { - } - count++; - if (count > 50) { -+ SHRINK; - GROW; - if (ctxt->instate == XML_PARSER_EOF) { - xmlFree(buf); -@@ -9783,6 +9788,7 @@ xmlParseCDSect(xmlParserCtxtPtr ctxt) { - sl = l; - count++; - if (count > 50) { -+ SHRINK; - GROW; - if (ctxt->instate == XML_PARSER_EOF) { - xmlFree(buf); --- -2.25.1 - diff --git a/patches/libxslt/0002-Fix-xml2-config-check-in-configure-script.patch b/patches/libxslt/0002-Fix-xml2-config-check-in-configure-script.patch new file mode 100644 index 0000000000..a882149aa3 --- /dev/null +++ b/patches/libxslt/0002-Fix-xml2-config-check-in-configure-script.patch @@ -0,0 +1,19 @@ +From: Nick Wellnhofer <wellnhofer@aevum.de> +Date: Fri, 15 Nov 2019 11:53:11 +0100 +Subject: [PATCH] Fix xml2-config check in configure script + +A 'print' option has never been supported. After a recent change to +libxml2, invalid options cause xml2-config to fail. +diff --git a/configure b/configure +index c63adc5..6061227 100755 +--- a/configure ++++ b/configure +@@ -14860,7 +14860,7 @@ PKG_CONFIG=$_save_PKG_CONFIG + fi + + +-if test "x$LIBXML_LIBS" = "x" && ${XML_CONFIG} --libs print > /dev/null 2>&1 ++if test "x$LIBXML_LIBS" = "x" && ${XML_CONFIG} --libs > /dev/null 2>&1 + then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for libxml libraries >= $LIBXML_REQUIRED_VERSION" >&5 + $as_echo_n "checking for libxml libraries >= $LIBXML_REQUIRED_VERSION... " >&6; } diff --git a/rakelib/cext-depend.rake b/rakelib/cext-depend.rake index d8247a8c6f..da02510f18 100644 --- a/rakelib/cext-depend.rake +++ b/rakelib/cext-depend.rake @@ -1,3 +1,4 @@ +# frozen_string_literal: true require "set" namespace "cext" do @@ -8,8 +9,11 @@ namespace "cext" do ext_dir = File.dirname(extconf) Dir.chdir(ext_dir) do puts "(in #{ext_dir})" - File.exist?("depend") or FileUtils.touch("depend") + File.exist?("depend") || FileUtils.touch("depend") sh "makedepend -f depend -Y -I. *.c 2> /dev/null" + dep = File.read("depend") + dep.gsub!(%r{ \./}, " $(srcdir)/") + File.open("depend", "w") { |f| f.write(dep) } end end end diff --git a/rakelib/check-manifest.rake b/rakelib/check-manifest.rake index d5eda72380..ff4c7e7afb 100644 --- a/rakelib/check-manifest.rake +++ b/rakelib/check-manifest.rake @@ -12,10 +12,11 @@ task :check_manifest do .github .vagrant .yardoc - concourse coverage doc gems + nokogumbo-import + oci-images patches pkg ports @@ -32,9 +33,8 @@ task :check_manifest do .cross_rubies .editorconfig .gitignore + .gitmodules .yardopts - appveyor.yml - nokogiri.gemspec CHANGELOG.md CODE_OF_CONDUCT.md CONTRIBUTING.md @@ -44,8 +44,13 @@ task :check_manifest do SECURITY.md STANDARD_RESPONSES.md Vagrantfile - [0-9]+-.* - [a-z.]+.(log|out) + [a-z]*.{log,out} + [0-9]* + appveyor.yml + gumbo-parser/test/* + lib/nokogiri/**/nokogiri.{jar,so} + lib/nokogiri/nokogiri.{jar,so} + nokogiri.gemspec ] intended_directories = Dir.children(".") @@ -54,10 +59,11 @@ task :check_manifest do intended_files = Dir.children(".") .select { |filename| File.file?(filename) } - .reject { |filename| ignore_files.any? { |ig| File.fnmatch?(ig, filename) } } + .reject { |filename| ignore_files.any? { |ig| File.fnmatch?(ig, filename, File::FNM_EXTGLOB) } } intended_files += Dir.glob(intended_directories.map { |d| File.join(d, "/**/*") }) .select { |filename| File.file?(filename) } + .reject { |filename| ignore_files.any? { |ig| File.fnmatch?(ig, filename, File::FNM_EXTGLOB) } } .sort spec_files = raw_gemspec.files.sort diff --git a/rakelib/clean.rake b/rakelib/clean.rake index 2ea8b962a0..098acde375 100644 --- a/rakelib/clean.rake +++ b/rakelib/clean.rake @@ -1,6 +1,5 @@ require "rake/clean" CLEAN.add( - "concourse/images/*.generated", "coverage", "ext/nokogiri/include", "lib/nokogiri/[0-9].[0-9]", diff --git a/rakelib/concourse.rake b/rakelib/concourse.rake deleted file mode 100644 index ae2ec124f2..0000000000 --- a/rakelib/concourse.rake +++ /dev/null @@ -1,7 +0,0 @@ -require "concourse" - -Concourse.new("nokogiri", fly_target: "ci", format: true) do |c| - c.add_pipeline "nokogiri", "nokogiri.yml", ytt: true - c.add_pipeline "nokogiri-pr", "nokogiri-pr.yml", ytt: true - c.add_pipeline "nokogiri-truffleruby", "nokogiri-truffleruby.yml", ytt: true -end diff --git a/rakelib/css-generate.rake b/rakelib/css-generate.rake index 80f3d67c95..0f436bcfea 100644 --- a/rakelib/css-generate.rake +++ b/rakelib/css-generate.rake @@ -22,5 +22,3 @@ end task "compile" => "css:generate" task "check_manifest" => "css:generate" - -task "generate" => "css:generate" # backwards compatibility, see concourse/tasks/rake-test/run.sh diff --git a/rakelib/docker.rake b/rakelib/docker.rake index 054558c04a..da4c8711a9 100644 --- a/rakelib/docker.rake +++ b/rakelib/docker.rake @@ -2,7 +2,14 @@ # docker docker docker # namespace "docker" do - IMAGE_DIR = "concourse/images" + IMAGE_DIR = "oci-images/nokogiri-test" + RUBIES = { + :mri=>["2.5", "2.6", "2.7", "3.0"], + :jruby=>["9.2"], + :rbx=>["latest"], + :windows=>["2.3", "2.4", "2.5", "2.6"], + :truffle=>["stable", "nightly"], + } def docker_tag_for(engine, version = nil) [engine, version].compact.join("-") @@ -19,9 +26,9 @@ namespace "docker" do def docker_files_each Dir[File.join(IMAGE_DIR, "Dockerfile.*.erb")].each do |template_path| tag_or_engine = File.basename(template_path).gsub(/Dockerfile\.(.*)\.erb/, '\1').to_sym - if Concourse::RUBIES.keys.include?(tag_or_engine) + if RUBIES.keys.include?(tag_or_engine) # engine - Concourse::RUBIES[tag_or_engine].each do |version| + RUBIES[tag_or_engine].each do |version| dockerfile_path = docker_file_for(tag_or_engine, version) yield File.read(template_path), dockerfile_path, version, docker_image_for(tag_or_engine, version) end @@ -35,6 +42,7 @@ namespace "docker" do desc "Generate Dockerfiles" task "generate" do + require "erb" docker_files_each do |template, dockerfile_path, version, _| puts "writing #{dockerfile_path} ..." File.open(dockerfile_path, "w") do |dockerfile| @@ -76,3 +84,5 @@ end desc "Build and push a docker image for testing" task "docker" => ["docker:generate", "docker:pull", "docker:build", "docker:push"] + +CLEAN.add("oci-images/nokogiri-test/*.generated") diff --git a/rakelib/extensions.rake b/rakelib/extensions.rake index bc5067e3f6..1fd24d4973 100644 --- a/rakelib/extensions.rake +++ b/rakelib/extensions.rake @@ -307,7 +307,9 @@ if java? jruby_home = RbConfig::CONFIG['prefix'] jars = ["#{jruby_home}/lib/jruby.jar"] + FileList['lib/*.jar'] + # Keep the extension C files because they have docstrings (and Java files don't) ext.gem_spec.files.reject! { |path| File.fnmatch?("ext/nokogiri/*.h", path) } + ext.gem_spec.files.reject! { |path| File.fnmatch?("gumbo-parser/**/*", path) } ext.ext_dir = 'ext/java' ext.lib_dir = 'lib/nokogiri' @@ -326,7 +328,7 @@ else dependencies = YAML.load_file("dependencies.yml") task gem_build_path do - NOKOGIRI_SPEC.files.reject! { |f| f =~ %r{\.(java|jar)$} } + NOKOGIRI_SPEC.files.reject! { |path| File.fnmatch?("**/*.{java,jar}", path, File::FNM_EXTGLOB) } ["libxml2", "libxslt"].each do |lib| version = dependencies[lib]["version"] @@ -345,7 +347,8 @@ else end Rake::ExtensionTask.new("nokogiri", NOKOGIRI_SPEC) do |ext| - ext.gem_spec.files.reject! { |f| f =~ %r{\.(java|jar)$} } + ext.source_pattern = "*.{c,cc,cpp,h}" + ext.gem_spec.files.reject! { |path| File.fnmatch?("**/*.{java,jar}", path, File::FNM_EXTGLOB) } ext.lib_dir = File.join(*['lib', 'nokogiri', ENV['FAT_DIR']].compact) ext.config_options << ENV['EXTOPTS'] @@ -354,6 +357,7 @@ else ext.cross_config_options << "--enable-cross-build" ext.cross_compiling do |spec| spec.files.reject! { |path| File.fnmatch?('ports/*', path) } + spec.files.reject! { |path| File.fnmatch?("gumbo-parser/**/*", path) } spec.dependencies.reject! { |dep| dep.name=='mini_portile2' } # when pre-compiling a native gem, package all the C headers sitting in ext/nokogiri/include diff --git a/rakelib/format.rake b/rakelib/format.rake index 9ed8542bef..306227c5a5 100644 --- a/rakelib/format.rake +++ b/rakelib/format.rake @@ -3,7 +3,7 @@ namespace "format" do def assert_astyle require "mkmf" - find_executable("astyle") || raise("Could not find command 'astyle'") + find_executable0("astyle") || raise("Could not find command 'astyle'") end def astyle_args @@ -13,6 +13,7 @@ namespace "format" do [ # indentation "--indent=spaces=2", + "--indent-switches", # brackets "--style=1tbs", @@ -37,11 +38,11 @@ namespace "format" do end def astyle_c_files - FileList.new("ext/nokogiri/*.[ch]") + FileList.new("ext/**/*.[ch]") end def astyle_java_files - FileList.new("ext/java/nokogiri/**/*.java") + FileList.new("ext/**/*.java") end desc "Format Nokogiri's C code" @@ -60,7 +61,6 @@ namespace "format" do CLEAN.add(astyle_c_files.map { |f| "#{f}.orig" }) CLEAN.add(astyle_java_files.map { |f| "#{f}.orig" }) - CLEAN.add("mkmf.log") # because of find_executable end task "format" => ["format:c", "format:java"] diff --git a/rakelib/gumbo.rake b/rakelib/gumbo.rake new file mode 100644 index 0000000000..b48e5f574b --- /dev/null +++ b/rakelib/gumbo.rake @@ -0,0 +1,29 @@ + +namespace "gumbo" do + gtest_pkg = "gumbo-parser/googletest" + gtest_lib = File.join(gtest_pkg, "make/gtest_main.a") + + file gtest_lib => gtest_pkg do + sh("make -C gumbo-parser/googletest/make gtest_main.a") + end + + file gtest_pkg do + sh(<<~EOF) + curl -L https://github.com/google/googletest/archive/release-1.8.0.tar.gz | \ + tar zxf - --strip-components 1 -C gumbo-parser googletest-release-1.8.0/googletest + EOF + end + + task "test" => gtest_lib do + sh("make -j2 -C gumbo-parser") + end + + task "clean" do + sh("make -j2 -C gumbo-parser clean") + end + + CLOBBER.add(gtest_pkg) +end + +task "gumbo" => "gumbo:test" +task "clean" => "gumbo:clean" diff --git a/rakelib/set-version-to-timestamp.rake b/rakelib/set-version-to-timestamp.rake index 5a028d3d40..a396636458 100644 --- a/rakelib/set-version-to-timestamp.rake +++ b/rakelib/set-version-to-timestamp.rake @@ -1,7 +1,7 @@ # frozen_string_literal: true desc "Temporarily set Nokogiri::VERSION to a unique timestamp" task "set-version-to-timestamp" do - # this task is used by concourse/tasks/gem-test/gem-build.sh + # this task is used by scripts/test-gem-build # to test building, packaging, and installing a Nokogiri gem version_constant_re = /^\s*VERSION\s*=\s*["'](.*)["']$/ diff --git a/rakelib/test.rake b/rakelib/test.rake index b9a64a9743..dcbabec066 100644 --- a/rakelib/test.rake +++ b/rakelib/test.rake @@ -75,6 +75,7 @@ def nokogiri_test_task_configuration(t) t.libs << "test" t.test_files = FileList["test/**/test_*.rb"] t.verbose = true + t.options = "-v" if ENV["CI"] end Rake::TestTask.new do |t| diff --git a/scripts/build-gems b/scripts/build-gems index b5a0f63d32..37b448d26c 100755 --- a/scripts/build-gems +++ b/scripts/build-gems @@ -39,3 +39,7 @@ cp -v pkg/nokogiri-*darwin*.gem gems # test those gem files! $(dirname $0)/test-gem-set gems/*.gem + +pushd gems + ls *.gem | sort | xargs sha256sum +popd diff --git a/scripts/generate-html5-entities b/scripts/generate-html5-entities new file mode 100755 index 0000000000..1d71784583 --- /dev/null +++ b/scripts/generate-html5-entities @@ -0,0 +1,96 @@ +#!/usr/bin/env ruby + +# Copyright 2018 Stephen Checkoway +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The output of this script should be compiled with ragel 6.10 which is the +# current stable version. Version 7 changes some stuff but what, precisely, +# isn't documented yet. + +require 'psych' +require 'open-uri' + +ENTITIES_URL = 'https://html.spec.whatwg.org/entities.json' + +# Modify the json to remove the characters item because it contains surrogate +# pairs that Psych errors on. +entities = URI.parse(ENTITIES_URL).read +entities.gsub!(/, "characters"[^}]*/, '') +entities = Psych.safe_load(entities) + +without_semicolon = {} +with_semicolon = {} +entities.each do |key, val| + if key.end_with?(';') + with_semicolon[key[1..-1]] = val['codepoints'] + else + without_semicolon[key[1..-1]] = val['codepoints'] + end +end + +without_semicolon.each do |entity, codepoints| + puts "#{entity} has the wrong number of codepoints" unless [1,2].include?(codepoints.length) + with = "#{entity};" + unless with_semicolon.key?(with) + puts("#{entity} but not #{with}") + next + end + if codepoints != with_semicolon[with] + puts("#{entity} and #{with} have different codepoints!") + end +end + +puts(<<-EOF) +// Generated from <#{ENTITIES_URL}> by entities.rb +#include "char_ref.h" +%%{ +machine named_char_ref; +named_char_ref := |* +EOF +with_semicolon.each do |entity, codepoints| + without = entity.chop + if without_semicolon.key?(without) + pattern = "'#{without}' . ';'?" + else + pattern = "'#{entity}'" + end + code = codepoints.map.with_index { |cp, idx| sprintf("output[%d]=%#04x", idx, cp) } + puts(" #{pattern} => {#{code.join('; ')}; fbreak;};") +end +puts(<<-EOF) +*|; +write data noerror nofinal noentry noprefix; +}%% + +size_t match_named_char_ref ( + const char *str, + size_t size, + int output[2] +) { + int cs; + int act; + const char *p = str; + const char *pe = str + size; + const char *const eof = pe; + const char *ts; + const char *te; + output[0] = output[1] = kGumboNoChar; + %% write init; + %% write exec; + (void)ts; + (void)act; + size = p - str; + return cs >= %%{ write first_final; }%%? size:0; +} +EOF diff --git a/scripts/test-exported-symbols b/scripts/test-exported-symbols index 7a0ef44552..27c2c97e88 100755 --- a/scripts/test-exported-symbols +++ b/scripts/test-exported-symbols @@ -31,11 +31,13 @@ noko_so = find_that_file("lib/**/nokogiri.so") libxml2_archive = find_that_file("ports/**/libxml2.a") libxslt_archive = find_that_file("ports/**/libxslt.a") libexslt_archive = find_that_file("ports/**/libexslt.a") +libgumbo_archive = find_that_file("tmp/**/libgumbo.a") symbols = external_symbols(noko_so) symbols -= external_symbols(libxml2_archive) symbols -= external_symbols(libxslt_archive) symbols -= external_symbols(libexslt_archive) +symbols -= external_symbols(libgumbo_archive) puts "#{noko_so} exports the following surprising symbols:" symbols.to_a.sort.each do |symbol| diff --git a/scripts/test-gem-build b/scripts/test-gem-build new file mode 100755 index 0000000000..243253ad7b --- /dev/null +++ b/scripts/test-gem-build @@ -0,0 +1,43 @@ +#! /usr/bin/env bash +# +# run as part of CI, see gem-install.yml +# +if [[ $# -lt 2 ]] ; then + echo "usage: $(basename $0) <output_dir> <platform>" + exit 1 +fi + +set -e -u + +OUTPUT_DIR=$1 +BUILD_NATIVE_GEM=$2 + +test -e /etc/os-release && cat /etc/os-release + +if [[ "${XRUBIES:-}" != "" ]] ; then + # normally part of rake-compiler-dock "runas" + . /etc/rubybashrc + ln -s /usr/local/rake-compiler "$HOME"/.rake-compiler + export RAKE_EXTENSION_TASK_NO_NATIVE=true +fi + +set -x + +bundle install --local || bundle install +bundle exec rake set-version-to-timestamp + +if [[ "${BUILD_NATIVE_GEM}" == "ruby" ]] ; then + # TODO we're only compiling so that we retrieve libxml2/libxslt tarballs, we can do better. + bundle exec rake clean compile + bundle exec rake gem +elif [[ "${BUILD_NATIVE_GEM}" == "java" ]] ; then + bundle exec rake java gem +else + bundle exec rake gem:${BUILD_NATIVE_GEM}:builder FORCE_CROSS_COMPILING=true +fi + +./scripts/test-gem-file-contents pkg/nokogiri*.gem + +mkdir -p ${OUTPUT_DIR} +cp -v pkg/nokogiri*.gem ${OUTPUT_DIR} +ls -l ${OUTPUT_DIR}/* diff --git a/scripts/test-gem-file-contents b/scripts/test-gem-file-contents index 08bee5ff73..c567bf40d5 100755 --- a/scripts/test-gem-file-contents +++ b/scripts/test-gem-file-contents @@ -78,7 +78,7 @@ describe File.basename(gemfile) do assert_operator(actual, :>, 60, "expected gemfile to contain more than #{actual} files") end - it "gemspec is a Gem::Specfication" do + it "gemspec is a Gem::Specification" do assert_equal(Gem::Specification, gemspec.class) end end @@ -93,6 +93,19 @@ describe File.basename(gemfile) do end describe "ruby platform" do + it "depends on mini_portile2" do + assert(gemspec.dependencies.find { |d| d.name == "mini_portile2" }) + end + + it "contains ext/nokogiri C and header files" do + assert_operator(gemfile_contents.grep(%r{^ext/nokogiri/.*\.c}).length, :>, 20) + assert_operator(gemfile_contents.grep(%r{^ext/nokogiri/.*\.h}).length, :>, 0) + end + + it "includes C files in extra_rdoc_files" do + assert_operator(gemspec.extra_rdoc_files.grep(%r{ext/nokogiri/.*\.c$}).length, :>, 10) + end + it "contains the port files" do actual_ports = gemfile_contents.grep(%r{^ports/}) assert_equal(1, actual_ports.grep(/libxml2-\d+\.\d+\.\d+\.tar\.gz/).length, @@ -106,31 +119,37 @@ describe File.basename(gemfile) do assert_operator(gemfile_contents.grep(%r{^patches/}).length, :>, 0) end - it "contains ext/nokogiri C and header files" do - assert_operator(gemfile_contents.grep(%r{^ext/nokogiri/.*\.c}).length, :>, 20) - assert_operator(gemfile_contents.grep(%r{^ext/nokogiri/.*\.h}).length, :>, 0) - end - it "does not contain packaged libraries' header files" do # these files are present after installation if the packaged libraries are used assert_empty(gemfile_contents.grep(%r{^ext/nokogiri/include/})) end + it "contains the gumbo parser source code" do + assert_includes(gemfile_contents, "gumbo-parser/src/Makefile") + assert_operator(gemfile_contents.grep(%r{^gumbo-parser/src/.*\.c}).length, :>, 10) + assert_operator(gemfile_contents.grep(%r{^gumbo-parser/src/.*\.h}).length, :>, 10) + end + it "does not contain java files" do assert_empty(gemfile_contents.grep(%r{^ext/java/})) assert_empty(gemfile_contents.grep(/.*\.jar$/)) end + end if gemspec.platform == Gem::Platform::RUBY - it "depends on mini_portile2" do - assert(gemspec.dependencies.find { |d| d.name == "mini_portile2" }) + describe "native platform" do + it "does not depend on mini_portile2" do + refute(gemspec.dependencies.find { |d| d.name == "mini_portile2" }) + end + + it "contains ext/nokogiri C and header files" do + assert_operator(gemfile_contents.grep(%r{^ext/nokogiri/.*\.c}).length, :>, 20) + assert_operator(gemfile_contents.grep(%r{^ext/nokogiri/.*\.h}).length, :>, 20) end it "includes C files in extra_rdoc_files" do assert_operator(gemspec.extra_rdoc_files.grep(%r{ext/nokogiri/.*\.c$}).length, :>, 10) end - end if gemspec.platform == Gem::Platform::RUBY - describe "native platform" do it "does not contain the port files" do assert_empty(gemfile_contents.grep(%r{^ports/})) end @@ -139,30 +158,21 @@ describe File.basename(gemfile) do assert_empty(gemfile_contents.grep(%r{^patches/})) end - it "contains ext/nokogiri C and header files" do - assert_operator(gemfile_contents.grep(%r{^ext/nokogiri/.*\.c}).length, :>, 20) - assert_operator(gemfile_contents.grep(%r{^ext/nokogiri/.*\.h}).length, :>, 20) - end - it "contains packaged libraries' header files" do assert_includes(gemfile_contents, "ext/nokogiri/include/libxml2/libxml/tree.h") assert_includes(gemfile_contents, "ext/nokogiri/include/libxslt/xslt.h") assert_includes(gemfile_contents, "ext/nokogiri/include/libexslt/exslt.h") end + it "does not contain the gumbo parser source code" do + assert_empty(gemfile_contents.grep(%r{^gumbo-parser/src/})) + end + it "does not contain java files" do assert_empty(gemfile_contents.grep(%r{^ext/java/})) assert_empty(gemfile_contents.grep(/.*\.jar$/)) end - it "does not depend on mini_portile2" do - refute(gemspec.dependencies.find { |d| d.name == "mini_portile2" }) - end - - it "includes C files in extra_rdoc_files" do - assert_operator(gemspec.extra_rdoc_files.grep(%r{ext/nokogiri/.*\.c$}).length, :>, 10) - end - it "contains expected shared library files " do native_ruby_versions.each do |version| actual = gemfile_contents.find do |p| @@ -184,15 +194,12 @@ describe File.basename(gemfile) do end if gemspec.platform.is_a?(Gem::Platform) && gemspec.platform.cpu describe "java platform" do - it "does not contain the port files" do - assert_empty(gemfile_contents.grep(%r{^ports/})) - end - - it "does not contain the patch files" do - assert_empty(gemfile_contents.grep(%r{^patches/})) + it "does not depend on mini_portile2" do + refute(gemspec.dependencies.find { |d| d.name == "mini_portile2" }) end it "contains ext/nokogiri C files" do + # Note: we keep the C files because they have docstrings and Java files don't assert_operator(gemfile_contents.grep(%r{^ext/nokogiri/.*\.c}).length, :>, 20) end @@ -200,10 +207,30 @@ describe File.basename(gemfile) do assert_empty(gemfile_contents.grep(%r{^ext/nokogiri/.*\.h})) end + it "includes C files in extra_rdoc_files" do + assert_operator(gemspec.extra_rdoc_files.grep(%r{ext/nokogiri/.*\.c$}).length, :>, 10) + end + + it "does not contain the port files" do + assert_empty(gemfile_contents.grep(%r{^ports/})) + end + + it "does not contain the patch files" do + assert_empty(gemfile_contents.grep(%r{^patches/})) + end + it "does not contain packaged libraries' header files" do assert_empty(gemfile_contents.grep(%r{^ext/nokogiri/include/})) end + it "does not contain the gumbo parser source code" do + assert_empty(gemfile_contents.grep(%r{^gumbo-parser/src/})) + end + + it "contains java source files" do + assert_operator(gemfile_contents.grep(%r{^ext/java/.*\.java}).length, :>, 20) + end + it "contains the java jar files" do actual_jars = gemfile_contents.grep(/.*\.jar$/) expected_jars = [ @@ -221,9 +248,5 @@ describe File.basename(gemfile) do assert_equal(1, actual_jars.grep(%r{/#{jar}\.jar$}).length, "expected to contain #{jar}.jar") end end - - it "does not depend on mini_portile2" do - refute(gemspec.dependencies.find { |d| d.name == "mini_portile2" }) - end end if gemspec.platform == Gem::Platform.new("java") end diff --git a/scripts/test-gem-install b/scripts/test-gem-install new file mode 100755 index 0000000000..c650ef5eeb --- /dev/null +++ b/scripts/test-gem-install @@ -0,0 +1,42 @@ +#! /usr/bin/env bash +# +# run as part of CI, see gem-install.yml +# +if [[ $# -lt 1 ]] ; then + echo "usage: $(basename $0) <gems_dir> [install_flags]" + exit 1 +fi + +GEMS_DIR=$1 +shift +INSTALL_FLAGS=$* + +test -e /etc/os-release && cat /etc/os-release + +set -e -x -u + +pushd $GEMS_DIR + + gemfile=$(ls *.gem | head -n1) + ls -l ${gemfile} + gem install --no-document ${gemfile} -- ${INSTALL_FLAGS} + gem list -d nokogiri + nokogiri -v + +popd + +if [ -n "${BUNDLE_APP_CONFIG:-}" ] ; then + export BUNDLE_CACHE_PATH="${BUNDLE_APP_CONFIG}/cache" +fi + +bundle install --local || bundle install + +rm -rf lib ext # ensure we don't use the local files +rake test + +./scripts/test-gem-installation + +# delete the Gemfile because that's confusing to older versions of rubygems (e.g., bionic32) +rm -f Gemfile Gemfile.lock +./scripts/test-nokogumbo-compatibility 2.0.4 +./scripts/test-nokogumbo-compatibility 2.0.5 diff --git a/scripts/test-gem-installation b/scripts/test-gem-installation index d2b64bdcc5..2e3745577e 100755 --- a/scripts/test-gem-installation +++ b/scripts/test-gem-installation @@ -13,7 +13,7 @@ # this line needs to come before the bundler bit, to assert that we're running against an # already-installed version (and not some other version that bundler/inline might install if it came # first) -gemspec = Gem::Specification.find_by_name('nokogiri') +gemspec = Gem::Specification.find_all_by_name("nokogiri").sort_by(&:version).last raise "could not find installed gem" unless gemspec require "bundler/inline" diff --git a/scripts/test-gem-set b/scripts/test-gem-set index 253a792d44..523cb36aa8 100755 --- a/scripts/test-gem-set +++ b/scripts/test-gem-set @@ -4,7 +4,7 @@ # - test-gem-file-contents # - conditionally, if the local system can do it, test-gem-installation # -source "$HOME/.rvm/scripts/rvm" +eval "$(rbenv init - bash)" set -o errexit set -o pipefail @@ -18,9 +18,10 @@ function remove_all_nokogiris { function install_and_test { gem=$1 if [[ $gem =~ "java" ]] ; then - rvm use jruby + jruby_version=$(rbenv versions --bare | fgrep jruby | sort | tail -1) + rbenv shell ${jruby_version} else - rvm use default + rbenv shell --unset fi remove_all_nokogiris gem install --local $gem @@ -35,7 +36,7 @@ function install_and_test { gems=$* -rvm use default +rbenv shell --unset for gem in $gems ; do ./scripts/test-gem-file-contents $gem diff --git a/scripts/test-nokogumbo-compatibility b/scripts/test-nokogumbo-compatibility index 3c57b31df7..6ccd53f5d4 100755 --- a/scripts/test-nokogumbo-compatibility +++ b/scripts/test-nokogumbo-compatibility @@ -17,23 +17,31 @@ end # this line needs to come before the bundler bit, to assert that we're running against an # already-installed version (and not some other version that bundler/inline might install if it came # first) -gemspec = Gem::Specification.find_by_name('nokogiri') +gemspec = Gem::Specification.find_all_by_name("nokogiri").sort_by(&:version).last raise "could not find installed gem" unless gemspec require "bundler/inline" +nokogumbo_version = if ARGV[0] =~ /\d+\.\d+\.\d+/ + ARGV[0] +end + gemfile(true) do source "https://rubygems.org" gem "minitest" gem "minitest-reporters" gem "nokogiri", "=#{gemspec.version}" - gem "nokogumbo" + if nokogumbo_version + gem "nokogumbo", "=#{nokogumbo_version}" + else + gem "nokogumbo" + end end -nokogumbo_gemspec = Gem::Specification.find_by_name('nokogumbo') +nokogumbo_gemspec = Gem::Specification.find_by_name("nokogumbo") -require 'nokogumbo' -require 'yaml' +require "nokogumbo" +require "yaml" if ARGV.include?("-v") puts "---------- Nokogiri version info ----------" @@ -54,6 +62,8 @@ Minitest::Reporters.use!([Minitest::Reporters::SpecReporter.new]) puts "Testing #{gemspec.full_name} installed in #{gemspec.base_dir}" describe gemspec.full_name do it "works" do - assert(Nokogiri::HTML5::Document.parse("<div>ahoy</div>")) + doc = Nokogiri::HTML5::Document.parse("<div>ahoy</div>") + assert(doc) + assert_equal("ahoy", doc.at_css("/html/body/div").text) end end diff --git a/suppressions/nokogiri_ruby-3.1.supp b/suppressions/nokogiri_ruby-3.1.supp new file mode 100644 index 0000000000..351a358f69 --- /dev/null +++ b/suppressions/nokogiri_ruby-3.1.supp @@ -0,0 +1,69 @@ +{ + ruby 3.1 startup + Memcheck:Addr4 + fun:iseq_peephole_optimize + fun:iseq_optimize* + fun:iseq_setup_insn + fun:iseq_setup_insn + fun:rb_iseq_compile_node + fun:rb_iseq_new_with_opt + fun:new_child_iseq + ... + # fun:iseq_compile_each0 + # fun:iseq_compile_each + # fun:iseq_compile_each0 + # fun:iseq_compile_each.constprop.1 + # fun:rb_iseq_compile_node + # fun:rb_iseq_new_with_opt + # fun:new_child_iseq + # fun:iseq_compile_each0 + # fun:iseq_compile_each + # fun:iseq_compile_each0 + # fun:iseq_compile_each.constprop.1 + # fun:rb_iseq_compile_node + # fun:rb_iseq_new_with_opt + # fun:rb_iseq_new_top + # fun:load_iseq_eval + # fun:require_internal + # fun:rb_require_string + # fun:vm_call_cfunc_with_frame + # fun:vm_sendish + # fun:vm_exec_core + # fun:rb_vm_exec + # fun:load_iseq_eval + # fun:require_internal + # fun:rb_require_string + # fun:vm_call_cfunc_with_frame + # fun:vm_call_method_each_type + # fun:vm_call_alias + # fun:vm_sendish + # fun:vm_exec_core + # fun:rb_vm_exec + # fun:load_iseq_eval + # fun:require_internal + # fun:rb_require_string + # fun:vm_call_cfunc_with_frame + # fun:vm_call_method_each_type + # fun:vm_call_alias + # fun:vm_sendish + # fun:vm_exec_core + # fun:rb_vm_exec + # fun:load_iseq_eval + # fun:require_internal + # fun:rb_require_string + # fun:vm_call_cfunc_with_frame + # fun:vm_call_method_each_type + # fun:vm_call_alias + # fun:vm_sendish + # fun:vm_exec_core + # fun:rb_vm_exec + # fun:load_iseq_eval + # fun:require_internal + # fun:rb_require_string + # fun:vm_call_cfunc_with_frame + # fun:vm_call_method_each_type + # fun:vm_call_alias + # fun:vm_sendish + # fun:vm_exec_core + # fun:rb_vm_exec +} diff --git a/test/files/huge-xpath-query.txt b/test/files/huge-xpath-query.txt new file mode 100644 index 0000000000..edcdf0dd43 --- /dev/null +++ b/test/files/huge-xpath-query.txt @@ -0,0 +1 @@ +( (((sum(ct:AIACapitalAllowancesInc) + sum(../ct:NotIncluded/ct:AIACapitalAllowancesNotInc))) - (( ( ceiling( ( number((round(date:seconds(normalize-space('2015-04-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2016-01-01')) div 86400))) * ( ( number((round(date:seconds(normalize-space('2015-04-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400))) * number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400) < round(date:seconds(normalize-space('2016-01-01')) div 86400))) * (round(date:seconds(normalize-space(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D')))) div 86400) - round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400)) ) + ( number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2015-04-01')) div 86400))) * number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400) < round(date:seconds(normalize-space('2016-01-01')) div 86400))) * (round(date:seconds(normalize-space(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D')))) div 86400) - round(date:seconds(normalize-space('2015-04-01')) div 86400)) ) + ( number((round(date:seconds(normalize-space('2015-04-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400))) * number((round(date:seconds(normalize-space('2016-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * (round(date:seconds(normalize-space('2016-01-01')) div 86400) - round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400)) ) + ( number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2015-04-01')) div 86400))) * number((round(date:seconds(normalize-space('2016-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * (round(date:seconds(normalize-space('2016-01-01')) div 86400) - round(date:seconds(normalize-space('2015-04-01')) div 86400)) ) ) * 500000 div (366 - number( ( ( (round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2015-04-01')) div 86400)) and (round(date:seconds(normalize-space('2015-04-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400)) ) or ( (round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2016-01-01')) div 86400)) and (round(date:seconds(normalize-space('2016-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400)) ) or ( (round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2019-01-01')) div 86400)) and (round(date:seconds(normalize-space('2019-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400)) ) or ( (round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2022-01-01')) div 86400)) and (round(date:seconds(normalize-space('2022-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400)) ) ) or ((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400) - round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400)) + 1 < 366) )) ) ) + ceiling( ( number((round(date:seconds(normalize-space('2016-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2019-01-01')) div 86400))) * ( ( number((round(date:seconds(normalize-space('2016-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400))) * number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400) < round(date:seconds(normalize-space('2019-01-01')) div 86400))) * (round(date:seconds(normalize-space(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D')))) div 86400) - round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400)) ) + ( number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2016-01-01')) div 86400))) * number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400) < round(date:seconds(normalize-space('2019-01-01')) div 86400))) * (round(date:seconds(normalize-space(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D')))) div 86400) - round(date:seconds(normalize-space('2016-01-01')) div 86400)) ) + ( number((round(date:seconds(normalize-space('2016-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400))) * number((round(date:seconds(normalize-space('2019-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * (round(date:seconds(normalize-space('2019-01-01')) div 86400) - round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400)) ) + ( number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2016-01-01')) div 86400))) * number((round(date:seconds(normalize-space('2019-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * (round(date:seconds(normalize-space('2019-01-01')) div 86400) - round(date:seconds(normalize-space('2016-01-01')) div 86400)) ) ) * 200000 div (366 - number( ( ( (round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2015-04-01')) div 86400)) and (round(date:seconds(normalize-space('2015-04-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400)) ) or ( (round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2016-01-01')) div 86400)) and (round(date:seconds(normalize-space('2016-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400)) ) or ( (round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2019-01-01')) div 86400)) and (round(date:seconds(normalize-space('2019-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400)) ) or ( (round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2022-01-01')) div 86400)) and (round(date:seconds(normalize-space('2022-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400)) ) ) or ((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400) - round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400)) + 1 < 366) )) ) ) + ceiling( ( number((round(date:seconds(normalize-space('2019-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2022-01-01')) div 86400))) * ( ( number((round(date:seconds(normalize-space('2019-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400))) * number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400) < round(date:seconds(normalize-space('2022-01-01')) div 86400))) * (round(date:seconds(normalize-space(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D')))) div 86400) - round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400)) ) + ( number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2019-01-01')) div 86400))) * number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400) < round(date:seconds(normalize-space('2022-01-01')) div 86400))) * (round(date:seconds(normalize-space(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D')))) div 86400) - round(date:seconds(normalize-space('2019-01-01')) div 86400)) ) + ( number((round(date:seconds(normalize-space('2019-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400))) * number((round(date:seconds(normalize-space('2022-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * (round(date:seconds(normalize-space('2022-01-01')) div 86400) - round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400)) ) + ( number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2019-01-01')) div 86400))) * number((round(date:seconds(normalize-space('2022-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * (round(date:seconds(normalize-space('2022-01-01')) div 86400) - round(date:seconds(normalize-space('2019-01-01')) div 86400)) ) ) * 1000000 div (366 - number( ( ( (round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2015-04-01')) div 86400)) and (round(date:seconds(normalize-space('2015-04-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400)) ) or ( (round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2016-01-01')) div 86400)) and (round(date:seconds(normalize-space('2016-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400)) ) or ( (round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2019-01-01')) div 86400)) and (round(date:seconds(normalize-space('2019-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400)) ) or ( (round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2022-01-01')) div 86400)) and (round(date:seconds(normalize-space('2022-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400)) ) ) or ((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400) - round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400)) + 1 < 366) )) ) ) + ceiling( ( number((round(date:seconds(normalize-space('2022-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * ( ( number((round(date:seconds(normalize-space('2022-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400))) * (round(date:seconds(normalize-space(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D')))) div 86400) - round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400)) ) + ( number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2022-01-01')) div 86400))) * (round(date:seconds(normalize-space(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D')))) div 86400) - round(date:seconds(normalize-space('2022-01-01')) div 86400)) ) ) * 200000 div (366 - number( ( ( (round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2015-04-01')) div 86400)) and (round(date:seconds(normalize-space('2015-04-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400)) ) or ( (round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2016-01-01')) div 86400)) and (round(date:seconds(normalize-space('2016-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400)) ) or ( (round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2019-01-01')) div 86400)) and (round(date:seconds(normalize-space('2019-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400)) ) or ( (round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2022-01-01')) div 86400)) and (round(date:seconds(normalize-space('2022-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400)) ) ) or ((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400) - round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400)) + 1 < 366) )) ) ) ) )) < 0.001) ) or ( ( date:day-in-month(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) = 1 and date:day-in-month(normalize-space(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1DT12H')))) = 1 ) and ( (((sum(ct:AIACapitalAllowancesInc) + sum(../ct:NotIncluded/ct:AIACapitalAllowancesNotInc))) - ((( ceiling( ( number((round(date:seconds(normalize-space('2015-04-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2016-01-01')) div 86400))) * ( ( number((round(date:seconds(normalize-space('2015-04-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400))) * number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400) < round(date:seconds(normalize-space('2016-01-01')) div 86400))) * (12 * (date:year(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D'))) - date:year(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) + date:month-in-year(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D'))) - date:month-in-year(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) ) + ( number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2015-04-01')) div 86400))) * number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400) < round(date:seconds(normalize-space('2016-01-01')) div 86400))) * (12 * (date:year(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D'))) - date:year('2015-04-01')) + date:month-in-year(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D'))) - date:month-in-year('2015-04-01')) ) + ( number((round(date:seconds(normalize-space('2015-04-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400))) * number((round(date:seconds(normalize-space('2016-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * (12 * (date:year('2016-01-01') - date:year(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) + date:month-in-year('2016-01-01') - date:month-in-year(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) ) + ( number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2015-04-01')) div 86400))) * number((round(date:seconds(normalize-space('2016-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * (12 * (date:year('2016-01-01') - date:year('2015-04-01')) + date:month-in-year('2016-01-01') - date:month-in-year('2015-04-01')) ) ) * 500000 div 12 ) ) + ceiling( ( number((round(date:seconds(normalize-space('2016-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2019-01-01')) div 86400))) * ( ( number((round(date:seconds(normalize-space('2016-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400))) * number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400) < round(date:seconds(normalize-space('2019-01-01')) div 86400))) * (12 * (date:year(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D'))) - date:year(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) + date:month-in-year(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D'))) - date:month-in-year(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) ) + ( number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2016-01-01')) div 86400))) * number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400) < round(date:seconds(normalize-space('2019-01-01')) div 86400))) * (12 * (date:year(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D'))) - date:year('2016-01-01')) + date:month-in-year(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D'))) - date:month-in-year('2016-01-01')) ) + ( number((round(date:seconds(normalize-space('2016-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400))) * number((round(date:seconds(normalize-space('2019-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * (12 * (date:year('2019-01-01') - date:year(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) + date:month-in-year('2019-01-01') - date:month-in-year(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) ) + ( number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2016-01-01')) div 86400))) * number((round(date:seconds(normalize-space('2019-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * (12 * (date:year('2019-01-01') - date:year('2016-01-01')) + date:month-in-year('2019-01-01') - date:month-in-year('2016-01-01')) ) ) * 200000 div 12 ) ) + ceiling( ( number((round(date:seconds(normalize-space('2019-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2022-01-01')) div 86400))) * ( ( number((round(date:seconds(normalize-space('2019-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400))) * number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400) < round(date:seconds(normalize-space('2022-01-01')) div 86400))) * (12 * (date:year(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D'))) - date:year(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) + date:month-in-year(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D'))) - date:month-in-year(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) ) + ( number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2019-01-01')) div 86400))) * number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400) < round(date:seconds(normalize-space('2022-01-01')) div 86400))) * (12 * (date:year(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D'))) - date:year('2019-01-01')) + date:month-in-year(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D'))) - date:month-in-year('2019-01-01')) ) + ( number((round(date:seconds(normalize-space('2019-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400))) * number((round(date:seconds(normalize-space('2022-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * (12 * (date:year('2022-01-01') - date:year(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) + date:month-in-year('2022-01-01') - date:month-in-year(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) ) + ( number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2019-01-01')) div 86400))) * number((round(date:seconds(normalize-space('2022-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * (12 * (date:year('2022-01-01') - date:year('2019-01-01')) + date:month-in-year('2022-01-01') - date:month-in-year('2019-01-01')) ) ) * 1000000 div 12 ) ) + ceiling( ( number((round(date:seconds(normalize-space('2022-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * ( ( number((round(date:seconds(normalize-space('2022-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400))) * (12 * (date:year(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D'))) - date:year(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) + date:month-in-year(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D'))) - date:month-in-year(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) ) + ( number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2022-01-01')) div 86400))) * (12 * (date:year(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D'))) - date:year('2022-01-01')) + date:month-in-year(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D'))) - date:month-in-year('2022-01-01')) ) ) * 200000 div 12 ) ) ) )) < 0.001) ) ) diff --git a/test/helper.rb b/test/helper.rb index eed661705c..ed96d3baaf 100644 --- a/test/helper.rb +++ b/test/helper.rb @@ -6,8 +6,8 @@ # - "normal" - normal GC functionality # - "minor" - force a minor GC cycle after each test # - "major" (default for Rubies without compaction) - force a major GC cycle after each test -# - "compact" (default for Rubies with compaction) - force a major GC and a compaction after each test -# - "verify" - force a major GC, a compaction, and verify references after each test +# - "compact" (default for Rubies with compaction) - force a major GC after each test and GC compaction after every 20 tests +# - "verify" - force a major GC after each test and verify references-after-compaction after every 20 tests # - "stress" - run tests with GC.stress set to true # - NOKOGIRI_GC: read more in test/test_memory_leak.rb # @@ -87,7 +87,11 @@ class TestCase < MiniTest::Spec warn "#{__FILE__}:#{__LINE__}: NOKOGIRI_TEST_GC_LEVEL: #{GC_LEVEL}" end + COMPACT_EVERY = 20 + @@test_count = 0 + def setup + @@test_count += 1 if Nokogiri.uses_libxml? @fake_error_handler_called = false Nokogiri::Test.__foreign_error_handler do @@ -110,10 +114,16 @@ def teardown when "major" GC.start(full_mark: true) when "compact" - GC.compact + if @@test_count % COMPACT_EVERY == 0 + GC.compact + else + GC.start(full_mark: true) + end when "verify" - # https://alanwu.space/post/check-compaction/ - GC.verify_compaction_references(double_heap: true, toward: :empty) + if @@test_count % COMPACT_EVERY == 0 + # https://alanwu.space/post/check-compaction/ + GC.verify_compaction_references(double_heap: true, toward: :empty) + end GC.start(full_mark: true) when "stress" GC.stress = false @@ -201,6 +211,15 @@ def i_am_running_in_valgrind # https://stackoverflow.com/questions/365458/how-can-i-detect-if-a-program-is-running-from-within-valgrind/62364698#62364698 ENV["LD_PRELOAD"] =~ /valgrind|vgpreload/ end + + def i_am_running_with_asan + # https://stackoverflow.com/questions/35012059/check-whether-sanitizer-like-addresssanitizer-is-active + begin + %x"ldd #{Gem.ruby}" =~ /libasan\.so/ + rescue + false + end + end end module SAX diff --git a/test/html/sax/test_parser.rb b/test/html4/sax/test_parser.rb similarity index 100% rename from test/html/sax/test_parser.rb rename to test/html4/sax/test_parser.rb diff --git a/test/html/sax/test_parser_context.rb b/test/html4/sax/test_parser_context.rb similarity index 100% rename from test/html/sax/test_parser_context.rb rename to test/html4/sax/test_parser_context.rb diff --git a/test/html/sax/test_parser_text.rb b/test/html4/sax/test_parser_text.rb similarity index 100% rename from test/html/sax/test_parser_text.rb rename to test/html4/sax/test_parser_text.rb diff --git a/test/html/sax/test_push_parser.rb b/test/html4/sax/test_push_parser.rb similarity index 80% rename from test/html/sax/test_push_parser.rb rename to test/html4/sax/test_push_parser.rb index dd64253907..f83440d272 100644 --- a/test/html/sax/test_push_parser.rb +++ b/test/html4/sax/test_push_parser.rb @@ -70,17 +70,6 @@ def test_chevron def test_default_options assert_equal(0, @parser.options) end - - def test_broken_encoding - skip_unless_libxml2("ultra hard to fix for pure Java version") - @parser.options |= XML::ParseOptions::RECOVER - # This is ISO_8859-1: - @parser.<< "<?xml version='1.0' encoding='UTF-8'?><r>Gau\337</r>" - @parser.finish - assert(@parser.document.errors.size >= 1) - assert_equal("Gau\337", @parser.document.data.join) - assert_equal([["r"], ["body"], ["html"]], @parser.document.end_elements) - end end end end diff --git a/test/html/test_attributes.rb b/test/html4/test_attributes.rb similarity index 100% rename from test/html/test_attributes.rb rename to test/html4/test_attributes.rb diff --git a/test/html/test_attributes_properly_escaped.rb b/test/html4/test_attributes_properly_escaped.rb similarity index 58% rename from test/html/test_attributes_properly_escaped.rb rename to test/html4/test_attributes_properly_escaped.rb index 91da4ccbe2..0debd26a4f 100755 --- a/test/html/test_attributes_properly_escaped.rb +++ b/test/html4/test_attributes_properly_escaped.rb @@ -5,7 +5,9 @@ module HTML class TestAttributesProperlyEscaped < Nokogiri::TestCase def test_attribute_macros_are_escaped - skip if Nokogiri::VersionInfo.instance.libxml2? && Nokogiri::VersionInfo.instance.libxml2_using_system? + if Nokogiri.uses_libxml? && !Nokogiri::VERSION_INFO["libxml"]["patches"]&.include?("0001-Remove-script-macro-support.patch") + skip("libxml2 has not been patched to be safe against attribute macros") + end html = "<p><i for=\"&{<test>}\"></i></p>" document = Nokogiri::HTML::Document.new @@ -15,7 +17,9 @@ def test_attribute_macros_are_escaped end def test_libxml_escapes_server_side_includes - skip if Nokogiri::VersionInfo.instance.libxml2? && Nokogiri::VersionInfo.instance.libxml2_using_system? + if Nokogiri.uses_libxml? && !Nokogiri::VERSION_INFO["libxml"]["patches"]&.include?("0002-Update-entities-to-remove-handling-of-ssi.patch") + skip("libxml2 has not been patched to be safe against SSI") + end original_html = %(<p><a href='<!--"><test>-->'></a></p>) document = Nokogiri::HTML::Document.new @@ -25,7 +29,9 @@ def test_libxml_escapes_server_side_includes end def test_libxml_escapes_server_side_includes_without_nested_quotes - skip if Nokogiri::VersionInfo.instance.libxml2? && Nokogiri::VersionInfo.instance.libxml2_using_system? + if Nokogiri.uses_libxml? && !Nokogiri::VERSION_INFO["libxml"]["patches"]&.include?("0002-Update-entities-to-remove-handling-of-ssi.patch") + skip("libxml2 has not been patched to be safe against SSI") + end original_html = %(<p><i for="<!--<test>-->"></i></p>) document = Nokogiri::HTML::Document.new diff --git a/test/html/test_builder.rb b/test/html4/test_builder.rb similarity index 100% rename from test/html/test_builder.rb rename to test/html4/test_builder.rb diff --git a/test/html/test_comments.rb b/test/html4/test_comments.rb similarity index 95% rename from test/html/test_comments.rb rename to test/html4/test_comments.rb index 476fc4b473..7d207815f5 100644 --- a/test/html/test_comments.rb +++ b/test/html4/test_comments.rb @@ -113,8 +113,7 @@ class TestComment < Nokogiri::TestCase let(:subject) { doc.at_css("div#under-test") } let(:inner_div) { doc.at_css("div#do-i-exist") } - if Nokogiri.uses_libxml? && Nokogiri::VersionInfo.instance.libxml2_using_packaged? - # see patches/libxml2/0006-htmlParseComment-treat-as-if-it-closed-the-comment.patch + if Nokogiri::VersionInfo.instance.libxml2_using_packaged? || (Nokogiri::VersionInfo.instance.libxml2_using_system? && Nokogiri.uses_libxml?(">=2.9.11")) it "behaves as if the comment is normally closed" do # COMPLIANT assert_equal 3, subject.children.length assert subject.children[0].comment? @@ -128,9 +127,7 @@ class TestComment < Nokogiri::TestCase end end - if Nokogiri.jruby? || Nokogiri::VersionInfo.instance.libxml2_using_system? - # this behavior may change to the above in libxml v2.9.11 depending on whether - # https://gitlab.gnome.org/GNOME/libxml2/-/merge_requests/82 is merged + if Nokogiri.jruby? || (Nokogiri::VersionInfo.instance.libxml2_using_system? && Nokogiri.uses_libxml?("<2.9.11")) it "behaves as if the comment encompasses the inner div" do # NON-COMPLIANT assert_equal 1, subject.children.length assert subject.children.first.comment? diff --git a/test/html/test_document.rb b/test/html4/test_document.rb similarity index 98% rename from test/html/test_document.rb rename to test/html4/test_document.rb index 29d87cf6d9..c6f0bfe862 100644 --- a/test/html/test_document.rb +++ b/test/html4/test_document.rb @@ -370,6 +370,15 @@ def test_to_xhtml assert_match("UTF-8", html.to_xhtml(encoding: "UTF-8")) end + def test_to_xhtml_self_closing_tags + # https://github.com/sparklemotion/nokogiri/issues/2324 + html = "<html><body><br><table><colgroup><col>" + doc = Nokogiri::HTML::Document.parse(html) + xhtml = doc.to_xhtml + assert_match(%r(<br ?/>), xhtml) + assert_match(%r(<col ?/>), xhtml) + end + def test_no_xml_header html = Nokogiri::HTML(<<~EOHTML) <html> diff --git a/test/html/test_document_encoding.rb b/test/html4/test_document_encoding.rb similarity index 100% rename from test/html/test_document_encoding.rb rename to test/html4/test_document_encoding.rb diff --git a/test/html/test_document_fragment.rb b/test/html4/test_document_fragment.rb similarity index 100% rename from test/html/test_document_fragment.rb rename to test/html4/test_document_fragment.rb diff --git a/test/html/test_element_description.rb b/test/html4/test_element_description.rb similarity index 100% rename from test/html/test_element_description.rb rename to test/html4/test_element_description.rb diff --git a/test/html4/test_html_module.rb b/test/html4/test_html_module.rb new file mode 100644 index 0000000000..6d142f016b --- /dev/null +++ b/test/html4/test_html_module.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true +require "helper" + +class Nokogiri::TestCase + describe Nokogiri::HTML do + it "is the same as Nokogiri::HTML4" do + assert_same(Nokogiri::HTML, Nokogiri::HTML4) + end + end + + describe "Nokogiri.HTML()" do + it "is the same as Nokogiri.HTML4()" do + assert_equal(Nokogiri.method(:HTML), Nokogiri.method(:HTML4)) + end + end +end diff --git a/test/html/test_named_characters.rb b/test/html4/test_named_characters.rb similarity index 100% rename from test/html/test_named_characters.rb rename to test/html4/test_named_characters.rb diff --git a/test/html/test_node.rb b/test/html4/test_node.rb similarity index 90% rename from test/html/test_node.rb rename to test/html4/test_node.rb index a55e2db13f..57165d8cdd 100644 --- a/test/html/test_node.rb +++ b/test/html4/test_node.rb @@ -187,21 +187,19 @@ def test_to_html_does_not_contain_entities nokogiri.at("p").to_html.gsub(/ /, '') end end - end - def test_GH_1042 - file = File.join(ASSETS_DIR, 'GH_1042.html'); - html = Nokogiri::HTML(File.read(file)) - table = html.xpath("//table")[1] - trs = table.xpath("tr").drop(1) - - # the jruby inplementation of drop uses dup() on the IRubyObject (which - # is NOT the same dup() method on the ruby Object) which produces a - # shallow clone. a shallow of valid XMLNode triggers several - # NullPointerException on inspect() since loads of invariants - # are not set. the fix for GH1042 ensures a proper working clone. - assert_nothing_raised do - trs.inspect + def test_GH_1042 + file = File.join(ASSETS_DIR, 'GH_1042.html'); + html = Nokogiri::HTML(File.read(file)) + table = html.xpath("//table")[1] + trs = table.xpath("tr").drop(1) + + # the jruby inplementation of drop uses dup() on the IRubyObject (which + # is NOT the same dup() method on the ruby Object) which produces a + # shallow clone. a shallow of valid XMLNode triggers several + # NullPointerException on inspect() since loads of invariants + # are not set. the fix for GH1042 ensures a proper working clone. + trs.inspect # assert_nothing_raised end end end diff --git a/test/html/test_node_encoding.rb b/test/html4/test_node_encoding.rb similarity index 100% rename from test/html/test_node_encoding.rb rename to test/html4/test_node_encoding.rb diff --git a/test/html5/test_api.rb b/test/html5/test_api.rb new file mode 100644 index 0000000000..c1c03652c4 --- /dev/null +++ b/test/html5/test_api.rb @@ -0,0 +1,178 @@ +# coding: utf-8 +# frozen_string_literal: true +require "helper" + +class TestHtml5API < Nokogiri::TestCase + def test_parse_convenience_methods + html = "<!DOCTYPE html><p>hi" + base = Nokogiri::HTML5::Document.parse(html) + html5_parse = Nokogiri::HTML5.parse(html) + html5 = Nokogiri::HTML5(html) + str = base.to_html + assert_equal(str, html5_parse.to_html) + assert_equal(str, html5.to_html) + end + + def test_fragment_convenience_methods + frag = "<div><p>hi</div>" + base = Nokogiri::HTML5::DocumentFragment.parse(frag) + html5_fragment = Nokogiri::HTML5.fragment(frag) + assert_equal(base.to_html, html5_fragment.to_html) + end + + def test_url + html = "<p>hi" + url = "http://example.com" + doc = Nokogiri::HTML5::Document.parse(html, url, max_errors: 1) + assert_equal(url, doc.errors[0].file) + + doc = Nokogiri::HTML5.parse(html, url, max_errors: 1) + assert_equal(url, doc.errors[0].file) + + doc = Nokogiri::HTML5(html, url, max_errors: 1) + assert_equal(url, doc.errors[0].file) + end + + def test_parse_encoding + utf8 = "<!DOCTYPE html><body><p>おはようございます" + shift_jis = utf8.encode(Encoding::SHIFT_JIS) + raw = shift_jis.dup + raw.force_encoding(Encoding::ASCII_8BIT) + + assert_match(/おはようございます/, Nokogiri::HTML5(utf8).to_s) + assert_match(/おはようございます/, Nokogiri::HTML5(shift_jis).to_s) + refute_match(/おはようございます/, Nokogiri::HTML5(raw).to_s) + + assert_match(/おはようございます/, Nokogiri::HTML5(raw, nil, Encoding::SHIFT_JIS).to_s) + assert_match(/おはようございます/, Nokogiri::HTML5.parse(raw, nil, Encoding::SHIFT_JIS).to_s) + assert_match(/おはようございます/, Nokogiri::HTML5::Document.parse(raw, nil, Encoding::SHIFT_JIS).to_s) + end + + def test_fragment_encoding + utf8 = "<div><p>おはようございます</div>" + shift_jis = utf8.encode(Encoding::SHIFT_JIS) + raw = shift_jis.dup + raw.force_encoding(Encoding::ASCII_8BIT) + + assert_match(/おはようございます/, Nokogiri::HTML5.fragment(utf8).to_s) + assert_match(/おはようございます/, Nokogiri::HTML5.fragment(shift_jis).to_s) + refute_match(/おはようございます/, Nokogiri::HTML5.fragment(raw).to_s) + + assert_match(/おはようございます/, Nokogiri::HTML5.fragment(raw, Encoding::SHIFT_JIS).to_s) + assert_match(/おはようございます/, Nokogiri::HTML5::DocumentFragment.parse(raw, Encoding::SHIFT_JIS).to_s) + end + + def test_fragment_serialization_encoding + frag = Nokogiri::HTML5.fragment("<span>아는 길도 물어가라</span>") + html = frag.serialize(encoding: "US-ASCII") + assert_equal("<span>아는 길도 물어가라</span>", html) + frag = Nokogiri::HTML5.fragment(html) + assert_equal("<span>아는 길도 물어가라</span>", frag.serialize) + end + + def test_serialization_encoding + html = "<!DOCUMENT html><span>ฉันไม่พูดภาษาไทย</span>" + doc = Nokogiri::HTML5(html) + span = doc.at("/html/body/span") + serialized = span.inner_html(encoding: "US-ASCII") + assert_match(/^(?:&#(?:\d+|x\h+);)*$/, serialized) + assert_equal("ฉันไม่พูดภาษาไทย".each_char.map(&:ord), + serialized.scan(/&#(\d+|x\h+);/).map do |s| + s = s.first + if s.start_with?("x") + s[1..-1].to_i(16) + else + s.to_i + end + end) + + doc2 = Nokogiri::HTML5(doc.serialize(encoding: "Big5")) + html2 = doc2.serialize(encoding: "UTF-8") + assert_match("ฉันไม่พูดภาษาไทย", html2) + end + + %w[pre listing textarea].each do |tag| + define_method("test_serialize_preserve_newline_#{tag}".to_sym) do + doc = Nokogiri::HTML5("<!DOCTYPE html><#{tag}>\n\nContent</#{tag}>") + html = doc.at("/html/body/#{tag}").serialize(preserve_newline: true) + assert_equal "<#{tag}>\n\nContent</#{tag}>", html + end + + define_method("test_inner_html_preserve_newline_#{tag}".to_sym) do + doc = Nokogiri::HTML5("<!DOCTYPE html><#{tag}>\n\nContent</#{tag}>") + html = doc.at("/html/body/#{tag}").inner_html(preserve_newline: true) + assert_equal "\n\nContent", html + end + end + + def test_document_io + html = StringIO.new("<!DOCTYPE html><span>test</span>", "r") + doc = Nokogiri::HTML5::Document.read_io(html) + refute_nil(doc.at_xpath("/html/body/span")) + end + + def test_document_memory + html = "<!DOCTYPE html><span>test</span>" + doc = Nokogiri::HTML5::Document.read_memory(html) + refute_nil(doc) + refute_nil(doc.at_xpath("/html/body/span")) + end + + def test_document_io_failure + html = "<!DOCTYPE html><span>test</span>" + assert_raises(ArgumentError) { Nokogiri::HTML5::Document.read_io(html) } + end + + def test_document_memory_failure + html = StringIO.new("<!DOCTYPE html><span>test</span>", "r") + assert_raises(ArgumentError) { Nokogiri::HTML5::Document.read_memory(html) } + end + + def test_document_parse_failure + html = ["Neither a string, nor I/O"] + assert_raises(ArgumentError) { Nokogiri::HTML5::Document.parse(html) } + end + + def test_ownership + # Test that we don't change the passed in string, even if we need to + # re-encode it. + html = "<!DOCTYPE html><html></html>" + refute_nil(Nokogiri::HTML5.parse(html)) + + iso8859_1 = html.encode(Encoding::ISO_8859_1).freeze + refute_nil(Nokogiri::HTML5.parse(iso8859_1)) + + ascii_8bit = html.encode(Encoding::ASCII_8BIT).freeze + refute_nil(Nokogiri::HTML5.parse(ascii_8bit)) + end + + def test_fragment_from_node + doc = Nokogiri.HTML5("<!DOCTYPE html><form><span></span></form>") + span = doc.at_xpath("/html/body/form/span") + refute_nil(span) + frag = span.fragment("<form>Nested forms should be ignored</form>") + assert(frag.is_a?(Nokogiri::HTML5::DocumentFragment)) + assert_equal(1, frag.children.length) + nested_form = frag.at_xpath("form") + assert_nil(nested_form) + assert(frag.children[0].text?) + end + + def test_fragment_from_node_no_form + doc = Nokogiri.HTML5("<!DOCTYPE html><span></span></form>") + span = doc.at_xpath("/html/body/span") + refute_nil(span) + frag = span.fragment("<form><span>Form should not be ignored</span></form>") + assert(frag.is_a?(Nokogiri::HTML5::DocumentFragment)) + assert_equal(1, frag.children.length) + form = frag.at_xpath("form") + refute_nil(form) + end + + def test_empty_fragment + doc = Nokogiri.HTML5("<!DOCTYPE html><body>") + frag = doc.fragment + assert(frag.is_a?(Nokogiri::HTML5::DocumentFragment)) + assert(frag.children.empty?) + end +end if Nokogiri.uses_gumbo? diff --git a/test/html5/test_encoding.rb b/test/html5/test_encoding.rb new file mode 100644 index 0000000000..bbe24771ba --- /dev/null +++ b/test/html5/test_encoding.rb @@ -0,0 +1,209 @@ +# coding: utf-8 +# frozen_string_literal: true +require "helper" + +class TestHtml5Encoding < Nokogiri::TestCase + if "".respond_to?("encoding") + def test_macroman_encoding + mac = String.new("<span>\xCA</span>").force_encoding("macroman") + doc = Nokogiri::HTML5(mac) + assert_equal("<span> </span>", doc.at("span").to_xml) + end + + def test_iso8859_encoding + iso8859 = String.new("<span>Se\xF1or</span>").force_encoding(Encoding::ASCII_8BIT) + doc = Nokogiri::HTML5(iso8859) + assert_equal("<span>Señor</span>", doc.at("span").to_xml) + end + + def test_charset_encoding + utf8 = String.new("<meta charset='utf-8'><span>Se\xC3\xB1or</span>") + .force_encoding(Encoding::ASCII_8BIT) + doc = Nokogiri::HTML5(utf8) + assert_equal("<span>Señor</span>", doc.at("span").to_xml) + end + + def test_bogus_encoding + bogus = String.new("<meta charset='bogus'><span>Se\xF1or</span>") + .force_encoding(Encoding::ASCII_8BIT) + doc = Nokogiri::HTML5(bogus) + assert_equal("<span>Señor</span>", doc.at("span").to_xml) + end + + def test_utf8_bom + utf8 = "\uFEFF<!DOCTYPE html><html></html>".encode("UTF-8") + doc = Nokogiri::HTML5(utf8, max_errors: 10) + assert_equal([], doc.errors) + end + + def test_utf16le_bom + utf16le = "\uFEFF<!DOCTYPE html><html></html>".encode("UTF-16LE") + doc = Nokogiri::HTML5(utf16le, max_errors: 10) + assert_equal([], doc.errors) + end + + def test_utf16be_bom + utf16be = "\uFEFF<!DOCTYPE html><html></html>".encode("UTF-16BE") + doc = Nokogiri::HTML5(utf16be, max_errors: 10) + assert_equal([], doc.errors) + end + + def test_utf8_bom_ascii + utf8 = "\uFEFF<!DOCTYPE html><html></html>".encode("UTF-8") + utf8.force_encoding(Encoding::ASCII_8BIT) + doc = Nokogiri::HTML5(utf8, max_errors: 10) + doc.errors.each { |err| puts(err) } + assert_equal([], doc.errors) + end + + def test_utf16le_bom_ascii + utf16le = "\uFEFF<!DOCTYPE html><html></html>".encode("UTF-16LE") + utf16le.force_encoding(Encoding::ASCII_8BIT) + doc = Nokogiri::HTML5(utf16le, max_errors: 10) + assert_equal([], doc.errors) + doc.errors.each { |err| puts(err) } + end + + def test_utf16be_bom_ascii + utf16be = "\uFEFF<!DOCTYPE html><html></html>".encode("UTF-16BE") + utf16be.force_encoding(Encoding::ASCII_8BIT) + doc = Nokogiri::HTML5(utf16be, max_errors: 10) + assert_equal([], doc.errors) + doc.errors.each { |err| puts(err) } + end + + def test_tag_after_utf8_bom + utf8 = "\uFEFF<b></b>".encode("UTF-8") + doc = Nokogiri::HTML5.fragment(utf8, max_errors: 10) + assert_equal([], doc.errors) + end + end + + # https://github.com/rubys/nokogumbo/issues/68 + def test_charset_sniff_to_html + html = <<-EOF.gsub(/^ /, "") + <!DOCTYPE html> + <html> + <head> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8; width=device-width"> + <style> .commit-tease, .user-profile-mini-avatar, .avatar, .vcard-details, .signup-prompt-bg { display: none !IMPORTANT; } </style> <script> document.addEventListener('DOMContentLoaded', function() { this.querySelectorAll('a').forEach(anchor => { anchor.addEventListener('click', e => { e.preventDefault(); const redact = new URLSearchParams(window.location.search).get('redact'); const hasExistingParams = anchor.href.includes('?'); window.location.href = anchor.href + (hasExistingParams ? `&redact=${redact}` : `?redact=${redact}`); }); }); }); </script> </head> + <body> + Hello! + </body> + </html> + EOF + doc = Nokogiri::HTML5(html, max_errors: 10) + assert_equal(0, doc.errors.length) + refute_equal("", doc.to_html) + end + + # https://encoding.spec.whatwg.org/#names-and-labels + # I chose these by looking at the Wikipedia page for each encoding, picked + # one of the languages it was supposed to encode, and then Googled for a + # proverb in the language. Apologies if these are ill-chosen or nonsensical. + # I'm happy to change them. I'm just pasting them in here so I'm pretty sure + # the right-to-left languages are backward. Corrections welcome. + ENCODINGS = [ + ["UTF-8", "Let's concatentate all of these for UTF-8"], # English + ["IBM866", "А дело бывало -- и коза волка съедала"], # Russian + ["ISO-8859-2", "Co můžeš udělat dnes, neodkládej na zítřek."], # Czech + ["ISO-8859-3", "Yukarda mavi gök, asağıda yağız yer yaratıldıkta"], # Turkish + ["ISO-8859-4", "Ceļš uz elli ir bruģēts ar labiem nodomiem."], # Latvian + ["ISO-8859-5", "Каде има сила, нема правдина."], # Macedonian + ["ISO-8859-6", "أباد الله خضراءهم ابذل لصديقك دمك ومالك"], # Arabic + ["ISO-8859-7", "Η καλύτερη άμυνα είναι η επίθεση."], # Greek + ["ISO-8859-8", "אין הנחתום מעיד על עיסתו"], # Hebrew + ["ISO-8859-8-I", "אל תסתכל בקנקן, אלא במה שבתוכו"], # Hebrew + ["ISO-8859-10", "Alla känner apan, men apan känner ingen."], # Swedish + ["ISO-8859-13", "Lašas po lašo ir akmenį pratašo."], # Lithuanian + ["ISO-8859-14", "ha bhòrd bòrd gun aran ach 's bòrd aran leis fhèin."], # Scottish Gaelic + ["ISO-8859-15", "This is essentially ISO 8859-1 but with € Š š Ž ž Œ œ Ÿ"], # English + ["ISO-8859-16", "Kiedy wszedłeś między wrony, musisz krakać jak i one."], # Polish + ["KOI8-R", "А дело бывало -- и коза волка съедала"], # Russian + ["KOI8-U", "Яблуко від яблуньки не далеко. Ґ, Є, І, Ї"], # Ukrainian + ["macroman", "Some good old Mac Roman œ∑´®†¥¨ˆøπå߃©"], # English + ["windows-874", "กระต่ายหมายจันทร์"], # Thai + ["windows-1250", "Addig nyújtózkodj, amíg a takaród ér."], # Hungarian + ["windows-1251", "Бързата работа - срам за майстора."], # Bulgarian + ["windows-1252", "Basically ISO 8859-1 with ‘differences’™ •"], # English + ["windows-1253", "Και οι τοίχοι έχουν αυτιά."], # Greek + ["windows-1254", "Baban nasılsa oğlu da öyledir."], # Turkish + ["windows-1255", "אל תקנה חתול בשק; ₪"], # Hebrew + ["windows-1256", "أبطأ من سلحفاة"], # Arabic + ["windows-1257", "Hommikune töö kuld, õhtune muld."], # Estonian + ["windows-1258", "Ăn theo thuở, ở theo thời."], # Vietnamese + ["macCyrillic", "А дело бывало -- и коза волка съедала"], # Russian + ["GBK", "不闻不若闻之，闻之不若见之，见之不若知之，知之不若行之；学至于行之而止矣"], # Simplified Chinese + ["gb18030", "不聞不若聞之，聞之不若見之，見之不若知之，知之不若行之；學至於行之而止矣"], # Traditional Chinese + ["Big5", "有其父必有其子"], # Traditional Chinese + ["EUC-JP", "猿も木から落ちる"], # Japanese + ["ISO-2022-JP", "井の中の蛙大海を知らず"], # Japanese + ["Shift_JIS", "鳥なき里の蝙蝠"], # Japanese + ["EUC-KR", "아는 길도 물어가라"], # Korean + ["replacement", "콩 심은데 콩나고, 팥 심은데 팥난다"], # Korean + ["UTF-16BE", "Everything had better be representable!"], # English + ["UTF-16LE", "Same as with UTF-16BE"], # English + ["US-ASCII", "Surprisingly not one of the required encodings"], # English + ].freeze + + def encodings_html + @encodings_html ||= + "<!DOCTYPE html><html><head> <style> .commit-tease, .user-profile-mini-avatar, .avatar, .vcard-details, .signup-prompt-bg { display: none !IMPORTANT; } </style> <script> document.addEventListener('DOMContentLoaded', function() { this.querySelectorAll('a').forEach(anchor => { anchor.addEventListener('click', e => { e.preventDefault(); const redact = new URLSearchParams(window.location.search).get('redact'); const hasExistingParams = anchor.href.includes('?'); window.location.href = anchor.href + (hasExistingParams ? `&redact=${redact}` : `?redact=${redact}`); }); }); }); </script> </head><body>" + + ENCODINGS.map { |enc| %(<span id="#{enc[0]}">#{enc[1]}</span>) }.join + + "</body></html>" + end + + def encodings_doc + @encodings_doc ||= Nokogiri::HTML5(encodings_html) + end + + def round_trip_through(str, enc) + begin + encoding = Encoding.find(enc) + rescue ArgumentError + skip("#{enc} not supported") + end + begin + encoded = str.encode(encoding) + rescue Encoding::ConverterNotFoundError + skip("Converting UTF-8 to #{enc} not supported") + end + begin + decoded = encoded.encode("UTF-8") + rescue Encoding::ConverterNotFoundError + skip("Converting #{enc} to UTF-8 not supported") + end + assert_equal(str, decoded, "'#{str}' did not round trip through #{enc[0]}") + encoded + end + + ENCODINGS.each do |enc| + define_method("test_parse_encoded_#{enc[0]}".to_sym) do + html = "<!DOCTYPE html><span>#{enc[1]}</span>" + encoded_html = round_trip_through(html, enc[0]) + doc = Nokogiri::HTML5(encoded_html, encoding: enc[0]) + span = doc.at("/html/body/span") + refute_nil span + assert_equal enc[1], span.content + end + + define_method("test_inner_html_encoded_#{enc[0]}".to_sym) do + encoded = round_trip_through(enc[1], enc[0]) + span = encodings_doc.at(%(/html/body/span[@id="#{enc[0]}"])) + refute_nil span + assert_equal encoded, span.inner_html(encoding: enc[0]) + end + + define_method("test_roundtrip_through_#{enc[0]}".to_sym) do + # https://bugs.ruby-lang.org/issues/15033 + # Ruby has a bug with the `:fallback` parameter passed to `#encode` when + # multiple conversions have to happen. I'm not sure it's worth working + # around. It impacts this test though. + skip "https://bugs.ruby-lang.org/issues/15033" if enc[0] == "ISO-2022-JP" + round_trip_through(enc[1], enc[0]) + encoded = encodings_doc.serialize(encoding: enc[0]) + doc = Nokogiri::HTML5(encoded, encoding: enc[0]) + assert_equal encodings_html, doc.serialize + end + end +end if Nokogiri.uses_gumbo? diff --git a/test/html5/test_monkey_patch.rb b/test/html5/test_monkey_patch.rb new file mode 100644 index 0000000000..42ff0ac6ab --- /dev/null +++ b/test/html5/test_monkey_patch.rb @@ -0,0 +1,15 @@ +# encoding: utf-8 +require 'helper' + +class TestHtml5SerializationMonkeyPatch < Nokogiri::TestCase + def test_to_xml + xml = Nokogiri.HTML5('<!DOCTYPE html><source>').to_xml + assert_match(/\A<\?xml version/, xml) + assert_match(/<source\s*\/>/, xml) + end + + def test_html4_fragment + frag = Nokogiri::HTML.fragment('<span></span>') + assert frag.is_a?(Nokogiri::HTML::DocumentFragment) + end +end if Nokogiri.uses_gumbo? diff --git a/test/html5/test_nokogumbo.rb b/test/html5/test_nokogumbo.rb new file mode 100644 index 0000000000..a49b9e9848 --- /dev/null +++ b/test/html5/test_nokogumbo.rb @@ -0,0 +1,353 @@ +# encoding: utf-8 +# frozen_string_literal: true +require "helper" + +class TestHtml5Nokogumbo < Nokogiri::TestCase + def test_element_text + doc = Nokogiri::HTML5(buffer) + assert_equal("content", doc.at("span").text) + end + + def test_element_cdata_textarea + doc = Nokogiri::HTML5(buffer) + assert_equal("foo<x>bar", doc.at("textarea").text.strip) + end + + def test_element_cdata_script + doc = Nokogiri::HTML5.fragment(buffer) + assert_equal(true, doc.document.html?) + assert_equal("<script> if (a < b) alert(1) </script>", doc.at("script").to_s) + end + + def test_attr_value + doc = Nokogiri::HTML5(buffer) + assert_equal("utf-8", doc.at("meta")["charset"]) + end + + def test_comment + doc = Nokogiri::HTML5(buffer) + assert_equal(" test comment ", doc.xpath("//comment()").text) + end + + def test_unknown_element + doc = Nokogiri::HTML5(buffer) + assert_equal("main", doc.at("main").name) + end + + def test_IO + require "stringio" + doc = Nokogiri::HTML5(StringIO.new(buffer)) + assert_equal("textarea", doc.at("form").element_children.first.name) + end + + def test_nil + doc = Nokogiri::HTML5(nil) + assert_equal(1, doc.search("body").count) + + fragment = Nokogiri::HTML5.fragment(nil) + assert_equal(0, fragment.errors.length) + end + + def test_html5_doctype + doc = Nokogiri::HTML5.parse("<!DOCTYPE html><html></html>") + assert_match(/<!DOCTYPE html>/, doc.to_html) + end + + def test_fragment_no_errors + doc = Nokogiri::HTML5.fragment("no missing DOCTYPE errors", max_errors: 10) + assert_equal(0, doc.errors.length) + end + + # This should be deleted when `:max_parse_errors` is removed. + def test_fragment_max_parse_errors + doc = Nokogiri::HTML5.fragment("testing deprecated :max_parse_errors", max_parse_errors: 10) + assert_equal(0, doc.errors.length) + end + + def test_fragment_head + doc = Nokogiri::HTML5.fragment(buffer[%r{<head>(.*?) <style> .commit-tease, .user-profile-mini-avatar, .avatar, .vcard-details, .signup-prompt-bg { display: none !IMPORTANT; } </style> <script> document.addEventListener('DOMContentLoaded', function() { this.querySelectorAll('a').forEach(anchor => { anchor.addEventListener('click', e => { e.preventDefault(); const redact = new URLSearchParams(window.location.search).get('redact'); const hasExistingParams = anchor.href.includes('?'); window.location.href = anchor.href + (hasExistingParams ? `&redact=${redact}` : `?redact=${redact}`); }); }); }); </script> </head>}m, 1]) + assert_equal("hello world", doc.xpath("title").text) + assert_equal("utf-8", doc.xpath("meta").first["charset"]) + end + + def test_fragment_body + doc = Nokogiri::HTML5.fragment(buffer[%r{<body>(.*?)</body>}m, 1]) + assert_equal("<span>content</span>", doc.xpath("main/span").to_xml) + assert_equal(" test comment ", doc.xpath("comment()").text) + end + + def test_xlink_attribute + source = <<-EOF.gsub(/^ {6}/, "") + <!DOCTYPE html> + <svg xmlns="http://www.w3.org/2000/svg"> + <a xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="#s1"/> + </svg> + EOF + doc = Nokogiri::HTML5.parse(source) + a = doc.at_xpath("/html/body/svg:svg/svg:a") + refute_nil(a) + refute_nil(a["xlink:href"]) + refute_nil(a["xmlns:xlink"]) + end + + def test_xlink_attribute_fragment + source = <<-EOF.gsub(/^ {6}/, "") + <svg xmlns="http://www.w3.org/2000/svg"> + <a xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="#s1"/> + </svg> + EOF + doc = Nokogiri::HTML5.fragment(source) + a = doc.at_xpath("svg:svg/svg:a") + refute_nil(a) + refute_nil(a["xlink:href"]) + refute_nil(a["xmlns:xlink"]) + end + + def test_template + source = <<-EOF.gsub(/^ {6}/, "") + <template id="productrow"> + <tr> + <td class="record"></td> + <td></td> + </tr> + </template> + EOF + doc = Nokogiri::HTML5.fragment(source) + template = doc.at("template") + assert_equal("productrow", template["id"]) + assert_equal("record", template.at("td")["class"]) + end + + def test_root_comments + doc = Nokogiri::HTML5("<!DOCTYPE html><!-- start --><html></html><!-- -->") + assert_equal(["html", "comment", "html", "comment"], doc.children.map(&:name)) + end + + def test_max_attributes + html = '<div id="i" class="c" title="t"><img src="s" alt="a"></div>' + + assert_raises(ArgumentError) { Nokogiri::HTML5(html, max_attributes: 2) } + assert_raises(ArgumentError) { Nokogiri::HTML5(html, max_attributes: 1) } + assert_raises(ArgumentError) { Nokogiri::HTML5(html, max_attributes: 0) } + + # -1 disables limit + doc = Nokogiri::HTML5(html, max_attributes: -1) + assert_equal({ "id" => "i", "class" => "c", "title" => "t" }, attributes(doc.at_css("div"))) + assert_equal({ "src" => "s", "alt" => "a" }, attributes(doc.at_css("img"))) + end + + def test_max_attributes_boolean + html = '<label><input checked type="checkbox" disabled name="cheese"> Cheese</label>' + + doc = Nokogiri::HTML5(html, max_attributes: 4) + assert_equal({ "checked" => "", "type" => "checkbox", "disabled" => "", "name" => "cheese" }, + attributes(doc.at_css("input"))) + + assert_raises(ArgumentError) { Nokogiri::HTML5(html, max_attributes: 3) } + assert_raises(ArgumentError) { Nokogiri::HTML5(html, max_attributes: 2) } + assert_raises(ArgumentError) { Nokogiri::HTML5(html, max_attributes: 1) } + assert_raises(ArgumentError) { Nokogiri::HTML5(html, max_attributes: 0) } + end + + def test_default_max_attributes + a = String.new("a") + attrs = 50_000.times.map { x = a.dup; a.succ!; x } + + # <div> contains 50,000 attributes, but default limit is 400. Parsing this would take ages if + # we were not enforcing any limit on attributes. All attributes are duplicated to make sure + # this doesn’t alter performance or end result. + html = <<~EOF + <div #{attrs.map.with_index { |x, i| "data-#{x}=#{i} data-#{x}=#{i}" }.join(" ")}> + hello + </div> + EOF + + assert_raises(ArgumentError) { Nokogiri::HTML5(html) } + end + + def test_fragment_max_attributes + html = '<div id="i" class="c" title="t"><img src="s" alt="a"></div>' + + assert_raises(ArgumentError) { Nokogiri::HTML5.fragment(html, max_attributes: 2) } + assert_raises(ArgumentError) { Nokogiri::HTML5.fragment(html, max_attributes: 1) } + assert_raises(ArgumentError) { Nokogiri::HTML5.fragment(html, max_attributes: 0) } + + # -1 disables limit + doc = Nokogiri::HTML5.fragment(html, max_attributes: -1) + assert_equal({ "id" => "i", "class" => "c", "title" => "t" }, attributes(doc.at_css("div"))) + assert_equal({ "src" => "s", "alt" => "a" }, attributes(doc.at_css("img"))) + end + + def test_fragment_default_max_attributes + a = String.new("a") + attrs = 50_000.times.map { x = a.dup; a.succ!; x } + + # <div> contains 50,000 attributes, but default limit is 400. Parsing this would take ages if + # we were not enforcing any limit on attributes. All attributes are duplicated to make sure + # this doesn’t alter performance or end result. + html = <<~EOF + <div #{attrs.map.with_index { |x, i| "data-#{x}=#{i} data-#{x}=#{i}" }.join(" ")}> + hello + </div> + EOF + + assert_raises(ArgumentError) { Nokogiri::HTML5.fragment(html) } + end + + def test_parse_errors + doc = Nokogiri::HTML5("<!DOCTYPE html><html><!-- <!-- --></a>", max_errors: 10) + assert_equal(doc.errors.length, 2) + doc = Nokogiri::HTML5("<!DOCTYPE html><html>", max_errors: 10) + assert_empty(doc.errors) + end + + def test_max_errors + # This document contains 2 parse errors, but we force limit to 1. + doc = Nokogiri::HTML5("<!DOCTYPE html><html><!-- -- --></a>", max_errors: 1) + assert_equal(1, doc.errors.length) + doc = Nokogiri::HTML5("<!DOCTYPE html><html>", max_errors: 1) + assert_empty(doc.errors) + end + + def test_default_max_errors + # This document contains 200 parse errors, but default limit is 0. + doc = Nokogiri::HTML5("<!DOCTYPE html><html>" + "</p>" * 200) + assert_equal(0, doc.errors.length) + end + + def test_parse_fragment_errors + doc = Nokogiri::HTML5.fragment("<\r\n", max_errors: 10) + refute_empty(doc.errors) + end + + def test_fragment_max_errors + # This fragment contains 2 parse errors, but we force limit to 1. + doc = Nokogiri::HTML5.fragment("<!-- <!-- --></span>", max_errors: 1) + assert_equal(1, doc.errors.length) + doc = Nokogiri::HTML5.fragment("<!-- <!-- --></span>", max_errors: 10) + assert_equal(2, doc.errors.length) + end + + def test_fragment_default_max_errors + # This fragment contains 200 parse errors, but default limit is 0. + doc = Nokogiri::HTML5.fragment("</p>" * 200) + assert_equal(0, Nokogiri::Gumbo::DEFAULT_MAX_ERRORS) + assert_equal(0, doc.errors.length) + end + + def test_default_max_depth_parse + assert_raises(ArgumentError) do + depth = Nokogiri::Gumbo::DEFAULT_MAX_TREE_DEPTH + 1 + Nokogiri::HTML5("<!DOCTYPE html><html><body>" + "<div>" * (depth - 2)) + end + end + + def test_default_max_depth_fragment + assert_raises(ArgumentError) do + depth = Nokogiri::Gumbo::DEFAULT_MAX_TREE_DEPTH + 1 + Nokogiri::HTML5.fragment("<div>" * depth) + end + end + + def test_max_depth_parse + depth = 10 + html = "<!DOCTYPE html><html><body>" + "<div>" * (depth - 2) + assert_raises(ArgumentError) do + Nokogiri::HTML5(html, max_tree_depth: depth - 1) + end + + begin + Nokogiri::HTML5(html, max_tree_depth: depth) + pass + rescue ArgumentError + flunk("Expected document parse to succeed") + end + end + + def test_max_depth_fragment + depth = 10 + html = "<div>" * depth + assert_raises(ArgumentError) do + Nokogiri::HTML5.fragment(html, max_tree_depth: depth - 1) + end + + begin + Nokogiri::HTML5.fragment(html, max_tree_depth: depth) + pass + rescue ArgumentError + flunk("Expected fragment parse to succeed") + end + end + + def test_document_encoding + html = <<-TEXT + <html> + <head> + <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"> + <style> .commit-tease, .user-profile-mini-avatar, .avatar, .vcard-details, .signup-prompt-bg { display: none !IMPORTANT; } </style> <script> document.addEventListener('DOMContentLoaded', function() { this.querySelectorAll('a').forEach(anchor => { anchor.addEventListener('click', e => { e.preventDefault(); const redact = new URLSearchParams(window.location.search).get('redact'); const hasExistingParams = anchor.href.includes('?'); window.location.href = anchor.href + (hasExistingParams ? `&redact=${redact}` : `?redact=${redact}`); }); }); }); </script> </head> + <body> + Кирилические символы + </body> + </html> + TEXT + doc = Nokogiri::HTML5.parse(html) + assert_equal("UTF-8", doc.encoding) + assert_equal("Кирилические символы", doc.at("body").text.gsub(/\n\s+/, "")) + end + + def test_line_text + doc = Nokogiri.HTML5("<!DOCTYPE html>\ntext node") + assert_equal(2, doc.at_xpath("/html/body/text()").line) + end + + def test_line_comment + doc = Nokogiri.HTML5("<!DOCTYPE html>\n\n<!-- comment -->") + assert_equal(3, doc.at_xpath("/comment()").line) + end + + def test_line_element + doc = Nokogiri.HTML5("<!DOCTYPE html>\n<p>") + assert_equal(2, doc.at_xpath("/html/body/p").line) + end + + def test_line_template + doc = Nokogiri.HTML5("<!DOCTYPE html>\n\n<template></template>") + assert_equal(3, doc.at_xpath("/html/head/template").line) + end + + def test_line_cdata + html = "<!DOCTYPE html>\n<svg>\n<script><![CDATA[ ]]></script></svg>" + doc = Nokogiri.HTML5(html) + node = doc.at_xpath("/html/body/svg:svg/svg:script/text()") + assert(node.cdata?) + assert_equal(3, node.line) + end + + private + + def buffer + <<-EOF.gsub(/^ /, "") + <html> + <head> + <meta charset="utf-8"/> + <title>hello world</title> + <script> if (a < b) alert(1) </script> + <style> .commit-tease, .user-profile-mini-avatar, .avatar, .vcard-details, .signup-prompt-bg { display: none !IMPORTANT; } </style> <script> document.addEventListener('DOMContentLoaded', function() { this.querySelectorAll('a').forEach(anchor => { anchor.addEventListener('click', e => { e.preventDefault(); const redact = new URLSearchParams(window.location.search).get('redact'); const hasExistingParams = anchor.href.includes('?'); window.location.href = anchor.href + (hasExistingParams ? `&redact=${redact}` : `?redact=${redact}`); }); }); }); </script> </head> + <body> + <h1>hello world</h1> + <main> + <span>content</span> + </main> + <!-- test comment --> + <form> + <textarea>foo<x>bar</textarea> + </form> + </body> + </html> + EOF + end + + def attributes(element) + element.attributes.map { |name, attribute| [name, attribute.value] }.to_h + end +end if Nokogiri.uses_gumbo? diff --git a/test/html5/test_null.rb b/test/html5/test_null.rb new file mode 100644 index 0000000000..d0a326c291 --- /dev/null +++ b/test/html5/test_null.rb @@ -0,0 +1,112 @@ +# encoding: utf-8 +# frozen_string_literal: true +require "helper" + +class TestHtml5Null < Nokogiri::TestCase + def fragment(s) + Nokogiri::HTML5.fragment(s, max_errors: 10) + end + + def test_null_char_ref + frag = fragment("�") + assert_equal(1, frag.errors.length) + end + + def test_data_state + frag = fragment("\u0000") + # 12.2.5.1 Data state: unexpected-null-character parse error + # 12.2.6.4.7 The "in body" insertion mode: Parse error + assert_equal(2, frag.errors.length) + end + + def test_data_rcdata_state + # 12.2.6.4.7 The "in body" insertion mode: textarea swiches to RCDATA + # state + frag = fragment("<textarea>\u0000</textarea>") + # 12.2.5.2 RCDATA state: unexpected-null-character parse error + assert_equal(1, frag.errors.length) + end + + def test_data_scriptdata_state + # 12.2.6.4.7 The "in body" insertion mode: Process "script" using rules + # for "in head" insertion mode + # 12.2.6.4.4 The "in head" insertion mode: "script" switches to script + # data state + frag = fragment("<script>\u0000</script>") + # 12.2.5.4 Script data state: unexpected-null-character parse error + assert_equal(1, frag.errors.length) + end + + def test_data_plaintext_state + frag = fragment("<plaintext>\u0000</plaintext>") + # 12.2.5.5 PLAINTEXT state: unexpected-null-character parse error + # EOF parse error because there's no way to switch out of plaintext! + assert_equal(2, frag.errors.length) + end + + def test_data_tag_name_state + frag = fragment("<x\u0000></x\ufffd>") + # 12.2.5.8 Tag name state: unexpected-null-character parse error + assert_equal(1, frag.errors.length) + end + + # XXX: There are 6 script states to test. + + def test_attribute_name_state + frag = fragment("<p \u0000>") + # 12.2.5.33 Attribute name state + assert_equal(1, frag.errors.length) + end + + def test_attribute_value_states + frag = fragment("<p x=\"\u0000\"><p x='\u0000'><p x=\u0000>") + # 12.2.5.36 Attribute value (double-quoted) state + # 12.2.5.37 Attribute value (single-quoted) state + # 12.2.5.38 Attribute value (unquoted) state + assert_equal(3, frag.errors.length) + end + + def test_bogus_comment_state + frag = fragment("<!\u0000>") + # 12.2.5.42 Markup declaration open state: incorrectly-opened-comment + # parse error + # 12.2.5.41 Bogus comment state: unexpected-null-character parse error + assert_equal(2, frag.errors.length) + end + + def test_comment_state + frag = fragment("<!-- \u0000 -->") + # 12.2.5.45 Comment state: unexpected-null-character parse error + assert_equal(1, frag.errors.length) + end + + def test_doctype_name_states + # There are two missing here for double quoted PUBLIC and SYSTEM values. + doc = Nokogiri::HTML5.parse("<!DOCTYPE \u0000\u0000 PUBLIC '\u0000' '\u0000' \u0000>", + max_errors: 10) + # 12.2.5.54 Before DOCTYPE name state: unexpected-null-character parse + # error + # 12.2.5.55 DOCTYPE name state: unexpected-null-character parse error + # 12.2.5.60 DOCTYPE public identifier (single-quoted) state: + # unexpected-null-character parse error + # 12.2.5.66 DOCTYPE system identifier (single-quoted) state: + # unexpected-null-character parse error + # 12.2.5.67 After DOCTYPE system identifier state: + # unexpected-character-after-doctype-system-identifier parse error + # 12.2.5.68 Bogus DOCTYPE state: unexpected-null-character parse error + # 12.2.6.4.1 The "initial" insertion mode: parse error + assert_equal(7, doc.errors.length) + end + + def test_cdata_section_state + frag = fragment("<script>//<![CDATA[\n\u0000\n//]]></script>") + # 12.2.6.5 The rules for parsing tokens in foreign content: parse error + assert_equal(1, frag.errors.length) + end + + def test_error_api_with_null + frag = fragment("<p \u0000>") + assert(frag.errors.any?) + assert_includes(frag.errors[0].to_s, "<p \u0000>") + end +end if Nokogiri.uses_gumbo? diff --git a/test/html5/test_serialize.rb b/test/html5/test_serialize.rb new file mode 100644 index 0000000000..32cd759061 --- /dev/null +++ b/test/html5/test_serialize.rb @@ -0,0 +1,506 @@ +# encoding: utf-8 +# frozen_string_literal: true +require "helper" + +class TestHtml5Serialize < Nokogiri::TestCase + # https://github.com/web-platform-tests/wpt/blob/master/html/syntax/serializing-html-fragments/initial-linefeed-pre.html + def initial_linefeed_pre + @initial_linefeed_pre ||= begin + html = <<~EOF.gsub(/^ /, "").freeze + <!DOCTYPE html> + <div id="outer"> + <div id="inner"> + <pre id="pre1"> + x</pre> + <pre id="pre2"> + + x</pre> + <textarea id="textarea1"> + x</textarea> + <textarea id="textarea2"> + + x</textarea> + <listing id="listing1"> + x</listing> + <listing id="listing2"> + + x</listing> + </div> + </div> + EOF + Nokogiri::HTML5(html) + end + @initial_linefeed_pre + end + + def test_initial_linefeed_pre_outer + expected = %{\n<div id="inner">\n<pre id="pre1">x</pre>\n<pre id="pre2">\nx</pre>\n<textarea id="textarea1">x</textarea>\n<textarea id="textarea2">\nx</textarea>\n<listing id="listing1">x</listing>\n<listing id="listing2">\nx</listing>\n</div>\n} + outer = initial_linefeed_pre.xpath('//div[@id="outer"]')[0] + refute_nil(outer) + assert_equal(expected, outer.inner_html) + end + + def test_initial_linefeed_pre_inner + expected = %{\n<pre id="pre1">x</pre>\n<pre id="pre2">\nx</pre>\n<textarea id="textarea1">x</textarea>\n<textarea id="textarea2">\nx</textarea>\n<listing id="listing1">x</listing>\n<listing id="listing2">\nx</listing>\n} + inner = initial_linefeed_pre.at('//div[@id="inner"]') + refute_nil(inner) + assert_equal(expected, inner.inner_html) + end + + %w[pre textarea listing].each do |tag| + define_method("test_initial_linefeed_#{tag}1".to_sym) do + elem = initial_linefeed_pre.at("//*[@id=\"#{tag}1\"]") + refute_nil elem + assert_equal "x", elem.inner_html + end + + define_method("test_initial_linefeed_#{tag}2".to_sym) do + elem = initial_linefeed_pre.at("//*[@id=\"#{tag}2\"]") + refute_nil elem + assert_equal "\nx", elem.inner_html + end + end + + # https://github.com/web-platform-tests/wpt/blob/master/html/syntax/serializing-html-fragments/outerHTML.html + ELEMENTS_WITH_END_TAG = [ + "a", + "abbr", + "address", + "article", + "aside", + "audio", + "b", + "bdi", + "bdo", + "blockquote", + "body", + "button", + "canvas", + "caption", + "cite", + "code", + "colgroup", + "command", + "datalist", + "dd", + "del", + "details", + "dfn", + "dialog", + "div", + "dl", + "dt", + "em", + "fieldset", + "figcaption", + "figure", + "footer", + "form", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "head", + "header", + "hgroup", + "html", + "i", + "iframe", + "ins", + "kbd", + "label", + "legend", + "li", + "map", + "mark", + "menu", + "meter", + "nav", + "noscript", + "object", + "ol", + "optgroup", + "option", + "output", + "p", + "pre", + "progress", + "q", + "rp", + "rt", + "ruby", + "s", + "samp", + "script", + "section", + "select", + "small", + "span", + "strong", + "style", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "textarea", + "tfoot", + "th", + "thead", + "time", + "title", + "tr", + "u", + "ul", + "var", + "video", + "data", + ].freeze + + ELEMENTS_WITHOUT_END_TAG = [ + "area", + "base", + "br", + "col", + "embed", + "hr", + "img", + "input", + "keygen", + "link", + "meta", + "param", + "source", + "track", + "wbr", + ] + + ELEMENTS_WITH_END_TAG.each do |tag| + define_method("test_outer_html_#{tag}".to_sym) do + doc = Nokogiri::HTML5::Document.new + child = Nokogiri::XML::Element.new(tag, doc) + assert_equal "<#{tag}></#{tag}>", child.serialize(save_with: 0) + end + end + + ELEMENTS_WITHOUT_END_TAG.each do |tag| + define_method("test_outer_html_#{tag}".to_sym) do + doc = Nokogiri::HTML5::Document.new + child = Nokogiri::XML::Element.new(tag, doc) + assert_equal "<#{tag}>", child.serialize(save_with: 0) + end + end + + # https://github.com/web-platform-tests/wpt/blob/master/html/syntax/serializing-html-fragments/serializing.html + def serializing_test_data + @serializing_test_data ||= begin + html = <<~EOF.gsub(/ /, "") + <!DOCTYPE html> + <div id="test" style="display:none"> + <span></span> + <span><a></a></span> + <span><a b=c></a></span> + <span><a b='c'></a></span> + <span><a b='&'></a></span> + <span><a b=' '></a></span> + <span><a b='"'></a></span> + <span><a b="<"></a></span> + <span><a b=">"></a></span> + <span><svg xlink:href="a"></svg></span> + <span><svg xmlns:svg="test"></svg></span> + <span>a</span> + <span>&</span> + <span> </span> + <span><</span> + <span>></span> + <span>"</span> + <span><style><&></style></span> + <span><script type="test"><&></script></span> + <script type="test"><&></script> + <span><xmp><&></xmp></span> + <span><iframe><&></iframe></span> + <span><noembed><&></noembed></span> + <span><noframes><&></noframes></span> + <span><noscript><&></noscript></span> + <span><!--data--></span> + <span><a><b><c></c></b><d>e</d><f><g>h</g></f></a></span> + <span b=c></span> + </div> + EOF + Nokogiri::HTML5(html).xpath("/html/body/div/*") + end + @serializing_test_data + end + + EXPECTED = [ + ["", "<span></span>"], + ["<a></a>", "<span><a></a></span>"], + ["<a b=\"c\"></a>", "<span><a b=\"c\"></a></span>"], + ["<a b=\"c\"></a>", "<span><a b=\"c\"></a></span>"], + ["<a b=\"&\"></a>", "<span><a b=\"&\"></a></span>"], + ["<a b=\" \"></a>", "<span><a b=\" \"></a></span>"], + ["<a b=\""\"></a>", "<span><a b=\""\"></a></span>"], + ["<a b=\"<\"></a>", "<span><a b=\"<\"></a></span>"], + ["<a b=\">\"></a>", "<span><a b=\">\"></a></span>"], + ["<svg xlink:href=\"a\"></svg>", "<span><svg xlink:href=\"a\"></svg></span>"], + ["<svg xmlns:svg=\"test\"></svg>", "<span><svg xmlns:svg=\"test\"></svg></span>"], + ["a", "<span>a</span>"], + ["&", "<span>&</span>"], + [" ", "<span> </span>"], + ["<", "<span><</span>"], + [">", "<span>></span>"], + ["\"", "<span>\"</span>"], + ["<style><&></style>", "<span><style><&></style></span>"], + ["<script type=\"test\"><&><\/script>", "<span><script type=\"test\"><&><\/script></span>"], + ["<&>", "<script type=\"test\"><&><\/script>"], + ["<xmp><&></xmp>", "<span><xmp><&></xmp></span>"], + ["<iframe><&></iframe>", "<span><iframe><&></iframe></span>"], + ["<noembed><&></noembed>", "<span><noembed><&></noembed></span>"], + ["<noframes><&></noframes>", "<span><noframes><&></noframes></span>"], + ["<noscript><&></noscript>", "<span><noscript><&></noscript></span>"], + ["<!--data-->", "<span><!--data--></span>"], + ["<a><b><c></c></b><d>e</d><f><g>h</g></f></a>", "<span><a><b><c></c></b><d>e</d><f><g>h</g></f></a></span>"], + ["", "<span b=\"c\"></span>"], + ].freeze + + DOM_TESTS = [ + ["Attribute in the XML namespace", + lambda do + doc = Nokogiri::HTML5::Document.new + span = Nokogiri::XML::Element.new("span", doc) + svg = Nokogiri::XML::Element.new("svg", doc) + span.add_child(svg) + svg.add_namespace("xml", "http://www.w3.org/XML/1998/namespace") + svg["xml:foo"] = "test" + span + end, + '<svg xml:foo="test"></svg>', + '<span><svg xml:foo="test"></svg></span>'], + + ["Attribute in the XML namespace with the prefix not set to xml:", + lambda do + doc = Nokogiri::HTML5::Document.new + span = Nokogiri::XML::Element.new("span", doc) + svg = Nokogiri::XML::Element.new("svg", doc) + span.add_child(svg) + svg["abc:foo"] = "test" + ns = svg.add_namespace("xml", "http://www.w3.org/XML/1998/namespace") + svg.attribute("abc:foo").namespace = ns + span + end, + '<svg xml:foo="test"></svg>', + '<span><svg xml:foo="test"></svg></span>'], + + ["Non-'xmlns' attribute in the xmlns namespace", + lambda do + doc = Nokogiri::HTML5::Document.new + span = Nokogiri::XML::Element.new("span", doc) + svg = Nokogiri::XML::Element.new("svg", doc) + span.add_child(svg) + svg.add_namespace("xmlns", "http://www.w3.org/2000/xmlns/") + svg["xmlns:foo"] = "test" + span + end, + '<svg xmlns:foo="test"></svg>', + '<span><svg xmlns:foo="test"></svg></span>'], + + ["'xmlns' attribute in the xmlns namespace", + lambda do + doc = Nokogiri::HTML5::Document.new + span = Nokogiri::XML::Element.new("span", doc) + svg = Nokogiri::XML::Element.new("svg", doc) + span.add_child(svg) + svg.add_namespace("xmlns", "http://www.w3.org/2000/xmlns/") + svg["xmlns"] = "test" + span + end, + '<svg xmlns="test"></svg>', + '<span><svg xmlns="test"></svg></span>'], + + ["Attribute in non-standard namespace", + lambda do + doc = Nokogiri::HTML5::Document.new + span = Nokogiri::XML::Element.new("span", doc) + svg = Nokogiri::XML::Element.new("svg", doc) + span.add_child(svg) + svg.add_namespace("abc", "fake_ns") + svg["abc:def"] = "test" + span + end, + '<svg abc:def="test"></svg>', + '<span><svg abc:def="test"></svg></span>'], + + ["<span> starting with U+000A", + lambda do + doc = Nokogiri::HTML5::Document.new + span = Nokogiri::XML::Element.new("span", doc) + text = Nokogiri::XML::Text.new("\x0A", doc) + span.add_child(text) + span + end, + "\x0A", + "<span>\x0A</span>"], + # TODO: Processing instructions + ] + + TEXT_ELEMENTS = %w[pre textarea listing] + TEXT_TESTS = [ + ["<%text> context starting with U+000A", + lambda do |tag| + doc = Nokogiri::HTML5::Document.new + elem = Nokogiri::XML::Element.new(tag, doc) + text = Nokogiri::XML::Text.new("\x0A", doc) + elem.add_child(text) + elem + end, + "\x0A", + "<%text>\x0A</%text>"], + + ["<%text> context not starting with U+000A", + lambda do |tag| + doc = Nokogiri::HTML5::Document.new + elem = Nokogiri::XML::Element.new(tag, doc) + text = Nokogiri::XML::Text.new("a\x0A", doc) + elem.add_child(text) + elem + end, + "a\x0A", + "<%text>a\x0A</%text>"], + + ["<%text> non-context starting with U+000A", + lambda do |tag| + doc = Nokogiri::HTML5::Document.new + elem = Nokogiri::XML::Element.new(tag, doc) + span = Nokogiri::XML::Element.new("span", doc) + text = Nokogiri::XML::Text.new("\x0A", doc) + elem.add_child(text) + span.add_child(elem) + span + end, + "<%text>\x0A</%text>", + "<span><%text>\x0A</%text></span>"], + + ["<%text> non-context not starting with U+000A", + lambda do |tag| + doc = Nokogiri::HTML5::Document.new + elem = Nokogiri::XML::Element.new(tag, doc) + span = Nokogiri::XML::Element.new("span", doc) + text = Nokogiri::XML::Text.new("a\x0A", doc) + elem.add_child(text) + span.add_child(elem) + span + end, + "<%text>a\x0A</%text>", + "<span><%text>a\x0A</%text></span>"], + ] + + VOID_ELEMENTS = [ + "area", "base", "basefont", "bgsound", "br", "col", "embed", + "frame", "hr", "img", "input", "keygen", "link", + "meta", "param", "source", "track", "wbr" + ] + VOID_TESTS = [ + ["Void context node", + lambda do |tag| + doc = Nokogiri::HTML5::Document.new + Nokogiri::XML::Element.new(tag, doc) + end, + "", + "<%void>"], + + ["void as first child with following siblings", + lambda do |tag| + doc = Nokogiri::HTML5::Document.new + span = Nokogiri::XML::Element.new("span", doc) + span.add_child(Nokogiri::XML::Element.new(tag, doc)) + span.add_child(Nokogiri::XML::Element.new("a", doc)) + .add_child(Nokogiri::XML::Text.new("test", doc)) + span.add_child(Nokogiri::XML::Element.new("b", doc)) + span + end, + "<%void><a>test</a><b></b>", + "<span><%void><a>test</a><b></b></span>"], + + ["void as second child with following siblings", + lambda do |tag| + doc = Nokogiri::HTML5::Document.new + span = Nokogiri::XML::Element.new("span", doc) + span.add_child(Nokogiri::XML::Element.new("a", doc)) + .add_child(Nokogiri::XML::Text.new("test", doc)) + span.add_child(Nokogiri::XML::Element.new(tag, doc)) + span.add_child(Nokogiri::XML::Element.new("b", doc)) + span + end, + "<a>test</a><%void><b></b>", + "<span><a>test</a><%void><b></b></span>"], + ["void as last child with preceding siblings", + lambda do |tag| + doc = Nokogiri::HTML5::Document.new + span = Nokogiri::XML::Element.new("span", doc) + span.add_child(Nokogiri::XML::Element.new("a", doc)) + .add_child(Nokogiri::XML::Text.new("test", doc)) + span.add_child(Nokogiri::XML::Element.new("b", doc)) + span.add_child(Nokogiri::XML::Element.new(tag, doc)) + span + end, + "<a>test</a><b></b><%void>", + "<span><a>test</a><b></b><%void></span>"], + ] + + # Generate tests + def self.cross_map(a1, a2) + rv = [] + a1.each do |a1_elem| + a2.each do |a2_elem| + rv << yield(a1_elem, a2_elem) + end + end + end + + EXPECTED.each_with_index do |item, i| + define_method("test_serializing_html_innerHTML_#{i}".to_sym) do + assert_equal item[0], serializing_test_data[i].inner_html + end + + define_method("test_serializing_html_outerHTML_#{i}".to_sym) do + assert_equal item[1], serializing_test_data[i].serialize + end + end + + DOM_TESTS.each do |test| + define_method("test_serializing_dom_innerHTML_#{test[0]}".to_sym) do + elem = test[1].call + refute_nil elem + assert_equal test[2], elem.inner_html + end + + define_method("test_serializing_dom_outerHTML_#{test[0]}".to_sym) do + elem = test[1].call + refute_nil elem + assert_equal test[3], elem.serialize + end + end + + cross_map(TEXT_TESTS, TEXT_ELEMENTS) do |test_data, tag| + define_method("test_serializing_text_innerHTML_#{test_data[0].gsub("%text", tag)}".to_sym) do + assert_equal test_data[2].gsub("%text", tag), test_data[1].call(tag).inner_html + end + + define_method("test_serialization_text_outerHTML_#{test_data[0].gsub("%text", tag)}".to_sym) do + assert_equal test_data[3].gsub("%text", tag), test_data[1].call(tag).serialize + end + end + + cross_map(VOID_TESTS, VOID_ELEMENTS) do |test_data, tag| + define_method("test_serializing_void_innerHTML_#{test_data[0]}_#{tag}".to_sym) do + assert_equal test_data[2].gsub("%void", tag), test_data[1].call(tag).inner_html + end + + define_method("test_serialization_void_outerHTML_#{test_data[0]}_#{tag}".to_sym) do + assert_equal test_data[3].gsub("%void", tag), test_data[1].call(tag).serialize + end + end +end if Nokogiri.uses_gumbo? diff --git a/test/html5/test_tree-construction.rb b/test/html5/test_tree-construction.rb new file mode 100644 index 0000000000..37fdee20d4 --- /dev/null +++ b/test/html5/test_tree-construction.rb @@ -0,0 +1,282 @@ +# encoding: utf-8 +# frozen_string_literal: true +require "helper" + +if Nokogiri.uses_gumbo? + def parse_test(test_data) + test = { script: :both } + index = /(?:^#errors\n|\n#errors\n)/ =~ test_data + abort("Expected #errors in\n#{test_data}") if index.nil? + skip_amount = $~[0].length + # Omit the final new line + test[:data] = test_data[0...index] + + # Process the rest line by line + lines = test_data[index + skip_amount..-1].split("\n") + index = lines.find_index do |line| + line == "#document-fragment" || + line == "#document" || + line == "#script-off" || + line == "#script-on" || + line == "#new-errors" + end + abort("Expected #document") if index.nil? + test[:errors] = lines[0...index] + test[:new_errors] = [] + if lines[index] == "#new-errors" + index += 1 + until %w[#document-fragment #document #script-off #script-on].include?(lines[index]) + test[:new_errors] << lines[index] + index += 1 + end + end + + if lines[index] == "#document-fragment" + test[:context] = lines[index + 1].chomp.split(" ", 2) + index += 2 + end + abort("failed to find fragment: #{index}: #{lines[index]}") if test_data.include?("#document-fragment") && test[:context].nil? + + if lines[index] =~ /#script-(on|off)/ + test[:script] = $~[1].to_sym + index += 1 + end + + abort("Expected #document, got #{lines[index]}") unless lines[index] == "#document" + index += 1 + + document = { + type: test[:context] ? :fragment : :document, + children: [], + } + open_nodes = [document] + while index < lines.length + abort("Expected '| ' but got #{lines[index]}") unless /^\| ( *)([^ ].*$)/ =~ lines[index] + depth = $~[1].length + if depth.odd? + abort("Invalid nesting depth") + else + depth /= 2 + end + abort("Too deep") if depth >= open_nodes.length + + node = {} + node_text = $~[2] + if node_text[0] == '"' + if node_text == '"' || node_text[-1] != '"' + loop do + index += 1 + node_text << "\n" + lines[index] + break if node_text[-1] == '"' + end + end + node[:type] = :text + node[:contents] = node_text[1..-2] + elsif /^<!DOCTYPE ([^ >]*)(?: "([^"]*)" "(.*)")?>$/ =~ node_text + node[:type] = :doctype + node[:name] = $~[1] + node[:public_id] = $~[2].nil? || $~[2].empty? ? nil : $~[2] + node[:system_id] = $~[3].nil? || $~[3].empty? ? nil : $~[3] + elsif node_text.start_with?('<!-- ') + loop do + break if lines[index].end_with?(' -->') + index += 1 + node_text << "\n" + lines[index] + end + node[:type] = :comment + node[:contents] = node_text[5..-5] + elsif /^<(svg |math )?(.+)>$/ =~ node_text + node[:type] = :element + node[:ns] = $~[1].nil? ? nil : $~[1].rstrip + node[:tag] = $~[2] + node[:attributes] = [] + node[:children] = [] + elsif /^([^ ]+ )?([^=]+)="(.*)"$/ =~ node_text + node[:type] = :attribute + node[:ns] = $~[1].nil? ? nil : $~[1].rstrip + node[:name] = $~[2] + node[:value] = $~[3] + elsif node_text == "content" + node[:type] = :template + else + abort("Unexpected node_text: #{node_text}") + end + + if node[:type] == :attribute + abort("depth #{depth} != #{open_nodes.length}") unless depth == open_nodes.length - 1 + abort("type :#{open_nodes[-1][:type]} != :element") unless open_nodes[-1][:type] == :element + abort("element has children") unless open_nodes[-1][:children].empty? + open_nodes[-1][:attributes] << node + elsif node[:type] == :template + abort("depth #{depth} != #{open_nodes.length}") unless depth == open_nodes.length - 1 + abort("type :#{open_nodes[-1][:type]} != :element") unless open_nodes[-1][:type] == :element + abort("tag :#{open_nodes[-1][:tag]} != template") unless open_nodes[-1][:tag] == "template" + abort("template has children before the 'content'") unless open_nodes[-1][:children].empty? + # Hack. We want the children of this template node to be reparented as + # children of the template element. + # XXX: Template contents are _not_ supposed to be children of the + # template, but we currently mishandle this. + open_nodes << open_nodes[-1] + else + open_nodes[depth][:children] << node + open_nodes[depth + 1..-1] = [] + if node[:type] == :element + open_nodes << node + end + end + index += 1 + end + test[:document] = document + test + end + + class TestHtml5TreeConstructionBase < Nokogiri::TestCase + def assert_equal_or_nil(exp, act) + if exp.nil? + assert_nil(act) + else + assert_equal(exp, act) + end + end + + def compare_nodes(node, ng_node) + case ng_node.type + when Nokogiri::XML::Node::ELEMENT_NODE + assert_equal(node[:type], :element) + if node[:ns] + refute_nil(ng_node.namespace) + assert_equal(node[:ns], ng_node.namespace.prefix) + end + assert_equal(node[:tag], ng_node.name) + attributes = ng_node.attributes + assert_equal(node[:attributes].length, attributes.length) + node[:attributes].each do |attr| + value = if attr[:ns] + ng_node["#{attr[:ns]}:#{attr[:name]}"] + else + attributes[attr[:name]].value + end + assert_equal(attr[:value], value) + end + assert_equal(node[:children].length, ng_node.children.length, + "Element <#{node[:tag]}> has wrong number of children #{ng_node.children.map { |c| c.name }} in #{@test[:data]}") + when Nokogiri::XML::Node::TEXT_NODE, Nokogiri::XML::Node::CDATA_SECTION_NODE + # We preserve the CDATA in the tree, but the tests represent it as text. + assert_equal(node[:type], :text) + assert_equal(node[:contents], ng_node.content) + when Nokogiri::XML::Node::COMMENT_NODE + assert_equal(node[:type], :comment) + assert_equal(node[:contents], ng_node.content) + when Nokogiri::XML::Node::HTML_DOCUMENT_NODE + assert_equal(node[:type], :document) + assert_equal(node[:children].length, ng_node.children.length) + when Nokogiri::XML::Node::DOCUMENT_FRAG_NODE + assert_equal(node[:type], :fragment) + assert_equal(node[:children].length, ng_node.children.length, + "Fragment node has wrong number of children #{ng_node.children.map { |c| c.name }} in #{@test[:data]}") + when Nokogiri::XML::Node::DTD_NODE + assert_equal(node[:type], :doctype) + assert_equal(node[:name], ng_node.name) + assert_equal_or_nil(node[:public_id], ng_node.external_id) + assert_equal_or_nil(node[:system_id], ng_node.system_id) + else + flunk("Unknown node type #{ng_node.type} (expected #{node[:type]})") + end + end + + def run_test + if @test[:context] + ctx = @test[:context].join(":") + doc = Nokogiri::HTML5::Document.new + doc = Nokogiri::HTML5::DocumentFragment.new(doc, @test[:data], ctx, max_errors: @test[:errors].length + 10) + else + doc = Nokogiri::HTML5.parse(@test[:data], max_errors: @test[:errors].length + 10) + end + # Walk the tree. + exp_nodes = [@test[:document]] + act_nodes = [doc] + children = [0] + compare_nodes(exp_nodes[0], doc) + while children.any? + child_index = children[-1] + exp = exp_nodes[-1] + act = act_nodes[-1] + if child_index == exp[:children].length + exp_nodes.pop + act_nodes.pop + children.pop + next + end + exp_child = exp[:children][child_index] + act_child = act.children[child_index] + compare_nodes(exp_child, act_child) + children[-1] = child_index + 1 + next unless exp_child.has_key?(:children) + exp_nodes << exp_child + act_nodes << act_child + children << 0 + end + + # Test the errors. + assert_equal(@test[:errors].length, doc.errors.length, "Wrong number of errors for #{@test[:data]}") + + # The new, standardized tokenizer errors live in @test[:new_errors]. Let's + # match each one to exactly one error in doc.errors. Unfortunately, the + # tests specify the column the error is detected, _not_ the column of the + # start of the problematic HTML (e.g., the start of a character reference + # or <![CDATA[) the way gumbo does. So check that Gumbo's column is no + # later than the error's column. + errors = doc.errors.map { |err| { line: err.line, column: err.column, code: err.str1 } } + errors.reject! { |err| err[:code] == "generic-parser" } + error_regex = /^\((?<line>\d+):(?<column>\d+)(?:-\d+:\d+)?\) (?<code>.*)$/ + @test[:new_errors].each do |err| + assert_match(error_regex, err, "New error format does not match: #{mu_pp(err)}") + m = err.match(error_regex) + line = m[:line].to_i + column = m[:column].to_i + code = m[:code] + idx = errors.index do |e| + e[:line] == line && + e[:code] == code && + e[:column] <= column + end + # This error should be the first error in the list. + # refute_nil(idx, "Expected to find error #{code} at #{line}:#{column}") + assert_equal(0, idx, "Expected to find error #{code} at #{line}:#{column} in #{@test[:data]}") + errors.delete_at(idx) + end + end + end + + tc_path = File.expand_path("../../html5lib-tests/tree-construction", __FILE__) + Dir[File.join(tc_path, "*.dat")].each do |path| + test_name = "TestHtml5TreeConstruction" + File.basename(path, ".dat") + .split(/[_-]/) + .map { |s| s.capitalize } + .join("") + tests = [] + File.open(path, "r", encoding: "UTF-8") do |f| + f.each("\n\n#data\n") do |test_data| + if test_data.start_with?("#data\n") + test_data = test_data[6..-1] + end + if test_data.end_with?("\n\n#data\n") + test_data = test_data[0..-9] + end + tests << parse_test(test_data) + end + end + + klass = Class.new(TestHtml5TreeConstructionBase) do + tests.each_with_index do |test, index| + next if test[:script] == :on + define_method "test_#{index}".to_sym do + @test = test + @index = index + run_test + end + end + end + Object.const_set(test_name, klass) + end +end diff --git a/test/html5lib-tests b/test/html5lib-tests new file mode 160000 index 0000000000..6030cb6e40 --- /dev/null +++ b/test/html5lib-tests @@ -0,0 +1 @@ +Subproject commit 6030cb6e40a0cf68ae38bf0001bb85b727b80a26 diff --git a/test/namespaces/test_namespaces_in_created_doc.rb b/test/namespaces/test_namespaces_in_created_doc.rb index 43dd7a9b8b..a1cc701ad1 100644 --- a/test/namespaces/test_namespaces_in_created_doc.rb +++ b/test/namespaces/test_namespaces_in_created_doc.rb @@ -7,7 +7,7 @@ def setup super @doc = Nokogiri::XML('<fruit xmlns="ns:fruit" xmlns:veg="ns:veg" xmlns:xlink="http://www.w3.org/1999/xlink"/>') pear = @doc.create_element('pear') - bosc = @doc.create_element('bosc') + bosc = @doc.create_element('bosc', {"veg:any" => "none"}) pear.add_child(bosc) @doc.root << pear @doc.root.add_child('<orange/>') @@ -38,6 +38,9 @@ def test_created_parent_default_ns def test_created_grandparent_default_ns assert_equal 'ns:fruit', check_namespace(@doc.root.elements[0].elements[0]) end + def test_created_attribute_with_ns + assert_equal 'ns:veg', check_namespace(@doc.root.elements[0].elements[0].attribute_nodes.first) + end def test_created_parent_nondefault_ns assert_equal 'ns:veg', check_namespace(@doc.root.elements[2]) end diff --git a/test/test_nokogiri.rb b/test/test_nokogiri.rb index e1a90b43f1..d44a6d5252 100644 --- a/test/test_nokogiri.rb +++ b/test/test_nokogiri.rb @@ -1,54 +1,55 @@ +# frozen_string_literal: true require "helper" -class TestNokogiri < Nokogiri::TestCase - def test_libxml_iconv - skip_unless_libxml2("this constant is only set in the C extension when libxml2 is used") - assert Nokogiri.const_defined?(:LIBXML_ICONV_ENABLED) - end - - def test_parse_with_io - doc = Nokogiri.parse( - StringIO.new("<html><head><title></title><body></body></html>") - ) - assert_instance_of Nokogiri::HTML::Document, doc - end - - def test_xml? - doc = Nokogiri.parse(File.read(XML_FILE)) - assert doc.xml? - assert !doc.html? - end - - def test_atom_is_xml? - doc = Nokogiri.parse(File.read(XML_ATOM_FILE)) - assert doc.xml? - assert !doc.html? - end - - def test_html? - doc = Nokogiri.parse(File.read(HTML_FILE)) - assert !doc.xml? - assert doc.html? - end - - def test_nokogiri_method_with_html - doc1 = Nokogiri(File.read(HTML_FILE)) - doc2 = Nokogiri.parse(File.read(HTML_FILE)) - assert_equal doc1.serialize, doc2.serialize - end - - def test_nokogiri_method_with_block - doc = Nokogiri { b "bold tag" } - assert_equal("<b>bold tag</b>", doc.to_html.chomp) - end - - def test_make_with_html - doc = Nokogiri.make("<b>bold tag</b>") - assert_equal("<b>bold tag</b>", doc.to_html.chomp) - end - - def test_make_with_block - doc = Nokogiri.make { b "bold tag" } - assert_equal("<b>bold tag</b>", doc.to_html.chomp) +class Nokogiri::TestCase + describe Nokogiri do + def test_libxml_iconv + skip_unless_libxml2("this constant is only set in the C extension when libxml2 is used") + assert(Nokogiri.const_defined?(:LIBXML_ICONV_ENABLED)) + end + + def test_parse_with_io + doc = Nokogiri.parse(StringIO.new("<html><head><title></title><body></body></html>")) + assert_instance_of(Nokogiri::HTML::Document, doc) + end + + def test_xml? + doc = Nokogiri.parse(File.read(XML_FILE)) + assert(doc.xml?) + assert(!doc.html?) + end + + def test_atom_is_xml? + doc = Nokogiri.parse(File.read(XML_ATOM_FILE)) + assert(doc.xml?) + assert(!doc.html?) + end + + def test_html? + doc = Nokogiri.parse(File.read(HTML_FILE)) + assert(!doc.xml?) + assert(doc.html?) + end + + def test_nokogiri_method_with_html + doc1 = Nokogiri(File.read(HTML_FILE)) + doc2 = Nokogiri.parse(File.read(HTML_FILE)) + assert_equal(doc1.serialize, doc2.serialize) + end + + def test_nokogiri_method_with_block + doc = Nokogiri { b("bold tag") } + assert_equal("<b>bold tag</b>", doc.to_html.chomp) + end + + def test_make_with_html + doc = Nokogiri.make("<b>bold tag</b>") + assert_equal("<b>bold tag</b>", doc.to_html.chomp) + end + + def test_make_with_block + doc = Nokogiri.make { b("bold tag") } + assert_equal("<b>bold tag</b>", doc.to_html.chomp) + end end end diff --git a/test/test_nokogumbo_contract.rb b/test/test_nokogumbo_contract.rb new file mode 100644 index 0000000000..83df5837ea --- /dev/null +++ b/test/test_nokogumbo_contract.rb @@ -0,0 +1,26 @@ +require "helper" + +describe "Nokogumbo contract expectations" do + # per https://github.com/rubys/nokogumbo/pull/171 + it "includes the HTML5 public interface" do + skip("Gumbo is not supported on this platform") unless Nokogiri.uses_gumbo? + + assert_includes(::Nokogiri.singleton_methods, :HTML5) + + assert_equal(defined?(::Nokogiri::HTML5), "constant") + assert_includes(::Nokogiri::HTML5.singleton_methods, :parse) + assert_includes(::Nokogiri::HTML5.singleton_methods, :fragment) + + assert_equal(defined?(::Nokogiri::HTML5::Node), "constant") + assert_equal(defined?(::Nokogiri::HTML5::Document), "constant") + assert_equal(defined?(::Nokogiri::HTML5::DocumentFragment), "constant") + end + + it "includes a replacement for the Nokogumbo private interface" do + skip("Gumbo is not supported on this platform") unless Nokogiri.uses_gumbo? + + assert_equal(defined?(::Nokogiri::Gumbo), "constant") + assert_includes(::Nokogiri::Gumbo.singleton_methods, :parse) + assert_includes(::Nokogiri::Gumbo.singleton_methods, :fragment) + end +end diff --git a/test/test_version.rb b/test/test_version.rb index dda8f392ba..e8f7468b6f 100644 --- a/test/test_version.rb +++ b/test/test_version.rb @@ -59,6 +59,7 @@ def test_version_info_for_libxml assert_equal("#{major}.#{minor}.#{bug}", Nokogiri::VERSION_INFO["libxml"]["loaded"]) assert(version_info["libxml"].key?("iconv_enabled")) + assert(version_info["libxslt"].key?("datetime_enabled")) end def test_version_info_for_libxslt diff --git a/test/test_xslt_transforms.rb b/test/test_xslt_transforms.rb index 708215dc80..1f34290b6a 100644 --- a/test/test_xslt_transforms.rb +++ b/test/test_xslt_transforms.rb @@ -1,372 +1,406 @@ # frozen_string_literal: true require "helper" -class TestXsltTransforms < Nokogiri::TestCase - def setup - super - @doc = Nokogiri::XML(File.open(XML_FILE)) - end +class Nokogiri::TestCase + describe Nokogiri::XSLT::Stylesheet do + let(:doc) { Nokogiri::XML(File.open(XML_FILE)) } - def test_class_methods - style = Nokogiri::XSLT(File.read(XSLT_FILE)) + def test_class_methods + style = Nokogiri::XSLT(File.read(XSLT_FILE)) - assert(result = style.apply_to(@doc, ['title', '"Grandma"'])) - assert_match(%r{<h1>Grandma</h1>}, result) - end + assert(result = style.apply_to(doc, ["title", '"Grandma"'])) + assert_match(%r{<h1>Grandma</h1>}, result) + end - def test_transform - assert(style = Nokogiri::XSLT.parse(File.read(XSLT_FILE))) - - assert(result = style.apply_to(@doc, ['title', '"Booyah"'])) - assert_match(%r{<h1>Booyah</h1>}, result) - assert_match(%r{<th.*Employee ID</th>}, result) - assert_match(%r{<th.*Name</th>}, result) - assert_match(%r{<th.*Position</th>}, result) - assert_match(%r{<th.*Salary</th>}, result) - assert_match(%r{<td>EMP0003</td>}, result) - assert_match(%r{<td>Margaret Martin</td>}, result) - assert_match(%r{<td>Computer Specialist</td>}, result) - assert_match(%r{<td>100,000</td>}, result) - assert_no_match(/Dallas|Texas/, result) - assert_no_match(/Female/, result) - - assert(result = style.apply_to(@doc, ['title', '"Grandma"'])) - assert_match(%r{<h1>Grandma</h1>}, result) - - assert(result = style.apply_to(@doc)) - assert_match(%r{<h1></h1>|<h1/>}, result) - end + def test_transform + assert(style = Nokogiri::XSLT.parse(File.read(XSLT_FILE))) + + assert(result = style.apply_to(doc, ["title", '"Booyah"'])) + assert_match(%r{<h1>Booyah</h1>}, result) + assert_match(%r{<th.*Employee ID</th>}, result) + assert_match(%r{<th.*Name</th>}, result) + assert_match(%r{<th.*Position</th>}, result) + assert_match(%r{<th.*Salary</th>}, result) + assert_match(%r{<td>EMP0003</td>}, result) + assert_match(%r{<td>Margaret Martin</td>}, result) + assert_match(%r{<td>Computer Specialist</td>}, result) + assert_match(%r{<td>100,000</td>}, result) + assert_no_match(/Dallas|Texas/, result) + assert_no_match(/Female/, result) + + assert(result = style.apply_to(doc, ["title", '"Grandma"'])) + assert_match(%r{<h1>Grandma</h1>}, result) + + assert(result = style.apply_to(doc)) + assert_match(%r{<h1></h1>|<h1/>}, result) + end - def test_xml_declaration - input_xml = <<~EOS - <?xml version="1.0" encoding="utf-8"?> - <report> - <title>My Report</title> - </report> - EOS - - input_xsl = <<~EOS - <?xml version="1.0" encoding="utf-8"?> - <xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> - <xsl:output method="xml" version="1.0" encoding="utf-8" indent="yes"/> - <xsl:template match="/"> - <html> - <head> - <title><xsl:value-of select="report/title"/></title> - <style> .commit-tease, .user-profile-mini-avatar, .avatar, .vcard-details, .signup-prompt-bg { display: none !IMPORTANT; } </style> <script> document.addEventListener('DOMContentLoaded', function() { this.querySelectorAll('a').forEach(anchor => { anchor.addEventListener('click', e => { e.preventDefault(); const redact = new URLSearchParams(window.location.search).get('redact'); const hasExistingParams = anchor.href.includes('?'); window.location.href = anchor.href + (hasExistingParams ? `&redact=${redact}` : `?redact=${redact}`); }); }); }); </script> </head> - <body> - <h1><xsl:value-of select="report/title"/></h1> - </body> - </html> - </xsl:template> - </xsl:stylesheet> - EOS + def test_xml_declaration + input_xml = <<~EOS + <?xml version="1.0" encoding="utf-8"?> + <report> + <title>My Report</title> + </report> + EOS + + input_xsl = <<~EOS + <?xml version="1.0" encoding="utf-8"?> + <xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> + <xsl:output method="xml" version="1.0" encoding="utf-8" indent="yes"/> + <xsl:template match="/"> + <html> + <head> + <title><xsl:value-of select="report/title"/></title> + <style> .commit-tease, .user-profile-mini-avatar, .avatar, .vcard-details, .signup-prompt-bg { display: none !IMPORTANT; } </style> <script> document.addEventListener('DOMContentLoaded', function() { this.querySelectorAll('a').forEach(anchor => { anchor.addEventListener('click', e => { e.preventDefault(); const redact = new URLSearchParams(window.location.search).get('redact'); const hasExistingParams = anchor.href.includes('?'); window.location.href = anchor.href + (hasExistingParams ? `&redact=${redact}` : `?redact=${redact}`); }); }); }); </script> </head> + <body> + <h1><xsl:value-of select="report/title"/></h1> + </body> + </html> + </xsl:template> + </xsl:stylesheet> + EOS - require 'nokogiri' + require "nokogiri" - xml = ::Nokogiri::XML(input_xml) - xsl = ::Nokogiri::XSLT(input_xsl) + xml = ::Nokogiri::XML(input_xml) + xsl = ::Nokogiri::XSLT(input_xsl) - assert_includes(xsl.apply_to(xml), '<?xml version="1.0" encoding="utf-8"?>') - end + assert_includes(xsl.apply_to(xml), '<?xml version="1.0" encoding="utf-8"?>') + end - def test_transform_with_output_style - xslt = "" - xslt = if Nokogiri.jruby? - Nokogiri::XSLT(<<~eoxslt) - <?xml version="1.0" encoding="ISO-8859-1"?> + def test_transform_with_output_style + xslt = "" + xslt = if Nokogiri.jruby? + Nokogiri::XSLT(<<~eoxslt) + <?xml version="1.0" encoding="ISO-8859-1"?> - <xsl:stylesheet version="1.0" - xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> - <xsl:output method="text" version="1.0" - encoding="iso-8859-1" indent="yes"/> - - <xsl:param name="title"/> - - <xsl:template match="/"> - <html> - <body> - <xsl:for-each select="staff/employee"> - <tr> - <td><xsl:value-of select="employeeId"/></td> - <td><xsl:value-of select="name"/></td> - <td><xsl:value-of select="position"/></td> - <td><xsl:value-of select="salary"/></td> - </tr> - </xsl:for-each> - </body> - </html> - </xsl:template> + <xsl:stylesheet version="1.0" + xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> + <xsl:output method="text" version="1.0" + encoding="iso-8859-1" indent="yes"/> - </xsl:stylesheet> - eoxslt - else - Nokogiri::XSLT(<<~eoxslt) - <?xml version="1.0" encoding="ISO-8859-1"?> + <xsl:param name="title"/> - <xsl:stylesheet version="1.0" - xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> - <xsl:output method="text" version="1.0" - encoding="iso-8859-1" indent="yes"/> - - <xsl:param name="title"/> - - <xsl:template match="/"> - <html> - <body> - <xsl:for-each select="staff/employee"> - <tr> - <td><xsl:value-of select="employeeId"/></td> - <td><xsl:value-of select="name"/></td> - <td><xsl:value-of select="position"/></td> - <td><xsl:value-of select="salary"/></td> - </tr> - </xsl:for-each> - </table> - </body> - </html> - </xsl:template> + <xsl:template match="/"> + <html> + <body> + <xsl:for-each select="staff/employee"> + <tr> + <td><xsl:value-of select="employeeId"/></td> + <td><xsl:value-of select="name"/></td> + <td><xsl:value-of select="position"/></td> + <td><xsl:value-of select="salary"/></td> + </tr> + </xsl:for-each> + </body> + </html> + </xsl:template> - </xsl:stylesheet> - eoxslt + </xsl:stylesheet> + eoxslt + else + Nokogiri::XSLT(<<~eoxslt) + <?xml version="1.0" encoding="ISO-8859-1"?> + + <xsl:stylesheet version="1.0" + xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> + <xsl:output method="text" version="1.0" + encoding="iso-8859-1" indent="yes"/> + + <xsl:param name="title"/> + + <xsl:template match="/"> + <html> + <body> + <xsl:for-each select="staff/employee"> + <tr> + <td><xsl:value-of select="employeeId"/></td> + <td><xsl:value-of select="name"/></td> + <td><xsl:value-of select="position"/></td> + <td><xsl:value-of select="salary"/></td> + </tr> + </xsl:for-each> + </table> + </body> + </html> + </xsl:template> + + </xsl:stylesheet> + eoxslt + end + result = xslt.apply_to(doc, ["title", "foo"]) + assert_no_match(/<td>/, result) + + # the entity-form is for systems with this bug with Encodings.properties + # https://issues.apache.org/jira/browse/XALANJ-2618 + # a.k.a. "Attempt to output character of integral value 48 that is not represented in specified output encoding of iso-8859-1." + assert_match( + /This is an adjacent|This is an adjacent/, result + ) end - result = xslt.apply_to(@doc, ['title', 'foo']) - assert_no_match(/<td>/, result) - - # the entity-form is for systems with this bug with Encodings.properties - # https://issues.apache.org/jira/browse/XALANJ-2618 - # a.k.a. "Attempt to output character of integral value 48 that is not represented in specified output encoding of iso-8859-1." - assert_match( - /This is an adjacent|This is an adjacent/, result - ) - end - def test_transform_arg_error - assert(style = Nokogiri::XSLT(File.read(XSLT_FILE))) - assert_raises(TypeError) do - style.transform(@doc, :foo) + def test_transform_arg_error + assert(style = Nokogiri::XSLT(File.read(XSLT_FILE))) + assert_raises(TypeError) do + style.transform(doc, :foo) + end end - end - def test_transform_with_hash - assert(style = Nokogiri::XSLT(File.read(XSLT_FILE))) - result = style.transform(@doc, { 'title' => '"Booyah"' }) - assert(result.html?) - assert_equal("Booyah", result.at_css("h1").content) - end + def test_transform_with_hash + assert(style = Nokogiri::XSLT(File.read(XSLT_FILE))) + result = style.transform(doc, { "title" => '"Booyah"' }) + assert(result.html?) + assert_equal("Booyah", result.at_css("h1").content) + end - def test_transform2 - assert(style = Nokogiri::XSLT(File.open(XSLT_FILE))) - assert(result_doc = style.transform(@doc)) - assert(result_doc.html?) - assert_equal("", result_doc.at_css("h1").content) + def test_transform2 + assert(style = Nokogiri::XSLT(File.open(XSLT_FILE))) + assert(result_doc = style.transform(doc)) + assert(result_doc.html?) + assert_equal("", result_doc.at_css("h1").content) - assert(style = Nokogiri::XSLT(File.read(XSLT_FILE))) - assert(result_doc = style.transform(@doc, ['title', '"Booyah"'])) - assert(result_doc.html?) - assert_equal("Booyah", result_doc.at_css("h1").content) + assert(style = Nokogiri::XSLT(File.read(XSLT_FILE))) + assert(result_doc = style.transform(doc, ["title", '"Booyah"'])) + assert(result_doc.html?) + assert_equal("Booyah", result_doc.at_css("h1").content) - assert(result_string = style.apply_to(@doc, ['title', '"Booyah"'])) - assert_equal(result_string, style.serialize(result_doc)) - end + assert(result_string = style.apply_to(doc, ["title", '"Booyah"'])) + assert_equal(result_string, style.serialize(result_doc)) + end - def test_transform_with_quote_params - assert(style = Nokogiri::XSLT(File.open(XSLT_FILE))) - assert(result_doc = style.transform(@doc, Nokogiri::XSLT.quote_params(['title', 'Booyah']))) - assert(result_doc.html?) - assert_equal("Booyah", result_doc.at_css("h1").content) + def test_transform_with_quote_params + assert(style = Nokogiri::XSLT(File.open(XSLT_FILE))) + assert(result_doc = style.transform(doc, Nokogiri::XSLT.quote_params(["title", "Booyah"]))) + assert(result_doc.html?) + assert_equal("Booyah", result_doc.at_css("h1").content) - assert(style = Nokogiri::XSLT.parse(File.read(XSLT_FILE))) - assert(result_doc = style.transform(@doc, Nokogiri::XSLT.quote_params({ 'title' => 'Booyah' }))) - assert(result_doc.html?) - assert_equal("Booyah", result_doc.at_css("h1").content) - end + assert(style = Nokogiri::XSLT.parse(File.read(XSLT_FILE))) + assert(result_doc = style.transform(doc, Nokogiri::XSLT.quote_params({ "title" => "Booyah" }))) + assert(result_doc.html?) + assert_equal("Booyah", result_doc.at_css("h1").content) + end - def test_quote_params - h = { - :sym => %{xxx}, - 'str' => %{"xxx"}, - :sym2 => %{'xxx'}, - 'str2' => %{x'x'x}, - :sym3 => %{x"x"x}, - } - hh = h.dup - result_hash = Nokogiri::XSLT.quote_params(h) - assert_equal(hh, h) # non-destructive - - a = h.to_a.flatten - result_array = Nokogiri::XSLT.quote_params(a) - assert_equal(h.to_a.flatten, a) # non-destructive - - assert_equal(result_array, result_hash) - end + def test_quote_params + h = { + :sym => %{xxx}, + "str" => %{"xxx"}, + :sym2 => %{'xxx'}, + "str2" => %{x'x'x}, + :sym3 => %{x"x"x}, + } + hh = h.dup + result_hash = Nokogiri::XSLT.quote_params(h) + assert_equal(hh, h) # non-destructive + + a = h.to_a.flatten + result_array = Nokogiri::XSLT.quote_params(a) + assert_equal(h.to_a.flatten, a) # non-destructive + + assert_equal(result_array, result_hash) + end - def test_exslt - # see http://yokolet.blogspot.com/2010/10/pure-java-nokogiri-xslt-extension.html") - skip_unless_libxml2("cannot get it working on JRuby") - - assert(doc = Nokogiri::XML.parse(File.read(EXML_FILE))) - assert(doc.xml?) - - assert(style = Nokogiri::XSLT.parse(File.read(EXSLT_FILE))) - params = { - p1: 'xxx', - p2: "x'x'x", - p3: 'x"x"x', - p4: '"xxx"', - } - result_doc = Nokogiri::XML.parse(style.apply_to(doc, + def test_exslt + # see http://yokolet.blogspot.com/2010/10/pure-java-nokogiri-xslt-extension.html") + skip_unless_libxml2("cannot get it working on JRuby") + + assert(doc = Nokogiri::XML.parse(File.read(EXML_FILE))) + assert(doc.xml?) + + assert(style = Nokogiri::XSLT.parse(File.read(EXSLT_FILE))) + params = { + p1: "xxx", + p2: "x'x'x", + p3: 'x"x"x', + p4: '"xxx"', + } + result_doc = Nokogiri::XML.parse(style.apply_to(doc, Nokogiri::XSLT.quote_params(params))) - assert_equal('func-result', result_doc.at('/root/function').content) - assert_equal(3, result_doc.at('/root/max').content.to_i) - assert_match( - /\d{4}-\d\d-\d\d([-|+]\d\d:\d\d)?/, - result_doc.at('/root/date').content - ) - result_doc.xpath('/root/params/*').each do |p| - assert_equal(p.content, params[p.name.intern]) - end - check_params(result_doc, params) - result_doc = Nokogiri::XML.parse(style.apply_to(doc, + assert_equal("func-result", result_doc.at("/root/function").content) + assert_equal(3, result_doc.at("/root/max").content.to_i) + if Nokogiri::VersionInfo.instance.libxslt_has_datetime? + assert_match( + /\d{4}-\d\d-\d\d([-|+]\d\d:\d\d)?/, + result_doc.at("/root/date").content + ) + end + result_doc.xpath("/root/params/*").each do |p| + assert_equal(p.content, params[p.name.intern]) + end + check_params(result_doc, params) + result_doc = Nokogiri::XML.parse(style.apply_to(doc, Nokogiri::XSLT.quote_params(params.to_a.flatten))) - check_params(result_doc, params) - end + check_params(result_doc, params) + end - def test_xslt_paramaters - # see http://yokolet.blogspot.com/2010/10/pure-java-nokogiri-xslt-extension.html") - skip_unless_libxml2("cannot get it working on JRuby") + def test_xslt_paramaters + # see http://yokolet.blogspot.com/2010/10/pure-java-nokogiri-xslt-extension.html") + skip_unless_libxml2("cannot get it working on JRuby") - xslt_str = <<~EOX - <xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" > - <xsl:template match="/"> - <xsl:value-of select="$foo" /> - </xsl:template> - </xsl:stylesheet> + xslt_str = <<~EOX + <xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" > + <xsl:template match="/"> + <xsl:value-of select="$foo" /> + </xsl:template> + </xsl:stylesheet> EOX - xslt = Nokogiri::XSLT(xslt_str) - doc = Nokogiri::XML("<root />") - assert_match(/bar/, xslt.transform(doc, Nokogiri::XSLT.quote_params('foo' => 'bar')).to_s) - end + xslt = Nokogiri::XSLT(xslt_str) + doc = Nokogiri::XML("<root />") + assert_match(/bar/, xslt.transform(doc, Nokogiri::XSLT.quote_params("foo" => "bar")).to_s) + end - def test_xslt_transform_error - # see http://yokolet.blogspot.com/2010/10/pure-java-nokogiri-xslt-extension.html") - skip_unless_libxml2("cannot get it working on JRuby") + def test_xslt_transform_error + # see http://yokolet.blogspot.com/2010/10/pure-java-nokogiri-xslt-extension.html") + skip_unless_libxml2("cannot get it working on JRuby") - xslt_str = <<~EOX - <xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" > - <xsl:template match="/"> - <xsl:value-of select="$foo" /> - </xsl:template> - </xsl:stylesheet> + xslt_str = <<~EOX + <xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" > + <xsl:template match="/"> + <xsl:value-of select="$foo" /> + </xsl:template> + </xsl:stylesheet> EOX - xslt = Nokogiri::XSLT(xslt_str) - doc = Nokogiri::XML("<root />") - assert_raises(RuntimeError) { xslt.transform(doc) } - end + xslt = Nokogiri::XSLT(xslt_str) + doc = Nokogiri::XML("<root />") + assert_raises(RuntimeError) { xslt.transform(doc) } + end - def test_xslt_parse_error - xslt_str = <<~EOX - <xsl:stylesheet version="1.0" - xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> - <!-- Not well-formed: --> - <xsl:template match="/"/> - <values> - <xsl:for-each select="//*"> - <value> - <xsl:value-of select="@id"/> - </value> - </xsl:for-each> - </values> - </xsl:template> - </xsl:stylesheet>} - EOX - assert_raises(RuntimeError) { Nokogiri::XSLT.parse(xslt_str) } - end + def test_xslt_parse_error + xslt_str = <<~EOX + <xsl:stylesheet version="1.0" + xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> + <!-- Not well-formed: --> + <xsl:template match="/"/> + <values> + <xsl:for-each select="//*"> + <value> + <xsl:value-of select="@id"/> + </value> + </xsl:for-each> + </values> + </xsl:template> + </xsl:stylesheet>} + EOX + assert_raises(RuntimeError) { Nokogiri::XSLT.parse(xslt_str) } + end - def test_passing_a_non_document_to_transform - xsl = Nokogiri::XSLT('<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform"></xsl:stylesheet>') - assert_raises(ArgumentError) { xsl.transform("<div></div>") } - assert_raises(ArgumentError) { xsl.transform(Nokogiri::HTML("").css("body")) } - end + def test_passing_a_non_document_to_transform + xsl = Nokogiri::XSLT('<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform"></xsl:stylesheet>') + assert_raises(ArgumentError) { xsl.transform("<div></div>") } + assert_raises(ArgumentError) { xsl.transform(Nokogiri::HTML("").css("body")) } + end - def check_params(result_doc, params) - result_doc.xpath('/root/params/*').each do |p| - assert_equal(p.content, params[p.name.intern]) + def check_params(result_doc, params) + result_doc.xpath("/root/params/*").each do |p| + assert_equal(p.content, params[p.name.intern]) + end end - end - def test_non_html_xslt_transform - xml = Nokogiri.XML(<<~EOXML) - <a> - <b> - <c>123</c> - </b> - </a> - EOXML - - xsl = Nokogiri.XSLT(<<~EOXSL) - <xsl:stylesheet version="1.0" - xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> - - <xsl:output encoding="UTF-8" indent="yes" method="xml" /> - - <xsl:template match="/"> - <a><xsl:value-of select="/a" /></a> - </xsl:template> - </xsl:stylesheet> - EOXSL - - result = xsl.transform(xml) - assert(!result.html?) - end + def test_non_html_xslt_transform + xml = Nokogiri.XML(<<~EOXML) + <a> + <b> + <c>123</c> + </b> + </a> + EOXML - it "should not crash when given XPath 2.0 features" do - # - # https://github.com/sparklemotion/nokogiri/issues/1802 - # - # note that here the XPath 2.0 feature is `decimal`. - # this test case is taken from the example provided in the original issue. - # - xml = <<~EOXML - <?xml version="1.0" encoding="UTF-8"?> - <Invoice xmlns="urn:oasis:names:specification:ubl:schema:xsd:Invoice-2" - xmlns:cac="urn:oasis:names:specification:ubl:schema:xsd:CommonAggregateComponents-2" - xmlns:cbc="urn:oasis:names:specification:ubl:schema:xsd:CommonBasicComponents-2" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="urn:oasis:names:specification:ubl:schema:xsd:Invoice-2"> - <cac:TaxTotal> - <cbc:TaxAmount currencyID="EUR">48.00</cbc:TaxAmount> - </cac:TaxTotal> - </Invoice> + xsl = Nokogiri.XSLT(<<~EOXSL) + <xsl:stylesheet version="1.0" + xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> + + <xsl:output encoding="UTF-8" indent="yes" method="xml" /> + + <xsl:template match="/"> + <a><xsl:value-of select="/a" /></a> + </xsl:template> + </xsl:stylesheet> + EOXSL + + result = xsl.transform(xml) + assert(!result.html?) + end + + it "should not crash when given XPath 2.0 features" do + # + # https://github.com/sparklemotion/nokogiri/issues/1802 + # + # note that here the XPath 2.0 feature is `decimal`. + # this test case is taken from the example provided in the original issue. + # + xml = <<~EOXML + <?xml version="1.0" encoding="UTF-8"?> + <Invoice xmlns="urn:oasis:names:specification:ubl:schema:xsd:Invoice-2" + xmlns:cac="urn:oasis:names:specification:ubl:schema:xsd:CommonAggregateComponents-2" + xmlns:cbc="urn:oasis:names:specification:ubl:schema:xsd:CommonBasicComponents-2" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="urn:oasis:names:specification:ubl:schema:xsd:Invoice-2"> + <cac:TaxTotal> + <cbc:TaxAmount currencyID="EUR">48.00</cbc:TaxAmount> + </cac:TaxTotal> + </Invoice> EOXML - xsl = <<~EOXSL - <?xml version="1.0" encoding="UTF-8" standalone="yes"?> - <xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" - xmlns:cbc="urn:oasis:names:specification:ubl:schema:xsd:CommonBasicComponents-2" - xmlns:cac="urn:oasis:names:specification:ubl:schema:xsd:CommonAggregateComponents-2" - xmlns:ubl="urn:oasis:names:specification:ubl:schema:xsd:Invoice-2" - xmlns:xs="http://www.w3.org/2001/XMLSchema" - version="1.0"> - <xsl:template match="/"> - <xsl:apply-templates select="/" mode="qwerty"/> - </xsl:template> - <xsl:template match="/ubl:Invoice/cac:TaxTotal" priority="1001" mode="qwerty"> - <xsl:choose> - <xsl:when test="(round(xs:decimal(child::cbc:TaxAmount)))"/> - </xsl:choose> - </xsl:template> - </xsl:stylesheet> + xsl = <<~EOXSL + <?xml version="1.0" encoding="UTF-8" standalone="yes"?> + <xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" + xmlns:cbc="urn:oasis:names:specification:ubl:schema:xsd:CommonBasicComponents-2" + xmlns:cac="urn:oasis:names:specification:ubl:schema:xsd:CommonAggregateComponents-2" + xmlns:ubl="urn:oasis:names:specification:ubl:schema:xsd:Invoice-2" + xmlns:xs="http://www.w3.org/2001/XMLSchema" + version="1.0"> + <xsl:template match="/"> + <xsl:apply-templates select="/" mode="qwerty"/> + </xsl:template> + <xsl:template match="/ubl:Invoice/cac:TaxTotal" priority="1001" mode="qwerty"> + <xsl:choose> + <xsl:when test="(round(xs:decimal(child::cbc:TaxAmount)))"/> + </xsl:choose> + </xsl:template> + </xsl:stylesheet> EOXSL - doc = Nokogiri::XML(xml) - xslt = Nokogiri::XSLT(xsl) - exception = assert_raise(RuntimeError) do - xslt.transform(doc) + doc = Nokogiri::XML(xml) + xslt = Nokogiri::XSLT(xsl) + exception = assert_raise(RuntimeError) do + xslt.transform(doc) + end + assert_match(/decimal/, exception.message) + end + + describe "DEFAULT_XSLT parse options" do + it "is the union of DEFAULT_XML and libxslt's XSLT_PARSE_OPTIONS" do + xslt_parse_options = Nokogiri::XML::ParseOptions.new.noent.dtdload.dtdattr.nocdata + expected = Nokogiri::XML::ParseOptions::DEFAULT_XML | xslt_parse_options.options + assert_equal(expected, Nokogiri::XML::ParseOptions::DEFAULT_XSLT) + end + + it "parses docs the same as xsltproc" do + skip_unless_libxml2("JRuby implementation disallows this edge case XSLT") + + # see https://github.com/sparklemotion/nokogiri/issues/1940 + xml = "<t></t>" + xsl = <<~EOF + <?xml version="1.0"?> + <xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> + <xsl:output method="text" omit-xml-declaration="no" /> + <xsl:template match="/"> + <xsl:text disable-output-escaping="yes"><![CDATA[<>]]></xsl:text> + </xsl:template> + </xsl:stylesheet> + EOF + + doc = Nokogiri::XML(xml) + stylesheet = Nokogiri::XSLT(xsl) + + # TODO: ideally I'd like to be able to access the parse options in the final object + # assert_equal(Nokogiri::XML::ParseOptions::DEFAULT_XSLT, stylesheet.document.parse_options) + + result = stylesheet.transform(doc) + assert_equal("<>", result.children.to_xml) + end end - assert_match(/decimal/, exception.message) end end diff --git a/test/xml/sax/test_parser.rb b/test/xml/sax/test_parser.rb index de078d6115..1a54ff638b 100644 --- a/test/xml/sax/test_parser.rb +++ b/test/xml/sax/test_parser.rb @@ -2,425 +2,462 @@ require "helper" -module Nokogiri - module XML - module SAX - class TestParser < Nokogiri::SAX::TestCase - def setup - super - @parser = XML::SAX::Parser.new(Doc.new) - end - - def test_parser_context_yielded_io - doc = Doc.new - parser = XML::SAX::Parser.new(doc) - xml = "<foo a='&b'/>" - - block_called = false - parser.parse(StringIO.new(xml)) do |ctx| - block_called = true - ctx.replace_entities = true - end - - assert(block_called) - - assert_equal([["foo", [["a", "&b"]]]], doc.start_elements) - end - - def test_parser_context_yielded_in_memory - doc = Doc.new - parser = XML::SAX::Parser.new(doc) - xml = "<foo a='&b'/>" - - block_called = false - parser.parse(xml) do |ctx| - block_called = true - ctx.replace_entities = true - end - - assert(block_called) - - assert_equal([["foo", [["a", "&b"]]]], doc.start_elements) - end - - def test_empty_decl - parser = XML::SAX::Parser.new(Doc.new) - - xml = "<root />" - parser.parse(xml) - assert(parser.document.start_document_called, xml) - assert_nil(parser.document.xmldecls, xml) - end - - def test_xml_decl - [ - ['<?xml version="1.0" ?>', ["1.0"]], - ['<?xml version="1.0" encoding="UTF-8" ?>', ["1.0", "UTF-8"]], - ['<?xml version="1.0" standalone="yes"?>', ["1.0", "yes"]], - ['<?xml version="1.0" standalone="no"?>', ["1.0", "no"]], - ['<?xml version="1.0" encoding="UTF-8" standalone="no"?>', ["1.0", "UTF-8", "no"]], - ['<?xml version="1.0" encoding="ISO-8859-1" standalone="yes"?>', ["1.0", "ISO-8859-1", "yes"]], - ].each do |decl, value| - parser = XML::SAX::Parser.new(Doc.new) - - xml = "#{decl}\n<root />" - parser.parse(xml) - assert(parser.document.start_document_called, xml) - assert_equal(value, parser.document.xmldecls, xml) - end - end - - def test_parse_empty - assert_raises(RuntimeError) do - @parser.parse("") - end - end - - def test_namespace_declaration_order_is_saved - @parser.parse(<<~eoxml) - <root xmlns:foo='http://foo.example.com/' xmlns='http://example.com/'> - <a foo:bar='hello' /> - </root> - eoxml - assert_equal(2, @parser.document.start_elements_namespace.length) - el = @parser.document.start_elements_namespace.first - namespaces = el.last - assert_equal(["foo", "http://foo.example.com/"], namespaces.first) - assert_equal([nil, "http://example.com/"], namespaces.last) - end - - def test_bad_document_calls_error_handler - @parser.parse("<foo><bar></foo>") - assert(@parser.document.errors) - assert(@parser.document.errors.length > 0) - end - - def test_namespace_are_super_fun_to_parse - @parser.parse(<<~eoxml) - <root xmlns:foo='http://foo.example.com/'> - <a foo:bar='hello' /> - <b xmlns:foo='http://bar.example.com/'> - <a foo:bar='hello' /> - </b> - <foo:bar>hello world</foo:bar> - </root> - eoxml - - assert(@parser.document.start_elements_namespace.length > 0) - el = @parser.document.start_elements_namespace[1] - assert_equal("a", el.first) - assert_equal(1, el[1].length) - - attribute = el[1].first - assert_equal("bar", attribute.localname) - assert_equal("foo", attribute.prefix) - assert_equal("hello", attribute.value) - assert_equal("http://foo.example.com/", attribute.uri) - end - - def test_sax_v1_namespace_attribute_declarations - @parser.parse(<<~eoxml) - <root xmlns:foo='http://foo.example.com/' xmlns='http://example.com/'> - <a foo:bar='hello' /> - <b xmlns:foo='http://bar.example.com/'> - <a foo:bar='hello' /> - </b> - <foo:bar>hello world</foo:bar> - </root> - eoxml - assert(@parser.document.start_elements.length > 0) - elm = @parser.document.start_elements.first - assert_equal("root", elm.first) - assert(elm[1].include?(["xmlns:foo", "http://foo.example.com/"])) - assert(elm[1].include?(["xmlns", "http://example.com/"])) - end - - def test_sax_v1_namespace_nodes - @parser.parse(<<~eoxml) - <root xmlns:foo='http://foo.example.com/' xmlns='http://example.com/'> - <a foo:bar='hello' /> - <b xmlns:foo='http://bar.example.com/'> - <a foo:bar='hello' /> - </b> - <foo:bar>hello world</foo:bar> - </root> - eoxml - assert_equal(5, @parser.document.start_elements.length) - assert(@parser.document.start_elements.map(&:first).include?("foo:bar")) - assert(@parser.document.end_elements.map(&:first).include?("foo:bar")) - end - - def test_start_is_called_without_namespace - @parser.parse(<<~eoxml) - <root xmlns:foo='http://foo.example.com/' xmlns='http://example.com/'> - <foo:f><bar></foo:f> - </root> - eoxml - assert_equal(["root", "foo:f", "bar"], - @parser.document.start_elements.map(&:first)) - end - - def test_parser_sets_encoding - parser = XML::SAX::Parser.new(Doc.new, "UTF-8") - assert_equal("UTF-8", parser.encoding) - end - - def test_errors_set_after_parsing_bad_dom - doc = Nokogiri::XML("<foo><bar></foo>") - assert(doc.errors) - - @parser.parse("<foo><bar></foo>") - assert(@parser.document.errors) - assert(@parser.document.errors.length > 0) - - doc.errors.each do |error| - assert_equal("UTF-8", error.message.encoding.name) - end - - # when using JRuby Nokogiri, more errors will be generated as the DOM - # parser continue to parse an ill formed document, while the sax parser - # will stop at the first error - unless Nokogiri.jruby? - assert_equal(doc.errors.length, @parser.document.errors.length) - end - end - - def test_parse_with_memory_argument - @parser.parse(File.read(XML_FILE)) - assert(@parser.document.cdata_blocks.length > 0) - end - - def test_parse_with_io_argument - File.open(XML_FILE, "rb") do |f| - @parser.parse(f) - end - assert(@parser.document.cdata_blocks.length > 0) - end - - def test_parse_io - call_parse_io_with_encoding("UTF-8") - end - - # issue #828 - def test_parse_io_lower_case_encoding - call_parse_io_with_encoding("utf-8") - end - - def call_parse_io_with_encoding(encoding) - File.open(XML_FILE, "rb") do |f| - @parser.parse_io(f, encoding) - end - assert(@parser.document.cdata_blocks.length > 0) - - called = false - @parser.document.start_elements.flatten.each do |thing| - assert_equal("UTF-8", thing.encoding.name) - called = true - end - assert(called) - - called = false - @parser.document.end_elements.flatten.each do |thing| - assert_equal("UTF-8", thing.encoding.name) - called = true - end - assert(called) - - called = false - @parser.document.data.each do |thing| - assert_equal("UTF-8", thing.encoding.name) - called = true - end - assert(called) - - called = false - @parser.document.comments.flatten.each do |thing| - assert_equal("UTF-8", thing.encoding.name) - called = true - end - assert(called) - - called = false - @parser.document.cdata_blocks.flatten.each do |thing| - assert_equal("UTF-8", thing.encoding.name) - called = true - end - assert(called) - end - - def test_parse_file - @parser.parse_file(XML_FILE) - - assert_raises(ArgumentError) do - @parser.parse_file(nil) - end - - assert_raises(Errno::ENOENT) do - @parser.parse_file("") - end - assert_raises(Errno::EISDIR) do - @parser.parse_file(File.expand_path(File.dirname(__FILE__))) - end - end - - def test_render_parse_nil_param - assert_raises(ArgumentError) { @parser.parse_memory(nil) } - end - - def test_bad_encoding_args - assert_raises(ArgumentError) { XML::SAX::Parser.new(Doc.new, "not an encoding") } - assert_raises(ArgumentError) { @parser.parse_io(StringIO.new("<root/>"), "not an encoding") } - end - - def test_ctag - @parser.parse_memory(<<~eoxml) - <p id="asdfasdf"> - <![CDATA[ This is a comment ]]> - Paragraph 1 - </p> - eoxml - assert_equal([" This is a comment "], @parser.document.cdata_blocks) - end - - def test_comment - @parser.parse_memory(<<~eoxml) - <p id="asdfasdf"> - <!-- This is a comment --> - Paragraph 1 - </p> - eoxml - assert_equal([" This is a comment "], @parser.document.comments) - end - - def test_characters - @parser.parse_memory(<<~eoxml) - <p id="asdfasdf">Paragraph 1</p> - eoxml - assert_equal(["Paragraph 1"], @parser.document.data) - end - - def test_end_document - @parser.parse_memory(<<~eoxml) - <p id="asdfasdf">Paragraph 1</p> - eoxml - assert(@parser.document.end_document_called) - end - - def test_end_element - @parser.parse_memory(<<~eoxml) - <p id="asdfasdf">Paragraph 1</p> - eoxml - assert_equal([["p"]], @parser.document.end_elements) - end - - def test_start_element_attrs - @parser.parse_memory(<<~eoxml) - <p id="asdfasdf">Paragraph 1</p> - eoxml - assert_equal([["p", [["id", "asdfasdf"]]]], @parser.document.start_elements) - end - - def test_start_element_attrs_include_namespaces - @parser.parse_memory(<<~eoxml) - <p xmlns:foo='http://foo.example.com/'>Paragraph 1</p> - eoxml - assert_equal([["p", [["xmlns:foo", "http://foo.example.com/"]]]], - @parser.document.start_elements) - end - - def test_processing_instruction - @parser.parse_memory(<<~eoxml) - <?xml-stylesheet href="a.xsl" type="text/xsl"?> - <?xml version="1.0"?> - eoxml - assert_equal([["xml-stylesheet", 'href="a.xsl" type="text/xsl"']], - @parser.document.processing_instructions) - end - - def test_parse_document - skip_unless_libxml2("JRuby SAXParser only parses well-formed XML documents") - @parser.parse_memory(<<~eoxml) - <p>Paragraph 1</p> - <p>Paragraph 2</p> - eoxml - end - - def test_parser_attributes - xml = <<~eoxml - <?xml version="1.0" ?><root><foo a="&b" c=">d" /></root> - eoxml - - block_called = false - @parser.parse(xml) do |ctx| - block_called = true - ctx.replace_entities = true - end - - assert(block_called) - - assert_equal([["root", []], ["foo", [["a", "&b"], ["c", ">d"]]]], @parser.document.start_elements) - end - - def test_recovery_from_incorrect_xml - xml = <<~eoxml - <?xml version="1.0" ?><Root><Data><?xml version='1.0'?><Item>hey</Item></Data><Data><Item>hey yourself</Item></Data></Root> - eoxml - - block_called = false - @parser.parse(xml) do |ctx| - block_called = true - ctx.recovery = true - end - - assert(block_called) - - assert_equal([["Root", []], ["Data", []], ["Item", []], ["Data", []], ["Item", []]], - @parser.document.start_elements) - end - - def test_square_bracket_in_text # issue 1261 - xml = <<~eoxml - <tu tuid="87dea04cf60af103ff09d1dba36ae820" segtype="block"> - <prop type="x-smartling-string-variant">en:#:home_page:#:stories:#:[6]:#:name</prop> - <tuv xml:lang="en-US"><seg>Sandy S.</seg></tuv> - </tu> - eoxml - @parser.parse(xml) - assert_includes(@parser.document.data, "en:#:home_page:#:stories:#:[6]:#:name") - end - - def test_large_cdata_is_handled - # see #2132 and https://gitlab.gnome.org/GNOME/libxml2/-/issues/200 - skip("Upstream libxml2 <= 2.9.10 needs to be patched") if Nokogiri::VersionInfo.instance.libxml2_using_system? - - template = <<~EOF - <?xml version="1.0" encoding="UTF-8"?> - <soapenv:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:ns="http://example.com"> - <soapenv:Header> - <AuthHeader xsi:type="ns:vAuthHeader"> - <userName xsi:type="xsd:string">gorilla</userName> - <password xsi:type="xsd:string">secret</password> - </AuthHeader> - </soapenv:Header> - <soapenv:Body> - <ns:checkToken soapenv:encodingStyle="http://schemas.xmlsoap.org/soap/encoding/"> - <checkToken xsi:type="xsd:string"><![CDATA[%s]]></checkToken> - </ns:checkToken> - </soapenv:Body> - </soapenv:Envelope> - EOF - - factor = 10 - huge_data = "a" * (1024 * 1024 * factor) - xml = StringIO.new(template % (huge_data)) - - handler = Nokogiri::SAX::TestCase::Doc.new - parser = Nokogiri::XML::SAX::Parser.new(handler) - parser.parse(xml) - - assert_predicate(handler.errors, :empty?) - end +class Nokogiri::SAX::TestCase + describe Nokogiri::XML::SAX::Parser do + let(:parser) { Nokogiri::XML::SAX::Parser.new(Doc.new) } + + it :test_parser_context_yielded_io do + doc = Doc.new + parser = Nokogiri::XML::SAX::Parser.new(doc) + xml = "<foo a='&b'/>" + + block_called = false + parser.parse(StringIO.new(xml)) do |ctx| + block_called = true + ctx.replace_entities = true end + + assert(block_called) + + assert_equal([["foo", [["a", "&b"]]]], doc.start_elements) + end + + it :test_parser_context_yielded_in_memory do + doc = Doc.new + parser = Nokogiri::XML::SAX::Parser.new(doc) + xml = "<foo a='&b'/>" + + block_called = false + parser.parse(xml) do |ctx| + block_called = true + ctx.replace_entities = true + end + + assert(block_called) + + assert_equal([["foo", [["a", "&b"]]]], doc.start_elements) + end + + it :test_empty_decl do + parser = Nokogiri::XML::SAX::Parser.new(Doc.new) + + xml = "<root />" + parser.parse(xml) + assert(parser.document.start_document_called, xml) + assert_nil(parser.document.xmldecls, xml) + end + + it :test_xml_decl do + [ + ['<?xml version="1.0" ?>', ["1.0"]], + ['<?xml version="1.0" encoding="UTF-8" ?>', ["1.0", "UTF-8"]], + ['<?xml version="1.0" standalone="yes"?>', ["1.0", "yes"]], + ['<?xml version="1.0" standalone="no"?>', ["1.0", "no"]], + ['<?xml version="1.0" encoding="UTF-8" standalone="no"?>', ["1.0", "UTF-8", "no"]], + ['<?xml version="1.0" encoding="ISO-8859-1" standalone="yes"?>', ["1.0", "ISO-8859-1", "yes"]], + ].each do |decl, value| + parser = Nokogiri::XML::SAX::Parser.new(Doc.new) + + xml = "#{decl}\n<root />" + parser.parse(xml) + assert(parser.document.start_document_called, xml) + assert_equal(value, parser.document.xmldecls, xml) + end + end + + it :test_parse_empty do + assert_raises(RuntimeError) do + parser.parse("") + end + end + + it :test_namespace_declaration_order_is_saved do + parser.parse(<<~EOF) + <root xmlns:foo='http://foo.example.com/' xmlns='http://example.com/'> + <a foo:bar='hello' /> + </root> + EOF + assert_equal(2, parser.document.start_elements_namespace.length) + el = parser.document.start_elements_namespace.first + namespaces = el.last + assert_equal(["foo", "http://foo.example.com/"], namespaces.first) + assert_equal([nil, "http://example.com/"], namespaces.last) + end + + it :test_bad_document_calls_error_handler do + parser.parse("<foo><bar></foo>") + assert(parser.document.errors) + assert(parser.document.errors.length > 0) + end + + it :test_namespace_are_super_fun_to_parse do + parser.parse(<<~EOF) + <root xmlns:foo='http://foo.example.com/'> + <a foo:bar='hello' /> + <b xmlns:foo='http://bar.example.com/'> + <a foo:bar='hello' /> + </b> + <foo:bar>hello world</foo:bar> + </root> + EOF + + assert(parser.document.start_elements_namespace.length > 0) + el = parser.document.start_elements_namespace[1] + assert_equal("a", el.first) + assert_equal(1, el[1].length) + + attribute = el[1].first + assert_equal("bar", attribute.localname) + assert_equal("foo", attribute.prefix) + assert_equal("hello", attribute.value) + assert_equal("http://foo.example.com/", attribute.uri) + end + + it :test_sax_v1_namespace_attribute_declarations do + parser.parse(<<~EOF) + <root xmlns:foo='http://foo.example.com/' xmlns='http://example.com/'> + <a foo:bar='hello' /> + <b xmlns:foo='http://bar.example.com/'> + <a foo:bar='hello' /> + </b> + <foo:bar>hello world</foo:bar> + </root> + EOF + assert(parser.document.start_elements.length > 0) + elm = parser.document.start_elements.first + assert_equal("root", elm.first) + assert(elm[1].include?(["xmlns:foo", "http://foo.example.com/"])) + assert(elm[1].include?(["xmlns", "http://example.com/"])) + end + + it :test_sax_v1_namespace_nodes do + parser.parse(<<~EOF) + <root xmlns:foo='http://foo.example.com/' xmlns='http://example.com/'> + <a foo:bar='hello' /> + <b xmlns:foo='http://bar.example.com/'> + <a foo:bar='hello' /> + </b> + <foo:bar>hello world</foo:bar> + </root> + EOF + assert_equal(5, parser.document.start_elements.length) + assert(parser.document.start_elements.map(&:first).include?("foo:bar")) + assert(parser.document.end_elements.map(&:first).include?("foo:bar")) + end + + it :test_start_is_called_without_namespace do + parser.parse(<<~EOF) + <root xmlns:foo='http://foo.example.com/' xmlns='http://example.com/'> + <foo:f><bar></foo:f> + </root> + EOF + assert_equal( + ["root", "foo:f", "bar"], + parser.document.start_elements.map(&:first) + ) + end + + it :test_parser_sets_encoding do + parser = Nokogiri::XML::SAX::Parser.new(Doc.new, "UTF-8") + assert_equal("UTF-8", parser.encoding) + end + + it :test_errors_set_after_parsing_bad_dom do + doc = Nokogiri::XML("<foo><bar></foo>") + assert(doc.errors) + + parser.parse("<foo><bar></foo>") + assert(parser.document.errors) + assert(parser.document.errors.length > 0) + + doc.errors.each do |error| + assert_equal("UTF-8", error.message.encoding.name) + end + + # when using JRuby Nokogiri, more errors will be generated as the DOM + # parser continue to parse an ill formed document, while the sax parser + # will stop at the first error + unless Nokogiri.jruby? + assert_equal(doc.errors.length, parser.document.errors.length) + end + end + + it :test_parse_with_memory_argument do + parser.parse(File.read(XML_FILE)) + assert(parser.document.cdata_blocks.length > 0) + end + + it :test_parse_with_io_argument do + File.open(XML_FILE, "rb") do |f| + parser.parse(f) + end + assert(parser.document.cdata_blocks.length > 0) + end + + it :test_parse_io do + call_parse_io_with_encoding("UTF-8") + end + + # issue #828 + it :test_parse_io_lower_case_encoding do + call_parse_io_with_encoding("utf-8") + end + + def call_parse_io_with_encoding(encoding) + File.open(XML_FILE, "rb") do |f| + parser.parse_io(f, encoding) + end + assert(parser.document.cdata_blocks.length > 0) + + called = false + parser.document.start_elements.flatten.each do |thing| + assert_equal("UTF-8", thing.encoding.name) + called = true + end + assert(called) + + called = false + parser.document.end_elements.flatten.each do |thing| + assert_equal("UTF-8", thing.encoding.name) + called = true + end + assert(called) + + called = false + parser.document.data.each do |thing| + assert_equal("UTF-8", thing.encoding.name) + called = true + end + assert(called) + + called = false + parser.document.comments.flatten.each do |thing| + assert_equal("UTF-8", thing.encoding.name) + called = true + end + assert(called) + + called = false + parser.document.cdata_blocks.flatten.each do |thing| + assert_equal("UTF-8", thing.encoding.name) + called = true + end + assert(called) + end + + it :test_parse_file do + parser.parse_file(XML_FILE) + + assert_raises(ArgumentError) do + parser.parse_file(nil) + end + + assert_raises(Errno::ENOENT) do + parser.parse_file("") + end + assert_raises(Errno::EISDIR) do + parser.parse_file(File.expand_path(File.dirname(__FILE__))) + end + end + + it :test_render_parse_nil_param do + assert_raises(ArgumentError) { parser.parse_memory(nil) } + end + + it :test_bad_encoding_args do + assert_raises(ArgumentError) { Nokogiri::XML::SAX::Parser.new(Doc.new, "not an encoding") } + assert_raises(ArgumentError) { parser.parse_io(StringIO.new("<root/>"), "not an encoding") } + end + + it :test_ctag do + parser.parse_memory(<<~EOF) + <p id="asdfasdf"> + <![CDATA[ This is a comment ]]> + Paragraph 1 + </p> + EOF + assert_equal([" This is a comment "], parser.document.cdata_blocks) + end + + it :test_comment do + parser.parse_memory(<<~EOF) + <p id="asdfasdf"> + <!-- This is a comment --> + Paragraph 1 + </p> + EOF + assert_equal([" This is a comment "], parser.document.comments) + end + + it :test_characters do + parser.parse_memory(<<~EOF) + <p id="asdfasdf">Paragraph 1</p> + EOF + assert_equal(["Paragraph 1"], parser.document.data) + end + + it :test_end_document do + parser.parse_memory(<<~EOF) + <p id="asdfasdf">Paragraph 1</p> + EOF + assert(parser.document.end_document_called) + end + + it :test_end_element do + parser.parse_memory(<<~EOF) + <p id="asdfasdf">Paragraph 1</p> + EOF + assert_equal([["p"]], parser.document.end_elements) + end + + it :test_start_element_attrs do + parser.parse_memory(<<~EOF) + <p id="asdfasdf">Paragraph 1</p> + EOF + assert_equal([["p", [["id", "asdfasdf"]]]], parser.document.start_elements) + end + + it :test_start_element_attrs_include_namespaces do + parser.parse_memory(<<~EOF) + <p xmlns:foo='http://foo.example.com/'>Paragraph 1</p> + EOF + assert_equal( + [["p", [["xmlns:foo", "http://foo.example.com/"]]]], + parser.document.start_elements + ) + end + + it :test_processing_instruction do + parser.parse_memory(<<~EOF) + <?xml-stylesheet href="a.xsl" type="text/xsl"?> + <?xml version="1.0"?> + EOF + assert_equal( + [["xml-stylesheet", 'href="a.xsl" type="text/xsl"']], + parser.document.processing_instructions + ) + end + + it :test_parse_document do + skip_unless_libxml2("JRuby SAXParser only parses well-formed XML documents") + parser.parse_memory(<<~EOF) + <p>Paragraph 1</p> + <p>Paragraph 2</p> + EOF + end + + it :test_parser_attributes do + xml = <<~EOF + <?xml version="1.0" ?><root><foo a="&b" c=">d" /></root> + EOF + + block_called = false + parser.parse(xml) do |ctx| + block_called = true + ctx.replace_entities = true + end + + assert(block_called) + + assert_equal( + [["root", []], ["foo", [["a", "&b"], ["c", ">d"]]]], parser.document.start_elements + ) + end + + it :test_recovery_from_incorrect_xml do + xml = <<~EOF + <?xml version="1.0" ?><Root><Data><?xml version='1.0'?><Item>hey</Item></Data><Data><Item>hey yourself</Item></Data></Root> + EOF + + block_called = false + parser.parse(xml) do |ctx| + block_called = true + ctx.recovery = true + end + + assert(block_called) + + assert_equal( + [["Root", []], ["Data", []], ["Item", []], ["Data", []], ["Item", []]], + parser.document.start_elements + ) + end + + it :test_square_bracket_in_text do + # issue 1261 + xml = <<~EOF + <tu tuid="87dea04cf60af103ff09d1dba36ae820" segtype="block"> + <prop type="x-smartling-string-variant">en:#:home_page:#:stories:#:[6]:#:name</prop> + <tuv xml:lang="en-US"><seg>Sandy S.</seg></tuv> + </tu> + EOF + parser.parse(xml) + assert_includes(parser.document.data, "en:#:home_page:#:stories:#:[6]:#:name") + end + + it :test_large_cdata_is_handled do + # see #2132 and https://gitlab.gnome.org/GNOME/libxml2/-/issues/200 + skip("Upstream libxml2 <= 2.9.10 needs to be patched") if Nokogiri::VersionInfo.instance.libxml2_using_system? + + template = <<~EOF + <?xml version="1.0" encoding="UTF-8"?> + <soapenv:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:ns="http://example.com"> + <soapenv:Header> + <AuthHeader xsi:type="ns:vAuthHeader"> + <userName xsi:type="xsd:string">gorilla</userName> + <password xsi:type="xsd:string">secret</password> + </AuthHeader> + </soapenv:Header> + <soapenv:Body> + <ns:checkToken soapenv:encodingStyle="http://schemas.xmlsoap.org/soap/encoding/"> + <checkToken xsi:type="xsd:string"><![CDATA[%s]]></checkToken> + </ns:checkToken> + </soapenv:Body> + </soapenv:Envelope> + EOF + + factor = 10 + huge_data = "a" * (1024 * 1024 * factor) + xml = StringIO.new(template % huge_data) + + handler = Nokogiri::SAX::TestCase::Doc.new + parser = Nokogiri::XML::SAX::Parser.new(handler) + parser.parse(xml) + + assert_predicate(handler.errors, :empty?) + end + + it "does not resolve entities by default" do + xml = <<~EOF + <?xml version="1.0" encoding="UTF-8"?> + <!DOCTYPE doc [ + <!ENTITY local SYSTEM "file:///#{File.expand_path(__FILE__)}"> + <!ENTITY custom "resolved> + ]> + <doc><foo>&local;</foo><foo>&custom;</foo></doc> + EOF + + doc = Doc.new + parser = Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(xml) + + assert_nil(doc.data) + end + + it "does not resolve network external entities by default" do + xml = <<~EOF + <?xml version="1.0" encoding="UTF-8"?> + <!DOCTYPE doc [ + <!ENTITY remote SYSTEM "http://0.0.0.0:8080/evil.dtd"> + ]> + <doc><foo>&remote;</foo></doc> + EOF + + doc = Doc.new + parser = Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(xml) + + assert_nil(doc.data) end end end diff --git a/test/xml/sax/test_push_parser.rb b/test/xml/sax/test_push_parser.rb index 587bd54a1a..dfab1793fa 100644 --- a/test/xml/sax/test_push_parser.rb +++ b/test/xml/sax/test_push_parser.rb @@ -1,266 +1,243 @@ # -*- coding: utf-8 -*- +# frozen_string_literal: true require "helper" -module Nokogiri - module XML - module SAX - class TestPushParser < Nokogiri::SAX::TestCase - def setup - super - @parser = XML::SAX::PushParser.new(Doc.new) - end - - def test_exception - assert_raises(SyntaxError) do - @parser << "<foo /><foo />" - end - - assert_raises(SyntaxError) do - @parser << nil - end - end +class Nokogiri::SAX::TestCase + describe Nokogiri::XML::SAX::PushParser do + let(:parser) { Nokogiri::XML::SAX::PushParser.new(Doc.new) } - def test_early_finish - @parser << "<foo>" - assert_raises(SyntaxError) do - @parser.finish - end - end + it :test_exception do + assert_raises(Nokogiri::XML::SyntaxError) do + parser << "<foo /><foo />" + end - def test_write_last_chunk - @parser << "<foo>" - @parser.write "</foo>", true - assert_equal [["foo", []]], @parser.document.start_elements - assert_equal [["foo"]], @parser.document.end_elements - end + assert_raises(Nokogiri::XML::SyntaxError) do + parser << nil + end + end - def test_empty_doc - @parser.options |= XML::ParseOptions::RECOVER - @parser.write "", true - assert_nil @parser.document.start_elements - assert_nil @parser.document.end_elements - end + it :test_early_finish do + parser << "<foo>" + assert_raises(Nokogiri::XML::SyntaxError) do + parser.finish + end + end + it :test_write_last_chunk do + parser << "<foo>" + parser.write("</foo>", true) + assert_equal [["foo", []]], parser.document.start_elements + assert_equal [["foo"]], parser.document.end_elements + end - def test_finish_should_rethrow_last_error - begin - @parser << "</foo>" - rescue => e - expected = e - end + it :test_empty_doc do + parser.options |= Nokogiri::XML::ParseOptions::RECOVER + parser.write("", true) + assert_nil parser.document.start_elements + assert_nil parser.document.end_elements + end - begin - @parser.finish - rescue => e - actual = e - end + it :test_finish_should_rethrow_last_error do + expected = assert_raise(Nokogiri::XML::SyntaxError) { parser << "</foo>" } + actual = assert_raise(Nokogiri::XML::SyntaxError) { parser.finish } + assert_equal actual.message, expected.message + end - assert_equal actual.message, expected.message + it :test_should_throw_error_returned_by_document do + doc = Doc.new + class << doc + def error(msg) + raise "parse error" end + end + parser = Nokogiri::XML::SAX::PushParser.new(doc) - def test_should_throw_error_returned_by_document - doc = Doc.new - class << doc - def error msg - raise "parse error" - end - end - - @parser = XML::SAX::PushParser.new(doc) - begin - @parser << "</foo>" - rescue => e - actual = e - end - fail 'PushParser should throw error when fed ill-formed data' if actual.nil? - - assert_equal actual.message, "parse error" - end + exception = assert_raise(RuntimeError) { parser << "</foo>" } + assert_equal exception.message, "parse error" + end - def test_writing_nil - assert_equal @parser.write(nil), @parser - end + it :test_writing_nil do + assert_equal parser.write(nil), parser + end - def test_end_document_called - @parser.<<(<<-eoxml) - <p id="asdfasdf"> - <!-- This is a comment --> - Paragraph 1 - </p> - eoxml - assert ! @parser.document.end_document_called - @parser.finish - assert @parser.document.end_document_called - end + it :test_end_document_called do + parser.<<(<<~EOF) + <p id="asdfasdf"> + <!-- This is a comment --> + Paragraph 1 + </p> + EOF + assert !parser.document.end_document_called + parser.finish + assert parser.document.end_document_called + end - def test_start_element - @parser.<<(<<-eoxml) - <p id="asdfasdf"> - eoxml - - assert_equal [["p", [["id", "asdfasdf"]]]], - @parser.document.start_elements - - @parser.<<(<<-eoxml) - <!-- This is a comment --> - Paragraph 1 - </p> - eoxml - assert_equal [' This is a comment '], @parser.document.comments - @parser.finish - end + it :test_start_element do + parser.<<(<<~EOF) + <p id="asdfasdf"> + EOF + + assert_equal [["p", [["id", "asdfasdf"]]]], + parser.document.start_elements + + parser.<<(<<~EOF) + <!-- This is a comment --> + Paragraph 1 + </p> + EOF + assert_equal [" This is a comment "], parser.document.comments + parser.finish + end - def test_start_element_with_namespaces - @parser.<<(<<-eoxml) - <p xmlns:foo="http://foo.example.com/"> - eoxml - - assert_equal [["p", [["xmlns:foo", "http://foo.example.com/"]]]], - @parser.document.start_elements - - @parser.<<(<<-eoxml) - <!-- This is a comment --> - Paragraph 1 - </p> - eoxml - assert_equal [' This is a comment '], @parser.document.comments - @parser.finish - end + it :test_start_element_with_namespaces do + parser.<<(<<~EOF) + <p xmlns:foo="http://foo.example.com/"> + EOF + + assert_equal [["p", [["xmlns:foo", "http://foo.example.com/"]]]], + parser.document.start_elements + + parser.<<(<<~EOF) + <!-- This is a comment --> + Paragraph 1 + </p> + EOF + assert_equal [" This is a comment "], parser.document.comments + parser.finish + end - def test_start_element_ns - @parser.<<(<<-eoxml) - <stream:stream xmlns='jabber:client' xmlns:stream='http://etherx.jabber.org/streams' version='1.0' size='large'></stream:stream> - eoxml + it :test_start_element_ns do + parser.<<(<<~EOF) + <stream:stream xmlns='jabber:client' xmlns:stream='http://etherx.jabber.org/streams' version='1.0' size='large'></stream:stream> + EOF - assert_equal 1, @parser.document.start_elements_namespace.length - el = @parser.document.start_elements_namespace.first + assert_equal 1, parser.document.start_elements_namespace.length + el = parser.document.start_elements_namespace.first - assert_equal 'stream', el.first - assert_equal 2, el[1].length - assert_equal [['version', '1.0'], ['size', 'large']], - el[1].map { |x| [x.localname, x.value] } + assert_equal "stream", el.first + assert_equal 2, el[1].length + assert_equal [["version", "1.0"], ["size", "large"]], + el[1].map { |x| [x.localname, x.value] } - assert_equal 'stream', el[2] - assert_equal 'http://etherx.jabber.org/streams', el[3] - @parser.finish - end + assert_equal "stream", el[2] + assert_equal "http://etherx.jabber.org/streams", el[3] + parser.finish + end - def test_end_element_ns - @parser.<<(<<-eoxml) - <stream:stream xmlns='jabber:client' xmlns:stream='http://etherx.jabber.org/streams' version='1.0'></stream:stream> - eoxml + it :test_end_element_ns do + parser.<<(<<~EOF) + <stream:stream xmlns='jabber:client' xmlns:stream='http://etherx.jabber.org/streams' version='1.0'></stream:stream> + EOF - assert_equal [['stream', 'stream', 'http://etherx.jabber.org/streams']], - @parser.document.end_elements_namespace - @parser.finish - end + assert_equal [["stream", "stream", "http://etherx.jabber.org/streams"]], + parser.document.end_elements_namespace + parser.finish + end - def test_chevron_partial_xml - @parser.<<(<<-eoxml) - <p id="asdfasdf"> - eoxml - - @parser.<<(<<-eoxml) - <!-- This is a comment --> - Paragraph 1 - </p> - eoxml - assert_equal [' This is a comment '], @parser.document.comments - @parser.finish - end + it :test_chevron_partial_xml do + parser.<<(<<~EOF) + <p id="asdfasdf"> + EOF + + parser.<<(<<~EOF) + <!-- This is a comment --> + Paragraph 1 + </p> + EOF + assert_equal [" This is a comment "], parser.document.comments + parser.finish + end - def test_chevron - @parser.<<(<<-eoxml) - <p id="asdfasdf"> - <!-- This is a comment --> - Paragraph 1 - </p> - eoxml - @parser.finish - assert_equal [' This is a comment '], @parser.document.comments - end + it :test_chevron do + parser.<<(<<~EOF) + <p id="asdfasdf"> + <!-- This is a comment --> + Paragraph 1 + </p> + EOF + parser.finish + assert_equal [" This is a comment "], parser.document.comments + end - def test_default_options - assert_equal 0, @parser.options - end + it :test_default_options do + assert_equal 0, parser.options + end - def test_recover - @parser.options |= XML::ParseOptions::RECOVER - @parser.<<(<<-eoxml) - <p> - Foo - <bar> - Bar - </p> - eoxml - @parser.finish - assert(@parser.document.errors.size >= 1) - assert_equal [["p", []], ["bar", []]], @parser.document.start_elements - assert_equal "FooBar", @parser.document.data.map { |x| - x.gsub(/\s/, '') - }.join - end + it :test_recover do + parser.options |= Nokogiri::XML::ParseOptions::RECOVER + parser.<<(<<~EOF) + <p> + Foo + <bar> + Bar + </p> + EOF + parser.finish + assert(parser.document.errors.size >= 1) + assert_equal [["p", []], ["bar", []]], parser.document.start_elements + assert_equal "FooBar", parser.document.data.map { |x| + x.gsub(/\s/, "") + }.join + end - def test_broken_encoding - skip_unless_libxml2("ultra hard to fix for pure Java version") - @parser.options |= XML::ParseOptions::RECOVER - # This is ISO_8859-1: - @parser.<< "<?xml version='1.0' encoding='UTF-8'?><r>Gau\337</r>" - @parser.finish - assert(@parser.document.errors.size >= 1) - assert_equal "Gau\337", @parser.document.data.join - assert_equal [["r"]], @parser.document.end_elements - end + it :test_broken_encoding do + skip_unless_libxml2("ultra hard to fix for pure Java version") + parser.options |= Nokogiri::XML::ParseOptions::RECOVER + # This is ISO_8859-1: + parser.<< "<?xml version='1.0' encoding='UTF-8'?><r>Gau\337</r>" + parser.finish + assert(parser.document.errors.size >= 1) + assert_equal "Gau\337", parser.document.data.join + assert_equal [["r"]], parser.document.end_elements + end - def test_replace_entities_attribute_behavior - if Nokogiri.uses_libxml? - # initially false - assert_equal false, @parser.replace_entities - - # can be set to true - @parser.replace_entities = true - assert_equal true, @parser.replace_entities - - # can be set to false - @parser.replace_entities = false - assert_equal false, @parser.replace_entities - else - # initially true - assert_equal true, @parser.replace_entities - - # ignore attempts to set to false - @parser.replace_entities = false # TODO: should we raise an exception here? - assert_equal true, @parser.replace_entities - end - end + it :test_replace_entities_attribute_behavior do + if Nokogiri.uses_libxml? + # initially false + assert_equal false, parser.replace_entities + + # can be set to true + parser.replace_entities = true + assert_equal true, parser.replace_entities + + # can be set to false + parser.replace_entities = false + assert_equal false, parser.replace_entities + else + # initially true + assert_equal true, parser.replace_entities + + # ignore attempts to set to false + parser.replace_entities = false # TODO: should we raise an exception here? + assert_equal true, parser.replace_entities + end + end - def test_untouched_entities - skip_unless_libxml2("entities are always replaced in pure Java version") - @parser.<<(<<-eoxml) - <p id="asdf&asdf"> - <!-- This is a comment --> - Paragraph 1 & 2 - </p> - eoxml - @parser.finish - assert_equal [["p", [["id", "asdf&asdf"]]]], @parser.document.start_elements - assert_equal "Paragraph 1 & 2", @parser.document.data.join.strip - end + it :test_untouched_entities do + skip_unless_libxml2("entities are always replaced in pure Java version") + parser.<<(<<~EOF) + <p id="asdf&asdf"> + <!-- This is a comment --> + Paragraph 1 & 2 + </p> + EOF + parser.finish + assert_equal [["p", [["id", "asdf&asdf"]]]], parser.document.start_elements + assert_equal "Paragraph 1 & 2", parser.document.data.join.strip + end - def test_replaced_entities - @parser.replace_entities = true - @parser.<<(<<-eoxml) - <p id="asdf&asdf"> - <!-- This is a comment --> - Paragraph 1 & 2 - </p> - eoxml - @parser.finish - assert_equal [["p", [["id", "asdf&asdf"]]]], @parser.document.start_elements - assert_equal "Paragraph 1 & 2", @parser.document.data.join.strip - end - end + it :test_replaced_entities do + parser.replace_entities = true + parser.<<(<<~EOF) + <p id="asdf&asdf"> + <!-- This is a comment --> + Paragraph 1 & 2 + </p> + EOF + parser.finish + assert_equal [["p", [["id", "asdf&asdf"]]]], parser.document.start_elements + assert_equal "Paragraph 1 & 2", parser.document.data.join.strip end end end diff --git a/test/xml/test_builder.rb b/test/xml/test_builder.rb index a17c0b5b9e..022a200970 100644 --- a/test/xml/test_builder.rb +++ b/test/xml/test_builder.rb @@ -107,6 +107,54 @@ def test_non_root_namespace assert_equal "one", b.doc.at("hello", "xmlns" => "one").namespace.href end + def test_builder_namespace_inheritance_true + # see https://github.com/sparklemotion/nokogiri/issues/2317 + result = Nokogiri::XML::Builder.new(encoding: 'utf-8') do |xml| + xml["soapenv"].Envelope("xmlns:soapenv" => "http://schemas.xmlsoap.org/soap/envelope/") do + xml.Header + end + end + assert(result.doc.namespace_inheritance) + assert( + result.doc.at_xpath("//soapenv:Header", "soapenv" => "http://schemas.xmlsoap.org/soap/envelope/"), + "header element should have a namespace" + ) + end + + def test_builder_namespace_inheritance_false + # see https://github.com/sparklemotion/nokogiri/issues/2317 + result = Nokogiri::XML::Builder.new(encoding: 'utf-8', namespace_inheritance: false) do |xml| + xml["soapenv"].Envelope("xmlns:soapenv" => "http://schemas.xmlsoap.org/soap/envelope/") do + xml.Header + end + end + refute(result.doc.namespace_inheritance) + assert( + result.doc.at_xpath("//Header"), + "header element should not have a namespace" + ) + end + + def test_builder_namespace_inheritance_false_part_deux + # see https://github.com/sparklemotion/nokogiri/issues/1712 + result = Nokogiri::XML::Builder.new(encoding: 'utf-8', namespace_inheritance: false) do |xml| + xml['soapenv'].Envelope('xmlns:soapenv' => 'http://schemas.xmlsoap.org/soap/envelope/', 'xmlns:emer' => 'http://dashcs.com/api/v1/emergency') do + xml['soapenv'].Header + xml['soapenv'].Body do + xml['emer'].validateLocation do + # these should not have a namespace + xml.location do + xml.address 'Some place over the rainbow' + end + end + end + end + end + assert(result.doc.at_xpath("//emer:validateLocation", {"emer" => "http://dashcs.com/api/v1/emergency"}), + "expected validateLocation node to have a namespace") + assert(result.doc.at_xpath("//location"), "expected location node to not have a namespace") + end + def test_specify_namespace b = Nokogiri::XML::Builder.new { |xml| xml.root("xmlns:foo" => "bar") do @@ -177,12 +225,18 @@ def test_specified_namespace_undeclared } end + def test_set_namespace_inheritance + assert(Nokogiri::XML::Builder.new.doc.namespace_inheritance) + refute(Nokogiri::XML::Builder.new(namespace_inheritance: false).doc.namespace_inheritance) + end + def test_set_encoding builder = Nokogiri::XML::Builder.new(:encoding => "UTF-8") do |xml| xml.root do xml.bar "blah" end end + assert_equal "UTF-8", builder.doc.encoding assert_match "UTF-8", builder.to_xml end diff --git a/test/xml/test_document.rb b/test/xml/test_document.rb index 2462ebab8b..48eed1d10e 100644 --- a/test/xml/test_document.rb +++ b/test/xml/test_document.rb @@ -15,6 +15,13 @@ class TestDocument < Nokogiri::TestCase let(:xml) { Nokogiri::XML.parse(File.read(XML_FILE), XML_FILE) } + def test_default_namespace_inheritance + doc = Nokogiri::XML::Document.new + refute(doc.namespace_inheritance) + doc.namespace_inheritance = true + assert(doc.namespace_inheritance) + end + def test_dtd_with_empty_internal_subset doc = Nokogiri::XML(<<~eoxml) <?xml version="1.0"?> @@ -518,7 +525,7 @@ def test_non_existant_function def test_xpath_syntax_error assert_raises(Nokogiri::XML::XPath::SyntaxError) do - xml.xpath('\\') + xml.xpath("\\") end end @@ -704,7 +711,7 @@ def test_parse_can_take_pathnames # an arbitrary assertion on the structure of the document assert_equal(20, doc.xpath("/xmlns:feed/xmlns:entry/xmlns:author", - "xmlns" => "http://www.w3.org/2005/Atom").length) + "xmlns" => "http://www.w3.org/2005/Atom").length) assert_equal(XML_ATOM_FILE, doc.url) end @@ -742,7 +749,7 @@ def test_document_search_with_multiple_queries def test_bad_xpath_raises_syntax_error assert_raises(XML::XPath::SyntaxError) do - xml.xpath('\\') + xml.xpath("\\") end end @@ -957,16 +964,16 @@ def test_can_be_closed rb_doc = Nokogiri::XML::Document.wrap(java_doc) assert_equal(0, rb_doc.root.children.length) - java_doc.getFirstChild().appendChild(java_doc.createElement("bar")) + java_doc.getFirstChild.appendChild(java_doc.createElement("bar")) assert_equal(1, rb_doc.root.children.length) end it "shares the same data structure as shown when modified via Nokogiri" do rb_doc = Nokogiri::XML::Document.wrap(java_doc) - assert_equal(0, java_doc.getFirstChild().getChildNodes().getLength()) + assert_equal(0, java_doc.getFirstChild.getChildNodes.getLength) rb_doc.root.add_child("<bar />") - assert_equal(1, java_doc.getFirstChild().getChildNodes().getLength()) + assert_equal(1, java_doc.getFirstChild.getChildNodes.getLength) end else it "does not have the method" do @@ -984,22 +991,22 @@ def test_can_be_closed it "returns the underlying java object" do java_doc = rb_doc.to_java assert_kind_of(org.w3c.dom.Document, java_doc) - assert_equal("foo", java_doc.getDocumentElement().getTagName()) + assert_equal("foo", java_doc.getDocumentElement.getTagName) end it "shares the same data structure as shown when modified via Nokogiri" do java_doc = rb_doc.to_java - assert_equal(1, java_doc.getFirstChild().getChildNodes().getLength()) + assert_equal(1, java_doc.getFirstChild.getChildNodes.getLength) rb_doc.root.add_child("<baz />") - assert_equal(2, java_doc.getFirstChild().getChildNodes().getLength()) + assert_equal(2, java_doc.getFirstChild.getChildNodes.getLength) end it "shares the same data structure as shown when modified via Java interop" do java_doc = rb_doc.to_java assert_equal(1, rb_doc.root.children.length) - java_doc.getFirstChild().appendChild(java_doc.createElement("baz")) + java_doc.getFirstChild.appendChild(java_doc.createElement("baz")) assert_equal(2, rb_doc.root.children.length) end else @@ -1145,9 +1152,9 @@ def initialize(*args) end it "doesn't leak the replaced node" do - skip("only run if NOKOGIRI_GC is set") unless ENV['NOKOGIRI_GC'] + skip("only run if NOKOGIRI_GC is set") unless ENV["NOKOGIRI_GC"] doc = Nokogiri::XML("<root>test</root>") - doc2 = Nokogiri::XML("<root>#{'x' * 5000000}</root>") + doc2 = Nokogiri::XML("<root>#{"x" * 5000000}</root>") doc2.root = doc.root end @@ -1164,7 +1171,19 @@ def initialize(*args) e = assert_raises(ArgumentError) do doc.root = node_set end - assert_equal("expected Nokogiri::XML::Node but received Nokogiri::XML::NodeSet", e.message); + assert_equal("expected Nokogiri::XML::Node but received Nokogiri::XML::NodeSet", e.message) + end + end + + describe "#path" do + it "should return '/'" do + xml = <<~EOF + <root></root> + EOF + + doc = Nokogiri::XML::Document.parse(xml) + assert_equal("/", doc.path) + assert_equal(doc, doc.at_xpath(doc.path)) # make sure we can round-trip end end end diff --git a/test/xml/test_document_fragment.rb b/test/xml/test_document_fragment.rb index b7de6f3444..ce5947a4a2 100644 --- a/test/xml/test_document_fragment.rb +++ b/test/xml/test_document_fragment.rb @@ -11,26 +11,26 @@ def test_replace_text_node html = "foo" doc = Nokogiri::XML::DocumentFragment.parse(html) doc.children[0].replace("bar") - assert_equal('bar', doc.children[0].content) + assert_equal("bar", doc.children[0].content) end def test_fragment_is_relative doc = Nokogiri::XML('<root><a xmlns="blah" /></root>') ctx = doc.root.child - fragment = Nokogiri::XML::DocumentFragment.new(doc, '<hello />', ctx) + fragment = Nokogiri::XML::DocumentFragment.new(doc, "<hello />", ctx) hello = fragment.child - assert_equal('hello', hello.name) + assert_equal("hello", hello.name) assert_equal(doc.root.child.namespace, hello.namespace) end def test_node_fragment_is_relative doc = Nokogiri::XML('<root><a xmlns="blah" /></root>') assert(doc.root.child) - fragment = doc.root.child.fragment('<hello />') + fragment = doc.root.child.fragment("<hello />") hello = fragment.child - assert_equal('hello', hello.name) + assert_equal("hello", hello.name) assert_equal(doc.root.child.namespace, hello.namespace) end @@ -45,7 +45,7 @@ def test_fragment_should_have_document def test_name fragment = Nokogiri::XML::DocumentFragment.new(xml) - assert_equal('#document-fragment', fragment.name) + assert_equal("#document-fragment", fragment.name) end def test_static_method @@ -112,12 +112,12 @@ def test_fragment_children_search fragment = Nokogiri::XML::Document.new.fragment( '<div><p id="content">hi</p></div>' ) - expected = fragment.children.xpath('.//p') + expected = fragment.children.xpath(".//p") assert_equal(1, expected.length) - css = fragment.children.css('p') - search_css = fragment.children.search('p') - search_xpath = fragment.children.search('.//p') + css = fragment.children.css("p") + search_css = fragment.children.search("p") + search_xpath = fragment.children.search(".//p") assert_equal(expected, css) assert_equal(expected, search_css) assert_equal(expected, search_xpath) @@ -130,14 +130,14 @@ def test_fragment_css_search_with_whitespace_and_node_removal fragment = Nokogiri::XML::DocumentFragment.parse(<<~EOXML) <p id="content">hi</p> x <!--y--> <p>another paragraph</p> EOXML - children = fragment.css('p') + children = fragment.css("p") assert_equal(2, children.length) # removing the last node instead does not yield the error. Probably the # node removal leaves around two consecutive text nodes which make the # css search crash? children.first.remove - assert_equal(1, fragment.xpath('.//p | self::p').length) - assert_equal(1, fragment.css('p').length) + assert_equal(1, fragment.xpath(".//p | self::p").length) + assert_equal(1, fragment.css("p").length) end def test_fragment_search_three_ways @@ -146,13 +146,13 @@ def test_fragment_search_three_ways assert_equal(2, expected.length) [ - [:css, '#content'], - [:search, '#content'], - [:search, './*[@id = \'content\']'], + [:css, "#content"], + [:search, "#content"], + [:search, "./*[@id = 'content']"], ].each do |method, query| result = frag.send(method, query) assert_equal(expected, result, - "fragment search with :#{method} using '#{query}' expected '#{expected}' got '#{result}'") + "fragment search with :#{method} using '#{query}' expected '#{expected}' got '#{result}'") end end @@ -169,9 +169,9 @@ def test_fragment_search_with_multiple_queries fragment = Nokogiri::XML.fragment(xml) assert_kind_of(Nokogiri::XML::DocumentFragment, fragment) - assert_equal(3, fragment.xpath('.//div', './/p').length) - assert_equal(3, fragment.css('.title', '.content', 'p').length) - assert_equal(3, fragment.search('.//div', 'p.blah').length) + assert_equal(3, fragment.xpath(".//div", ".//p").length) + assert_equal(3, fragment.css(".title", ".content", "p").length) + assert_equal(3, fragment.search(".//div", "p.blah").length) end def test_fragment_without_a_namespace_does_not_get_a_namespace @@ -215,7 +215,7 @@ def awesome! util_decorate(xml, x) fragment = Nokogiri::XML::DocumentFragment.new(xml, "<div>a</div><div>b</div>") - assert(node_set = fragment.css('div')) + assert(node_set = fragment.css("div")) assert(node_set.respond_to?(:awesome!)) node_set.each do |node| assert(node.respond_to?(:awesome!), node.class) @@ -234,8 +234,8 @@ def awesome! end def test_add_node_to_doc_fragment_segfault - frag = Nokogiri::XML::DocumentFragment.new(xml, '<p>hello world</p>') - Nokogiri::XML::Comment.new(frag, 'moo') + frag = Nokogiri::XML::DocumentFragment.new(xml, "<p>hello world</p>") + Nokogiri::XML::Comment.new(frag, "moo") end def test_issue_1077_parsing_of_frozen_strings @@ -363,6 +363,25 @@ def initialize(*args) end end end + + describe "#path" do + it "should return '?'" do + # see https://github.com/sparklemotion/nokogiri/issues/2250 + # this behavior is clearly undesirable, but is what libxml <= 2.9.10 returned, and so we + # do this for now to preserve the behavior across libxml2 versions. + xml = <<~EOF + <root1></root1> + <root2></root2> + EOF + + frag = Nokogiri::XML::DocumentFragment.parse(xml) + assert_equal "?", frag.path + + # # TODO: we should circle back and fix both the `#path` behavior and the `#xpath` + # # behavior so we can round-trip and get the DocumentFragment back again. + # assert_equal(frag, frag.at_xpath(doc.path)) # make sure we can round-trip + end + end end end end diff --git a/test/xml/test_node.rb b/test/xml/test_node.rb index 5995055cb7..f125d41d54 100644 --- a/test/xml/test_node.rb +++ b/test/xml/test_node.rb @@ -334,7 +334,7 @@ def test_namespace_definitions_when_no_exist assert_equal(0, namespace_definitions.length) end - def test_namespace_goes_to_children + def test_default_namespace_goes_to_children fruits = Nokogiri::XML(<<~eoxml) <Fruit xmlns='www.fruits.org'> </Fruit> @@ -347,6 +347,22 @@ def test_namespace_goes_to_children assert(fruits.at('//fruit:Apple', { 'fruit' => 'www.fruits.org' })) end + def test_parent_namespace_is_not_inherited + fruits = Nokogiri::XML(<<-eoxml) + <fruit xmlns:fruit="http://fruits.org"> + <fruit:apple /> + </fruit> + eoxml + + apple = fruits.at_xpath('//fruit:apple', {"fruit" => "http://fruits.org"}) + assert(apple) + + orange = Nokogiri::XML::Node.new('orange', fruits) + apple.add_child(orange) + + assert_equal(orange, fruits.at_xpath('//orange')) + end + def test_description assert_nil(xml.at('employee').description) end @@ -1049,18 +1065,24 @@ def test_namespace_without_an_href_on_html_node # describe how we handle microsoft word's HTML formatting. # this test is descriptive, not prescriptive. # - skip_unless_libxml2("Xerces handles this edge case completely differently") - - xml = Nokogiri::HTML.parse(<<~EOF) + html = Nokogiri::HTML.parse(<<~EOF) <div><o:p>foo</o:p></div> EOF - - node = xml.at("p") + node = html.at("div").children.first assert_not_nil(node) - assert_equal(1, node.namespaces.keys.size) - assert(node.namespaces.has_key?('xmlns:o')) - assert_nil(node.namespaces['xmlns:o']) + if Nokogiri.uses_libxml?(">= 2.9.12") + assert_empty(node.namespaces.keys) + assert_equal("<p>foo</p>", node.to_html) + elsif Nokogiri.uses_libxml? + assert_equal(1, node.namespaces.keys.size) + assert(node.namespaces.has_key?('xmlns:o')) + assert_nil(node.namespaces['xmlns:o']) + assert_equal("<p>foo</p>", node.to_html) + else # jruby + assert_empty(node.namespaces.keys) + assert_equal("<o:p>foo</o:p>", node.to_html) + end end def test_xpath_results_have_document_and_are_decorated diff --git a/test/xml/test_node_reparenting.rb b/test/xml/test_node_reparenting.rb index 2bd80cf9f0..4bb9f43bd8 100644 --- a/test/xml/test_node_reparenting.rb +++ b/test/xml/test_node_reparenting.rb @@ -360,6 +360,30 @@ def coerce(data) assert_match(%r{<first xmlns=\"http://tenderlovemaking.com/\">}, @doc.to_xml) end end + + describe "and a child node has a namespaced attribute" do + # https://github.com/sparklemotion/nokogiri/issues/2228 + it "should not lose attribute namespace" do + source_doc = Nokogiri::XML::Document.parse(<<~EOXML) + <pre1:root xmlns:pre1="ns1" xmlns:pre2="ns2"> + <pre1:child pre2:attr="attrval"> + </pre1:root> + EOXML + assert(source_node = source_doc.at_xpath("//pre1:child", {"pre1" => "ns1"})) + assert_equal("attrval", source_node.attribute_with_ns("attr", "ns2")&.value) + + dest_doc = Nokogiri::XML::Document.parse(<<~EOXML) + <pre1:root xmlns:pre1="ns1" xmlns:pre2="ns2"> + </pre1:root> + EOXML + assert(dest_node = dest_doc.at_xpath("//pre1:root", {"pre1" => "ns1"})) + + inserted = dest_node.add_child(source_node) + + assert_equal("attrval", inserted.attribute_with_ns("attr", "ns2")&.value, + "inserted node attribute should be namespaced") + end + end end describe "given a parent node with a default and non-default namespace" do @@ -533,6 +557,35 @@ def coerce(data) end end end + + describe "given a parent node with a non-default namespace" do + let(:doc) do + Nokogiri::XML(<<~EOF) + <root xmlns:foo="http://nokogiri.org/default_ns/test/foo"> + <foo:parent> + </foo:parent> + </root> + EOF + end + let(:parent) { doc.at_xpath("//foo:parent", "foo" => "http://nokogiri.org/default_ns/test/foo") } + + describe "and namespace_inheritance is off" do + it "inserts a child node that does not inherit the parent's namespace" do + refute(doc.namespace_inheritance) + child = parent.add_child("<child></child>").first + assert_nil(child.namespace) + end + end + + describe "and namespace_inheritance is on" do + it "inserts a child node that inherits the parent's namespace" do + doc.namespace_inheritance = true + child = parent.add_child("<child></child>").first + assert_not_nil(child.namespace) + assert_equal("http://nokogiri.org/default_ns/test/foo", child.namespace.href) + end + end + end end describe "#add_previous_sibling" do diff --git a/test/xml/test_node_set.rb b/test/xml/test_node_set.rb index be5cdc8eb0..c386bc8278 100644 --- a/test/xml/test_node_set.rb +++ b/test/xml/test_node_set.rb @@ -740,6 +740,7 @@ def awesome(ns) skip("cannot use Ruby 2.5 Enumerator#each in a systemd container") end skip("enumerators confuse valgrind") if i_am_running_in_valgrind + skip("enumerators confuse ASan") if i_am_running_with_asan employees = xml.search("//employee") enum = employees.each diff --git a/test/xml/test_schema.rb b/test/xml/test_schema.rb index 6cba39abe7..f09678b579 100644 --- a/test/xml/test_schema.rb +++ b/test/xml/test_schema.rb @@ -214,6 +214,21 @@ def test_xsd_with_dtd end end + def test_xsd_import_with_no_systemid + # https://github.com/sparklemotion/nokogiri/pull/2296 + xsd = <<~EOF + <?xml version="1.0" encoding="UTF-8"?> + <xs:schema + xmlns:xs="http://www.w3.org/2001/XMLSchema" + xmlns="http://www.w3.org/1998/Math/MathML" + targetNamespace="http://www.w3.org/1998/Math/MathML" + > + <xs:import/> + </xs:schema> + EOF + Nokogiri::XML::Schema(xsd) # assert_nothing_raised + end + describe "CVE-2020-26247" do # https://github.com/sparklemotion/nokogiri/security/advisories/GHSA-vr8q-g5c7-m54m let(:schema) do diff --git a/test/xml/test_xpath.rb b/test/xml/test_xpath.rb index 271412a70f..504a30caa8 100644 --- a/test/xml/test_xpath.rb +++ b/test/xml/test_xpath.rb @@ -340,7 +340,7 @@ def name_equals(nodeset, name, *args) # long list of long arguments, to apply GC pressure during # ruby_funcall argument marshalling xpath = ["//tool[name_equals(.,'hammer'"] - 1000.times { xpath << "'unused argument #{'x' * 1000}'" } + 500.times { xpath << "'unused argument #{'x' * 1000}'" } xpath << "'unused argument')]" xpath = xpath.join(',') @@ -470,6 +470,27 @@ def test_xpath_syntax_error end end + def test_huge_xpath_query + if Nokogiri.uses_libxml?("~>2.9.11") && !Nokogiri::VERSION_INFO["libxml"]["patches"]&.include?("0007-Fix-XPath-recursion-limit.patch") + skip("libxml2 under test is broken with respect to xpath query recusion depth") + end + + # real world example + # from https://github.com/sparklemotion/nokogiri/issues/2257 + query = File.read(File.join(ASSETS_DIR, 'huge-xpath-query.txt')) + + doc = Nokogiri::XML::Document.parse("<root></root>") + handler = Class.new do + def seconds(context) + 42 + end + def add(context, rhs) + 42 + end + end + doc.xpath(query, {"ct" => "https://test.nokogiri.org/ct", "date" => "https://test.nokogiri.org/date"}, handler.new) + end + describe "nokogiri-builtin:css-class xpath function" do before do @doc = Nokogiri::HTML::Document.parse("<html></html>") diff --git a/test/xslt/test_custom_functions.rb b/test/xslt/test_custom_functions.rb index 01daba28dd..34f509e56c 100644 --- a/test/xslt/test_custom_functions.rb +++ b/test/xslt/test_custom_functions.rb @@ -41,15 +41,15 @@ def capitalize(nodes) xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:f="http://e.org/functions" extension-element-prefixes="f"> - <xsl:template match="text()"> - <xsl:copy-of select="f:capitalize(.)"/> - </xsl:template> <xsl:template match="@*|node()"> <xsl:copy> <xsl:apply-templates select="@*|node()"/> <xsl:apply-imports/> </xsl:copy> </xsl:template> + <xsl:template match="text()"> + <xsl:copy-of select="f:capitalize(.)"/> + </xsl:template> </xsl:stylesheet> EOXSL result = xsl.transform(@xml) @@ -108,15 +108,15 @@ def america(nodes) xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:f="http://e.org/functions" extension-element-prefixes="f"> - <xsl:template match="text()"> - <xsl:copy-of select="f:america(.)"/> - </xsl:template> <xsl:template match="@*|node()"> <xsl:copy> <xsl:apply-templates select="@*|node()"/> <xsl:apply-imports/> </xsl:copy> </xsl:template> + <xsl:template match="text()"> + <xsl:copy-of select="f:america(.)"/> + </xsl:template> </xsl:stylesheet> EOXSL result = xsl.transform(@xml)
\r\na
", ToString(token_.original_text)); + EXPECT_EQ(15, token_.position.offset); +} + +TEST_F(GumboTokenizerTest, SelfClosingStartTag) { + SetInput(""); + NextStartTag(GUMBO_TAG_BR); + EXPECT_EQ(0, token_.position.offset); + EXPECT_EQ("", ToString(token_.original_text)); + + GumboTokenStartTag* start_tag = &token_.v.start_tag; + EXPECT_EQ(0, start_tag->attributes.length); + EXPECT_TRUE(start_tag->is_self_closing); +} + +TEST_F(GumboTokenizerTest, SelfClosingEndTag) { + SetInput(""); + NextEndTag(GUMBO_TAG_P, true); + Error(GUMBO_ERR_END_TAG_WITH_TRAILING_SOLIDUS); + EXPECT_EQ(0, token_.position.offset); + EXPECT_EQ("", ToString(token_.original_text)); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, OpenTagWithAttributes) { + SetInput(""); + NextStartTag(GUMBO_TAG_A); + + GumboTokenStartTag* start_tag = &token_.v.start_tag; + EXPECT_FALSE(start_tag->is_self_closing); + ASSERT_EQ(2, start_tag->attributes.length); + + GumboAttribute* href = + static_cast(start_tag->attributes.data[0]); + EXPECT_STREQ("href", href->name); + EXPECT_EQ("href", ToString(href->original_name)); + EXPECT_STREQ("/search?q=foo&hl=en", href->value); + EXPECT_EQ("'/search?q=foo&hl=en'", ToString(href->original_value)); + + GumboAttribute* id = + static_cast(start_tag->attributes.data[1]); + EXPECT_STREQ("id", id->name); + EXPECT_EQ("id", ToString(id->original_name)); + EXPECT_STREQ("link", id->value); + EXPECT_EQ("link", ToString(id->original_value)); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, BogusComment1) { + SetInput("Text"); + NextComment("?xml is bogus-comment", true); + Error(GUMBO_ERR_UNEXPECTED_QUESTION_MARK_INSTEAD_OF_TAG_NAME); + NextChar('T'); + NextChar('e'); + NextChar('x'); + NextChar('t'); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, BogusComment2) { + SetInput(""); + NextStartTag(GUMBO_TAG_UNKNOWN); + + GumboTokenStartTag* start_tag = &token_.v.start_tag; + EXPECT_TRUE(start_tag->is_self_closing); + ASSERT_EQ(1, start_tag->attributes.length); + + GumboAttribute* long_attr = + static_cast(start_tag->attributes.data[0]); + EXPECT_STREQ("long_attr", long_attr->name); + EXPECT_EQ("long_attr", ToString(long_attr->original_name)); + EXPECT_STREQ( + "SomeCode;\n" + " calls_a_big_long_function();\n" + " return true;", + long_attr->value); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, DoubleAmpersand) { + SetInput(""); + NextStartTag(GUMBO_TAG_SPAN); + GumboTokenStartTag* start_tag = &token_.v.start_tag; + EXPECT_FALSE(start_tag->is_self_closing); + ASSERT_EQ(1, start_tag->attributes.length); + + GumboAttribute* jsif = + static_cast(start_tag->attributes.data[0]); + EXPECT_STREQ("jsif", jsif->name); + EXPECT_EQ("jsif", ToString(jsif->original_name)); + EXPECT_STREQ("foo && bar", jsif->value); + EXPECT_EQ("\"foo && bar\"", ToString(jsif->original_value)); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, MatchedTagPair) { + SetInput("a"); + NextStartTag(GUMBO_TAG_DIV, true); + Error(GUMBO_ERR_UNEXPECTED_CHARACTER_IN_UNQUOTED_ATTRIBUTE_VALUE); + EXPECT_EQ(0, token_.position.offset); + + GumboTokenStartTag* start_tag = &token_.v.start_tag; + EXPECT_FALSE(start_tag->is_self_closing); + ASSERT_EQ(2, start_tag->attributes.length); + + GumboAttribute* id = + static_cast(start_tag->attributes.data[0]); + EXPECT_STREQ("id", id->name); + EXPECT_EQ("id", ToString(id->original_name)); + EXPECT_EQ(1, id->name_start.line); + EXPECT_EQ(5, id->name_start.offset); + EXPECT_EQ(6, id->name_start.column); + EXPECT_EQ(8, id->name_end.column); + EXPECT_STREQ("dash<-Dash", id->value); + EXPECT_EQ("dash<-Dash", ToString(id->original_value)); + EXPECT_EQ(9, id->value_start.column); + EXPECT_EQ(19, id->value_end.column); + + GumboAttribute* data_attr = + static_cast(start_tag->attributes.data[1]); + EXPECT_STREQ("data-test", data_attr->name); + EXPECT_EQ("data-test", ToString(data_attr->original_name)); + EXPECT_EQ(20, data_attr->name_start.column); + EXPECT_EQ(29, data_attr->name_end.column); + EXPECT_STREQ("bar", data_attr->value); + EXPECT_EQ("\"bar\"", ToString(data_attr->original_value)); + EXPECT_EQ(30, data_attr->value_start.column); + EXPECT_EQ(35, data_attr->value_end.column); + + NextChar('a'); + EXPECT_EQ(35, token_.position.offset); + NextEndTag(GUMBO_TAG_DIV); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, BogusEndTag) { + // According to the spec, the correct parse of this is an end tag token for + // "" (notice the ending bracket) with the attribute "th=th" (ignored + // because end tags don't take attributes). + SetInput(""); + NextEndTag(GUMBO_TAG_UNKNOWN, true); + Error(GUMBO_ERR_UNEXPECTED_SOLIDUS_IN_TAG); + Error(GUMBO_ERR_END_TAG_WITH_ATTRIBUTES); + EXPECT_STREQ("div<", token_.v.end_tag.name); + EXPECT_EQ(0, token_.position.offset); + EXPECT_EQ("", ToString(token_.original_text)); +} + +TEST_F(GumboTokenizerTest, Whitespace) { + SetInput(" "); + NextChar(0xa0); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, NumericHex) { + SetInput("ካ"); + NextChar(0x12ab); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, NumericDecimal) { + SetInput("Ӓ"); + NextChar(1234); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, NumericInvalidDigit) { + SetInput("&#google"); + NextChar('&', true); + Error(GUMBO_ERR_ABSENCE_OF_DIGITS_IN_NUMERIC_CHARACTER_REFERENCE); + NextChar('#'); + NextChar('g'); + NextChar('o'); + NextChar('o'); + NextChar('g'); + NextChar('l'); + NextChar('e'); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, NumericNoSemicolon) { + SetInput("Ӓgoogle"); + NextChar(1234, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + NextChar('g'); + NextChar('o'); + NextChar('o'); + NextChar('g'); + NextChar('l'); + NextChar('e'); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, NamedReplacement) { + SetInput("<"); + NextChar('<'); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, NamedReplacementNoSemicolon) { + SetInput(">"); + NextChar('>', true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, NamedReplacementWithInvalidUtf8) { + SetInput("&\xc3\xa5"); + NextChar('&'); + NextChar(229); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, NamedReplacementInvalid) { + SetInput("&google;"); + // This is an error on the semicolon since that's where the error occurs in + // the spec. Anything other than a semicolon would cause no error. + NextChar('&'); + NextChar('g'); + NextChar('o'); + NextChar('o'); + NextChar('g'); + NextChar('l'); + NextChar('e'); + NextChar(';', true); + Error(GUMBO_ERR_UNKNOWN_NAMED_CHARACTER_REFERENCE); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, NamedReplacementInvalidNoSemicolon) { + SetInput("&google"); + NextChar('&'); + NextChar('g'); + NextChar('o'); + NextChar('o'); + NextChar('g'); + NextChar('l'); + NextChar('e'); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, InAttribute) { + SetInput(""); + NextStartTag(GUMBO_TAG_SPAN); + GumboTokenStartTag* start_tag = &token_.v.start_tag; + ASSERT_EQ(1, start_tag->attributes.length); + GumboAttribute* attr = static_cast(start_tag->attributes.data[0]); + EXPECT_STREQ("¬ed", attr->value); + NextEndTag(GUMBO_TAG_SPAN); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, MultiChars) { + SetInput("⋵̸"); + NextChar(0x22F5); + NextChar(0x0338); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, CharAfter) { + SetInput("<x"); + NextChar('<'); + NextChar('x'); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, MaxNumericCharRef) { + SetInput("FOOZOO"); + NextChar('F'); + NextChar('O'); + NextChar('O'); + NextChar(0x10FFFF, true); + Error(GUMBO_ERR_NONCHARACTER_CHARACTER_REFERENCE); + NextChar('Z'); + NextChar('O'); + NextChar('O'); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, InvalidRef) { + SetInput("ZZ&prod_id=23"); + NextChar('Z'); + NextChar('Z'); + NextChar('&'); + NextChar('p'); + NextChar('r'); + NextChar('o'); + NextChar('d'); + NextChar('_'); + NextChar('i'); + NextChar('d'); + NextChar('='); + NextChar('2'); + NextChar('3'); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, InvalidRefInAttr) { + SetInput(""); + NextStartTag(GUMBO_TAG_SPAN); + GumboTokenStartTag* start_tag = &token_.v.start_tag; + ASSERT_EQ(1, start_tag->attributes.length); + GumboAttribute* attr = static_cast(start_tag->attributes.data[0]); + EXPECT_STREQ("ZZ&prod_id=23", attr->value); + NextEndTag(GUMBO_TAG_SPAN); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, NumericTooLarge) { + SetInput("�"); + NextChar(0xFFFD, true); + Error(GUMBO_ERR_CHARACTER_REFERENCE_OUTSIDE_UNICODE_RANGE); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, CdataWithBrackets) { + SetInput("y"); + Foreign(); + Next(); + EXPECT_EQ(GUMBO_TOKEN_CDATA, token_.type); + EXPECT_EQ('x', token_.v.character); + EXPECT_EQ(10, token_.position.column); + EXPECT_EQ(1, token_.position.line); + EXPECT_EQ(9, token_.position.offset); + + Next(); + EXPECT_EQ(GUMBO_TOKEN_CDATA, token_.type); + EXPECT_EQ(']', token_.v.character); + EXPECT_EQ(11, token_.position.column); + EXPECT_EQ(1, token_.position.line); + EXPECT_EQ(10, token_.position.offset); + + Next(); + EXPECT_EQ(GUMBO_TOKEN_CDATA, token_.type); + EXPECT_EQ(']', token_.v.character); + EXPECT_EQ(12, token_.position.column); + EXPECT_EQ(1, token_.position.line); + EXPECT_EQ(11, token_.position.offset); + + Next(); + EXPECT_EQ(GUMBO_TOKEN_CDATA, token_.type); + EXPECT_EQ(']', token_.v.character); + EXPECT_EQ(13, token_.position.column); + EXPECT_EQ(1, token_.position.line); + EXPECT_EQ(12, token_.position.offset); + + NextChar('y'); + EXPECT_EQ(17, token_.position.column); + EXPECT_EQ(1, token_.position.line); + EXPECT_EQ(16, token_.position.offset); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, EscapedScriptStates) { + SetInput(""); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('/'); + NextChar('>'); + NextChar('<'); + NextChar('/'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextEndTag(GUMBO_TAG_SCRIPT); +} + +TEST_F(GumboTokenizerTest, EscapedScriptStates2) { + SetInput(""); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextChar('-'); + NextChar('-'); + NextChar('>'); + NextEndTag(GUMBO_TAG_SCRIPT); +} + +TEST_F(GumboTokenizerTest, ReadOutOfBounds) { + SetInput("⋵̸", 6); + NextChar(0x00AC, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + NextChar('i'); + NextChar('n'); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, ControlCharRefs) { + SetInput("€‚ƒ„…†‡ˆ‰" + "Š‹ŒŽ‘’“”•" + "–—˜™š›œžŸ"); + NextChar(0x20AC, true); + NextChar(0x201A, true); + NextChar(0x0192, true); + NextChar(0x201E, true); + NextChar(0x2026, true); + NextChar(0x2020, true); + NextChar(0x2021, true); + NextChar(0x02C6, true); + NextChar(0x2030, true); + NextChar(0x0160, true); + NextChar(0x2039, true); + NextChar(0x0152, true); + NextChar(0x017D, true); + NextChar(0x2018, true); + NextChar(0x2019, true); + NextChar(0x201C, true); + NextChar(0x201D, true); + NextChar(0x2022, true); + NextChar(0x2013, true); + NextChar(0x2014, true); + NextChar(0x02DC, true); + NextChar(0x2122, true); + NextChar(0x0161, true); + NextChar(0x203A, true); + NextChar(0x0153, true); + NextChar(0x017E, true); + NextChar(0x0178, true); + for (int i = 0; i < 27; ++i) + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + AtEnd(); +} + +// The tests below here are designed to test all of the tokenizers +// transitions. The name of each test has a specific form (for the most part): +// StartState_Rule1_Rule2_..._Rulen +// The parser sets the tokenizer in just a handful of states: plaintext, +// rawtext, rcdata, script data, and data. The tests are divided into those 5 +// categories and should cover every possible transition from each state. +// (Some transitions are covered more than once, and I'm sure some transitions +// are missing and should be added when found.) +// +// When a character is reconsumed in a new state, the test name _should_ +// include which rule gets matched for the new state, but not all of them do. +// That's just an oversight and should be fixed for consistency. +// +// Many states require additional characters in order to emit a token (or do +// so without an error). The rules used to match those tokens are omitted from +// the state name. + +// Starting in the PLAINTEXT state +TEST_F(GumboTokenizerTest, Plaintext_NULL) { + SetInput("", 1); + SetState(GUMBO_LEX_PLAINTEXT); + NextChar(0xFFFD, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); +} + +TEST_F(GumboTokenizerTest, Plaintext_EOF) { + SetInput(""); + SetState(GUMBO_LEX_PLAINTEXT); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Plaintext_AnythingElse) { + SetInput("a\xce\xb2\xd7\x92</plaintext>¬"); + NextStartTag(GUMBO_TAG_PLAINTEXT); + SetState(GUMBO_LEX_PLAINTEXT); + NextChar(0x61); + NextChar(0x03B2); + NextChar(0x05D2); + NextChar('<'); + NextChar('/'); + NextChar('p'); + NextChar('l'); + NextChar('a'); + NextChar('i'); + NextChar('n'); + NextChar('t'); + NextChar('e'); + NextChar('x'); + NextChar('t'); + NextChar('>'); + NextChar('&'); + NextChar('n'); + NextChar('o'); + NextChar('t'); + NextChar(';'); +} + +// Starting in the RAWTEXT state +TEST_F(GumboTokenizerTest, Rawtext_LT_Slash_Alpha_InappropriateTab) { + SetInput("<iframe></i\tx"); + NextStartTag(GUMBO_TAG_IFRAME); + SetState(GUMBO_LEX_RAWTEXT); + NextChar('<'); + NextChar('/'); + NextChar('i'); + NextSpace('\t'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rawtext_LT_Slash_Alpha_InappropriateLF) { + SetInput("<noframes></n\nx"); + NextStartTag(GUMBO_TAG_NOFRAMES); + SetState(GUMBO_LEX_RAWTEXT); + NextChar('<'); + NextChar('/'); + NextChar('n'); + NextSpace('\n'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rawtext_LT_Slash_Alpha_InappropriateFF) { + SetInput("<iframe></i\fx"); + NextStartTag(GUMBO_TAG_IFRAME); + SetState(GUMBO_LEX_RAWTEXT); + NextChar('<'); + NextChar('/'); + NextChar('i'); + NextSpace('\f'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rawtext_LT_Slash_Alpha_InappropriateSpace) { + SetInput("<noframes></n x"); + NextStartTag(GUMBO_TAG_NOFRAMES); + SetState(GUMBO_LEX_RAWTEXT); + NextChar('<'); + NextChar('/'); + NextChar('n'); + NextSpace(' '); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rawtext_LT_Slash_Alpha_InappropriateSlash) { + SetInput("<iframe></i/x"); + NextStartTag(GUMBO_TAG_IFRAME); + SetState(GUMBO_LEX_RAWTEXT); + NextChar('<'); + NextChar('/'); + NextChar('i'); + NextChar('/'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rawtext_LT_Slash_Alpha_InappropriateGT) { + SetInput("<noframes></n>x"); + NextStartTag(GUMBO_TAG_NOFRAMES); + SetState(GUMBO_LEX_RAWTEXT); + NextChar('<'); + NextChar('/'); + NextChar('n'); + NextChar('>'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rawtext_LT_Slash_Alpha_AppropriateLF) { + SetInput("<noframes></nofRAmes\n>x"); + NextStartTag(GUMBO_TAG_NOFRAMES); + SetState(GUMBO_LEX_RAWTEXT); + NextEndTag(GUMBO_TAG_NOFRAMES); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rawtext_LT_Slash_Alpha_AppropriateFF) { + SetInput("<iframe></ifrAMe\f foo=bar>x"); + NextStartTag(GUMBO_TAG_IFRAME); + SetState(GUMBO_LEX_RAWTEXT); + NextEndTag(GUMBO_TAG_IFRAME, true); + Error(GUMBO_ERR_END_TAG_WITH_ATTRIBUTES); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rawtext_LT_Slash_Alpha_AppropriateSpace) { + SetInput("<noframes></Noframes >x"); + NextStartTag(GUMBO_TAG_NOFRAMES); + SetState(GUMBO_LEX_RAWTEXT); + NextEndTag(GUMBO_TAG_NOFRAMES); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rawtext_LT_Slash_Alpha_AppropriateSlash_GT) { + SetInput("<iframe></IFRAME/>x"); + NextStartTag(GUMBO_TAG_IFRAME); + SetState(GUMBO_LEX_RAWTEXT); + NextEndTag(GUMBO_TAG_IFRAME, true); + Error(GUMBO_ERR_END_TAG_WITH_TRAILING_SOLIDUS); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rawtext_LT_Slash_Alpha_AppropriateSlash_EOF) { + SetInput("<iframe></iframe/"); + NextStartTag(GUMBO_TAG_IFRAME); + SetState(GUMBO_LEX_RAWTEXT); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_TAG); +} + +TEST_F(GumboTokenizerTest, Rawtext_LT_Slash_Alpha_AppropriateSlash_AnythingElse) { + SetInput("<iframe></iframe/ >x"); + NextStartTag(GUMBO_TAG_IFRAME); + SetState(GUMBO_LEX_RAWTEXT); + NextEndTag(GUMBO_TAG_IFRAME, true); + Error(GUMBO_ERR_UNEXPECTED_SOLIDUS_IN_TAG); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rawtext_LT_Slash_Alpha_AppropriateGT) { + SetInput("<noframes></noframes>x"); + NextStartTag(GUMBO_TAG_NOFRAMES); + SetState(GUMBO_LEX_RAWTEXT); + NextEndTag(GUMBO_TAG_NOFRAMES); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rawtext_LT_Slash_Alpha_AnythingElse) { + SetInput("<iframe></if\xce\xb2"); + NextStartTag(GUMBO_TAG_IFRAME); + SetState(GUMBO_LEX_RAWTEXT); + NextChar('<'); + NextChar('/'); + NextChar('i'); + NextChar('f'); + NextChar(0x03B2); +} + +TEST_F(GumboTokenizerTest, Rawtext_LT_Slash_AnythingElse) { + SetInput("</0"); + SetState(GUMBO_LEX_RAWTEXT); + NextChar('<'); + NextChar('/'); + NextChar('0'); +} + +TEST_F(GumboTokenizerTest, Rawtext_LT_AnythingElse) { + SetInput("<p>"); + SetState(GUMBO_LEX_RAWTEXT); + NextChar('<'); + NextChar('p'); + NextChar('>'); +} + +TEST_F(GumboTokenizerTest, Rawtext_NULL) { + SetInput("", 1); + SetState(GUMBO_LEX_RAWTEXT); + NextChar(0xFFFD, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); +} + +TEST_F(GumboTokenizerTest, Rawtext_EOF) { + SetInput(""); + SetState(GUMBO_LEX_RAWTEXT); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Rawtext_AnythingElse) { + SetInput("a\xce\xb2\xd7\x92>¬"); + SetState(GUMBO_LEX_RAWTEXT); + NextChar(0x61); + NextChar(0x03B2); + NextChar(0x05D2); + NextChar('>'); + NextChar('&'); + NextChar('n'); + NextChar('o'); + NextChar('t'); + NextChar(';'); +} + +// Starting in the RCDATA state +TEST_F(GumboTokenizerTest, Rcdata_Amp_Alnum_Match_Semicolon) { + SetInput("fjx"); + SetState(GUMBO_LEX_RCDATA); + NextChar(0x0066); + NextChar(0x006A); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_Alnum_Match) { + SetInput("áx"); + SetState(GUMBO_LEX_RCDATA); + NextChar(0x00E1, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_Alnum_Otherwise_Alnum_Semicolon) { + SetInput("&1f;x"); + SetState(GUMBO_LEX_RCDATA); + NextChar('&'); + NextChar('1'); + NextChar('f'); + NextChar(';', true); + Error(GUMBO_ERR_UNKNOWN_NAMED_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_Alnum_Otherwise_Alnum_AnythingElse) { + SetInput("&1f.x"); + SetState(GUMBO_LEX_RCDATA); + NextChar('&'); + NextChar('1'); + NextChar('f'); + NextChar('.'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_NumberSign_X_Hex_Semicolon_Zero) { + SetInput("�x"); + SetState(GUMBO_LEX_RCDATA); + NextChar(0xFFFD, true); + Error(GUMBO_ERR_NULL_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_NumberSign_X_Hex_Semicolon_TooLarge) { + SetInput("�x"); + SetState(GUMBO_LEX_RCDATA); + NextChar(0xFFFD, true); + Error(GUMBO_ERR_CHARACTER_REFERENCE_OUTSIDE_UNICODE_RANGE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_NumberSign_X_Hex_Semicolon_Surrogate) { + SetInput("�x"); + SetState(GUMBO_LEX_RCDATA); + NextChar(0xFFFD, true); + Error(GUMBO_ERR_SURROGATE_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_NumberSign_x_Hex_Semicolon_Noncharacter) { + SetInput("x"); + SetState(GUMBO_LEX_RCDATA); + NextChar(0xFDD8, true); + Error(GUMBO_ERR_NONCHARACTER_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_NumberSign_x_Hex_Semicolon_Noncharacter2) { + SetInput("x"); + SetState(GUMBO_LEX_RCDATA); + NextChar(0x7FFFE, true); + Error(GUMBO_ERR_NONCHARACTER_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_NumberSign_x_Hex_Semicolon_C0Control) { + SetInput("x"); + SetState(GUMBO_LEX_RCDATA); + NextChar(3, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_NumberSign_x_Hex_Semicolon_NL) { + SetInput("
x"); + SetState(GUMBO_LEX_RCDATA); + NextSpace('\n'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_NumberSign_x_Hex_Semicolon_FF) { + SetInput("x"); + SetState(GUMBO_LEX_RCDATA); + NextSpace('\f'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_NumberSign_x_Hex_Semicolon_CR) { + SetInput("
x"); + SetState(GUMBO_LEX_RCDATA); + NextSpace('\r', true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_NumberSign_X_Hex_Semicolon_Control) { + SetInput("x"); + SetState(GUMBO_LEX_RCDATA); + NextChar(0x81, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_NumberSign_x_Hex_Semicolon_Control2) { + SetInput("‚x"); + SetState(GUMBO_LEX_RCDATA); + NextChar(0x201A, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_NumberSign_x_Hex_AnythingElse) { + SetInput("⼚x"); + SetState(GUMBO_LEX_RCDATA); + NextChar(0x2F1A, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_NumberSign_x_Hex_AnythingElse2) { + SetInput("⼚-"); + SetState(GUMBO_LEX_RCDATA); + NextChar(0x2F1A, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + NextChar('-'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_NumberSign_x_AnythingElse) { + SetInput("&#xG"); + SetState(GUMBO_LEX_RCDATA); + NextChar('&', true); + Error(GUMBO_ERR_ABSENCE_OF_DIGITS_IN_NUMERIC_CHARACTER_REFERENCE); + NextChar('#'); + NextChar('x'); + NextChar('G'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_NumberSign_AnythingElse_Digit_Semicolon) { + SetInput("Ӓx"); + SetState(GUMBO_LEX_RCDATA); + NextChar(1234); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_NumberSign_AnythingElse_Digit_Semicolon_Control) { + SetInput("€x"); + SetState(GUMBO_LEX_RCDATA); + NextChar(0x20AC, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_NumberSign_AnythingElse_Digit_AnythingElse) { + SetInput("Ӓa"); + SetState(GUMBO_LEX_RCDATA); + NextChar(1234, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + NextChar('a'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_NumberSign_AnythingElse_Digit_AnythingElse_Control) { + SetInput("€a"); + SetState(GUMBO_LEX_RCDATA); + NextChar(0x20AC, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + NextChar('a'); +} + +TEST_F(GumboTokenizerTest, Rcdata_Amp_AnythingElse) { + SetInput("&#x"); + SetState(GUMBO_LEX_RCDATA); + NextChar('&', true); + Error(GUMBO_ERR_ABSENCE_OF_DIGITS_IN_NUMERIC_CHARACTER_REFERENCE); + NextChar('#'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_LT_Slash_Alpha_InappropriateTab) { + SetInput("<title></t\tx"); + NextStartTag(GUMBO_TAG_TITLE); + SetState(GUMBO_LEX_RCDATA); + NextChar('<'); + NextChar('/'); + NextChar('t'); + NextSpace('\t'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_LT_Slash_Alpha_InappropriateLF) { + SetInput("<textarea></t\nx"); + NextStartTag(GUMBO_TAG_TEXTAREA); + SetState(GUMBO_LEX_RCDATA); + NextChar('<'); + NextChar('/'); + NextChar('t'); + NextSpace('\n'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_LT_Slash_Alpha_InappropriateFF) { + SetInput("<title></t\fx"); + NextStartTag(GUMBO_TAG_TITLE); + SetState(GUMBO_LEX_RCDATA); + NextChar('<'); + NextChar('/'); + NextChar('t'); + NextSpace('\f'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_LT_Slash_Alpha_InappropriateSpace) { + SetInput("<textarea></t x"); + NextStartTag(GUMBO_TAG_TEXTAREA); + SetState(GUMBO_LEX_RCDATA); + NextChar('<'); + NextChar('/'); + NextChar('t'); + NextSpace(' '); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_LT_Slash_Alpha_InappropriateSlash) { + SetInput("<title></t/x"); + NextStartTag(GUMBO_TAG_TITLE); + SetState(GUMBO_LEX_RCDATA); + NextChar('<'); + NextChar('/'); + NextChar('t'); + NextChar('/'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_LT_Slash_Alpha_InappropriateGT) { + SetInput("<textarea></t>x"); + NextStartTag(GUMBO_TAG_TEXTAREA); + SetState(GUMBO_LEX_RCDATA); + NextChar('<'); + NextChar('/'); + NextChar('t'); + NextChar('>'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_LT_Slash_Alpha_AppropriateLF) { + SetInput("<textarea></teXTarea\n>x"); + NextStartTag(GUMBO_TAG_TEXTAREA); + SetState(GUMBO_LEX_RCDATA); + NextEndTag(GUMBO_TAG_TEXTAREA); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_LT_Slash_Alpha_AppropriateFF) { + SetInput("<title></titlE\f foo=bar>x"); + NextStartTag(GUMBO_TAG_TITLE); + SetState(GUMBO_LEX_RCDATA); + NextEndTag(GUMBO_TAG_TITLE, true); + Error(GUMBO_ERR_END_TAG_WITH_ATTRIBUTES); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_LT_Slash_Alpha_AppropriateSpace) { + SetInput("<textarea></Textarea >x"); + NextStartTag(GUMBO_TAG_TEXTAREA); + SetState(GUMBO_LEX_RCDATA); + NextEndTag(GUMBO_TAG_TEXTAREA); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_LT_Slash_Alpha_AppropriateSlash_GT) { + SetInput("<title></TITLE/>x"); + NextStartTag(GUMBO_TAG_TITLE); + SetState(GUMBO_LEX_RCDATA); + NextEndTag(GUMBO_TAG_TITLE, true); + Error(GUMBO_ERR_END_TAG_WITH_TRAILING_SOLIDUS); + NextChar('x'); +} +TEST_F(GumboTokenizerTest, Rcdata_LT_Slash_Alpha_AppropriateSlash_EOF) { + SetInput("<title></tITle/"); + NextStartTag(GUMBO_TAG_TITLE); + SetState(GUMBO_LEX_RCDATA); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_TAG); +} + +TEST_F(GumboTokenizerTest, Rcdata_LT_Slash_Alpha_AppropriateSlash_AnythingElse) { + SetInput("<title></title/ >x"); + NextStartTag(GUMBO_TAG_TITLE); + SetState(GUMBO_LEX_RCDATA); + NextEndTag(GUMBO_TAG_TITLE, true); + Error(GUMBO_ERR_UNEXPECTED_SOLIDUS_IN_TAG); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_LT_Slash_Alpha_AppropriateGT) { + SetInput("<textarea></textarea>x"); + NextStartTag(GUMBO_TAG_TEXTAREA); + SetState(GUMBO_LEX_RCDATA); + NextEndTag(GUMBO_TAG_TEXTAREA); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Rcdata_LT_Slash_Alpha_AnythingElse) { + SetInput("<title></ti\xce\xb2"); + NextStartTag(GUMBO_TAG_TITLE); + SetState(GUMBO_LEX_RCDATA); + NextChar('<'); + NextChar('/'); + NextChar('t'); + NextChar('i'); + NextChar(0x03B2); +} + +TEST_F(GumboTokenizerTest, Rcdata_LT_Slash_AnythingElse) { + SetInput("</0"); + SetState(GUMBO_LEX_RCDATA); + NextChar('<'); + NextChar('/'); + NextChar('0'); +} + +TEST_F(GumboTokenizerTest, Rcdata_LT_AnythingElse) { + SetInput("<p>"); + SetState(GUMBO_LEX_RCDATA); + NextChar('<'); + NextChar('p'); + NextChar('>'); +} + +TEST_F(GumboTokenizerTest, Rcdata_NULL) { + SetInput("", 1); + SetState(GUMBO_LEX_RCDATA); + NextChar(0xFFFD, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); +} + +TEST_F(GumboTokenizerTest, Rcdata_EOF) { + SetInput(""); + SetState(GUMBO_LEX_RCDATA); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Rcdata_AnythingElse) { + SetInput("a\xce\xb2\xd7\x92"); + SetState(GUMBO_LEX_RCDATA); + NextChar(0x61); + NextChar(0x03B2); + NextChar(0x05D2); +} + +// Starting in the script data state +TEST_F(GumboTokenizerTest, ScriptData_LT_Slash_Alpha_InappropriateTab) { + SetInput("<script></s\tx"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('/'); + NextChar('s'); + NextSpace('\t'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Slash_Alpha_InappropriateLF) { + SetInput("<script></s\nx"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('/'); + NextChar('s'); + NextSpace('\n'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Slash_Alpha_InappropriateFF) { + SetInput("<script></s\fx"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('/'); + NextChar('s'); + NextSpace('\f'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Slash_Alpha_InappropriateSpace) { + SetInput("<script></s x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('/'); + NextChar('s'); + NextSpace(' '); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Slash_Alpha_InappropriateSlash) { + SetInput("<script></s/x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('/'); + NextChar('s'); + NextChar('/'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Slash_Alpha_InappropriateGT) { + SetInput("<script></s>x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('/'); + NextChar('s'); + NextChar('>'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Slash_Alpha_AppropriateLF) { + SetInput("<script></scRIpt\n>x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextEndTag(GUMBO_TAG_SCRIPT); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Slash_Alpha_AppropriateFF) { + SetInput("<script></scripT\f foo=bar>x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextEndTag(GUMBO_TAG_SCRIPT, true); + Error(GUMBO_ERR_END_TAG_WITH_ATTRIBUTES); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Slash_Alpha_AppropriateSpace) { + SetInput("<script></Script >x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextEndTag(GUMBO_TAG_SCRIPT); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Slash_Alpha_AppropriateSlash_GT) { + SetInput("<script></SCRIPT/>x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextEndTag(GUMBO_TAG_SCRIPT, true); + Error(GUMBO_ERR_END_TAG_WITH_TRAILING_SOLIDUS); + NextChar('x'); +} +TEST_F(GumboTokenizerTest, ScriptData_LT_Slash_Alpha_AppropriateSlash_EOF) { + SetInput("<script></script/"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_TAG); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Slash_Alpha_AppropriateSlash_AnythingElse) { + SetInput("<script></script/ >x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextEndTag(GUMBO_TAG_SCRIPT, true); + Error(GUMBO_ERR_UNEXPECTED_SOLIDUS_IN_TAG); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Slash_Alpha_AppropriateGT) { + SetInput("<script></script>x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextEndTag(GUMBO_TAG_SCRIPT); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Slash_Alpha_AnythingElse) { + SetInput("<script></sc</script>x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('/'); + NextChar('s'); + NextChar('c'); + NextEndTag(GUMBO_TAG_SCRIPT); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Slash_AnythingElse) { + SetInput("</0"); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('/'); + NextChar('0'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_Dash_LT_Slash_Alpha_InappropriateTab_EOF) { + SetInput("<script><!---</s\t"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('/'); + NextChar('s'); + NextSpace('\t'); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_SCRIPT_HTML_COMMENT_LIKE_TEXT); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Slash_Alpha_InappropriateLF_NULL) { + SetInput("<script><!--</s\n\x00", sizeof("<script><!--</s\n\x00")-1); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('/'); + NextChar('s'); + NextSpace('\n'); + NextChar(0xFFFD, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Slash_Alpha_InappropriateFF_AnythingElse) { + SetInput("<script><!--</s\fx"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('/'); + NextChar('s'); + NextSpace('\f'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Slash_Alpha_InappropriateSpace) { + SetInput("<script><!--</s x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('/'); + NextChar('s'); + NextSpace(' '); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Slash_Alpha_InappropriateSlash) { + SetInput("<script><!--</s/x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('/'); + NextChar('s'); + NextChar('/'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Slash_Alpha_InappropriateGT) { + SetInput("<script><!--</s>x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('/'); + NextChar('s'); + NextChar('>'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Slash_Alpha_AppropriateTab) { + SetInput("<script><!--</sCrIpT\t>x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextEndTag(GUMBO_TAG_SCRIPT); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Slash_Alpha_AppropriateLF) { + SetInput("<script><!--</Script\n>x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextEndTag(GUMBO_TAG_SCRIPT); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Slash_Alpha_AppropriateFF) { + SetInput("<script><!--</SCRIPT\f>x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextEndTag(GUMBO_TAG_SCRIPT); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Slash_Alpha_AppropriateSpace) { + SetInput("<script><!--</script >x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextEndTag(GUMBO_TAG_SCRIPT); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Slash_Alpha_AppropriateSlash_GT) { + SetInput("<script><!--</script/>x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextEndTag(GUMBO_TAG_SCRIPT, true); + Error(GUMBO_ERR_END_TAG_WITH_TRAILING_SOLIDUS); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Slash_Alpha_AppropriateSlash_EOF) { + SetInput("<script><!--</script/"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_TAG); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Slash_Alpha_AppropriateSlash_AnythingElse) { + SetInput("<script><!--</script/ >x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextEndTag(GUMBO_TAG_SCRIPT, true); + Error(GUMBO_ERR_UNEXPECTED_SOLIDUS_IN_TAG); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Slash_Alpha_AppropriateGT) { + SetInput("<script><!--</scrIPT>x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextEndTag(GUMBO_TAG_SCRIPT); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Slash_Alpha_AnythingElse) { + SetInput("<script><!--</s.x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('/'); + NextChar('s'); + NextChar('.'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Slash_AnythingElse) { + SetInput("<!--</>"); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('/'); + NextChar('>'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptTab_Dash_Dash_Dash_NULL_EOF) { + SetInput("<!--<scRIpt\t---\x00", sizeof("<!--<scRIpt\t---\x00")-1); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('R'); + NextChar('I'); + NextChar('p'); + NextChar('t'); + NextSpace('\t'); + NextChar('-'); + NextChar('-'); + NextChar('-'); + NextChar(0xFFFD, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_SCRIPT_HTML_COMMENT_LIKE_TEXT); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptGT_Dash_Dash_LT) { + SetInput("<script><!--<script>--</script></script>"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('/'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextEndTag(GUMBO_TAG_SCRIPT); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptGT_Dash_Dash_EOF) { + SetInput("<script><!--<script>--"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextChar('-'); + NextChar('-'); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_SCRIPT_HTML_COMMENT_LIKE_TEXT); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptGT_Dash_Dash_AnythingElse) { + SetInput("<script><!--<script>--X--></script>"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextChar('-'); + NextChar('-'); + NextChar('X'); + NextChar('-'); + NextChar('-'); + NextChar('>'); + NextEndTag(GUMBO_TAG_SCRIPT); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptTab_Dash_NULL) { + SetInput("<!--<scRIpt\t-\x00", sizeof("<!--<scRIpt\t---\x00")-1); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('R'); + NextChar('I'); + NextChar('p'); + NextChar('t'); + NextSpace('\t'); + NextChar('-'); + NextChar(0xFFFD, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptGT_Dash_LT) { + SetInput("<script><!--<script>-</script></script>"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextChar('-'); + NextChar('<'); + NextChar('/'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextEndTag(GUMBO_TAG_SCRIPT); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptGT_Dash_EOF) { + SetInput("<script><!--<script>-"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextChar('-'); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_SCRIPT_HTML_COMMENT_LIKE_TEXT); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptGT_Dash_AnythingElse) { + SetInput("<script><!--<script>-X--></script>"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextChar('-'); + NextChar('X'); + NextChar('-'); + NextChar('-'); + NextChar('>'); + NextEndTag(GUMBO_TAG_SCRIPT); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptNL_LT_Slash_ScriptTab) { + SetInput("<script><!--<script\n</Script\tx"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextSpace('\n'); + NextChar('<'); + NextChar('/'); + NextChar('S'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextSpace('\t'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptFF_LT_Slash_ScriptNL) { + SetInput("<script><!--<script\f</Script\nx"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextSpace('\f'); + NextChar('<'); + NextChar('/'); + NextChar('S'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextSpace('\n'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptSpace_LT_Slash_ScriptFF) { + SetInput("<script><!--<script </Script\fx"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextSpace(' '); + NextChar('<'); + NextChar('/'); + NextChar('S'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextSpace('\f'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptSlash_LT_Slash_ScriptSpace) { + SetInput("<script><!--<script/</Script x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('/'); + NextChar('<'); + NextChar('/'); + NextChar('S'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextSpace(' '); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptGT_LT_Slash_ScriptSlash) { + SetInput("<script><!--<script></Script/x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextChar('<'); + NextChar('/'); + NextChar('S'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('/'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptGT_LT_Slash_ScriptGT) { + SetInput("<script><!--<script></Script>x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextChar('<'); + NextChar('/'); + NextChar('S'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptGT_LT_Slash_Tab) { + SetInput("<script><!--<script></\t>x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextChar('<'); + NextChar('/'); + NextSpace('\t'); + NextChar('>'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptGT_LT_Slash_LF) { + SetInput("<script><!--<script></\n>x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextChar('<'); + NextChar('/'); + NextSpace('\n'); + NextChar('>'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptGT_LT_Slash_FF) { + SetInput("<script><!--<script></\f>x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextChar('<'); + NextChar('/'); + NextSpace('\f'); + NextChar('>'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptGT_LT_Slash_Slash) { + SetInput("<script><!--<script><//>x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextChar('<'); + NextChar('/'); + NextChar('/'); + NextChar('>'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptGT_LT_Slash_AnythingElse) { + SetInput("<script><!--<script></.>x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextChar('<'); + NextChar('/'); + NextChar('.'); + NextChar('>'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptGT_AnythingElse) { + SetInput("<script><!--<script>x"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_ScriptGT_NULL) { + SetInput("<script><!--<script>\x00", sizeof("<script><!--<script>\x00")-1); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('c'); + NextChar('r'); + NextChar('i'); + NextChar('p'); + NextChar('t'); + NextChar('>'); + NextChar(0xFFFD, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_Tab_LT_Slash_Alpha_AppropriateTab) { + SetInput("<script><!--<s\t</SCRIPT\t>"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextSpace('\t'); + NextEndTag(GUMBO_TAG_SCRIPT); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_LF_LT_Slash_Alpha_AppropriateLF) { + SetInput("<script><!--<s\n</SCRIPT\n>"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextSpace('\n'); + NextEndTag(GUMBO_TAG_SCRIPT); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_FF_LT_Slash_Alpha_AppropriateFF) { + SetInput("<script><!--<s\f</SCRIPT\f>"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextSpace('\f'); + NextEndTag(GUMBO_TAG_SCRIPT); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_Space_LT_Slash_Alpha_AppropriateSpace) { + SetInput("<script><!--<s </SCRIPT >"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextSpace(' '); + NextEndTag(GUMBO_TAG_SCRIPT); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_Slash_LT_Slash_Alpha_AppropriateSlash) { + SetInput("<script><!--<s/</SCRIPT/>"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('/'); + NextEndTag(GUMBO_TAG_SCRIPT, true); + Error(GUMBO_ERR_END_TAG_WITH_TRAILING_SOLIDUS); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_GT_LT_Slash_Alpha_AppropriateGT) { + SetInput("<script><!--<s></SCRIPT>"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('>'); + NextEndTag(GUMBO_TAG_SCRIPT); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_Alpha_AnythingElse_LT_Slash_Alpha_AnythingElse) { + SetInput("<script><!--<s.</S."); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('s'); + NextChar('.'); + NextChar('<'); + NextChar('/'); + NextChar('S'); + NextChar('.'); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_Dash_LT_AnythingElse_NULL_AnythingElse_EOF) { + SetInput("<script><!--<\x00.", sizeof("<script><!--<\x00.")-1); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar(0xFFFD, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + NextChar('.'); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_SCRIPT_HTML_COMMENT_LIKE_TEXT); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_Bang_Dash_AnythingElse) { + SetInput("<script><!-x</script>"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('x'); + NextEndTag(GUMBO_TAG_SCRIPT); +} + +TEST_F(GumboTokenizerTest, ScriptData_Bang_Dash_Dash_LT_AnythingElse_Dash_Dash_NULL_AnythingElse_EOF) { + SetInput("<script><!--<.--\x00.", sizeof("<script><!--<.--\x00.")-1); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('<'); + NextChar('.'); + NextChar('-'); + NextChar('-'); + NextChar(0xFFFD, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + NextChar('.'); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_SCRIPT_HTML_COMMENT_LIKE_TEXT); +} + +TEST_F(GumboTokenizerTest, ScriptData_Bang_Dash_Dash_EOF) { + SetInput("<script><!--"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_SCRIPT_HTML_COMMENT_LIKE_TEXT); +} + +TEST_F(GumboTokenizerTest, ScriptData_Bang_Dash_Dash_GT_EOF) { + SetInput("<script><!-->"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('>'); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, ScriptData_Bang_Dash_Dash_AnythingElse_Dash_LT) { + SetInput("<script><!--X-<</script>"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('X'); + NextChar('-'); + NextChar('<'); + NextEndTag(GUMBO_TAG_SCRIPT); +} + +TEST_F(GumboTokenizerTest, ScriptData_Bang_Dash_Dash_AnythingElse_Dash_NULL) { + SetInput("<script><!--X-\x00</script>", sizeof("<script><!--X-\x00</script>")-1); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('X'); + NextChar('-'); + NextChar(0xFFFD, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + NextEndTag(GUMBO_TAG_SCRIPT); +} + +TEST_F(GumboTokenizerTest, ScriptData_Bang_Dash_Dash_AnythingElse_Dash_EOF) { + SetInput("<script><!--X-"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('X'); + NextChar('-'); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_SCRIPT_HTML_COMMENT_LIKE_TEXT); +} + +TEST_F(GumboTokenizerTest, ScriptData_Bang_Dash_Dash_AnythingElse_Dash_AnythingElse) { + SetInput("<script><!--X-X</script>"); + NextStartTag(GUMBO_TAG_SCRIPT); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('!'); + NextChar('-'); + NextChar('-'); + NextChar('X'); + NextChar('-'); + NextChar('X'); + NextEndTag(GUMBO_TAG_SCRIPT); +} + +TEST_F(GumboTokenizerTest, ScriptData_LT_AnythingElse) { + SetInput("<p>"); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar('<'); + NextChar('p'); + NextChar('>'); +} + +TEST_F(GumboTokenizerTest, ScriptData_NULL) { + SetInput("", 1); + SetState(GUMBO_LEX_SCRIPT_DATA); + NextChar(0xFFFD, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); +} + +TEST_F(GumboTokenizerTest, Data_Amp_Alnum_Otherwise_Alnum_Semicolon) { + SetInput("&1f;x"); + NextChar('&'); + NextChar('1'); + NextChar('f'); + NextChar(';', true); + Error(GUMBO_ERR_UNKNOWN_NAMED_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_Alnum_Otherwise_Alnum_AnythingElse) { + SetInput("&1f.x"); + NextChar('&'); + NextChar('1'); + NextChar('f'); + NextChar('.'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_NumberSign_X_Hex_Semicolon_Zero) { + SetInput("�x"); + NextChar(0xFFFD, true); + Error(GUMBO_ERR_NULL_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_NumberSign_X_Hex_Semicolon_TooLarge) { + SetInput("�x"); + NextChar(0xFFFD, true); + Error(GUMBO_ERR_CHARACTER_REFERENCE_OUTSIDE_UNICODE_RANGE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_NumberSign_X_Hex_Semicolon_Surrogate) { + SetInput("�x"); + NextChar(0xFFFD, true); + Error(GUMBO_ERR_SURROGATE_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_NumberSign_x_Hex_Semicolon_Noncharacter) { + SetInput("x"); + NextChar(0xFDD8, true); + Error(GUMBO_ERR_NONCHARACTER_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_NumberSign_x_Hex_Semicolon_Noncharacter2) { + SetInput("x"); + NextChar(0x7FFFE, true); + Error(GUMBO_ERR_NONCHARACTER_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_NumberSign_x_Hex_Semicolon_C0Control) { + SetInput("x"); + NextChar(3, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_NumberSign_x_Hex_Semicolon_NL) { + SetInput("
x"); + NextSpace('\n'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_NumberSign_x_Hex_Semicolon_FF) { + SetInput("x"); + NextSpace('\f'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_NumberSign_x_Hex_Semicolon_CR) { + SetInput("
x"); + NextSpace('\r', true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_NumberSign_X_Hex_Semicolon_Control) { + SetInput("x"); + NextChar(0x81, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_NumberSign_x_Hex_Semicolon_Control2) { + SetInput("‚x"); + NextChar(0x201A, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_NumberSign_x_Hex_AnythingElse) { + SetInput("⼚x"); + NextChar(0x2F1A, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_NumberSign_x_Hex_AnythingElse2) { + SetInput("⼚-"); + NextChar(0x2F1A, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + NextChar('-'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_NumberSign_x_AnythingElse) { + SetInput("&#xG"); + NextChar('&', true); + Error(GUMBO_ERR_ABSENCE_OF_DIGITS_IN_NUMERIC_CHARACTER_REFERENCE); + NextChar('#'); + NextChar('x'); + NextChar('G'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_NumberSign_AnythingElse_Digit_Semicolon) { + SetInput("Ӓx"); + NextChar(1234); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_NumberSign_AnythingElse_Digit_Semicolon_TooLarge) { + SetInput("�x"); + NextChar(0xFFFD, true); + Error(GUMBO_ERR_CHARACTER_REFERENCE_OUTSIDE_UNICODE_RANGE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_NumberSign_AnythingElse_Digit_Semicolon_Control) { + SetInput("€x"); + NextChar(0x20AC, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_NumberSign_AnythingElse_Digit_AnythingElse) { + SetInput("Ӓa"); + NextChar(1234, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + NextChar('a'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_NumberSign_AnythingElse_Digit_AnythingElse_Control) { + SetInput("€a"); + NextChar(0x20AC, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + NextChar('a'); +} + +TEST_F(GumboTokenizerTest, Data_Amp_AnythingElse) { + SetInput("&#x"); + NextChar('&', true); + Error(GUMBO_ERR_ABSENCE_OF_DIGITS_IN_NUMERIC_CHARACTER_REFERENCE); + NextChar('#'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_DashDash_Dash_Dash_GT) { + SetInput("<!---->"); + NextComment(""); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_DashDash_Dash_Dash_Bang_Dash_EOF) { + SetInput("<!----!-"); + NextComment("--!", true); + Error(GUMBO_ERR_EOF_IN_COMMENT); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_DashDash_Dash_Dash_Bang_GT) { + SetInput("<!----!>"); + NextComment("", true); + Error(GUMBO_ERR_INCORRECTLY_CLOSED_COMMENT); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_DashDash_Dash_Dash_Bang_EOF) { + SetInput("<!----!"); + NextComment("", true); + Error(GUMBO_ERR_EOF_IN_COMMENT); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_DashDash_Dash_Dash_Bang_AnythingElse_AnythingElse_Dash_EOF) { + SetInput("<!----!x-"); + NextComment("--!x", true); + Error(GUMBO_ERR_EOF_IN_COMMENT); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_DashDash_Dash_Dash_Dash_EOF) { + SetInput("<!-----"); + NextComment("-", true); + Error(GUMBO_ERR_EOF_IN_COMMENT); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_DashDash_Dash_Dash_AnythingElse_AnythingElse_EOF) { + SetInput("<!----x"); + NextComment("--x", true); + Error(GUMBO_ERR_EOF_IN_COMMENT); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_DashDash_Dash_GT) { + SetInput("<!--->"); + NextComment("", true); + Error(GUMBO_ERR_ABRUPT_CLOSING_OF_EMPTY_COMMENT); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_DashDash_Dash_EOF) { + SetInput("<!---"); + NextComment("", true); + Error(GUMBO_ERR_EOF_IN_COMMENT); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_DashDash_Dash_AnythingElse) { + SetInput("<!---x-->"); + NextComment("-x"); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_DashDash_GT) { + SetInput("<!-->"); + NextComment("", true); + Error(GUMBO_ERR_ABRUPT_CLOSING_OF_EMPTY_COMMENT); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_DashDash_AnythingElse_LT_Bang_Dash_Dash_GT_GT) { + SetInput("<!--<!-->"); + NextComment("<!"); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_DashDash_AnythingElse_LT_Bang_Dash_Dash_EOF) { + SetInput("<!--<!--"); + NextComment("<!", true); + Error(GUMBO_ERR_EOF_IN_COMMENT); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_DashDash_AnythingElse_LT_Bang_Dash_Dash_AnythingElse) { + SetInput("<!--<!--x-->"); + NextComment("<!--x", true); + Error(GUMBO_ERR_NESTED_COMMENT); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_DashDash_AnythingElse_LT_Bang_Dash_AnythingElse) { + SetInput("<!--<!-x-->"); + NextComment("<!-x"); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_DashDash_AnythingElse_LT_Bang_AnythingElse) { + SetInput("<!--<!x-->"); + NextComment("<!x"); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_DashDash_AnythingElse_LT_LT_AnythingElse) { + SetInput("<!--<<-->"); + NextComment("<<"); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_DashDash_Dash_AnythingElse_AnythingElse_NULL) { + SetInput("<!---x\x00-->", sizeof("<!---x\x00-->") - 1); + NextComment("-x\xEF\xBF\xBD", true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_DashDash_AnythingElse_AnythingElse_NULL_EOF) { + SetInput("<!--0\x00", sizeof("<!--0\x00")-1); + NextComment("0\xEF\xBF\xBD", true); // Two errors. + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + Error(GUMBO_ERR_EOF_IN_COMMENT); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Tab_Tab_LF_FF_Space_Alpha_Tab_Tab_LF_FF_Space_GT) { + SetInput("<!DoCtyPE\t\t\n\f H\t\t\n\f >"); + NextDoctype("h", NULL, NULL); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_LF_Tab_LF_FF_Space_Alpha_LF_Tab_LF_FF_Space_GT) { + SetInput("<!DoCtyPE\n\t\n\f H\n\t\n\f >"); + NextDoctype("h", NULL, NULL); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_FF_Tab_LF_FF_Space_Alpha_FF_Tab_LF_FF_Space_GT) { + SetInput("<!DoCtyPE\f\t\n\f H\f\t\n\f >"); + NextDoctype("h", NULL, NULL); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Tab_LF_FF_Space_Alpha_Space_Tab_LF_FF_Space_GT) { + SetInput("<!DoCtyPE \t\n\f H \t\n\f >"); + NextDoctype("h", NULL, NULL); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_NULL_GT) { + SetInput("<!DOCTYPE \x00>", sizeof("<!DOCTYPE \x00>")-1); + NextDoctype("\xEF\xBF\xBD", NULL, NULL, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_GT) { + SetInput("<!DOCTYPE >"); + NextDoctype(NULL, NULL, NULL, true); + Error(GUMBO_ERR_MISSING_DOCTYPE_NAME); + EXPECT_TRUE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_EOF) { + SetInput("<!DOCTYPE "); + NextDoctype(NULL, NULL, NULL, true); + Error(GUMBO_ERR_EOF_IN_DOCTYPE); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_AnythingElse_NULL) { + SetInput("<!doctype ht\x00ml>", sizeof("<!doctype ht\x00ml>")-1); + NextDoctype("ht\xEF\xBF\xBDml", NULL, NULL, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_AnythingElse_EOF) { + SetInput("<!doctype html"); + NextDoctype("html", NULL, NULL, true); + Error(GUMBO_ERR_EOF_IN_DOCTYPE); + EXPECT_TRUE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_EOF) { + SetInput("<!DOCtype HTML "); + NextDoctype("html", NULL, NULL, true); + Error(GUMBO_ERR_EOF_IN_DOCTYPE); + EXPECT_TRUE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Tab_Tab_LF_FF_Space_Quote_Quote_Tab_Tab_LF_FF_Space_GT) { + SetInput("<!docTYPE HTML PUBLIC\t\t\n\f \"\"\t\t\n\f >"); + NextDoctype("html", "", NULL); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_LF_Tab_LF_FF_Space_Quote_Quote_LF_Tab_LF_FF_Space_GT) { + SetInput("<!docTYPE HTML PUBLIC\n\t\n\f \"\"\n\t\n\f >"); + NextDoctype("html", "", NULL); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_FF_Tab_LF_FF_Space_Quote_Quote_FF_Tab_LF_FF_Space_GT) { + SetInput("<!docTYPE HTML PUBLIC\f\t\n\f \"\"\f\t\n\f >"); + NextDoctype("html", "", NULL); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Tab_LF_FF_Space_Quote_Quote_Space_Tab_LF_FF_Space_GT) { + SetInput("<!docTYPE HTML PUBLIC \t\n\f \"\" \t\n\f >"); + NextDoctype("html", "", NULL); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_Quote_Space_Quote_Quote_Tab_LF_FF_Space_GT) { + SetInput("<!DOCTYPE html pubLIC \"\" \"\"\t\n\f >"); + NextDoctype("html", "", ""); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_Quote_Space_Quote_Quote_EOF) { + SetInput("<!DOCTYPE html pubLIC \"\" \"\""); + NextDoctype("html", "", "", true); + Error(GUMBO_ERR_EOF_IN_DOCTYPE); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_Quote_Space_Quote_Quote_AnythingElse_GT) { + SetInput("<!DOCTYPE html public \"\" \"\"x>"); + NextDoctype("html", "", "", true); + Error(GUMBO_ERR_UNEXPECTED_CHARACTER_AFTER_DOCTYPE_SYSTEM_IDENTIFIER); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_Quote_Space_Quote_Quote_AnythingElse_NULL_GT) { + SetInput("<!DOCTYPE html public \"\" \"\"\x00>", sizeof("<!DOCTYPE html public \"\" \"\"\x00>")-1); + NextDoctype("html", "", "", true); // Two errors! + Error(GUMBO_ERR_UNEXPECTED_CHARACTER_AFTER_DOCTYPE_SYSTEM_IDENTIFIER); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_Quote_Space_Quote_Quote_AnythingElse_EOF) { + SetInput("<!DOCTYPE html public \"\" \"\"x"); + NextDoctype("html", "", "", true); // Only one error. + Error(GUMBO_ERR_UNEXPECTED_CHARACTER_AFTER_DOCTYPE_SYSTEM_IDENTIFIER); + EXPECT_FALSE(token_.v.doc_type.force_quirks); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_Quote_Space_Quote_NULL) { + SetInput("<!DOCTYPE html public \"\" \"\x00\">", sizeof("<!DOCTYPE html public \"\" \"\x00\">")-1); + NextDoctype("html", "", "\xEF\xBF\xBD", true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_Quote_Space_Quote_GT) { + SetInput("<!DOCTYPE html public \"\" \">"); + NextDoctype("html", "", "", true); + Error(GUMBO_ERR_ABRUPT_DOCTYPE_SYSTEM_IDENTIFIER); + EXPECT_TRUE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_Quote_Space_Quote_EOF) { + SetInput("<!DOCTYPE html public \"\" \""); + NextDoctype("html", "", "", true); + Error(GUMBO_ERR_EOF_IN_DOCTYPE); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_Quote_Space_Quote_AnythingElse) { + SetInput("<!DOCTYPE html public \"\" \"SyS\">"); + NextDoctype("html", "", "SyS"); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_Quote_Space_Apos_AnythingElse) { + SetInput("<!DOCTYPE html public \"\" 'sys'>"); + NextDoctype("html", "", "sys"); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_Quote_Space_EOF) { + SetInput("<!DOCTYPE html public \"\" "); + NextDoctype("html", "", NULL, true); + Error(GUMBO_ERR_EOF_IN_DOCTYPE); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_Quote_Space_AnythingElse) { + SetInput("<!DOCTYPE html public \"\" x>z"); + NextDoctype("html", "", NULL, true); + Error(GUMBO_ERR_MISSING_QUOTE_BEFORE_DOCTYPE_SYSTEM_IDENTIFIER); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_Quote_GT) { + SetInput("<!DOCTYPE html public \"\">"); + NextDoctype("html", "", NULL); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_Quote_Quote_Quote) { + SetInput("<!DOCTYPE html public \"\"\"\">"); + NextDoctype("html", "", "", true); + Error(GUMBO_ERR_MISSING_WHITESPACE_BETWEEN_DOCTYPE_PUBLIC_AND_SYSTEM_IDENTIFIERS); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_Quote_Apos) { + SetInput("<!DOCTYPE html public \"\"''>"); + NextDoctype("html", "", "", true); + Error(GUMBO_ERR_MISSING_WHITESPACE_BETWEEN_DOCTYPE_PUBLIC_AND_SYSTEM_IDENTIFIERS); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_Quote_EOF) { + SetInput("<!DOCTYPE html public \"\""); + NextDoctype("html", "", NULL, true); + Error(GUMBO_ERR_EOF_IN_DOCTYPE); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_Quote_AnythingElse) { + SetInput("<!DOCTYPE html public \"\"x>"); + NextDoctype("html", "", NULL, true); + Error(GUMBO_ERR_MISSING_QUOTE_BEFORE_DOCTYPE_SYSTEM_IDENTIFIER); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_NULL) { + SetInput("<!DOCTYPE html public \"\x00\">", sizeof("<!DOCTYPE html public \"\x00\">")-1); + NextDoctype("html", "\xEF\xBF\xBD", NULL, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_GT) { + SetInput("<!DOCTYPE html public \">"); + NextDoctype("html", "", NULL, true); + Error(GUMBO_ERR_ABRUPT_DOCTYPE_PUBLIC_IDENTIFIER); + EXPECT_TRUE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Quote_EOF) { + SetInput("<!DOCTYPE html public \"foo"); + NextDoctype("html", "foo", NULL, true); + Error(GUMBO_ERR_EOF_IN_DOCTYPE); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Apos_Apos) { + SetInput("<!DOCTYPE html public ''>"); + NextDoctype("html", "", NULL); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Apos_NULL) { + SetInput("<!DOCTYPE html public '\x00'>", sizeof("<!DOCTYPE html public '\x00'>")-1); + NextDoctype("html", "\xEF\xBF\xBD", NULL, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Apos_GT) { + SetInput("<!DOCTYPE html public 'foo>"); + NextDoctype("html", "foo", NULL, true); + Error(GUMBO_ERR_ABRUPT_DOCTYPE_PUBLIC_IDENTIFIER); + EXPECT_TRUE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_Apos_EOF) { + SetInput("<!DOCTYPE html public 'foo"); + NextDoctype("html", "foo", NULL, true); + Error(GUMBO_ERR_EOF_IN_DOCTYPE); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_GT) { + SetInput("<!DOCTYPE HTML PUblic >"); + NextDoctype("html", NULL, NULL, true); + Error(GUMBO_ERR_MISSING_DOCTYPE_PUBLIC_IDENTIFIER); + EXPECT_TRUE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_EOF) { + SetInput("<!DOCTYPE HTML PUblic "); + NextDoctype("html", NULL, NULL, true); + Error(GUMBO_ERR_EOF_IN_DOCTYPE); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Space_AnythingElse) { + SetInput("<!DOCTYPE HTML PUblic x>"); + NextDoctype("html", NULL, NULL, true); + Error(GUMBO_ERR_MISSING_QUOTE_BEFORE_DOCTYPE_PUBLIC_IDENTIFIER); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Quote) { + SetInput("<!DOCTYPE html public\"foo\">"); + NextDoctype("html", "foo", NULL, true); + Error(GUMBO_ERR_MISSING_WHITESPACE_AFTER_DOCTYPE_PUBLIC_KEYWORD); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_Apos) { + SetInput("<!DOCTYPE html public''>"); + NextDoctype("html", "", NULL, true); + Error(GUMBO_ERR_MISSING_WHITESPACE_AFTER_DOCTYPE_PUBLIC_KEYWORD); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_GT) { + SetInput("<!DOCTYPE html public>"); + NextDoctype("html", NULL, NULL, true); + Error(GUMBO_ERR_MISSING_DOCTYPE_PUBLIC_IDENTIFIER); + EXPECT_TRUE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_EOF) { + SetInput("<!DOCTYPE html public"); + NextDoctype("html", NULL, NULL, true); + Error(GUMBO_ERR_EOF_IN_DOCTYPE); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Public_AnythingElse) { + SetInput("<!DOCTYPE html publicX>"); + NextDoctype("html", NULL, NULL, true); + Error(GUMBO_ERR_MISSING_QUOTE_BEFORE_DOCTYPE_PUBLIC_IDENTIFIER); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_System_Tab_Tab_LF_FF_Space_Quote_Quote_GT) { + SetInput("<!DOCTYPE HtMl SySTem\t\t\n\f \"\">"); + NextDoctype("html", NULL, ""); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_System_LF_Apos_Apos_GT) { + SetInput("<!docTYPE hTmL sYstEM\n''>"); + NextDoctype("html", NULL, ""); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_System_FF_Apos_Apos_GT) { + SetInput("<!docTYPE hTmL sYstEM\f''>"); + NextDoctype("html", NULL, ""); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_System_Space_Apos_NULL) { + SetInput("<!DOCTYPE html SYSTEM '\x00'>", sizeof("<!DOCTYPE html SYSTEM '\x00'>")-1); + NextDoctype("html", NULL, "\xEF\xBF\xBD", true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + EXPECT_FALSE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_System_Space_Apos_GT) { + SetInput("<!DOCTYPE html SYSTEM '>"); + NextDoctype("html", NULL, "", true); + Error(GUMBO_ERR_ABRUPT_DOCTYPE_SYSTEM_IDENTIFIER); + EXPECT_TRUE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_System_Space_Apos_EOF) { + SetInput("<!DOCTYPE html SYSTEM '"); + NextDoctype("html", NULL, "", true); + Error(GUMBO_ERR_EOF_IN_DOCTYPE); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_System_Space_GT) { + SetInput("<!docTYPE hTmL sYstEM >"); + NextDoctype("html", NULL, NULL, true); + Error(GUMBO_ERR_MISSING_DOCTYPE_SYSTEM_IDENTIFIER); + EXPECT_TRUE(token_.v.doc_type.force_quirks); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_System_Space_EOF) { + SetInput("<!docTYPE hTmL sYstEM "); + NextDoctype("html", NULL, NULL, true); + Error(GUMBO_ERR_EOF_IN_DOCTYPE); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_System_Space_AnythingElse) { + SetInput("<!docTYPE hTmL sYstEM x>z"); + NextDoctype("html", NULL, NULL, true); + Error(GUMBO_ERR_MISSING_QUOTE_BEFORE_DOCTYPE_SYSTEM_IDENTIFIER); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_System_Quote) { + SetInput("<!DOCTYPE foo system\"bar\">z"); + NextDoctype("foo", NULL, "bar", true); + Error(GUMBO_ERR_MISSING_WHITESPACE_AFTER_DOCTYPE_SYSTEM_KEYWORD); + EXPECT_FALSE(token_.v.doc_type.force_quirks); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_System_Apos) { + SetInput("<!DOCTYPE foo system'bar'>z"); + NextDoctype("foo", NULL, "bar", true); + Error(GUMBO_ERR_MISSING_WHITESPACE_AFTER_DOCTYPE_SYSTEM_KEYWORD); + EXPECT_FALSE(token_.v.doc_type.force_quirks); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_System_GT) { + SetInput("<!DOCTYPE foo system>z"); + NextDoctype("foo", NULL, NULL, true); + Error(GUMBO_ERR_MISSING_DOCTYPE_SYSTEM_IDENTIFIER); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_System_EOF) { + SetInput("<!DOCTYPE foo system"); + NextDoctype("foo", NULL, NULL, true); + Error(GUMBO_ERR_EOF_IN_DOCTYPE); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_System_AnythingElse) { + SetInput("<!DOCTYPE foo systemX>z"); + NextDoctype("foo", NULL, NULL, true); + Error(GUMBO_ERR_MISSING_QUOTE_BEFORE_DOCTYPE_SYSTEM_IDENTIFIER); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_Space_Bogus) { + SetInput("<!DOCTYPE html foobar>z"); + NextDoctype("html", NULL, NULL, true); + Error(GUMBO_ERR_INVALID_CHARACTER_SEQUENCE_AFTER_DOCTYPE_NAME); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_GT) { + SetInput("<!DOCTYPE html>z"); + NextDoctype("html", NULL, NULL); + EXPECT_FALSE(token_.v.doc_type.force_quirks); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_NULL) { + SetInput("<!DOCTYPE html\x00>z", sizeof("<!DOCTYPE html\x00>z")-1); + NextDoctype("html\xEF\xBF\xBD", NULL, NULL, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + EXPECT_FALSE(token_.v.doc_type.force_quirks); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_Alpha_EOF) { + SetInput("<!DOCTYPE html"); + NextDoctype("html", NULL, NULL, true); + Error(GUMBO_ERR_EOF_IN_DOCTYPE); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_Space_NULL) { + SetInput("<!DOCTYPE \x00>z", sizeof("<!DOCTYPE \x00>z")-1); + NextDoctype("\xEF\xBF\xBD", NULL, NULL, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + EXPECT_FALSE(token_.v.doc_type.force_quirks); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_GT) { + SetInput("<!DOCtype>z"); + NextDoctype(NULL, NULL, NULL, true); + Error(GUMBO_ERR_MISSING_DOCTYPE_NAME); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_EOF) { + SetInput("<!DOCTYPE"); + NextDoctype(NULL, NULL, NULL, true); + Error(GUMBO_ERR_EOF_IN_DOCTYPE); + EXPECT_TRUE(token_.v.doc_type.force_quirks); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Doctype_AnythingElse) { + SetInput("<!DOCTYPEhtml>z"); + NextDoctype("html", NULL, NULL, true); + Error(GUMBO_ERR_MISSING_WHITESPACE_BEFORE_DOCTYPE_NAME); + EXPECT_FALSE(token_.v.doc_type.force_quirks); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Cdata_Bracket_Bracket_Bracket_GT) { + SetInput("<![CDATA[]]]>x"); + Foreign(); + NextCdata(']'); + NextChar('x'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Cdata_Bracket_Bracket_AnythingElse) { + SetInput("<![CDATA[]]x]]>z"); + Foreign(); + NextCdata(']'); + NextCdata(']'); + NextCdata('x'); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Cdata_Bracket_AnythingElse) { + SetInput("<![CDATA[]>x]]>z"); + Foreign(); + NextCdata(']'); + NextCdata('>'); + NextCdata('x'); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Cdata_AnythingElse_EOF) { + SetInput("<![CDATA[x"); + Foreign(); + NextCdata('x'); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_CDATA); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_CdataHtml_GT) { + SetInput("<![CDATA[bogus>z"); + NextComment("[CDATA[bogus", true); + Error(GUMBO_ERR_CDATA_IN_HTML_CONTENT); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_CdataHtml_EOF) { + SetInput("<![CDATA[bogus"); + NextComment("[CDATA[bogus", true); + Error(GUMBO_ERR_CDATA_IN_HTML_CONTENT); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_Cdata_Html_NULL_AnythingElse) { + SetInput("<![CDATA[bogus\x00""comment>z", sizeof("<![CDATA[bogus\x00""comment>z")-1); + NextComment("[CDATA[bogus\xEF\xBF\xBD""comment", true); // Two errors. + Error(GUMBO_ERR_CDATA_IN_HTML_CONTENT); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Bang_AnythingElse) { + SetInput("<!4asdf>z"); + NextComment("4asdf", true); + Error(GUMBO_ERR_INCORRECTLY_OPENED_COMMENT); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Slash_Alpha_Tab_Tab_Slash_Slash_GT) { + SetInput("</p\t\t/>z"); + NextEndTag(GUMBO_TAG_P, true); + Error(GUMBO_ERR_END_TAG_WITH_TRAILING_SOLIDUS); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Slash_Alpha_LF_LF_Slash_Slash_EOF) { + SetInput("</p\n\n/"); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_TAG); +} + +TEST_F(GumboTokenizerTest, Data_LT_Slash_Alpha_FF_FF_GT) { + SetInput("</p\f\f>z"); + NextEndTag(GUMBO_TAG_P); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Slash_Alpha_Space_Space_GT) { + SetInput("</p >z"); + NextEndTag(GUMBO_TAG_P); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Slash_Alpha_Space_Alpha_EQ_Alpha_GT) { + SetInput("</span foo=bar>z"); + NextEndTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_END_TAG_WITH_ATTRIBUTES); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Slash_GT) { + SetInput("</>z"); + NextChar('z', true); + Error(GUMBO_ERR_MISSING_END_TAG_NAME); +} + +TEST_F(GumboTokenizerTest, Data_LT_Slash_EOF) { + SetInput("</"); + NextChar('<', true); + Error(GUMBO_ERR_EOF_BEFORE_TAG_NAME); + NextChar('/'); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_Slash_AnythingElse) { + SetInput("</?>z"); + NextComment("?", true); + Error(GUMBO_ERR_INVALID_FIRST_CHARACTER_OF_TAG_NAME); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Tab_Tab_Slash_Slash_GT) { + SetInput("<hr\t\t/>z"); + NextStartTag(GUMBO_TAG_HR); + EXPECT_TRUE(token_.v.start_tag.is_self_closing); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_LF_LF_GT) { + SetInput("<hr\n\n>z"); + NextStartTag(GUMBO_TAG_HR); + EXPECT_FALSE(token_.v.start_tag.is_self_closing); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_FF_FF_Slash_Slash_GT) { + SetInput("<hr\f\f/>z"); + NextStartTag(GUMBO_TAG_HR); + EXPECT_TRUE(token_.v.start_tag.is_self_closing); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_Space_GT) { + SetInput("<hr >z"); + NextStartTag(GUMBO_TAG_HR); + EXPECT_FALSE(token_.v.start_tag.is_self_closing); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_EOF) { + SetInput("<span "); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_TAG); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_EQ_GT) { + SetInput("<span =>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_UNEXPECTED_EQUALS_SIGN_BEFORE_ATTRIBUTE_NAME); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("=", attr->name); + EXPECT_STREQ("", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_Tab_Tab_GT) { + SetInput("<body X\t>z"); + NextStartTag(GUMBO_TAG_BODY); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("x", attr->name); + EXPECT_STREQ("", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_AnythingElse_LF_LF_GT) { + SetInput("<body ?\n>z"); + NextStartTag(GUMBO_TAG_BODY); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("?", attr->name); + EXPECT_STREQ("", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_AnythingElse_FF_FF_GT) { + SetInput("<body &$\f>z"); + NextStartTag(GUMBO_TAG_BODY); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("&$", attr->name); + EXPECT_STREQ("", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_Space_Space_GT) { + SetInput("<body xml:lang >z"); + NextStartTag(GUMBO_TAG_BODY); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("xml:lang", attr->name); + EXPECT_STREQ("", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_Slash_GT) { + SetInput("<br foo/>z"); + NextStartTag(GUMBO_TAG_BR); + EXPECT_TRUE(token_.v.start_tag.is_self_closing); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_GT) { + SetInput("<br foo>z"); + NextStartTag(GUMBO_TAG_BR); + EXPECT_FALSE(token_.v.start_tag.is_self_closing); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EOF) { + SetInput("<br foo"); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_TAG); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Tab_LF_FF_Space_Quote_Quote_Slash) { + SetInput("<span foo=\t\n\f \"\"/>z"); + NextStartTag(GUMBO_TAG_SPAN); + EXPECT_TRUE(token_.v.start_tag.is_self_closing); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Tab_LF_FF_Space_Quote_Quote_GT) { + SetInput("<span foo=\t\n\f \"\">z"); + NextStartTag(GUMBO_TAG_SPAN); + EXPECT_FALSE(token_.v.start_tag.is_self_closing); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Space_Quote_Quote_EOF) { + SetInput("<span foo= \"\""); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_TAG); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Space_Quote_Quote_AnythingElse) { + SetInput("<span foo= \"\"bar=''>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_MISSING_WHITESPACE_BETWEEN_ATTRIBUTES); + EXPECT_FALSE(token_.v.start_tag.is_self_closing); + ASSERT_EQ(2, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("", attr->value); + + attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[1]); + EXPECT_STREQ("bar", attr->name); + EXPECT_STREQ("", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_Alnum_Match_Semicolon) { + SetInput("<span foo=\"fjx\">z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\x66\x6Ax", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_Alnum_Match_Alpha) { + SetInput("<span foo=\"áx\">z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("áx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_Alnum_Match_Num) { + SetInput("<span foo=\"á0\">z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("á0", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_Alnum_Match_EQ) { + SetInput("<span foo=\"á=\">z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("á=", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_Alnum_Match_Otherwise) { + SetInput("<span foo=\"á\">z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xC3\xA1", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_Alnum_Otherwise_Alnum_Semicolon) { + SetInput("<span foo=\"&1f;x\">z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_UNKNOWN_NAMED_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("&1f;x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_Alnum_Otherwise_Alnum_AnythingElse) { + SetInput("<span foo=\"&1f.x\">z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("&1f.x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_NumberSign_X_Hex_Semicolon_Zero) { + SetInput("<span foo=\"�x\">z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_NULL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xEF\xBF\xBDx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_NumberSign_X_Hex_Semicolon_TooLarge) { + SetInput("<span foo=\"�x\">z"); + NextStartTag(GUMBO_TAG_SPAN, true); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + Error(GUMBO_ERR_CHARACTER_REFERENCE_OUTSIDE_UNICODE_RANGE); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xEF\xBF\xBDx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_NumberSign_X_Hex_Semicolon_Surrogate) { + SetInput("<span foo=\"�x\">z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_SURROGATE_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xEF\xBF\xBDx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_NumberSign_x_Hex_Semicolon_Noncharacter) { + SetInput("<span foo=\"x\">z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_NONCHARACTER_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xEF\xB7\x98x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_NumberSign_x_Hex_Semicolon_Noncharacter2) { + SetInput("<span foo=\"x\">z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_NONCHARACTER_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xF1\xBF\xBF\xBEx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_NumberSign_x_Hex_Semicolon_C0Control) { + SetInput("<span foo=\"x\">z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\x03x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_NumberSign_x_Hex_Semicolon_NL) { + SetInput("<span foo=\"
x\">z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\nx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_NumberSign_x_Hex_Semicolon_FF) { + SetInput("<span foo=\"x\">z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\fx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_NumberSign_x_Hex_Semicolon_CR) { + SetInput("<span foo=\"
x\">z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\rx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_NumberSign_X_Hex_Semicolon_Control) { + SetInput("<span foo=\"x\">z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xC2\x81x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_NumberSign_x_Hex_Semicolon_Control2) { + SetInput("<span foo=\"‚x\">z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xE2\x80\x9Ax", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_NumberSign_x_Hex_AnythingElse) { + SetInput("<span foo=\"⼚x\">z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xE2\xBC\x9Ax", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_NumberSign_x_Hex_AnythingElse2) { + SetInput("<span foo=\"⼚-\">z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xE2\xBC\x9A-", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_NumberSign_x_AnythingElse) { + SetInput("<span foo=\"&#xG\">z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_ABSENCE_OF_DIGITS_IN_NUMERIC_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("&#xG", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_NumberSign_AnythingElse_Digit_Semicolon) { + SetInput("<span foo=\"Ӓx\">z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xD3\x92x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_NumberSign_AnythingElse_Digit_Semicolon_Control) { + SetInput("<span foo=\"€x\">z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xE2\x82\xACx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_NumberSign_AnythingElse_Digit_AnythingElse) { + SetInput("<span foo=\"Ӓa\">z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xD3\x92""a", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_NumberSign_AnythingElse_Digit_AnythingElse_Control) { + SetInput("<span foo=\"€a\">z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xE2\x82\xAC""a", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_Amp_AnythingElse) { + SetInput("<span foo=\"&#x\">z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_ABSENCE_OF_DIGITS_IN_NUMERIC_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("&#x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_NULL) { + SetInput("<span foo=\"\x00\">z", sizeof("<span foo=\"\x00\">z")-1); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xEF\xBF\xBD", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_EOF) { + SetInput("<span foo=\""); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_TAG); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Quote_AnythingElse) { + SetInput("<span foo=\"b@#$'\t\n\f =<>!\">z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("b@#$'\t\n\f =<>!", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_Alnum_Match_Semicolon) { + SetInput("<span foo='fjx'>z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\x66\x6Ax", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_Alnum_Match_Alpha) { + SetInput("<span foo='áx'>z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("áx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_Alnum_Match_Num) { + SetInput("<span foo='á0'>z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("á0", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_Alnum_Match_EQ) { + SetInput("<span foo='á='>z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("á=", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_Alnum_Match_Otherwise) { + SetInput("<span foo='á'>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xC3\xA1", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_Alnum_Otherwise_Alnum_Semicolon) { + SetInput("<span foo='&1f;x'>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_UNKNOWN_NAMED_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("&1f;x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_Alnum_Otherwise_Alnum_AnythingElse) { + SetInput("<span foo='&1f.x'>z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("&1f.x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_NumberSign_X_Hex_Semicolon_Zero) { + SetInput("<span foo='�x'>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_NULL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xEF\xBF\xBDx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_NumberSign_X_Hex_Semicolon_TooLarge) { + SetInput("<span foo='�x'>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_CHARACTER_REFERENCE_OUTSIDE_UNICODE_RANGE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xEF\xBF\xBDx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_NumberSign_X_Hex_Semicolon_Surrogate) { + SetInput("<span foo='�x'>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_SURROGATE_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xEF\xBF\xBDx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_NumberSign_x_Hex_Semicolon_Noncharacter) { + SetInput("<span foo='x'>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_NONCHARACTER_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xEF\xB7\x98x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_NumberSign_x_Hex_Semicolon_Noncharacter2) { + SetInput("<span foo='x'>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_NONCHARACTER_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xF1\xBF\xBF\xBEx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_NumberSign_x_Hex_Semicolon_C0Control) { + SetInput("<span foo='x'>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\x03x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_NumberSign_x_Hex_Semicolon_NL) { + SetInput("<span foo='
x'>z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\nx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_NumberSign_x_Hex_Semicolon_FF) { + SetInput("<span foo='x'>z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\fx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_NumberSign_x_Hex_Semicolon_CR) { + SetInput("<span foo='
x'>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\rx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_NumberSign_X_Hex_Semicolon_Control) { + SetInput("<span foo='x'>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xC2\x81x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_NumberSign_x_Hex_Semicolon_Control2) { + SetInput("<span foo='‚x'>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xE2\x80\x9Ax", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_NumberSign_x_Hex_AnythingElse) { + SetInput("<span foo='⼚x'>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xE2\xBC\x9Ax", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_NumberSign_x_Hex_AnythingElse2) { + SetInput("<span foo='⼚-'>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xE2\xBC\x9A-", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_NumberSign_x_AnythingElse) { + SetInput("<span foo='&#xG'>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_ABSENCE_OF_DIGITS_IN_NUMERIC_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("&#xG", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_NumberSign_AnythingElse_Digit_Semicolon) { + SetInput("<span foo='Ӓx'>z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xD3\x92x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_NumberSign_AnythingElse_Digit_Semicolon_Control) { + SetInput("<span foo='€x'>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xE2\x82\xACx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_NumberSign_AnythingElse_Digit_AnythingElse) { + SetInput("<span foo='Ӓa'>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xD3\x92""a", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_NumberSign_AnythingElse_Digit_AnythingElse_Control) { + SetInput("<span foo='€a'>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xE2\x82\xAC""a", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_Amp_AnythingElse) { + SetInput("<span foo='&#x'>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_ABSENCE_OF_DIGITS_IN_NUMERIC_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("&#x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_NULL) { + SetInput("<span foo='\x00'>z", sizeof("<span foo='\x00'>z")-1); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xEF\xBF\xBD", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_EOF) { + SetInput("<span foo='"); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_TAG); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_Apos_AnythingElse) { + SetInput("<span foo='b@#$\"\t\n\f =<>!'>z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("b@#$\"\t\n\f =<>!", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_GT) { + SetInput("<span foo=>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_MISSING_ATTRIBUTE_VALUE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_Alnum_Match_Semicolon) { + SetInput("<span foo=fjx>z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\x66\x6Ax", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_Alnum_Match_Alpha) { + SetInput("<span foo=áx>z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("áx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_Alnum_Match_Num) { + SetInput("<span foo=á0>z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("á0", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_Alnum_Match_Otherwise) { + SetInput("<span foo=á>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xC3\xA1", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_Alnum_Otherwise_Alnum_Semicolon) { + SetInput("<span foo=&1f;x>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_UNKNOWN_NAMED_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("&1f;x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_Alnum_Otherwise_Alnum_AnythingElse) { + SetInput("<span foo=&1f.x>z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("&1f.x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_NumberSign_X_Hex_Semicolon_Zero) { + SetInput("<span foo=�x>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_NULL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xEF\xBF\xBDx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_NumberSign_X_Hex_Semicolon_TooLarge) { + SetInput("<span foo=�x>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_CHARACTER_REFERENCE_OUTSIDE_UNICODE_RANGE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xEF\xBF\xBDx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_NumberSign_X_Hex_Semicolon_Surrogate) { + SetInput("<span foo=�x>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_SURROGATE_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xEF\xBF\xBDx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_NumberSign_x_Hex_Semicolon_Noncharacter) { + SetInput("<span foo=x>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_NONCHARACTER_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xEF\xB7\x98x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_NumberSign_x_Hex_Semicolon_Noncharacter2) { + SetInput("<span foo=x>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_NONCHARACTER_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xF1\xBF\xBF\xBEx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_NumberSign_x_Hex_Semicolon_C0Control) { + SetInput("<span foo=x>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\x03x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_NumberSign_x_Hex_Semicolon_NL) { + SetInput("<span foo=
x>z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\nx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_NumberSign_x_Hex_Semicolon_FF) { + SetInput("<span foo=x>z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\fx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_NumberSign_x_Hex_Semicolon_CR) { + SetInput("<span foo=
x>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\rx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_NumberSign_X_Hex_Semicolon_Control) { + SetInput("<span foo=x>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xC2\x81x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_NumberSign_x_Hex_Semicolon_Control2) { + SetInput("<span foo=‚x>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xE2\x80\x9Ax", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_NumberSign_x_Hex_AnythingElse) { + SetInput("<span foo=⼚x>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xE2\xBC\x9Ax", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_NumberSign_x_Hex_AnythingElse2) { + SetInput("<span foo=⼚->z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xE2\xBC\x9A-", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_NumberSign_x_AnythingElse) { + SetInput("<span foo=&#xG>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_ABSENCE_OF_DIGITS_IN_NUMERIC_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("&#xG", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_NumberSign_AnythingElse_Digit_Semicolon) { + SetInput("<span foo=Ӓx>z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xD3\x92x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_NumberSign_AnythingElse_Digit_Semicolon_Control) { + SetInput("<span foo=€x>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xE2\x82\xACx", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_NumberSign_AnythingElse_Digit_AnythingElse) { + SetInput("<span foo=Ӓa>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xD3\x92""a", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_NumberSign_AnythingElse_Digit_AnythingElse_Control) { + SetInput("<span foo=€a>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_MISSING_SEMICOLON_AFTER_CHARACTER_REFERENCE); + Error(GUMBO_ERR_CONTROL_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xE2\x82\xAC""a", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_Amp_AnythingElse) { + SetInput("<span foo=&#x>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_ABSENCE_OF_DIGITS_IN_NUMERIC_CHARACTER_REFERENCE); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("&#x", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_NULL) { + SetInput("<span foo=\x00>z", sizeof("<span foo=\x00>z")-1); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("\xEF\xBF\xBD", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_EOF) { + SetInput("<span foo="); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_TAG); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_EQ_AnythingElse_AnythingElse) { + SetInput("<span foo=b@#$!>z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("b@#$!", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Alpha_Space_AnythingElse) { + SetInput("<span foo bar=''>z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(2, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("", attr->value); + + attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[1]); + EXPECT_STREQ("bar", attr->name); + EXPECT_STREQ("", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_NULL) { + SetInput("<span \x00=''>z", sizeof("<span \x00=''>z")-1); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("\xEF\xBF\xBD", attr->name); + EXPECT_STREQ("", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Quote) { + SetInput("<span \"=''>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_UNEXPECTED_CHARACTER_IN_ATTRIBUTE_NAME); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("\"", attr->name); + EXPECT_STREQ("", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_Apos) { + SetInput("<span '=''>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_UNEXPECTED_CHARACTER_IN_ATTRIBUTE_NAME); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("'", attr->name); + EXPECT_STREQ("", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Space_AnythingElse_LT) { + SetInput("<span <=''>z"); + NextStartTag(GUMBO_TAG_SPAN, true); + Error(GUMBO_ERR_UNEXPECTED_CHARACTER_IN_ATTRIBUTE_NAME); + ASSERT_EQ(1, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("<", attr->name); + EXPECT_STREQ("", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_MultipleAttributes) { + SetInput("<span foo=\"123\"\tbar='456'\nbaz=789\f>z"); + NextStartTag(GUMBO_TAG_SPAN); + ASSERT_EQ(3, token_.v.start_tag.attributes.length); + GumboAttribute *attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[0]); + EXPECT_STREQ("foo", attr->name); + EXPECT_STREQ("123", attr->value); + + attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[1]); + EXPECT_STREQ("bar", attr->name); + EXPECT_STREQ("456", attr->value); + + attr = static_cast<GumboAttribute*>(token_.v.start_tag.attributes.data[2]); + EXPECT_STREQ("baz", attr->name); + EXPECT_STREQ("789", attr->value); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Slash_GT) { + SetInput("<br/>z"); + NextStartTag(GUMBO_TAG_BR); + EXPECT_TRUE(token_.v.start_tag.is_self_closing); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Slash_EOF) { + SetInput("<br/"); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_TAG); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_Slash_AnythingElse) { + SetInput("<br/ >z"); + NextStartTag(GUMBO_TAG_BR, true); + Error(GUMBO_ERR_UNEXPECTED_SOLIDUS_IN_TAG); + EXPECT_FALSE(token_.v.start_tag.is_self_closing); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_GT) { + SetInput("<mATh>z"); + NextStartTag(GUMBO_TAG_MATH); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_NULL) { + SetInput("<s\x00pan>z", sizeof("<s\x00pan>z")-1); + Next(true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + ASSERT_EQ(GUMBO_TOKEN_START_TAG, token_.type); + EXPECT_EQ(GUMBO_TAG_UNKNOWN, token_.v.start_tag.tag); + EXPECT_STREQ("s\xEF\xBF\xBDpan", token_.v.start_tag.name); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_EOF) { + SetInput("<sp"); + AtEnd(true); + Error(GUMBO_ERR_EOF_IN_TAG); +} + +TEST_F(GumboTokenizerTest, Data_LT_Alpha_AnythingElse) { + SetInput("<h5?-&!]>z"); + Next(); + ASSERT_EQ(GUMBO_TOKEN_START_TAG, token_.type); + EXPECT_EQ(GUMBO_TAG_UNKNOWN, token_.v.start_tag.tag); + EXPECT_STREQ("h5?-&!]", token_.v.start_tag.name); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_Question) { + SetInput("<?php ?>z"); + NextComment("?php ?", true); + Error(GUMBO_ERR_UNEXPECTED_QUESTION_MARK_INSTEAD_OF_TAG_NAME); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_LT_EOF) { + SetInput("<"); + NextChar('<', true); + Error(GUMBO_ERR_EOF_BEFORE_TAG_NAME); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_LT_AnythingElse) { + SetInput("<0z"); + NextChar('<', true); + Error(GUMBO_ERR_INVALID_FIRST_CHARACTER_OF_TAG_NAME); + NextChar('0'); + NextChar('z'); +} + +TEST_F(GumboTokenizerTest, Data_NULL) { + SetInput("", 1); + Next(true); + Error(GUMBO_ERR_UNEXPECTED_NULL_CHARACTER); + ASSERT_EQ(GUMBO_TOKEN_NULL, token_.type); + EXPECT_EQ(0, token_.v.character); +} + +TEST_F(GumboTokenizerTest, Data_EOF) { + SetInput(""); + AtEnd(); +} + +TEST_F(GumboTokenizerTest, Data_AnythingElse) { + SetInput("a\xce\xb2\xd7\x92"); + NextChar(0x61); + NextChar(0x03B2); + NextChar(0x05D2); +} + +TEST_F(GumboTokenizerTest, UTF8_BOM) { + SetInput("\xEF\xBB\xBF<b>"); + Next(); + ASSERT_EQ(GUMBO_TOKEN_START_TAG, token_.type); + EXPECT_EQ(GUMBO_TAG_B, token_.v.start_tag.tag); + AtEnd(); +} + +} // namespace +// vim: set sw=2 sts=2 ts=8 et: diff --git a/gumbo-parser/test/utf8.cc b/gumbo-parser/test/utf8.cc new file mode 100644 index 0000000000..7235342d1f --- /dev/null +++ b/gumbo-parser/test/utf8.cc @@ -0,0 +1,623 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: jdtang@google.com (Jonathan Tang) + +#include "utf8.h" + +#include <string.h> + +#include "gtest/gtest.h" +#include "error.h" +#include "gumbo.h" +#include "test_utils.h" + +namespace { + +// Tests for utf8.c +class Utf8Test : public GumboTest { + protected: + void Advance(int num_chars) { + for (int i = 0; i < num_chars; ++i) { + utf8iterator_next(&input_); + } + } + + void ResetText(const char* text) { + text_ = text; + utf8iterator_init(&parser_, text, strlen(text), &input_); + } + + GumboError* GetFirstError() { + return static_cast<GumboError*>(parser_._output->errors.data[0]); + } + + int GetNumErrors() { return parser_._output->errors.length; } + + Utf8Iterator input_; +}; + +TEST_F(Utf8Test, EmptyString) { + ResetText(""); + EXPECT_EQ(-1, utf8iterator_current(&input_)); +} + +TEST_F(Utf8Test, GetPosition_EmptyString) { + ResetText(""); + GumboSourcePosition pos; + + utf8iterator_get_position(&input_, &pos); + EXPECT_EQ(1, pos.line); + EXPECT_EQ(1, pos.column); + EXPECT_EQ(0, pos.offset); +} + +TEST_F(Utf8Test, Null) { + // Can't use ResetText, as the implicit strlen will choke on the null. + text_ = "\0f"; + utf8iterator_init(&parser_, text_, 2, &input_); + + EXPECT_EQ(0, utf8iterator_current(&input_)); + EXPECT_EQ('\0', *utf8iterator_get_char_pointer(&input_)); + utf8iterator_next(&input_); + EXPECT_EQ('f', utf8iterator_current(&input_)); + EXPECT_EQ('f', *utf8iterator_get_char_pointer(&input_)); +} + +TEST_F(Utf8Test, OneByteChar) { + ResetText("a"); + + EXPECT_EQ(0, GetNumErrors()); + EXPECT_EQ('a', utf8iterator_current(&input_)); + EXPECT_EQ('a', *utf8iterator_get_char_pointer(&input_)); + + GumboSourcePosition pos; + utf8iterator_get_position(&input_, &pos); + EXPECT_EQ(1, pos.line); + EXPECT_EQ(1, pos.column); + EXPECT_EQ(0, pos.offset); + + utf8iterator_next(&input_); + EXPECT_EQ(-1, utf8iterator_current(&input_)); +} + +TEST_F(Utf8Test, ContinuationByte) { + ResetText("\x85"); + + EXPECT_EQ(1, GetNumErrors()); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + EXPECT_EQ('\x85', *utf8iterator_get_char_pointer(&input_)); + + errors_are_expected_ = true; + GumboError* error = GetFirstError(); + EXPECT_EQ(GUMBO_ERR_UTF8_INVALID, error->type); + EXPECT_EQ('\x85', *error->original_text.data); + // This isn't a valid code point so it doesn't matter what value it has. + // EXPECT_EQ(0x85, error->v.tokenizer.codepoint); + + utf8iterator_next(&input_); + EXPECT_EQ(-1, utf8iterator_current(&input_)); +} + +TEST_F(Utf8Test, MultipleContinuationBytes) { + ResetText("a\x85\xA0\xC2x\x9A"); + EXPECT_EQ('a', utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ('x', utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ(-1, utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ(4, GetNumErrors()); + errors_are_expected_ = true; +} + +TEST_F(Utf8Test, OverlongEncoding) { + // \xC0\x75 = 11000000 01110101. + ResetText("\xC0\x75"); + + ASSERT_EQ(1, GetNumErrors()); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + EXPECT_EQ('\xC0', *utf8iterator_get_char_pointer(&input_)); + + errors_are_expected_ = true; + GumboError* error = GetFirstError(); + EXPECT_EQ(GUMBO_ERR_UTF8_INVALID, error->type); + EXPECT_EQ(1, error->position.line); + EXPECT_EQ(1, error->position.column); + EXPECT_EQ(0, error->position.offset); + EXPECT_EQ('\xC0', *error->original_text.data); + // This isn't a valid code point so it doesn't matter what value it has. + // EXPECT_EQ(0xC0, error->v.tokenizer.codepoint); + + utf8iterator_next(&input_); + EXPECT_EQ(0x75, utf8iterator_current(&input_)); + EXPECT_EQ('\x75', *utf8iterator_get_char_pointer(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ(-1, utf8iterator_current(&input_)); +} + +TEST_F(Utf8Test, OverlongEncodingWithContinuationByte) { + // \xC0\x85 = 11000000 10000101. + ResetText("\xC0\x85"); + + ASSERT_EQ(1, GetNumErrors()); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + EXPECT_EQ('\xC0', *utf8iterator_get_char_pointer(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + + errors_are_expected_ = true; + GumboError* error = GetFirstError(); + EXPECT_EQ(GUMBO_ERR_UTF8_INVALID, error->type); + EXPECT_EQ(1, error->position.line); + EXPECT_EQ(1, error->position.column); + EXPECT_EQ(0, error->position.offset); + EXPECT_EQ('\xC0', *error->original_text.data); + // This isn't a valid code point so it doesn't matter what value it has. + // EXPECT_EQ(0xC0, error->v.tokenizer.codepoint); + + utf8iterator_next(&input_); + EXPECT_EQ(-1, utf8iterator_current(&input_)); +} + +TEST_F(Utf8Test, TwoByteChar) { + // \xC3\xA5 = 11000011 10100101. + ResetText("\xC3\xA5o"); + + EXPECT_EQ(0, GetNumErrors()); + // Codepoint = 000 11100101 = 0xE5. + EXPECT_EQ(0xE5, utf8iterator_current(&input_)); + EXPECT_EQ('\xC3', *utf8iterator_get_char_pointer(&input_)); + + GumboSourcePosition pos; + utf8iterator_get_position(&input_, &pos); + EXPECT_EQ(1, pos.line); + EXPECT_EQ(1, pos.column); + EXPECT_EQ(0, pos.offset); + + utf8iterator_next(&input_); + EXPECT_EQ('o', utf8iterator_current(&input_)); + + utf8iterator_get_position(&input_, &pos); + EXPECT_EQ(1, pos.line); + EXPECT_EQ(2, pos.column); + EXPECT_EQ(2, pos.offset); +} + +TEST_F(Utf8Test, TwoByteChar2) { + // \xC2\xA5 = 11000010 10100101. + ResetText("\xC2\xA5"); + + EXPECT_EQ(0, GetNumErrors()); + // Codepoint = 000 10100101 = 0xA5. + EXPECT_EQ(0xA5, utf8iterator_current(&input_)); + EXPECT_EQ('\xC2', *utf8iterator_get_char_pointer(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ(-1, utf8iterator_current(&input_)); +} + +TEST_F(Utf8Test, ThreeByteChar) { + // \xE3\xA7\xA7 = 11100011 10100111 10100111 + ResetText("\xE3\xA7\xA7\xB0"); + + EXPECT_EQ(0, GetNumErrors()); + // Codepoint = 00111001 11100111 = 0x39E7 + EXPECT_EQ(0x39E7, utf8iterator_current(&input_)); + EXPECT_EQ('\xE3', *utf8iterator_get_char_pointer(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ(1, GetNumErrors()); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + EXPECT_EQ('\xB0', *utf8iterator_get_char_pointer(&input_)); + + GumboSourcePosition pos; + utf8iterator_get_position(&input_, &pos); + EXPECT_EQ(1, pos.line); + EXPECT_EQ(2, pos.column); + EXPECT_EQ(3, pos.offset); + + utf8iterator_next(&input_); + EXPECT_EQ(-1, utf8iterator_current(&input_)); + errors_are_expected_ = true; +} + +TEST_F(Utf8Test, FourByteChar) { + // \xC3\x9A = 11000011 10011010 + // \xF1\xA7\xA7\xA7 = 11110001 10100111 10100111 10100111 + ResetText("\xC3\x9A\xF1\xA7\xA7\xA7"); + + // Codepoint = 000 11011010 = 0xDA. + EXPECT_EQ(0xDA, utf8iterator_current(&input_)); + EXPECT_EQ('\xC3', *utf8iterator_get_char_pointer(&input_)); + + utf8iterator_next(&input_); + // Codepoint = 00110 01111001 11100111 = 0x679E7. + EXPECT_EQ(0x679E7, utf8iterator_current(&input_)); + EXPECT_EQ('\xF1', *utf8iterator_get_char_pointer(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ(-1, utf8iterator_current(&input_)); +} + +TEST_F(Utf8Test, FourByteCharWithoutContinuationChars) { + // \xF1\xA7\xA7\xA7 = 11110001 10100111 10100111 10100111 + ResetText("\xF1\xA7\xA7-"); + + EXPECT_EQ(1, GetNumErrors()); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + EXPECT_EQ('\xF1', *utf8iterator_get_char_pointer(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ('-', utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ(-1, utf8iterator_current(&input_)); + errors_are_expected_ = true; +} + +TEST_F(Utf8Test, FiveByteCharIsError) { + ResetText("\xF6\xA7\xA7\xA7\xA7x"); + + EXPECT_EQ(1, GetNumErrors()); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + utf8iterator_next(&input_); + utf8iterator_next(&input_); + utf8iterator_next(&input_); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + utf8iterator_next(&input_); + EXPECT_EQ('x', utf8iterator_current(&input_)); + errors_are_expected_ = true; +} + +TEST_F(Utf8Test, SixByteCharIsError) { + ResetText("\xF8\xA7\xA7\xA7\xA7\xA7x"); + + EXPECT_EQ(1, GetNumErrors()); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + utf8iterator_next(&input_); + utf8iterator_next(&input_); + utf8iterator_next(&input_); + utf8iterator_next(&input_); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + utf8iterator_next(&input_); + EXPECT_EQ('x', utf8iterator_current(&input_)); + errors_are_expected_ = true; +} + +TEST_F(Utf8Test, SevenByteCharIsError) { + ResetText("\xFC\xA7\xA7\xA7\xA7\xA7\xA7x"); + + EXPECT_EQ(1, GetNumErrors()); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + utf8iterator_next(&input_); + utf8iterator_next(&input_); + utf8iterator_next(&input_); + utf8iterator_next(&input_); + utf8iterator_next(&input_); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + utf8iterator_next(&input_); + EXPECT_EQ('x', utf8iterator_current(&input_)); + errors_are_expected_ = true; +} + +TEST_F(Utf8Test, 0xFFIsError) { + ResetText("\xFFx"); + + EXPECT_EQ(1, GetNumErrors()); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ('x', utf8iterator_current(&input_)); + errors_are_expected_ = true; +} + +TEST_F(Utf8Test, InvalidControlCharIsNotReplaced) { + ResetText("\x1Bx"); + + EXPECT_EQ(1, GetNumErrors()); + EXPECT_EQ(0x001b, utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ('x', utf8iterator_current(&input_)); + errors_are_expected_ = true; +} + +TEST_F(Utf8Test, TruncatedInput) { + ResetText("\xF1\xA7"); + + EXPECT_EQ(1, GetNumErrors()); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + + errors_are_expected_ = true; + GumboError* error = GetFirstError(); + EXPECT_EQ(GUMBO_ERR_UTF8_TRUNCATED, error->type); + EXPECT_EQ(1, error->position.line); + EXPECT_EQ(1, error->position.column); + EXPECT_EQ(0, error->position.offset); + EXPECT_EQ('\xF1', *error->original_text.data); + // This isn't a valid code point so it doesn't matter what value it has. + // EXPECT_EQ(0xF1A7, error->v.tokenizer.codepoint); + + utf8iterator_next(&input_); + EXPECT_EQ(-1, utf8iterator_current(&input_)); +} + +TEST_F(Utf8Test, Html5SpecExample) { + // This example has since been removed from the spec, and the spec has been + // changed to reference the Unicode Standard 6.2, 5.22 "Best practices for + // U+FFFD substitution." + errors_are_expected_ = true; + ResetText("\x41\x98\xBA\x42\xE2\x98\x43\xE2\x98\xBA\xE2\x98"); + + EXPECT_EQ('A', utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ('B', utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ('C', utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + // \xE2\x98\xBA = 11100010 10011000 10111010 + // Codepoint = 00100110 00111010 = 0x263A + EXPECT_EQ(0x263A, utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ(0xFFFD, utf8iterator_current(&input_)); + utf8iterator_next(&input_); +} + +TEST_F(Utf8Test, MultipleEOFReads) { + ResetText("a"); + Advance(2); + EXPECT_EQ(-1, utf8iterator_current(&input_)); + + utf8iterator_next(&input_); + EXPECT_EQ(-1, utf8iterator_current(&input_)); +} + +TEST_F(Utf8Test, AsciiOnly) { + ResetText("hello"); + Advance(4); + + EXPECT_EQ('o', utf8iterator_current(&input_)); + EXPECT_EQ('o', *utf8iterator_get_char_pointer(&input_)); + + GumboSourcePosition pos; + utf8iterator_get_position(&input_, &pos); + EXPECT_EQ(1, pos.line); + EXPECT_EQ(5, pos.column); + EXPECT_EQ(4, pos.offset); + + Advance(1); + EXPECT_EQ(-1, utf8iterator_current(&input_)); +} + +TEST_F(Utf8Test, NewlinePosition) { + ResetText("a\nnewline"); + Advance(1); + + // Newline itself should register as being at the end of a line. + GumboSourcePosition pos; + utf8iterator_get_position(&input_, &pos); + EXPECT_EQ(1, pos.line); + EXPECT_EQ(2, pos.column); + EXPECT_EQ(1, pos.offset); + + // The next character should be at the next line. + Advance(1); + utf8iterator_get_position(&input_, &pos); + EXPECT_EQ(2, pos.line); + EXPECT_EQ(1, pos.column); + EXPECT_EQ(2, pos.offset); +} + +TEST_F(Utf8Test, TabPositionFreshTabstop) { + ResetText("a\n\ttab"); + Advance(sizeof("a\n\t") - 1); + + GumboSourcePosition pos; + utf8iterator_get_position(&input_, &pos); + EXPECT_EQ(2, pos.line); + EXPECT_EQ(8, pos.column); + EXPECT_EQ(3, pos.offset); +} + +TEST_F(Utf8Test, TabPositionMidTabstop) { + ResetText("a tab\tinline"); + Advance(sizeof("a tab\t") - 1); + + GumboSourcePosition pos; + utf8iterator_get_position(&input_, &pos); + EXPECT_EQ(1, pos.line); + EXPECT_EQ(8, pos.column); + EXPECT_EQ(6, pos.offset); +} + +TEST_F(Utf8Test, ConfigurableTabstop) { + options_.tab_stop = 4; + ResetText("a\n\ttab"); + Advance(sizeof("a\n\t") - 1); + + GumboSourcePosition pos; + utf8iterator_get_position(&input_, &pos); + EXPECT_EQ(2, pos.line); + EXPECT_EQ(4, pos.column); + EXPECT_EQ(3, pos.offset); +} + +TEST_F(Utf8Test, CRLF) { + ResetText("Windows\r\nlinefeeds"); + Advance(sizeof("Windows") - 1); + + EXPECT_EQ('\n', utf8iterator_current(&input_)); + EXPECT_EQ('\n', *utf8iterator_get_char_pointer(&input_)); + + GumboSourcePosition pos; + utf8iterator_get_position(&input_, &pos); + EXPECT_EQ(1, pos.line); + // The carriage return should be ignore in column calculations, treating the + // CRLF combination as one character. + EXPECT_EQ(8, pos.column); + // However, it should not be ignored in computing offsets, which are often + // used by other tools to index into the original buffer. We don't expect + // other unicode-aware tools to have the same \r\n handling as HTML5. + EXPECT_EQ(8, pos.offset); +} + +TEST_F(Utf8Test, SkipBom) { + ResetText("\xEF\xBB\xBFX"); + + EXPECT_EQ(0, GetNumErrors()); + EXPECT_EQ('X', utf8iterator_current(&input_)); + + GumboSourcePosition pos; + utf8iterator_get_position(&input_, &pos); + EXPECT_EQ(1, pos.line); + EXPECT_EQ(1, pos.column); + EXPECT_EQ(3, pos.offset); +} + +TEST_F(Utf8Test, CarriageReturn) { + ResetText("Mac\rlinefeeds"); + Advance(sizeof("Mac") - 1); + + EXPECT_EQ('\n', utf8iterator_current(&input_)); + // We don't change the original pointer, which is part of the const input + // buffer. original_text pointers will see a carriage return as original + // written. + EXPECT_EQ('\r', *utf8iterator_get_char_pointer(&input_)); + + GumboSourcePosition pos; + utf8iterator_get_position(&input_, &pos); + EXPECT_EQ(1, pos.line); + EXPECT_EQ(4, pos.column); + EXPECT_EQ(3, pos.offset); + + Advance(1); + EXPECT_EQ('l', utf8iterator_current(&input_)); + EXPECT_EQ('l', *utf8iterator_get_char_pointer(&input_)); + + utf8iterator_get_position(&input_, &pos); + EXPECT_EQ(2, pos.line); + EXPECT_EQ(1, pos.column); + EXPECT_EQ(4, pos.offset); +} + +TEST_F(Utf8Test, Matches) { + ResetText("\xC2\xA5goobar"); + Advance(1); + EXPECT_TRUE(utf8iterator_maybe_consume_match(&input_, "goo", 3, true)); + EXPECT_EQ('b', utf8iterator_current(&input_)); +} + +TEST_F(Utf8Test, MatchesOverflow) { + ResetText("goo"); + EXPECT_FALSE(utf8iterator_maybe_consume_match(&input_, "goobar", 6, true)); + EXPECT_EQ('g', utf8iterator_current(&input_)); +} + +TEST_F(Utf8Test, MatchesEof) { + ResetText("goo"); + EXPECT_TRUE(utf8iterator_maybe_consume_match(&input_, "goo", 3, true)); + EXPECT_EQ(-1, utf8iterator_current(&input_)); +} + +TEST_F(Utf8Test, MatchesCaseSensitivity) { + ResetText("gooBAR"); + EXPECT_FALSE(utf8iterator_maybe_consume_match(&input_, "goobar", 6, true)); + EXPECT_EQ('g', utf8iterator_current(&input_)); +} + +TEST_F(Utf8Test, MatchesCaseInsensitive) { + ResetText("gooBAR"); + EXPECT_TRUE(utf8iterator_maybe_consume_match(&input_, "goobar", 6, false)); + EXPECT_EQ(-1, utf8iterator_current(&input_)); +} + +TEST_F(Utf8Test, MatchFollowedByNullByte) { + // Can't use ResetText, as the implicit strlen will choke on the null. + text_ = "CDATA\0f"; + utf8iterator_init(&parser_, text_, 7, &input_); + + EXPECT_TRUE(utf8iterator_maybe_consume_match( + &input_, "cdata", sizeof("cdata") - 1, false)); + + EXPECT_EQ(0, utf8iterator_current(&input_)); + EXPECT_EQ('\0', *utf8iterator_get_char_pointer(&input_)); + utf8iterator_next(&input_); + EXPECT_EQ('f', utf8iterator_current(&input_)); + EXPECT_EQ('f', *utf8iterator_get_char_pointer(&input_)); +} + +TEST_F(Utf8Test, MarkReset) { + ResetText("this is a test"); + Advance(5); + EXPECT_EQ('i', utf8iterator_current(&input_)); + utf8iterator_mark(&input_); + + Advance(3); + EXPECT_EQ('a', utf8iterator_current(&input_)); + + GumboSourcePosition position = utf8iterator_get_mark_position(&input_); + const char* original_text = utf8iterator_get_mark_pointer(&input_); + EXPECT_EQ('i', *original_text); + EXPECT_EQ(1, position.line); + EXPECT_EQ(6, position.column); + EXPECT_EQ(5, position.offset); + + utf8iterator_reset(&input_); + EXPECT_EQ('i', utf8iterator_current(&input_)); + EXPECT_EQ('i', *utf8iterator_get_char_pointer(&input_)); + + utf8iterator_get_position(&input_, &position); + EXPECT_EQ(1, position.line); + EXPECT_EQ(6, position.column); + EXPECT_EQ(5, position.offset); +} + +} // namespace diff --git a/gumbo-parser/test/vector.cc b/gumbo-parser/test/vector.cc new file mode 100644 index 0000000000..33b15c3621 --- /dev/null +++ b/gumbo-parser/test/vector.cc @@ -0,0 +1,129 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Author: jdtang@google.com (Jonathan Tang) + +#include <stddef.h> +#include "vector.h" +#include "gtest/gtest.h" +#include "test_utils.h" + +namespace { + +class GumboVectorTest : public GumboTest { + protected: + GumboVectorTest(): one_(1), two_(2), three_(3) { + gumbo_vector_init(2, &vector_); + } + + ~GumboVectorTest() { gumbo_vector_destroy(&vector_); } + + GumboVector vector_; + + // dummy ints that we can use to take addresses of. + int one_; + int two_; + int three_; +}; + +TEST_F(GumboVectorTest, Init) { + EXPECT_EQ(0, vector_.length); + EXPECT_EQ(2, vector_.capacity); +} + +TEST_F(GumboVectorTest, InitZeroCapacity) { + gumbo_vector_destroy(&vector_); + gumbo_vector_init(0, &vector_); + + gumbo_vector_add(&one_, &vector_); + EXPECT_EQ(1, vector_.length); + EXPECT_EQ(1, *(static_cast<int*>(vector_.data[0]))); +} + +TEST_F(GumboVectorTest, Add) { + gumbo_vector_add(&one_, &vector_); + EXPECT_EQ(1, vector_.length); + EXPECT_EQ(1, *(static_cast<int*>(vector_.data[0]))); + EXPECT_EQ(0, gumbo_vector_index_of(&vector_, &one_)); + EXPECT_EQ(-1, gumbo_vector_index_of(&vector_, &two_)); +} + +TEST_F(GumboVectorTest, AddMultiple) { + gumbo_vector_add(&one_, &vector_); + gumbo_vector_add(&two_, &vector_); + EXPECT_EQ(2, vector_.length); + EXPECT_EQ(2, *(static_cast<int*>(vector_.data[1]))); + EXPECT_EQ(1, gumbo_vector_index_of(&vector_, &two_)); +} + +TEST_F(GumboVectorTest, Realloc) { + gumbo_vector_add(&one_, &vector_); + gumbo_vector_add(&two_, &vector_); + gumbo_vector_add(&three_, &vector_); + EXPECT_EQ(3, vector_.length); + EXPECT_EQ(4, vector_.capacity); + EXPECT_EQ(3, *(static_cast<int*>(vector_.data[2]))); +} + +TEST_F(GumboVectorTest, Pop) { + gumbo_vector_add(&one_, &vector_); + int result = *static_cast<int*>(gumbo_vector_pop(&vector_)); + EXPECT_EQ(1, result); + EXPECT_EQ(0, vector_.length); +} + +TEST_F(GumboVectorTest, PopEmpty) { + EXPECT_EQ(NULL, gumbo_vector_pop(&vector_)); +} + +TEST_F(GumboVectorTest, InsertAtFirst) { + gumbo_vector_add(&one_, &vector_); + gumbo_vector_add(&two_, &vector_); + gumbo_vector_insert_at(&three_, 0, &vector_); + EXPECT_EQ(3, vector_.length); + int result = *static_cast<int*>(vector_.data[0]); + EXPECT_EQ(3, result); +} + +TEST_F(GumboVectorTest, InsertAtLast) { + gumbo_vector_add(&one_, &vector_); + gumbo_vector_add(&two_, &vector_); + gumbo_vector_insert_at(&three_, 2, &vector_); + EXPECT_EQ(3, vector_.length); + int result = *static_cast<int*>(vector_.data[2]); + EXPECT_EQ(3, result); +} + +TEST_F(GumboVectorTest, Remove) { + gumbo_vector_add(&one_, &vector_); + gumbo_vector_add(&two_, &vector_); + gumbo_vector_add(&three_, &vector_); + gumbo_vector_remove(&two_, &vector_); + EXPECT_EQ(2, vector_.length); + int three = *static_cast<int*>(vector_.data[1]); + EXPECT_EQ(3, three); +} + +TEST_F(GumboVectorTest, RemoveAt) { + gumbo_vector_add(&one_, &vector_); + gumbo_vector_add(&two_, &vector_); + gumbo_vector_add(&three_, &vector_); + int result = *static_cast<int*>(gumbo_vector_remove_at(1, &vector_)); + EXPECT_EQ(2, result); + EXPECT_EQ(2, vector_.length); + int three = *static_cast<int*>(vector_.data[1]); + EXPECT_EQ(3, three); +} + +} // namespace diff --git a/lib/nokogiri.rb b/lib/nokogiri.rb index bf5663dad4..b396f47e16 100644 --- a/lib/nokogiri.rb +++ b/lib/nokogiri.rb @@ -2,38 +2,29 @@ # frozen_string_literal: true # Modify the PATH on windows so that the external DLLs will get loaded. -require 'rbconfig' +require "rbconfig" if defined?(RUBY_ENGINE) && RUBY_ENGINE == "jruby" - require 'nokogiri/jruby/dependencies' + require_relative "nokogiri/jruby/dependencies" end -require 'nokogiri/extension' - -require 'nokogiri/version' -require 'nokogiri/syntax_error' -require 'nokogiri/xml' -require 'nokogiri/xslt' -require 'nokogiri/html' -require 'nokogiri/decorators/slop' -require 'nokogiri/css' -require 'nokogiri/html/builder' +require_relative "nokogiri/extension" # Nokogiri parses and searches XML/HTML very quickly, and also has # correctly implemented CSS3 selector support as well as XPath 1.0 # support. # # Parsing a document returns either a Nokogiri::XML::Document, or a -# Nokogiri::HTML::Document depending on the kind of document you parse. +# Nokogiri::HTML4::Document depending on the kind of document you parse. # # Here is an example: # # require 'nokogiri' # require 'open-uri' # -# # Get a Nokogiri::HTML:Document for the page we’re interested in... +# # Get a Nokogiri::HTML4::Document for the page we’re interested in... # -# doc = Nokogiri::HTML(URI.open('http://www.google.com/search?q=tenderlove')) +# doc = Nokogiri::HTML4(URI.open('http://www.google.com/search?q=tenderlove')) # # # Do funky things with it using Nokogiri::XML::Node methods... # @@ -49,27 +40,27 @@ module Nokogiri class << self ### # Parse an HTML or XML document. +string+ contains the document. - def parse string, url = nil, encoding = nil, options = nil + def parse(string, url = nil, encoding = nil, options = nil) if string.respond_to?(:read) || /^\s*<(?:!DOCTYPE\s+)?html[\s>]/i === string[0, 512] # Expect an HTML indicator to appear within the first 512 # characters of a document. (<?xml ?> + <?xml-stylesheet ?> # shouldn't be that long) - Nokogiri.HTML(string, url, encoding, + Nokogiri.HTML4(string, url, encoding, options || XML::ParseOptions::DEFAULT_HTML) else Nokogiri.XML(string, url, encoding, options || XML::ParseOptions::DEFAULT_XML) - end.tap { |doc| + end.tap do |doc| yield doc if block_given? - } + end end ### # Create a new Nokogiri::XML::DocumentFragment - def make input = nil, opts = {}, &blk + def make(input = nil, opts = {}, &blk) if input - Nokogiri::HTML.fragment(input).children.first + Nokogiri::HTML4.fragment(input).children.first else Nokogiri(&blk) end @@ -98,10 +89,10 @@ def install_default_aliases # Make sure to support some popular encoding aliases not known by # all iconv implementations. { - 'Windows-31J' => 'CP932', # Windows-31J is the IANA registered name of CP932. - }.each { |alias_name, name| + "Windows-31J" => "CP932", # Windows-31J is the IANA registered name of CP932. + }.each do |alias_name, name| EncodingHandler.alias(name, alias_name) if EncodingHandler[alias_name].nil? - } + end end end @@ -109,15 +100,26 @@ def install_default_aliases end ### -# Parse a document contained in +args+. Nokogiri will try to guess what -# type of document you are attempting to parse. For more information, see -# Nokogiri.parse +# Parse a document contained in +args+. Nokogiri will try to guess what type of document you are +# attempting to parse. For more information, see Nokogiri.parse # -# To specify the type of document, use Nokogiri.XML or Nokogiri.HTML. +# To specify the type of document, use {Nokogiri.XML}, {Nokogiri.HTML4}, or {Nokogiri.HTML5}. def Nokogiri(*args, &block) if block_given? - Nokogiri::HTML::Builder.new(&block).doc.root + Nokogiri::HTML4::Builder.new(&block).doc.root else Nokogiri.parse(*args) end end + +require_relative "nokogiri/version" +require_relative "nokogiri/syntax_error" +require_relative "nokogiri/xml" +require_relative "nokogiri/xslt" +require_relative "nokogiri/html4" +require_relative "nokogiri/html" +require_relative "nokogiri/decorators/slop" +require_relative "nokogiri/css" +require_relative "nokogiri/html4/builder" + +require_relative "nokogiri/html5" if Nokogiri.uses_gumbo? diff --git a/lib/nokogiri/css.rb b/lib/nokogiri/css.rb index 786aa5e239..1b8fbdca24 100644 --- a/lib/nokogiri/css.rb +++ b/lib/nokogiri/css.rb @@ -1,28 +1,28 @@ # frozen_string_literal: true -require 'nokogiri/css/node' -require 'nokogiri/css/xpath_visitor' -x = $-w -$-w = false -require 'nokogiri/css/parser' -$-w = x - -require 'nokogiri/css/tokenizer' -require 'nokogiri/css/syntax_error' - module Nokogiri module CSS class << self ### # Parse this CSS selector in +selector+. Returns an AST. - def parse selector - Parser.new.parse selector + def parse(selector) + Parser.new.parse(selector) end ### # Get the XPath for +selector+. - def xpath_for selector, options={} - Parser.new(options[:ns] || {}).xpath_for selector, options + def xpath_for(selector, options = {}) + Parser.new(options[:ns] || {}).xpath_for(selector, options) end end end end + +require_relative "css/node" +require_relative "css/xpath_visitor" +x = $-w +$-w = false +require_relative "css/parser" +$-w = x + +require_relative "css/tokenizer" +require_relative "css/syntax_error" diff --git a/lib/nokogiri/css/parser.rb b/lib/nokogiri/css/parser.rb index 52b96826c9..29bdf59e79 100644 --- a/lib/nokogiri/css/parser.rb +++ b/lib/nokogiri/css/parser.rb @@ -8,7 +8,7 @@ require 'racc/parser.rb' -require 'nokogiri/css/parser_extras' +require_relative "parser_extras" module Nokogiri module CSS diff --git a/lib/nokogiri/css/parser.y b/lib/nokogiri/css/parser.y index 4ac8f9d69b..9ed97c8559 100644 --- a/lib/nokogiri/css/parser.y +++ b/lib/nokogiri/css/parser.y @@ -253,7 +253,7 @@ end ---- header -require 'nokogiri/css/parser_extras' +require_relative "parser_extras" ---- inner diff --git a/lib/nokogiri/css/syntax_error.rb b/lib/nokogiri/css/syntax_error.rb index db3e2e583e..7b48806468 100644 --- a/lib/nokogiri/css/syntax_error.rb +++ b/lib/nokogiri/css/syntax_error.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true -require 'nokogiri/syntax_error' +require_relative "../syntax_error" module Nokogiri module CSS class SyntaxError < ::Nokogiri::SyntaxError diff --git a/lib/nokogiri/extension.rb b/lib/nokogiri/extension.rb index b7de485bb5..a7e8ca8637 100644 --- a/lib/nokogiri/extension.rb +++ b/lib/nokogiri/extension.rb @@ -2,8 +2,9 @@ # load the C or Java extension begin + # native precompiled gems package shared libraries in <gem_dir>/lib/nokogiri/<ruby_version> ::RUBY_VERSION =~ /(\d+\.\d+)/ - require "nokogiri/#{Regexp.last_match(1)}/nokogiri" + require_relative "#{Regexp.last_match(1)}/nokogiri" rescue LoadError => e if e.message =~ /GLIBC/ warn(<<~EOM) @@ -22,5 +23,9 @@ EOM raise e end - require 'nokogiri/nokogiri' + + # use "require" instead of "require_relative" because non-native gems will place C extension files + # in Gem::BasicSpecification#extension_dir after compilation (during normal installation), which + # is in $LOAD_PATH but not necessarily relative to this file (see #2300) + require "nokogiri/nokogiri" end diff --git a/lib/nokogiri/gumbo.rb b/lib/nokogiri/gumbo.rb new file mode 100644 index 0000000000..d6f7403ec3 --- /dev/null +++ b/lib/nokogiri/gumbo.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true +module Nokogiri + module Gumbo + # The default maximum number of attributes per element. + DEFAULT_MAX_ATTRIBUTES = 400 + + # The default maximum number of errors for parsing a document or a fragment. + DEFAULT_MAX_ERRORS = 0 + + # The default maximum depth of the DOM tree produced by parsing a document + # or fragment. + DEFAULT_MAX_TREE_DEPTH = 400 + end +end diff --git a/lib/nokogiri/html.rb b/lib/nokogiri/html.rb index 21ce61a8dd..4fbed9bccc 100644 --- a/lib/nokogiri/html.rb +++ b/lib/nokogiri/html.rb @@ -1,38 +1,42 @@ # frozen_string_literal: true -require 'nokogiri/html/entity_lookup' -require 'nokogiri/html/document' -require 'nokogiri/html/document_fragment' -require 'nokogiri/html/sax/parser_context' -require 'nokogiri/html/sax/parser' -require 'nokogiri/html/sax/push_parser' -require 'nokogiri/html/element_description' -require 'nokogiri/html/element_description_defaults' +require_relative "html4" module Nokogiri - class << self - ### - # Parse HTML. Convenience method for Nokogiri::HTML::Document.parse - def HTML thing, url = nil, encoding = nil, options = XML::ParseOptions::DEFAULT_HTML, &block - Nokogiri::HTML::Document.parse(thing, url, encoding, options, &block) - end - end + HTML = Nokogiri::HTML4 + + # @!method HTML(input, url = nil, encoding = nil, options = XML::ParseOptions::DEFAULT_HTML, &block) + # Parse HTML. Convenience method for Nokogiri::HTML4::Document.parse + # @!scope class + define_singleton_method(:HTML, Nokogiri.method(:HTML4)) + # @note This module/namespace is an alias for {Nokogiri::HTML4} as of v1.12.0. Before v1.12.0, + # {Nokogiri::HTML4} did not exist, and this was the module/namespace for all HTML-related + # classes. module HTML - class << self - ### - # Parse HTML. Convenience method for Nokogiri::HTML::Document.parse - def parse thing, url = nil, encoding = nil, options = XML::ParseOptions::DEFAULT_HTML, &block - Document.parse(thing, url, encoding, options, &block) + # @note This class is an alias for {Nokogiri::HTML4::Document} as of v1.12.0. + class Document < Nokogiri::XML::Document + end + + # @note This class is an alias for {Nokogiri::HTML4::DocumentFragment} as of v1.12.0. + class DocumentFragment < Nokogiri::XML::DocumentFragment + end + + # @note This class is an alias for {Nokogiri::HTML4::Builder} as of v1.12.0. + class Builder < Nokogiri::XML::Builder + end + + module SAX + # @note This class is an alias for {Nokogiri::HTML4::SAX::Parser} as of v1.12.0. + class Parser < Nokogiri::XML::SAX::Parser end - #### - # Parse a fragment from +string+ in to a NodeSet. - def fragment string, encoding = nil - HTML::DocumentFragment.parse string, encoding + # @note This class is an alias for {Nokogiri::HTML4::SAX::ParserContext} as of v1.12.0. + class ParserContext < Nokogiri::XML::SAX::ParserContext end - end - # Instance of Nokogiri::HTML::EntityLookup - NamedCharacters = EntityLookup.new + # @note This class is an alias for {Nokogiri::HTML4::SAX::PushParser} as of v1.12.0. + class PushParser + end + end end end diff --git a/lib/nokogiri/html/sax/parser_context.rb b/lib/nokogiri/html/sax/parser_context.rb deleted file mode 100644 index 5eb48ca329..0000000000 --- a/lib/nokogiri/html/sax/parser_context.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true -module Nokogiri - module HTML - module SAX - ### - # Context for HTML SAX parsers. This class is usually not instantiated - # by the user. Instead, you should be looking at - # Nokogiri::HTML::SAX::Parser - class ParserContext < Nokogiri::XML::SAX::ParserContext - def self.new thing, encoding = 'UTF-8' - [:read, :close].all? { |x| thing.respond_to?(x) } ? super : - memory(thing, encoding) - end - end - end - end -end diff --git a/lib/nokogiri/html4.rb b/lib/nokogiri/html4.rb new file mode 100644 index 0000000000..b2fe513509 --- /dev/null +++ b/lib/nokogiri/html4.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true +module Nokogiri + class << self + ### + # Parse HTML. Convenience method for Nokogiri::HTML4::Document.parse + def HTML4(input, url = nil, encoding = nil, options = XML::ParseOptions::DEFAULT_HTML, &block) + Nokogiri::HTML4::Document.parse(input, url, encoding, options, &block) + end + end + + # @since v1.12.0 + # @note Before v1.12.0, {Nokogiri::HTML4} did not exist, and {Nokogiri::HTML} was the module/namespace for parsing HTML. + module HTML4 + class << self + ### + # Parse HTML. Convenience method for Nokogiri::HTML4::Document.parse + def parse(input, url = nil, encoding = nil, options = XML::ParseOptions::DEFAULT_HTML, &block) + Document.parse(input, url, encoding, options, &block) + end + + #### + # Parse a fragment from +string+ in to a NodeSet. + def fragment(string, encoding = nil) + HTML4::DocumentFragment.parse(string, encoding) + end + end + + # Instance of Nokogiri::HTML4::EntityLookup + NamedCharacters = EntityLookup.new + end +end + +require_relative "html4/entity_lookup" +require_relative "html4/document" +require_relative "html4/document_fragment" +require_relative "html4/sax/parser_context" +require_relative "html4/sax/parser" +require_relative "html4/sax/push_parser" +require_relative "html4/element_description" +require_relative "html4/element_description_defaults" diff --git a/lib/nokogiri/html/builder.rb b/lib/nokogiri/html4/builder.rb similarity index 93% rename from lib/nokogiri/html/builder.rb rename to lib/nokogiri/html4/builder.rb index 098588ae87..a875e4eedc 100644 --- a/lib/nokogiri/html/builder.rb +++ b/lib/nokogiri/html4/builder.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true module Nokogiri - module HTML + module HTML4 ### # Nokogiri HTML builder is used for building HTML documents. It is very # similar to the Nokogiri::XML::Builder. In fact, you should go read the @@ -12,7 +12,7 @@ module HTML # Create an HTML document with a body that has an onload attribute, and a # span tag with a class of "bold" that has content of "Hello world". # - # builder = Nokogiri::HTML::Builder.new do |doc| + # builder = Nokogiri::HTML4::Builder.new do |doc| # doc.html { # doc.body(:onload => 'some_func();') { # doc.span.bold { diff --git a/lib/nokogiri/html/document.rb b/lib/nokogiri/html4/document.rb similarity index 97% rename from lib/nokogiri/html/document.rb rename to lib/nokogiri/html4/document.rb index c54676c92f..07f4d0968e 100644 --- a/lib/nokogiri/html/document.rb +++ b/lib/nokogiri/html4/document.rb @@ -3,7 +3,7 @@ require 'pathname' module Nokogiri - module HTML + module HTML4 class Document < Nokogiri::XML::Document ### # Get the meta tag encoding for this document. If there is no meta tag, @@ -268,12 +268,12 @@ def self.detect_encoding(chunk) m = chunk.match(/(<meta\s)(.*)(charset\s*=\s*([\w-]+))(.*)/i) and return m[4] catch(:encoding_found) { - Nokogiri::HTML::SAX::Parser.new(JumpSAXHandler.new(:encoding_found)).parse(chunk) + Nokogiri::HTML4::SAX::Parser.new(JumpSAXHandler.new(:encoding_found)).parse(chunk) nil } else handler = SAXHandler.new - parser = Nokogiri::HTML::SAX::PushParser.new(handler) + parser = Nokogiri::HTML4::SAX::PushParser.new(handler) parser << chunk rescue Nokogiri::SyntaxError handler.encoding end @@ -286,7 +286,7 @@ def initialize(io) end # This method is used by the C extension so that - # Nokogiri::HTML::Document#read_io() does not leak memory when + # Nokogiri::HTML4::Document#read_io() does not leak memory when # EncodingFound is raised. attr_reader :encoding_found diff --git a/lib/nokogiri/html/document_fragment.rb b/lib/nokogiri/html4/document_fragment.rb similarity index 90% rename from lib/nokogiri/html/document_fragment.rb rename to lib/nokogiri/html4/document_fragment.rb index ce85cd78e3..09f9cdc8d0 100644 --- a/lib/nokogiri/html/document_fragment.rb +++ b/lib/nokogiri/html4/document_fragment.rb @@ -1,11 +1,11 @@ # frozen_string_literal: true module Nokogiri - module HTML + module HTML4 class DocumentFragment < Nokogiri::XML::DocumentFragment #### # Create a Nokogiri::XML::DocumentFragment from +tags+, using +encoding+ def self.parse(tags, encoding = nil) - doc = HTML::Document.new + doc = HTML4::Document.new encoding ||= if tags.respond_to?(:encoding) encoding = tags.encoding @@ -39,7 +39,7 @@ def initialize(document, tags = nil, ctx = nil) "/html/body/node()" end - temp_doc = HTML::Document.parse("<html><body>#{tags}", nil, document.encoding) + temp_doc = HTML4::Document.parse("<html><body>#{tags}", nil, document.encoding) temp_doc.xpath(path).each { |child| child.parent = self } self.errors = temp_doc.errors end diff --git a/lib/nokogiri/html/element_description.rb b/lib/nokogiri/html4/element_description.rb similarity index 96% rename from lib/nokogiri/html/element_description.rb rename to lib/nokogiri/html4/element_description.rb index 6c415774da..b025a2ea9e 100644 --- a/lib/nokogiri/html/element_description.rb +++ b/lib/nokogiri/html4/element_description.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true module Nokogiri - module HTML + module HTML4 class ElementDescription ### # Is this element a block element? diff --git a/lib/nokogiri/html/element_description_defaults.rb b/lib/nokogiri/html4/element_description_defaults.rb similarity index 99% rename from lib/nokogiri/html/element_description_defaults.rb rename to lib/nokogiri/html4/element_description_defaults.rb index 6763cd496e..f336eec612 100644 --- a/lib/nokogiri/html/element_description_defaults.rb +++ b/lib/nokogiri/html4/element_description_defaults.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true module Nokogiri - module HTML + module HTML4 class ElementDescription # Methods are defined protected by method_defined? because at diff --git a/lib/nokogiri/html/entity_lookup.rb b/lib/nokogiri/html4/entity_lookup.rb similarity index 94% rename from lib/nokogiri/html/entity_lookup.rb rename to lib/nokogiri/html4/entity_lookup.rb index bc66e41afe..a6034e278c 100644 --- a/lib/nokogiri/html/entity_lookup.rb +++ b/lib/nokogiri/html4/entity_lookup.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true module Nokogiri - module HTML + module HTML4 class EntityDescription < Struct.new(:value, :name, :description); end class EntityLookup diff --git a/lib/nokogiri/html/sax/parser.rb b/lib/nokogiri/html4/sax/parser.rb similarity index 75% rename from lib/nokogiri/html/sax/parser.rb rename to lib/nokogiri/html4/sax/parser.rb index 16853e6957..356f37c96c 100644 --- a/lib/nokogiri/html/sax/parser.rb +++ b/lib/nokogiri/html4/sax/parser.rb @@ -1,18 +1,15 @@ # frozen_string_literal: true module Nokogiri - module HTML + module HTML4 ### - # Nokogiri lets you write a SAX parser to process HTML but get HTML - # correction features. + # Nokogiri lets you write a SAX parser to process HTML but get HTML correction features. # - # See Nokogiri::HTML::SAX::Parser for a basic example of using a - # SAX parser with HTML. + # See Nokogiri::HTML4::SAX::Parser for a basic example of using a SAX parser with HTML. # # For more information on SAX parsers, see Nokogiri::XML::SAX module SAX ### - # This class lets you perform SAX style parsing on HTML with HTML - # error correction. + # This class lets you perform SAX style parsing on HTML with HTML error correction. # # Here is a basic usage example: # @@ -22,40 +19,40 @@ module SAX # end # end # - # parser = Nokogiri::HTML::SAX::Parser.new(MyDoc.new) + # parser = Nokogiri::HTML4::SAX::Parser.new(MyDoc.new) # parser.parse(File.read(ARGV[0], mode: 'rb')) # # For more information on SAX parsers, see Nokogiri::XML::SAX class Parser < Nokogiri::XML::SAX::Parser ### # Parse html stored in +data+ using +encoding+ - def parse_memory data, encoding = 'UTF-8' + def parse_memory(data, encoding = "UTF-8") raise ArgumentError unless data return unless data.length > 0 ctx = ParserContext.memory(data, encoding) yield ctx if block_given? - ctx.parse_with self + ctx.parse_with(self) end ### # Parse given +io+ - def parse_io io, encoding = 'UTF-8' + def parse_io(io, encoding = "UTF-8") check_encoding(encoding) @encoding = encoding ctx = ParserContext.io(io, ENCODINGS[encoding]) yield ctx if block_given? - ctx.parse_with self + ctx.parse_with(self) end ### # Parse a file with +filename+ - def parse_file filename, encoding = 'UTF-8' + def parse_file(filename, encoding = "UTF-8") raise ArgumentError unless filename raise Errno::ENOENT unless File.exist?(filename) raise Errno::EISDIR if File.directory?(filename) ctx = ParserContext.file(filename, encoding) yield ctx if block_given? - ctx.parse_with self + ctx.parse_with(self) end end end diff --git a/lib/nokogiri/html4/sax/parser_context.rb b/lib/nokogiri/html4/sax/parser_context.rb new file mode 100644 index 0000000000..6d964a3059 --- /dev/null +++ b/lib/nokogiri/html4/sax/parser_context.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true +module Nokogiri + module HTML4 + module SAX + ### + # Context for HTML SAX parsers. This class is usually not instantiated by the user. Instead, + # you should be looking at Nokogiri::HTML4::SAX::Parser + class ParserContext < Nokogiri::XML::SAX::ParserContext + def self.new(thing, encoding = "UTF-8") + if [:read, :close].all? { |x| thing.respond_to?(x) } + super + else + memory(thing, encoding) + end + end + end + end + end +end diff --git a/lib/nokogiri/html/sax/push_parser.rb b/lib/nokogiri/html4/sax/push_parser.rb similarity index 70% rename from lib/nokogiri/html/sax/push_parser.rb rename to lib/nokogiri/html4/sax/push_parser.rb index 6054e00a27..2794757199 100644 --- a/lib/nokogiri/html/sax/push_parser.rb +++ b/lib/nokogiri/html4/sax/push_parser.rb @@ -1,17 +1,17 @@ # frozen_string_literal: true module Nokogiri - module HTML + module HTML4 module SAX class PushParser - # The Nokogiri::HTML::SAX::Document on which the PushParser will be + # The Nokogiri::HTML4::SAX::Document on which the PushParser will be # operating attr_accessor :document - def initialize(doc = HTML::SAX::Document.new, file_name = nil, encoding = 'UTF-8') + def initialize(doc = HTML4::SAX::Document.new, file_name = nil, encoding = 'UTF-8') @document = doc @encoding = encoding - @sax_parser = HTML::SAX::Parser.new(doc, @encoding) + @sax_parser = HTML4::SAX::Parser.new(doc, @encoding) ## Create our push parser context initialize_native(@sax_parser, file_name, encoding) @@ -27,7 +27,7 @@ def write chunk, last_chunk = false ### # Finish the parsing. This method is only necessary for - # Nokogiri::HTML::SAX::Document#end_document to be called. + # Nokogiri::HTML4::SAX::Document#end_document to be called. def finish write '', true end diff --git a/lib/nokogiri/html5.rb b/lib/nokogiri/html5.rb new file mode 100644 index 0000000000..86b4bba717 --- /dev/null +++ b/lib/nokogiri/html5.rb @@ -0,0 +1,473 @@ +# coding: utf-8 +# frozen_string_literal: true +# +# Copyright 2013-2021 Sam Ruby, Stephen Checkoway +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require_relative 'html5/document' +require_relative 'html5/document_fragment' +require_relative 'html5/node' + +module Nokogiri + # @since v1.12.0 + # @note HTML5 functionality is not available when running JRuby. + # Parse an HTML5 document. Convenience method for {Nokogiri::HTML5::Document.parse} + def self.HTML5(input, url = nil, encoding = nil, **options, &block) + Nokogiri::HTML5::Document.parse(input, url, encoding, **options, &block) + end + + # == Usage + # + # Parse an HTML5 document: + # + # doc = Nokogiri.HTML5(string) + # + # Parse an HTML5 fragment: + # + # fragment = Nokogiri::HTML5.fragment(string) + # + # == Parsing options + # + # The document and fragment parsing methods support options that are different from Nokogiri's. + # + # - <tt>Nokogiri.HTML5(html, url = nil, encoding = nil, options = {})</tt> + # - <tt>Nokogiri::HTML5.parse(html, url = nil, encoding = nil, options = {})</tt> + # - <tt>Nokogiri::HTML5::Document.parse(html, url = nil, encoding = nil, options = {})</tt> + # - <tt>Nokogiri::HTML5.fragment(html, encoding = nil, options = {})</tt> + # - <tt>Nokogiri::HTML5::DocumentFragment.parse(html, encoding = nil, options = {})</tt> + # + # The three currently supported options are +:max_errors+, +:max_tree_depth+ and + # +:max_attributes+, described below. + # + # === Error reporting + # + # Nokogiri contains an experimental HTML5 parse error reporting facility. By default, no parse + # errors are reported but this can be configured by passing the +:max_errors+ option to + # {HTML5.parse} or {HTML5.fragment}. + # + # For example, this script: + # + # doc = Nokogiri::HTML5.parse('<span/>Hi there!</span foo=bar />', max_errors: 10) + # doc.errors.each do |err| + # puts(err) + # end + # + # Emits: + # + # 1:1: ERROR: Expected a doctype token + # <span/>Hi there!</span foo=bar /> + # ^ + # 1:1: ERROR: Start tag of nonvoid HTML element ends with '/>', use '>'. + # <span/>Hi there!</span foo=bar /> + # ^ + # 1:17: ERROR: End tag ends with '/>', use '>'. + # <span/>Hi there!</span foo=bar /> + # ^ + # 1:17: ERROR: End tag contains attributes. + # <span/>Hi there!</span foo=bar /> + # ^ + # + # Using <tt>max_errors: -1</tt> results in an unlimited number of errors being returned. + # + # The errors returned by {HTML5::Document#errors} are instances of {Nokogiri::XML::SyntaxError}. + # + # The {https://html.spec.whatwg.org/multipage/parsing.html#parse-errors HTML standard} defines a + # number of standard parse error codes. These error codes only cover the "tokenization" stage of + # parsing HTML. The parse errors in the "tree construction" stage do not have standardized error + # codes (yet). + # + # As a convenience to Nokogiri users, the defined error codes are available via + # {Nokogiri::XML::SyntaxError#str1} method. + # + # doc = Nokogiri::HTML5.parse('<span/>Hi there!</span foo=bar />', max_errors: 10) + # doc.errors.each do |err| + # puts("#{err.line}:#{err.column}: #{err.str1}") + # end + # # => 1:1: generic-parser + # # 1:1: non-void-html-element-start-tag-with-trailing-solidus + # # 1:17: end-tag-with-trailing-solidus + # # 1:17: end-tag-with-attributes + # + # Note that the first error is +generic-parser+ because it's an error from the tree construction + # stage and doesn't have a standardized error code. + # + # For the purposes of semantic versioning, the error messages, error locations, and error codes + # are not part of Nokogiri's public API. That is, these are subject to change without Nokogiri's + # major version number changing. These may be stabilized in the future. + # + # === Maximum tree depth + # + # The maximum depth of the DOM tree parsed by the various parsing methods is configurable by the + # +:max_tree_depth+ option. If the depth of the tree would exceed this limit, then an + # {::ArgumentError} is thrown. + # + # This limit (which defaults to <tt>Nokogiri::Gumbo::DEFAULT_MAX_TREE_DEPTH = 400</tt>) can be + # removed by giving the option <tt>max_tree_depth: -1</tt>. + # + # html = '<!DOCTYPE html>' + '<div>' * 1000 + # doc = Nokogiri.HTML5(html) + # # raises ArgumentError: Document tree depth limit exceeded + # doc = Nokogiri.HTML5(html, max_tree_depth: -1) + # + # === Attribute limit per element + # + # The maximum number of attributes per DOM element is configurable by the +:max_attributes+ + # option. If a given element would exceed this limit, then an {::ArgumentError} is thrown. + # + # This limit (which defaults to <tt>Nokogiri::Gumbo::DEFAULT_MAX_ATTRIBUTES = 400</tt>) can be + # removed by giving the option <tt>max_attributes: -1</tt>. + # + # html = '<!DOCTYPE html><div ' + (1..1000).map { |x| "attr-#{x}" }.join(' ') + '>' + # # "<!DOCTYPE html><div attr-1 attr-2 attr-3 ... attr-1000>" + # doc = Nokogiri.HTML5(html) + # # raises ArgumentError: Attributes per element limit exceeded + # doc = Nokogiri.HTML5(html, max_attributes: -1) + # + # == HTML Serialization + # + # After parsing HTML, it may be serialized using any of the {Nokogiri::XML::Node} serialization + # methods. In particular, {XML::Node#serialize}, {XML::Node#to_html}, and {XML::Node#to_s} will + # serialize a given node and its children. (This is the equivalent of JavaScript's + # +Element.outerHTML+.) Similarly, {XML::Node#inner_html} will serialize the children of a given + # node. (This is the equivalent of JavaScript's +Element.innerHTML+.) + # + # doc = Nokogiri::HTML5("<!DOCTYPE html><span>Hello world!</span>") + # puts doc.serialize + # # => <!DOCTYPE html><html><head> <style> .commit-tease, .user-profile-mini-avatar, .avatar, .vcard-details, .signup-prompt-bg { display: none !IMPORTANT; } </style> <script> document.addEventListener('DOMContentLoaded', function() { this.querySelectorAll('a').forEach(anchor => { anchor.addEventListener('click', e => { e.preventDefault(); const redact = new URLSearchParams(window.location.search).get('redact'); const hasExistingParams = anchor.href.includes('?'); window.location.href = anchor.href + (hasExistingParams ? `&redact=${redact}` : `?redact=${redact}`); }); }); }); </script> </head><body><span>Hello world!</span></body></html> + # + # Due to quirks in how HTML is parsed and serialized, it's possible for a DOM tree to be + # serialized and then re-parsed, resulting in a different DOM. Mostly, this happens with DOMs + # produced from invalid HTML. Unfortunately, even valid HTML may not survive serialization and + # re-parsing. + # + # In particular, a newline at the start of +pre+, +listing+, and +textarea+ elements is ignored by + # the parser. + # + # doc = Nokogiri::HTML5(<<-EOF) + # <!DOCTYPE html> + # <pre> + # Content</pre> + # EOF + # puts doc.at('/html/body/pre').serialize + # # => <pre>Content</pre> + # + # In this case, the original HTML is semantically equivalent to the serialized version. If the + # +pre+, +listing+, or +textarea+ content starts with two newlines, the first newline will be + # stripped on the first parse and the second newline will be stripped on the second, leading to + # semantically different DOMs. Passing the parameter <tt>preserve_newline: true</tt> will cause + # two or more newlines to be preserved. (A single leading newline will still be removed.) + # + # doc = Nokogiri::HTML5(<<-EOF) + # <!DOCTYPE html> + # <listing> + # + # Content</listing> + # EOF + # puts doc.at('/html/body/listing').serialize(preserve_newline: true) + # # => <listing> + # # + # # Content</listing> + # + # == Encodings + # + # Nokogiri always parses HTML5 using {https://en.wikipedia.org/wiki/UTF-8 UTF-8}; however, the + # encoding of the input can be explicitly selected via the optional +encoding+ parameter. This is + # most useful when the input comes not from a string but from an IO object. + # + # When serializing a document or node, the encoding of the output string can be specified via the + # +:encoding+ options. Characters that cannot be encoded in the selected encoding will be encoded + # as {https://en.wikipedia.org/wiki/List_of_XML_and_HTML_character_entity_references HTML numeric + # entities}. + # + # frag = Nokogiri::HTML5.fragment('<span>아는 길도 물어가라</span>') + # html = frag.serialize(encoding: 'US-ASCII') + # puts html + # # => <span>아는 길도 물어가라</span> + # frag = Nokogiri::HTML5.fragment(html) + # puts frag.serialize + # # => <span>아는 길도 물어가라</span> + # + # (There's a {https://bugs.ruby-lang.org/issues/15033 bug} in all current versions of Ruby that + # can cause the entity encoding to fail. Of the mandated supported encodings for HTML, the only + # encoding I'm aware of that has this bug is <tt>'ISO-2022-JP'</tt>. We recommend avoiding this + # encoding.) + # + # == Notes + # + # * The {Nokogiri::HTML5.fragment} function takes a string and parses it + # as a HTML5 document. The +<html>+, +<head>+, and +<body>+ elements are + # removed from this document, and any children of these elements that remain + # are returned as a {Nokogiri::HTML5::DocumentFragment}. + # + # * The {Nokogiri::HTML5.parse} function takes a string and passes it to the + # <code>gumbo_parse_with_options</code> method, using the default options. + # The resulting Gumbo parse tree is then walked. + # + # * Instead of uppercase element names, lowercase element names are produced. + # + # * Instead of returning +unknown+ as the element name for unknown tags, the + # original tag name is returned verbatim. + # + # @since v1.12.0 + # @note HTML5 functionality is not available when running JRuby. + module HTML5 + # HTML uses the XHTML namespace. + HTML_NAMESPACE = 'http://www.w3.org/1999/xhtml'.freeze + MATHML_NAMESPACE = 'http://www.w3.org/1998/Math/MathML'.freeze + SVG_NAMESPACE = 'http://www.w3.org/2000/svg'.freeze + XLINK_NAMESPACE = 'http://www.w3.org/1999/xlink'.freeze + XML_NAMESPACE = 'http://www.w3.org/XML/1998/namespace'.freeze + XMLNS_NAMESPACE = 'http://www.w3.org/2000/xmlns/'.freeze + + # Parse an HTML 5 document. Convenience method for {Nokogiri::HTML5::Document.parse} + def self.parse(string, url = nil, encoding = nil, **options, &block) + Document.parse(string, url, encoding, **options, &block) + end + + # Parse a fragment from +string+. Convenience method for + # {Nokogiri::HTML5::DocumentFragment.parse}. + def self.fragment(string, encoding = nil, **options) + DocumentFragment.parse(string, encoding, options) + end + + # Fetch and parse a HTML document from the web, following redirects, + # handling https, and determining the character encoding using HTML5 + # rules. +uri+ may be a +String+ or a +URI+. +options+ contains + # http headers and special options. Everything which is not a + # special option is considered a header. Special options include: + # * :follow_limit => number of redirects which are followed + # * :basic_auth => [username, password] + def self.get(uri, options={}) + warn("Nokogiri::HTML5.get is deprecated and will be removed in a future version of Nokogiri.", + uplevel: 1, category: :deprecated) + get_impl(uri, options) + end + + private + + def self.get_impl(uri, options={}) + headers = options.clone + headers = {:follow_limit => headers} if Numeric === headers # deprecated + limit=headers[:follow_limit] ? headers.delete(:follow_limit).to_i : 10 + + require 'net/http' + uri = URI(uri) unless URI === uri + + http = Net::HTTP.new(uri.host, uri.port) + + # TLS / SSL support + http.use_ssl = true if uri.scheme == 'https' + + # Pass through Net::HTTP override values, which currently include: + # :ca_file, :ca_path, :cert, :cert_store, :ciphers, + # :close_on_empty_response, :continue_timeout, :key, :open_timeout, + # :read_timeout, :ssl_timeout, :ssl_version, :use_ssl, + # :verify_callback, :verify_depth, :verify_mode + options.each do |key, value| + http.send "#{key}=", headers.delete(key) if http.respond_to? "#{key}=" + end + + request = Net::HTTP::Get.new(uri.request_uri) + + # basic authentication + auth = headers.delete(:basic_auth) + auth ||= [uri.user, uri.password] if uri.user && uri.password + request.basic_auth auth.first, auth.last if auth + + # remaining options are treated as headers + headers.each {|key, value| request[key.to_s] = value.to_s} + + response = http.request(request) + + case response + when Net::HTTPSuccess + doc = parse(reencode(response.body, response['content-type']), options) + doc.instance_variable_set('@response', response) + doc.class.send(:attr_reader, :response) + doc + when Net::HTTPRedirection + response.value if limit <= 1 + location = URI.join(uri, response['location']) + get_impl(location, options.merge(:follow_limit => limit-1)) + else + response.value + end + end + + def self.read_and_encode(string, encoding) + # Read the string with the given encoding. + if string.respond_to?(:read) + if encoding.nil? + string = string.read + else + string = string.read(encoding: encoding) + end + else + # Otherwise the string has the given encoding. + string = string.to_s + if encoding + string = string.dup + string.force_encoding(encoding) + end + end + + # convert to UTF-8 + if string.encoding != Encoding::UTF_8 + string = reencode(string) + end + string + end + + # Charset sniffing is a complex and controversial topic that understandably isn't done _by + # default_ by the Ruby Net::HTTP library. This being said, it is a very real problem for + # consumers of HTML as the default for HTML is iso-8859-1, most "good" producers use utf-8, and + # the Gumbo parser *only* supports utf-8. + # + # Accordingly, Nokogiri::HTML4::Document.parse provides limited encoding detection. Following + # this lead, Nokogiri::HTML5 attempts to do likewise, while attempting to more closely follow + # the HTML5 standard. + # + # http://bugs.ruby-lang.org/issues/2567 + # http://www.w3.org/TR/html5/syntax.html#determining-the-character-encoding + # + def self.reencode(body, content_type=nil) + if body.encoding == Encoding::ASCII_8BIT + encoding = nil + + # look for a Byte Order Mark (BOM) + initial_bytes = body[0..2].bytes + if initial_bytes[0..2] == [0xEF, 0xBB, 0xBF] + encoding = Encoding::UTF_8 + elsif initial_bytes[0..1] == [0xFE, 0xFF] + encoding = Encoding::UTF_16BE + elsif initial_bytes[0..1] == [0xFF, 0xFE] + encoding = Encoding::UTF_16LE + end + + # look for a charset in a content-encoding header + if content_type + encoding ||= content_type[/charset=["']?(.*?)($|["';\s])/i, 1] + end + + # look for a charset in a meta tag in the first 1024 bytes + if not encoding + data = body[0..1023].gsub(/<!--.*?(-->|\Z)/m, '') + data.scan(/<meta.*?>/m).each do |meta| + encoding ||= meta[/charset=["']?([^>]*?)($|["'\s>])/im, 1] + end + end + + # if all else fails, default to the official default encoding for HTML + encoding ||= Encoding::ISO_8859_1 + + # change the encoding to match the detected or inferred encoding + body = body.dup + begin + body.force_encoding(encoding) + rescue ArgumentError + body.force_encoding(Encoding::ISO_8859_1) + end + end + + body.encode(Encoding::UTF_8) + end + + def self.serialize_node_internal(current_node, io, encoding, options) + case current_node.type + when XML::Node::ELEMENT_NODE + ns = current_node.namespace + ns_uri = ns.nil? ? nil : ns.href + # XXX(sfc): attach namespaces to all nodes, even html? + if ns_uri.nil? || ns_uri == HTML_NAMESPACE || ns_uri == MATHML_NAMESPACE || ns_uri == SVG_NAMESPACE + tagname = current_node.name + else + tagname = "#{ns.prefix}:#{current_node.name}" + end + io << '<' << tagname + current_node.attribute_nodes.each do |attr| + attr_ns = attr.namespace + if attr_ns.nil? + attr_name = attr.name + else + ns_uri = attr_ns.href + if ns_uri == XML_NAMESPACE + attr_name = 'xml:' + attr.name.sub(/^[^:]*:/, '') + elsif ns_uri == XMLNS_NAMESPACE && attr.name.sub(/^[^:]*:/, '') == 'xmlns' + attr_name = 'xmlns' + elsif ns_uri == XMLNS_NAMESPACE + attr_name = 'xmlns:' + attr.name.sub(/^[^:]*:/, '') + elsif ns_uri == XLINK_NAMESPACE + attr_name = 'xlink:' + attr.name.sub(/^[^:]*:/, '') + else + attr_name = "#{attr_ns.prefix}:#{attr.name}" + end + end + io << ' ' << attr_name << '="' << escape_text(attr.content, encoding, true) << '"' + end + io << '>' + if !%w[area base basefont bgsound br col embed frame hr img input keygen + link meta param source track wbr].include?(current_node.name) + io << "\n" if options[:preserve_newline] && prepend_newline?(current_node) + current_node.children.each do |child| + # XXX(sfc): Templates handled specially? + serialize_node_internal(child, io, encoding, options) + end + io << '</' << tagname << '>' + end + when XML::Node::TEXT_NODE + parent = current_node.parent + if parent.element? && %w[style script xmp iframe noembed noframes plaintext noscript].include?(parent.name) + io << current_node.content + else + io << escape_text(current_node.content, encoding, false) + end + when XML::Node::CDATA_SECTION_NODE + io << '<![CDATA[' << current_node.content << ']]>' + when XML::Node::COMMENT_NODE + io << '<!--' << current_node.content << '-->' + when XML::Node::PI_NODE + io << '<?' << current_node.content << '>' + when XML::Node::DOCUMENT_TYPE_NODE, XML::Node::DTD_NODE + io << '<!DOCTYPE ' << current_node.name << '>' + when XML::Node::HTML_DOCUMENT_NODE, XML::Node::DOCUMENT_FRAG_NODE + current_node.children.each do |child| + serialize_node_internal(child, io, encoding, options) + end + else + raise "Unexpected node '#{current_node.name}' of type #{current_node.type}" + end + end + + def self.escape_text(text, encoding, attribute_mode) + if attribute_mode + text = text.gsub(/[&\u00a0"]/, + '&' => '&', "\u00a0" => ' ', '"' => '"') + else + text = text.gsub(/[&\u00a0<>]/, + '&' => '&', "\u00a0" => ' ', '<' => '<', '>' => '>') + end + # Not part of the standard + text.encode(encoding, fallback: lambda { |c| "&\#x#{c.ord.to_s(16)};" }) + end + + def self.prepend_newline?(node) + return false unless %w[pre textarea listing].include?(node.name) && !node.children.empty? + first_child = node.children[0] + first_child.text? && first_child.content.start_with?("\n") + end + end +end + +require_relative 'gumbo' diff --git a/lib/nokogiri/html5/document.rb b/lib/nokogiri/html5/document.rb new file mode 100644 index 0000000000..fe74d99b75 --- /dev/null +++ b/lib/nokogiri/html5/document.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true +# +# Copyright 2013-2021 Sam Ruby, Stephen Checkoway +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require_relative "../html4/document" + +module Nokogiri + module HTML5 + # @since v1.12.0 + # @note HTML5 functionality is not available when running JRuby. + class Document < Nokogiri::HTML4::Document + def self.parse(string_or_io, url = nil, encoding = nil, **options, &block) + yield options if block_given? + string_or_io = '' unless string_or_io + + if string_or_io.respond_to?(:encoding) && string_or_io.encoding.name != 'ASCII-8BIT' + encoding ||= string_or_io.encoding.name + end + + if string_or_io.respond_to?(:read) && string_or_io.respond_to?(:path) + url ||= string_or_io.path + end + unless string_or_io.respond_to?(:read) || string_or_io.respond_to?(:to_str) + raise ArgumentError.new("not a string or IO object") + end + do_parse(string_or_io, url, encoding, options) + end + + def self.read_io(io, url = nil, encoding = nil, **options) + raise ArgumentError.new("io object doesn't respond to :read") unless io.respond_to?(:read) + do_parse(io, url, encoding, options) + end + + def self.read_memory(string, url = nil, encoding = nil, **options) + raise ArgumentError.new("string object doesn't respond to :to_str") unless string.respond_to?(:to_str) + do_parse(string, url, encoding, options) + end + + def fragment(tags = nil) + DocumentFragment.new(self, tags, self.root) + end + + def to_xml(options = {}, &block) + # Bypass XML::Document#to_xml which doesn't add + # XML::Node::SaveOptions::AS_XML like XML::Node#to_xml does. + XML::Node.instance_method(:to_xml).bind(self).call(options, &block) + end + + private + def self.do_parse(string_or_io, url, encoding, options) + string = HTML5.read_and_encode(string_or_io, encoding) + max_attributes = options[:max_attributes] || Nokogiri::Gumbo::DEFAULT_MAX_ATTRIBUTES + max_errors = options[:max_errors] || options[:max_parse_errors] || Nokogiri::Gumbo::DEFAULT_MAX_ERRORS + max_depth = options[:max_tree_depth] || Nokogiri::Gumbo::DEFAULT_MAX_TREE_DEPTH + doc = Nokogiri::Gumbo.parse(string, url, max_attributes, max_errors, max_depth) + doc.encoding = 'UTF-8' + doc + end + end + end +end diff --git a/lib/nokogiri/html5/document_fragment.rb b/lib/nokogiri/html5/document_fragment.rb new file mode 100644 index 0000000000..66c6b413a9 --- /dev/null +++ b/lib/nokogiri/html5/document_fragment.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true +# +# Copyright 2013-2021 Sam Ruby, Stephen Checkoway +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require_relative "../html4/document_fragment" + +module Nokogiri + module HTML5 + # @since v1.12.0 + # @note HTML5 functionality is not available when running JRuby. + class DocumentFragment < Nokogiri::HTML4::DocumentFragment + attr_accessor :document + attr_accessor :errors + + # Create a document fragment. + def initialize(doc, tags = nil, ctx = nil, options = {}) + self.document = doc + self.errors = [] + return self unless tags + + max_attributes = options[:max_attributes] || Nokogiri::Gumbo::DEFAULT_MAX_ATTRIBUTES + max_errors = options[:max_errors] || Nokogiri::Gumbo::DEFAULT_MAX_ERRORS + max_depth = options[:max_tree_depth] || Nokogiri::Gumbo::DEFAULT_MAX_TREE_DEPTH + tags = Nokogiri::HTML5.read_and_encode(tags, nil) + Nokogiri::Gumbo.fragment(self, tags, ctx, max_attributes, max_errors, max_depth) + end + + def serialize(options = {}, &block) + # Bypass XML::Document.serialize which doesn't support options even + # though XML::Node.serialize does! + XML::Node.instance_method(:serialize).bind(self).call(options, &block) + end + + # Parse a document fragment from +tags+, returning a Nodeset. + def self.parse(tags, encoding = nil, options = {}) + doc = HTML5::Document.new + tags = HTML5.read_and_encode(tags, encoding) + doc.encoding = "UTF-8" + new(doc, tags, nil, options) + end + + def extract_params(params) # :nodoc: + handler = params.find do |param| + ![Hash, String, Symbol].include?(param.class) + end + params -= [handler] if handler + + hashes = [] + while Hash === params.last || params.last.nil? + hashes << params.pop + break if params.empty? + end + ns, binds = hashes.reverse + + ns ||= + begin + ns = {} + children.each { |child| ns.merge!(child.namespaces) } + ns + end + + [params, handler, ns, binds] + end + end + end +end +# vim: set shiftwidth=2 softtabstop=2 tabstop=8 expandtab: diff --git a/lib/nokogiri/html5/node.rb b/lib/nokogiri/html5/node.rb new file mode 100644 index 0000000000..92cf1b9470 --- /dev/null +++ b/lib/nokogiri/html5/node.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true +# +# Copyright 2013-2021 Sam Ruby, Stephen Checkoway +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require_relative "../xml/node" + +module Nokogiri + module HTML5 + # @since v1.12.0 + # @note HTML5 functionality is not available when running JRuby. + module Node + def inner_html(options = {}) + return super(options) unless document.is_a?(HTML5::Document) + result = options[:preserve_newline] && HTML5.prepend_newline?(self) ? String.new("\n") : String.new + result << children.map { |child| child.to_html(options) }.join + result + end + + def write_to(io, *options) + return super(io, *options) unless document.is_a?(HTML5::Document) + options = options.first.is_a?(Hash) ? options.shift : {} + encoding = options[:encoding] || options[0] + if Nokogiri.jruby? + save_options = options[:save_with] || options[1] + indent_times = options[:indent] || 0 + else + save_options = options[:save_with] || options[1] || XML::Node::SaveOptions::FORMAT + indent_times = options[:indent] || 2 + end + indent_string = (options[:indent_text] || " ") * indent_times + + config = XML::Node::SaveOptions.new(save_options.to_i) + yield config if block_given? + + config_options = config.options + if config_options & (XML::Node::SaveOptions::AS_XML | XML::Node::SaveOptions::AS_XHTML) != 0 + # Use Nokogiri's serializing code. + native_write_to(io, encoding, indent_string, config_options) + else + # Serialize including the current node. + encoding ||= document.encoding || Encoding::UTF_8 + internal_ops = { + preserve_newline: options[:preserve_newline] || false, + } + HTML5.serialize_node_internal(self, io, encoding, internal_ops) + end + end + + def fragment(tags) + return super(tags) unless document.is_a?(HTML5::Document) + DocumentFragment.new(document, tags, self) + end + + private + + # HTML elements can have attributes that contain colons. + # Nokogiri::XML::Node#[]= treats names with colons as a prefixed QName + # and tries to create an attribute in a namespace. This is especially + # annoying with attribute names like xml:lang since libxml2 will + # actually create the xml namespace if it doesn't exist already. + def add_child_node_and_reparent_attrs(node) + return super(node) unless document.is_a?(HTML5::Document) + # I'm not sure what this method is supposed to do. Reparenting + # namespaces is handled by libxml2, including child namespaces which + # this method wouldn't handle. + # https://github.com/sparklemotion/nokogiri/issues/1790 + add_child_node(node) + # node.attribute_nodes.find_all { |a| a.namespace }.each do |attr| + # attr.remove + # ns = attr.namespace + # a["#{ns.prefix}:#{attr.name}"] = attr.value + # end + end + end + # Monkey patch + XML::Node.prepend(HTML5::Node) + end +end + +# vim: set shiftwidth=2 softtabstop=2 tabstop=8 expandtab: diff --git a/lib/nokogiri/version/constant.rb b/lib/nokogiri/version/constant.rb index bb3e4ee505..c6b1a69c42 100644 --- a/lib/nokogiri/version/constant.rb +++ b/lib/nokogiri/version/constant.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module Nokogiri # The version of Nokogiri you are using - VERSION = "1.11.3" + VERSION = "1.12.5" end diff --git a/lib/nokogiri/version/info.rb b/lib/nokogiri/version/info.rb index 1bfea42908..460408d186 100644 --- a/lib/nokogiri/version/info.rb +++ b/lib/nokogiri/version/info.rb @@ -52,6 +52,10 @@ def libxml2_has_iconv? defined?(Nokogiri::LIBXML_ICONV_ENABLED) && Nokogiri::LIBXML_ICONV_ENABLED end + def libxslt_has_datetime? + defined?(Nokogiri::LIBXSLT_DATETIME_ENABLED) && Nokogiri::LIBXSLT_DATETIME_ENABLED + end + def libxml2_using_packaged? libxml2? && Nokogiri::PACKAGED_LIBRARIES end @@ -137,6 +141,7 @@ def to_hash else libxml["source"] = "system" end + libxml["memory_management"] = Nokogiri::LIBXML_MEMORY_MANAGEMENT libxml["iconv_enabled"] = libxml2_has_iconv? libxml["compiled"] = compiled_libxml_version.to_s libxml["loaded"] = loaded_libxml_version.to_s @@ -150,6 +155,7 @@ def to_hash else libxslt["source"] = "system" end + libxslt["datetime_enabled"] = libxslt_has_datetime? libxslt["compiled"] = compiled_libxslt_version.to_s libxslt["loaded"] = loaded_libxslt_version.to_s end @@ -190,15 +196,19 @@ def self.uses_libxml?(requirement = nil) # :nodoc: Gem::Requirement.new(requirement).satisfied_by?(VersionInfo.instance.loaded_libxml_version) end + def self.uses_gumbo? + uses_libxml? # TODO: replace with Gumbo functionality + end + def self.jruby? # :nodoc: VersionInfo.instance.jruby? end # Ensure constants used in this file are loaded - see #1896 if Nokogiri.jruby? - require "nokogiri/jruby/dependencies" + require_relative "../jruby/dependencies" end - require "nokogiri/extension" + require_relative "../extension" # More complete version information about libxml VERSION_INFO = VersionInfo.instance.to_hash diff --git a/lib/nokogiri/xml.rb b/lib/nokogiri/xml.rb index 93a898b951..e1e21c05ea 100644 --- a/lib/nokogiri/xml.rb +++ b/lib/nokogiri/xml.rb @@ -1,38 +1,9 @@ # frozen_string_literal: true -require 'nokogiri/xml/pp' -require 'nokogiri/xml/parse_options' -require 'nokogiri/xml/sax' -require 'nokogiri/xml/searchable' -require 'nokogiri/xml/node' -require 'nokogiri/xml/attribute_decl' -require 'nokogiri/xml/element_decl' -require 'nokogiri/xml/element_content' -require 'nokogiri/xml/character_data' -require 'nokogiri/xml/namespace' -require 'nokogiri/xml/attr' -require 'nokogiri/xml/dtd' -require 'nokogiri/xml/cdata' -require 'nokogiri/xml/text' -require 'nokogiri/xml/document' -require 'nokogiri/xml/document_fragment' -require 'nokogiri/xml/processing_instruction' -require 'nokogiri/xml/node_set' -require 'nokogiri/xml/syntax_error' -require 'nokogiri/xml/xpath' -require 'nokogiri/xml/xpath_context' -require 'nokogiri/xml/builder' -require 'nokogiri/xml/reader' -require 'nokogiri/xml/notation' -require 'nokogiri/xml/entity_decl' -require 'nokogiri/xml/entity_reference' -require 'nokogiri/xml/schema' -require 'nokogiri/xml/relax_ng' - module Nokogiri class << self ### # Parse XML. Convenience method for Nokogiri::XML::Document.parse - def XML thing, url = nil, encoding = nil, options = XML::ParseOptions::DEFAULT_XML, &block + def XML(thing, url = nil, encoding = nil, options = XML::ParseOptions::DEFAULT_XML, &block) Nokogiri::XML::Document.parse(thing, url, encoding, options, &block) end end @@ -41,20 +12,19 @@ module XML # Original C14N 1.0 spec canonicalization XML_C14N_1_0 = 0 # Exclusive C14N 1.0 spec canonicalization - XML_C14N_EXCLUSIVE_1_0 = 1 + XML_C14N_EXCLUSIVE_1_0 = 1 # C14N 1.1 spec canonicalization XML_C14N_1_1 = 2 class << self ### # Parse an XML document using the Nokogiri::XML::Reader API. See # Nokogiri::XML::Reader for mor information - def Reader string_or_io, url = nil, encoding = nil, options = ParseOptions::STRICT - + def Reader(string_or_io, url = nil, encoding = nil, options = ParseOptions::STRICT) options = Nokogiri::XML::ParseOptions.new(options) if Integer === options # Give the options to the user yield options if block_given? - if string_or_io.respond_to? :read + if string_or_io.respond_to?(:read) return Reader.from_io(string_or_io, url, encoding, options.to_i) end Reader.from_memory(string_or_io, url, encoding, options.to_i) @@ -62,15 +32,44 @@ def Reader string_or_io, url = nil, encoding = nil, options = ParseOptions::STRI ### # Parse XML. Convenience method for Nokogiri::XML::Document.parse - def parse thing, url = nil, encoding = nil, options = ParseOptions::DEFAULT_XML, &block + def parse(thing, url = nil, encoding = nil, options = ParseOptions::DEFAULT_XML, &block) Document.parse(thing, url, encoding, options, &block) end #### # Parse a fragment from +string+ in to a NodeSet. - def fragment string + def fragment(string) XML::DocumentFragment.parse(string) end end end end + +require_relative "xml/pp" +require_relative "xml/parse_options" +require_relative "xml/sax" +require_relative "xml/searchable" +require_relative "xml/node" +require_relative "xml/attribute_decl" +require_relative "xml/element_decl" +require_relative "xml/element_content" +require_relative "xml/character_data" +require_relative "xml/namespace" +require_relative "xml/attr" +require_relative "xml/dtd" +require_relative "xml/cdata" +require_relative "xml/text" +require_relative "xml/document" +require_relative "xml/document_fragment" +require_relative "xml/processing_instruction" +require_relative "xml/node_set" +require_relative "xml/syntax_error" +require_relative "xml/xpath" +require_relative "xml/xpath_context" +require_relative "xml/builder" +require_relative "xml/reader" +require_relative "xml/notation" +require_relative "xml/entity_decl" +require_relative "xml/entity_reference" +require_relative "xml/schema" +require_relative "xml/relax_ng" diff --git a/lib/nokogiri/xml/builder.rb b/lib/nokogiri/xml/builder.rb index 1c7ed087fe..d08ca83f31 100644 --- a/lib/nokogiri/xml/builder.rb +++ b/lib/nokogiri/xml/builder.rb @@ -196,6 +196,41 @@ module XML # # Note the "foo:object" tag. # + # === Namespace inheritance + # + # In the Builder context, children will inherit their parent's namespace. This is the same + # behavior as if the underlying {XML::Document} set +namespace_inheritance+ to +true+: + # + # result = Nokogiri::XML::Builder.new do |xml| + # xml["soapenv"].Envelope("xmlns:soapenv" => "http://schemas.xmlsoap.org/soap/envelope/") do + # xml.Header + # end + # end + # result.doc.to_xml + # # => <?xml version="1.0" encoding="utf-8"?> + # # <soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/"> + # # <soapenv:Header/> + # # </soapenv:Envelope> + # + # Users may turn this behavior off by passing a keyword argument +namespace_inheritance:false+ + # to the initializer: + # + # result = Nokogiri::XML::Builder.new(namespace_inheritance: false) do |xml| + # xml["soapenv"].Envelope("xmlns:soapenv" => "http://schemas.xmlsoap.org/soap/envelope/") do + # xml.Header + # xml["soapenv"].Body # users may explicitly opt into the namespace + # end + # end + # result.doc.to_xml + # # => <?xml version="1.0" encoding="utf-8"?> + # # <soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/"> + # # <Header/> + # # <soapenv:Body/> + # # </soapenv:Envelope> + # + # For more information on namespace inheritance, please see {XML::Document#namespace_inheritance} + # + # # == Document Types # # To create a document type (DTD), access use the Builder#doc method to get @@ -226,6 +261,8 @@ module XML # </root> # class Builder + DEFAULT_DOCUMENT_OPTIONS = {namespace_inheritance: true} + # The current Document object being built attr_accessor :doc @@ -282,6 +319,7 @@ def initialize(options = {}, root = nil, &block) @arity = nil @ns = nil + options = DEFAULT_DOCUMENT_OPTIONS.merge(options) options.each do |k, v| @doc.send(:"#{k}=", v) end diff --git a/lib/nokogiri/xml/document.rb b/lib/nokogiri/xml/document.rb index d09c0e41ad..4a4e5087a1 100644 --- a/lib/nokogiri/xml/document.rb +++ b/lib/nokogiri/xml/document.rb @@ -113,9 +113,55 @@ def self.parse string_or_io, url = nil, encoding = nil, options = ParseOptions:: # A list of Nokogiri::XML::SyntaxError found when parsing a document attr_accessor :errors + # When true, reparented elements without a namespace will inherit their new parent's + # namespace (if one exists). Defaults to +false+. + # + # @example Default behavior of namespace inheritance + # xml = <<~EOF + # <root xmlns:foo="http://nokogiri.org/default_ns/test/foo"> + # <foo:parent> + # </foo:parent> + # </root> + # EOF + # doc = Nokogiri::XML(xml) + # parent = doc.at_xpath("//foo:parent", "foo" => "http://nokogiri.org/default_ns/test/foo") + # parent.add_child("<child></child>") + # doc.to_xml + # # => <?xml version="1.0"?> + # # <root xmlns:foo="http://nokogiri.org/default_ns/test/foo"> + # # <foo:parent> + # # <child/> + # # </foo:parent> + # # </root> + # + # @example Setting namespace inheritance to +true+ + # xml = <<~EOF + # <root xmlns:foo="http://nokogiri.org/default_ns/test/foo"> + # <foo:parent> + # </foo:parent> + # </root> + # EOF + # doc = Nokogiri::XML(xml) + # doc.namespace_inheritance = true + # parent = doc.at_xpath("//foo:parent", "foo" => "http://nokogiri.org/default_ns/test/foo") + # parent.add_child("<child></child>") + # doc.to_xml + # # => <?xml version="1.0"?> + # # <root xmlns:foo="http://nokogiri.org/default_ns/test/foo"> + # # <foo:parent> + # # <foo:child/> + # # </foo:parent> + # # </root> + # + # @return [Boolean] + # + # @since v1.12.4 + attr_accessor :namespace_inheritance + def initialize *args # :nodoc: @errors = [] @decorators = nil + @namespace_inheritance = false end ## diff --git a/lib/nokogiri/xml/node.rb b/lib/nokogiri/xml/node.rb index 44f94ccd49..a06ba5fa56 100644 --- a/lib/nokogiri/xml/node.rb +++ b/lib/nokogiri/xml/node.rb @@ -1,7 +1,6 @@ # encoding: UTF-8 # frozen_string_literal: true require "stringio" -require "nokogiri/xml/node/save_options" module Nokogiri module XML @@ -837,7 +836,7 @@ def parse(string_or_io, options = nil) node_set = in_context(contents, options.to_i) if (node_set.empty? && (document.errors.length > error_count)) if options.recover? - fragment = Nokogiri::HTML::DocumentFragment.parse contents + fragment = Nokogiri::HTML4::DocumentFragment.parse contents node_set = fragment.children else raise document.errors[error_count] @@ -883,7 +882,7 @@ def xml? type == DOCUMENT_NODE end - # Returns true if this is an HTML::Document node + # Returns true if this is an HTML4::Document node def html? type == HTML_DOCUMENT_NODE end @@ -909,11 +908,11 @@ def fragment? end ### - # Fetch the Nokogiri::HTML::ElementDescription for this node. Returns + # Fetch the Nokogiri::HTML4::ElementDescription for this node. Returns # nil on XML documents and on unknown tags. def description return nil if document.xml? - Nokogiri::HTML::ElementDescription[name] + Nokogiri::HTML4::ElementDescription[name] end ### @@ -1235,3 +1234,5 @@ def add_child_node_and_reparent_attrs(node) end end end + +require_relative "node/save_options" diff --git a/lib/nokogiri/xml/node/save_options.rb b/lib/nokogiri/xml/node/save_options.rb index 6da00134bb..8afeef345a 100644 --- a/lib/nokogiri/xml/node/save_options.rb +++ b/lib/nokogiri/xml/node/save_options.rb @@ -34,7 +34,7 @@ class SaveOptions DEFAULT_HTML = FORMAT | NO_DECLARATION | NO_EMPTY_TAGS | AS_HTML end # the default for XHTML document - DEFAULT_XHTML = FORMAT | NO_DECLARATION | NO_EMPTY_TAGS | AS_XHTML + DEFAULT_XHTML = FORMAT | NO_DECLARATION | AS_XHTML # Integer representation of the SaveOptions attr_reader :options diff --git a/lib/nokogiri/xml/parse_options.rb b/lib/nokogiri/xml/parse_options.rb index a266d5ba07..36186ab7b9 100644 --- a/lib/nokogiri/xml/parse_options.rb +++ b/lib/nokogiri/xml/parse_options.rb @@ -71,6 +71,8 @@ class ParseOptions # the default options used for parsing XML documents DEFAULT_XML = RECOVER | NONET + # the default options used for parsing XSLT stylesheets + DEFAULT_XSLT = RECOVER | NONET | NOENT | DTDLOAD | DTDATTR | NOCDATA # the default options used for parsing HTML documents DEFAULT_HTML = RECOVER | NOERROR | NOWARNING | NONET # the default options used for parsing XML schemas diff --git a/lib/nokogiri/xml/pp.rb b/lib/nokogiri/xml/pp.rb index 1b0f4df850..591bc8fa79 100644 --- a/lib/nokogiri/xml/pp.rb +++ b/lib/nokogiri/xml/pp.rb @@ -1,3 +1,3 @@ # frozen_string_literal: true -require 'nokogiri/xml/pp/node' -require 'nokogiri/xml/pp/character_data' +require_relative "pp/node" +require_relative "pp/character_data" diff --git a/lib/nokogiri/xml/sax.rb b/lib/nokogiri/xml/sax.rb index e965f677b0..f4c35b7137 100644 --- a/lib/nokogiri/xml/sax.rb +++ b/lib/nokogiri/xml/sax.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true -require 'nokogiri/xml/sax/document' -require 'nokogiri/xml/sax/parser_context' -require 'nokogiri/xml/sax/parser' -require 'nokogiri/xml/sax/push_parser' +require_relative "sax/document" +require_relative "sax/parser_context" +require_relative "sax/parser" +require_relative "sax/push_parser" diff --git a/lib/nokogiri/xml/sax/document.rb b/lib/nokogiri/xml/sax/document.rb index 0f32e8930a..8336b74789 100644 --- a/lib/nokogiri/xml/sax/document.rb +++ b/lib/nokogiri/xml/sax/document.rb @@ -2,20 +2,19 @@ module Nokogiri module XML ### - # SAX Parsers are event driven parsers. Nokogiri provides two different - # event based parsers when dealing with XML. If you want to do SAX style - # parsing using HTML, check out Nokogiri::HTML::SAX. + # SAX Parsers are event driven parsers. Nokogiri provides two different event based parsers when + # dealing with XML. If you want to do SAX style parsing using HTML, check out + # Nokogiri::HTML4::SAX. # - # The basic way a SAX style parser works is by creating a parser, - # telling the parser about the events we're interested in, then giving - # the parser some XML to process. The parser will notify you when - # it encounters events you said you would like to know about. + # The basic way a SAX style parser works is by creating a parser, telling the parser about the + # events we're interested in, then giving the parser some XML to process. The parser will notify + # you when it encounters events you said you would like to know about. # - # To register for events, you simply subclass Nokogiri::XML::SAX::Document, - # and implement the methods for which you would like notification. + # To register for events, you simply subclass Nokogiri::XML::SAX::Document, and implement the + # methods for which you would like notification. # - # For example, if I want to be notified when a document ends, and when an - # element starts, I would write a class like this: + # For example, if I want to be notified when a document ends, and when an element starts, I + # would write a class like this: # # class MyDocument < Nokogiri::XML::SAX::Document # def end_document @@ -27,8 +26,7 @@ module XML # end # end # - # Then I would instantiate a SAX parser with this document, and feed the - # parser some XML + # Then I would instantiate a SAX parser with this document, and feed the parser some XML # # # Create a new parser # parser = Nokogiri::XML::SAX::Parser.new(MyDocument.new) @@ -36,25 +34,21 @@ module XML # # Feed the parser some XML # parser.parse(File.open(ARGV[0])) # - # Now my document handler will be called when each node starts, and when - # then document ends. To see what kinds of events are available, take - # a look at Nokogiri::XML::SAX::Document. + # Now my document handler will be called when each node starts, and when then document ends. To + # see what kinds of events are available, take a look at Nokogiri::XML::SAX::Document. # - # Two SAX parsers for XML are available, a parser that reads from a string - # or IO object as it feels necessary, and a parser that lets you spoon - # feed it XML. If you want to let Nokogiri deal with reading your XML, - # use the Nokogiri::XML::SAX::Parser. If you want to have fine grain + # Two SAX parsers for XML are available, a parser that reads from a string or IO object as it + # feels necessary, and a parser that lets you spoon feed it XML. If you want to let Nokogiri + # deal with reading your XML, use the Nokogiri::XML::SAX::Parser. If you want to have fine grain # control over the XML input, use the Nokogiri::XML::SAX::PushParser. module SAX ### - # This class is used for registering types of events you are interested - # in handling. All of the methods on this class are available as - # possible events while parsing an XML document. To register for any - # particular event, just subclass this class and implement the methods - # you are interested in knowing about. + # This class is used for registering types of events you are interested in handling. All of + # the methods on this class are available as possible events while parsing an XML document. To + # register for any particular event, just subclass this class and implement the methods you + # are interested in knowing about. # - # To only be notified about start and end element events, write a class - # like this: + # To only be notified about start and end element events, write a class like this: # # class MyDocument < Nokogiri::XML::SAX::Document # def start_element name, attrs = [] @@ -66,8 +60,8 @@ module SAX # end # end # - # You can use this event handler for any SAX style parser included with - # Nokogiri. See Nokogiri::XML::SAX, and Nokogiri::HTML::SAX. + # You can use this event handler for any SAX style parser included with Nokogiri. See + # Nokogiri::XML::SAX, and Nokogiri::HTML4::SAX. class Document ### # Called when an XML declaration is parsed @@ -129,7 +123,7 @@ def end_element_namespace name, prefix = nil, uri = nil end ### - # Characters read between a tag. This method might be called multiple + # Characters read between a tag. This method might be called multiple # times given one contiguous string of characters. # # +string+ contains the character data diff --git a/lib/nokogiri/xml/xpath.rb b/lib/nokogiri/xml/xpath.rb index e2f30fe0e4..4caba97f8d 100644 --- a/lib/nokogiri/xml/xpath.rb +++ b/lib/nokogiri/xml/xpath.rb @@ -1,9 +1,9 @@ # frozen_string_literal: true -require 'nokogiri/xml/xpath/syntax_error' - module Nokogiri module XML module XPath end end end + +require_relative "xpath/syntax_error" diff --git a/lib/nokogiri/xslt.rb b/lib/nokogiri/xslt.rb index 50cdcaf9ae..87e87839c5 100644 --- a/lib/nokogiri/xslt.rb +++ b/lib/nokogiri/xslt.rb @@ -1,6 +1,4 @@ # frozen_string_literal: true -require 'nokogiri/xslt/stylesheet' - module Nokogiri class << self ### @@ -22,32 +20,32 @@ module XSLT class << self ### # Parse the stylesheet in +string+, register any +modules+ - def parse string, modules = {} + def parse(string, modules = {}) modules.each do |url, klass| - XSLT.register url, klass + XSLT.register(url, klass) end + doc = XML::Document.parse(string, nil, nil, XML::ParseOptions::DEFAULT_XSLT) if Nokogiri.jruby? - Stylesheet.parse_stylesheet_doc(XML.parse(string), string) + Stylesheet.parse_stylesheet_doc(doc, string) else - Stylesheet.parse_stylesheet_doc(XML.parse(string)) + Stylesheet.parse_stylesheet_doc(doc) end end ### # Quote parameters in +params+ for stylesheet safety - def quote_params params + def quote_params(params) parray = (params.instance_of?(Hash) ? params.to_a.flatten : params).dup - parray.each_with_index do |v,i| - if i % 2 > 0 - parray[i]= - if v =~ /'/ - "concat('#{ v.gsub(/'/, %q{', "'", '}) }')" - else - "'#{v}'"; - end + parray.each_with_index do |v, i| + parray[i] = if i % 2 > 0 + if v =~ /'/ + "concat('#{v.gsub(/'/, %q{', "'", '})}')" + else + "'#{v}'" + end else - parray[i] = v.to_s + v.to_s end end parray.flatten @@ -55,3 +53,5 @@ def quote_params params end end end + +require_relative "xslt/stylesheet" diff --git a/lib/nokogiri/xslt/stylesheet.rb b/lib/nokogiri/xslt/stylesheet.rb index acdd9cb3f1..4d445d7946 100644 --- a/lib/nokogiri/xslt/stylesheet.rb +++ b/lib/nokogiri/xslt/stylesheet.rb @@ -18,7 +18,7 @@ class Stylesheet # Apply an XSLT stylesheet to an XML::Document. # +params+ is an array of strings used as XSLT parameters. # returns serialized document - def apply_to document, params = [] + def apply_to(document, params = []) serialize(transform(document, params)) end end diff --git a/nokogiri.gemspec b/nokogiri.gemspec index cfbd69c50b..9a36de9f96 100644 --- a/nokogiri.gemspec +++ b/nokogiri.gemspec @@ -27,6 +27,9 @@ Gem::Specification.new do |spec| "Akinori MUSHA", "John Shahid", "Karol Bucek", + "Sam Ruby", + "Craig Barnes", + "Stephen Checkoway", "Lars Kanis", "Sergio Arbeo", "Timothy Elliott", @@ -56,11 +59,11 @@ Gem::Specification.new do |spec| "bin/nokogiri", "dependencies.yml", "ext/java/nokogiri/EncodingHandler.java", - "ext/java/nokogiri/HtmlDocument.java", - "ext/java/nokogiri/HtmlElementDescription.java", - "ext/java/nokogiri/HtmlEntityLookup.java", - "ext/java/nokogiri/HtmlSaxParserContext.java", - "ext/java/nokogiri/HtmlSaxPushParser.java", + "ext/java/nokogiri/Html4Document.java", + "ext/java/nokogiri/Html4ElementDescription.java", + "ext/java/nokogiri/Html4EntityLookup.java", + "ext/java/nokogiri/Html4SaxParserContext.java", + "ext/java/nokogiri/Html4SaxPushParser.java", "ext/java/nokogiri/NokogiriService.java", "ext/java/nokogiri/XmlAttr.java", "ext/java/nokogiri/XmlAttributeDecl.java", @@ -144,11 +147,11 @@ Gem::Specification.new do |spec| "ext/java/nokogiri/internals/dom2dtm/DOM2DTMdefaultNamespaceDeclarationNode.java", "ext/nokogiri/depend", "ext/nokogiri/extconf.rb", - "ext/nokogiri/html_document.c", - "ext/nokogiri/html_element_description.c", - "ext/nokogiri/html_entity_lookup.c", - "ext/nokogiri/html_sax_parser_context.c", - "ext/nokogiri/html_sax_push_parser.c", + "ext/nokogiri/html4_document.c", + "ext/nokogiri/html4_element_description.c", + "ext/nokogiri/html4_entity_lookup.c", + "ext/nokogiri/html4_sax_parser_context.c", + "ext/nokogiri/html4_sax_push_parser.c", "ext/nokogiri/libxml2_backwards_compat.c", "ext/nokogiri/nokogiri.c", "ext/nokogiri/nokogiri.h", @@ -178,6 +181,51 @@ Gem::Specification.new do |spec| "ext/nokogiri/xml_text.c", "ext/nokogiri/xml_xpath_context.c", "ext/nokogiri/xslt_stylesheet.c", + "gumbo-parser/CHANGES.md", + "gumbo-parser/Makefile", + "gumbo-parser/THANKS", + "gumbo-parser/src/Makefile", + "gumbo-parser/src/README.md", + "gumbo-parser/src/ascii.c", + "gumbo-parser/src/ascii.h", + "gumbo-parser/src/attribute.c", + "gumbo-parser/src/attribute.h", + "gumbo-parser/src/char_ref.c", + "gumbo-parser/src/char_ref.h", + "gumbo-parser/src/char_ref.rl", + "gumbo-parser/src/error.c", + "gumbo-parser/src/error.h", + "gumbo-parser/src/foreign_attrs.c", + "gumbo-parser/src/foreign_attrs.gperf", + "gumbo-parser/src/gumbo.h", + "gumbo-parser/src/insertion_mode.h", + "gumbo-parser/src/macros.h", + "gumbo-parser/src/parser.c", + "gumbo-parser/src/parser.h", + "gumbo-parser/src/replacement.h", + "gumbo-parser/src/string_buffer.c", + "gumbo-parser/src/string_buffer.h", + "gumbo-parser/src/string_piece.c", + "gumbo-parser/src/svg_attrs.c", + "gumbo-parser/src/svg_attrs.gperf", + "gumbo-parser/src/svg_tags.c", + "gumbo-parser/src/svg_tags.gperf", + "gumbo-parser/src/tag.c", + "gumbo-parser/src/tag_lookup.c", + "gumbo-parser/src/tag_lookup.gperf", + "gumbo-parser/src/tag_lookup.h", + "gumbo-parser/src/token_buffer.c", + "gumbo-parser/src/token_buffer.h", + "gumbo-parser/src/token_type.h", + "gumbo-parser/src/tokenizer.c", + "gumbo-parser/src/tokenizer.h", + "gumbo-parser/src/tokenizer_states.h", + "gumbo-parser/src/utf8.c", + "gumbo-parser/src/utf8.h", + "gumbo-parser/src/util.c", + "gumbo-parser/src/util.h", + "gumbo-parser/src/vector.c", + "gumbo-parser/src/vector.h", "lib/isorelax.jar", "lib/jing.jar", "lib/nekodtd.jar", @@ -194,16 +242,22 @@ Gem::Specification.new do |spec| "lib/nokogiri/css/xpath_visitor.rb", "lib/nokogiri/decorators/slop.rb", "lib/nokogiri/extension.rb", + "lib/nokogiri/gumbo.rb", "lib/nokogiri/html.rb", - "lib/nokogiri/html/builder.rb", - "lib/nokogiri/html/document.rb", - "lib/nokogiri/html/document_fragment.rb", - "lib/nokogiri/html/element_description.rb", - "lib/nokogiri/html/element_description_defaults.rb", - "lib/nokogiri/html/entity_lookup.rb", - "lib/nokogiri/html/sax/parser.rb", - "lib/nokogiri/html/sax/parser_context.rb", - "lib/nokogiri/html/sax/push_parser.rb", + "lib/nokogiri/html4.rb", + "lib/nokogiri/html4/builder.rb", + "lib/nokogiri/html4/document.rb", + "lib/nokogiri/html4/document_fragment.rb", + "lib/nokogiri/html4/element_description.rb", + "lib/nokogiri/html4/element_description_defaults.rb", + "lib/nokogiri/html4/entity_lookup.rb", + "lib/nokogiri/html4/sax/parser.rb", + "lib/nokogiri/html4/sax/parser_context.rb", + "lib/nokogiri/html4/sax/push_parser.rb", + "lib/nokogiri/html5.rb", + "lib/nokogiri/html5/document.rb", + "lib/nokogiri/html5/document_fragment.rb", + "lib/nokogiri/html5/node.rb", "lib/nokogiri/jruby/dependencies.rb", "lib/nokogiri/syntax_error.rb", "lib/nokogiri/version.rb", @@ -263,10 +317,9 @@ Gem::Specification.new do |spec| spec.rdoc_options = ["--main", "README.md"] spec.add_runtime_dependency("racc", "~> 1.4") - spec.add_runtime_dependency("mini_portile2", "~> 2.5.0") unless java_p # keep version in sync with extconf.rb + spec.add_runtime_dependency("mini_portile2", "~> 2.6.1") unless java_p # keep version in sync with extconf.rb spec.add_development_dependency("bundler", "~> 2.2") - spec.add_development_dependency("concourse", "~> 0.41") spec.add_development_dependency("hoe-markdown", "~> 1.4") spec.add_development_dependency("minitest", "~> 5.8") spec.add_development_dependency("minitest-reporters", "~> 1.4") diff --git a/concourse/images/Dockerfile.alpine.erb b/oci-images/nokogiri-test/Dockerfile.alpine.erb similarity index 100% rename from concourse/images/Dockerfile.alpine.erb rename to oci-images/nokogiri-test/Dockerfile.alpine.erb diff --git a/concourse/images/Dockerfile.alpine32.erb b/oci-images/nokogiri-test/Dockerfile.alpine32.erb similarity index 100% rename from concourse/images/Dockerfile.alpine32.erb rename to oci-images/nokogiri-test/Dockerfile.alpine32.erb diff --git a/concourse/images/Dockerfile.bionic.erb b/oci-images/nokogiri-test/Dockerfile.bionic.erb similarity index 100% rename from concourse/images/Dockerfile.bionic.erb rename to oci-images/nokogiri-test/Dockerfile.bionic.erb diff --git a/concourse/images/Dockerfile.bionic32.erb b/oci-images/nokogiri-test/Dockerfile.bionic32.erb similarity index 100% rename from concourse/images/Dockerfile.bionic32.erb rename to oci-images/nokogiri-test/Dockerfile.bionic32.erb diff --git a/concourse/images/Dockerfile.jruby.erb b/oci-images/nokogiri-test/Dockerfile.jruby.erb similarity index 100% rename from concourse/images/Dockerfile.jruby.erb rename to oci-images/nokogiri-test/Dockerfile.jruby.erb diff --git a/concourse/images/Dockerfile.mri.erb b/oci-images/nokogiri-test/Dockerfile.mri.erb similarity index 100% rename from concourse/images/Dockerfile.mri.erb rename to oci-images/nokogiri-test/Dockerfile.mri.erb diff --git a/concourse/images/Dockerfile.truffle.erb b/oci-images/nokogiri-test/Dockerfile.truffle.erb similarity index 100% rename from concourse/images/Dockerfile.truffle.erb rename to oci-images/nokogiri-test/Dockerfile.truffle.erb diff --git a/concourse/images/bundle-install.step b/oci-images/nokogiri-test/bundle-install.step similarity index 100% rename from concourse/images/bundle-install.step rename to oci-images/nokogiri-test/bundle-install.step diff --git a/concourse/images/debian-libxml-et-al.step b/oci-images/nokogiri-test/debian-libxml-et-al.step similarity index 100% rename from concourse/images/debian-libxml-et-al.step rename to oci-images/nokogiri-test/debian-libxml-et-al.step diff --git a/concourse/images/debian-prelude.step b/oci-images/nokogiri-test/debian-prelude.step similarity index 100% rename from concourse/images/debian-prelude.step rename to oci-images/nokogiri-test/debian-prelude.step diff --git a/concourse/images/debian-ruby.step b/oci-images/nokogiri-test/debian-ruby.step similarity index 100% rename from concourse/images/debian-ruby.step rename to oci-images/nokogiri-test/debian-ruby.step diff --git a/concourse/images/debian-valgrind.step b/oci-images/nokogiri-test/debian-valgrind.step similarity index 100% rename from concourse/images/debian-valgrind.step rename to oci-images/nokogiri-test/debian-valgrind.step diff --git a/concourse/images/java-opts.step b/oci-images/nokogiri-test/java-opts.step similarity index 100% rename from concourse/images/java-opts.step rename to oci-images/nokogiri-test/java-opts.step diff --git a/concourse/images/update-bundler.step b/oci-images/nokogiri-test/update-bundler.step similarity index 100% rename from concourse/images/update-bundler.step rename to oci-images/nokogiri-test/update-bundler.step diff --git a/patches/libxml2/0002-Remove-script-macro-support.patch b/patches/libxml2/0001-Remove-script-macro-support.patch similarity index 100% rename from patches/libxml2/0002-Remove-script-macro-support.patch rename to patches/libxml2/0001-Remove-script-macro-support.patch diff --git a/patches/libxml2/0001-Revert-Do-not-URI-escape-in-server-side-includes.patch b/patches/libxml2/0001-Revert-Do-not-URI-escape-in-server-side-includes.patch deleted file mode 100644 index e3ef2f2a68..0000000000 --- a/patches/libxml2/0001-Revert-Do-not-URI-escape-in-server-side-includes.patch +++ /dev/null @@ -1,78 +0,0 @@ -From c5538465c08a8ea248a370bf55bc39cd3385e4af Mon Sep 17 00:00:00 2001 -From: Mike Dalessio <mike.dalessio@gmail.com> -Date: Thu, 29 Mar 2018 14:09:00 -0400 -Subject: [PATCH] Revert "Do not URI escape in server side includes" - -This reverts commit 960f0e275616cadc29671a218d7fb9b69eb35588. ---- - HTMLtree.c | 49 +++++++++++-------------------------------------- - 1 file changed, 11 insertions(+), 38 deletions(-) - -diff --git a/HTMLtree.c b/HTMLtree.c -index 2fd0c9c..67160c5 100644 ---- a/HTMLtree.c -+++ b/HTMLtree.c -@@ -717,49 +717,22 @@ htmlAttrDumpOutput(xmlOutputBufferPtr buf, xmlDocPtr doc, xmlAttrPtr cur, - (!xmlStrcasecmp(cur->name, BAD_CAST "src")) || - ((!xmlStrcasecmp(cur->name, BAD_CAST "name")) && - (!xmlStrcasecmp(cur->parent->name, BAD_CAST "a"))))) { -+ xmlChar *escaped; - xmlChar *tmp = value; -- /* xmlURIEscapeStr() escapes '"' so it can be safely used. */ -- xmlBufCCat(buf->buffer, "\""); - - while (IS_BLANK_CH(*tmp)) tmp++; - -- /* URI Escape everything, except server side includes. */ -- for ( ; ; ) { -- xmlChar *escaped; -- xmlChar endChar; -- xmlChar *end = NULL; -- xmlChar *start = (xmlChar *)xmlStrstr(tmp, BAD_CAST "<!--"); -- if (start != NULL) { -- end = (xmlChar *)xmlStrstr(tmp, BAD_CAST "-->"); -- if (end != NULL) { -- *start = '\0'; -- } -- } -- -- /* Escape the whole string, or until start (set to '\0'). */ -- escaped = xmlURIEscapeStr(tmp, BAD_CAST"@/:=?;#%&,+"); -- if (escaped != NULL) { -- xmlBufCat(buf->buffer, escaped); -- xmlFree(escaped); -- } else { -- xmlBufCat(buf->buffer, tmp); -- } -- -- if (end == NULL) { /* Everything has been written. */ -- break; -- } -- -- /* Do not escape anything within server side includes. */ -- *start = '<'; /* Restore the first character of "<!--". */ -- end += 3; /* strlen("-->") */ -- endChar = *end; -- *end = '\0'; -- xmlBufCat(buf->buffer, start); -- *end = endChar; -- tmp = end; -+ /* -+ * the < and > have already been escaped at the entity level -+ * And doing so here breaks server side includes -+ */ -+ escaped = xmlURIEscapeStr(tmp, BAD_CAST"@/:=?;#%&,+<>"); -+ if (escaped != NULL) { -+ xmlBufWriteQuotedString(buf->buffer, escaped); -+ xmlFree(escaped); -+ } else { -+ xmlBufWriteQuotedString(buf->buffer, value); - } -- -- xmlBufCCat(buf->buffer, "\""); - } else { - xmlBufWriteQuotedString(buf->buffer, value); - } --- -2.9.5 - diff --git a/patches/libxml2/0003-Update-entities-to-remove-handling-of-ssi.patch b/patches/libxml2/0002-Update-entities-to-remove-handling-of-ssi.patch similarity index 100% rename from patches/libxml2/0003-Update-entities-to-remove-handling-of-ssi.patch rename to patches/libxml2/0002-Update-entities-to-remove-handling-of-ssi.patch diff --git a/patches/libxml2/0004-libxml2.la-is-in-top_builddir.patch b/patches/libxml2/0003-libxml2.la-is-in-top_builddir.patch similarity index 94% rename from patches/libxml2/0004-libxml2.la-is-in-top_builddir.patch rename to patches/libxml2/0003-libxml2.la-is-in-top_builddir.patch index 30c81dc77c..b6add83769 100644 --- a/patches/libxml2/0004-libxml2.la-is-in-top_builddir.patch +++ b/patches/libxml2/0003-libxml2.la-is-in-top_builddir.patch @@ -16,7 +16,7 @@ index cf96d41..1372d8b 100644 } -libxml2.la: $(libxml2_la_OBJECTS) $(libxml2_la_DEPENDENCIES) $(EXTRA_libxml2_la_DEPENDENCIES) -+$(top_builddir)/libxml2.la: $(libxml2_la_OBJECTS) $(libxml2_la_DEPENDENCIES) $(EXTRA_libxml2_la_DEPENDENCIES) ++$(top_builddir)/libxml2.la: $(libxml2_la_OBJECTS) $(libxml2_la_DEPENDENCIES) $(EXTRA_libxml2_la_DEPENDENCIES) $(AM_V_CCLD)$(libxml2_la_LINK) -rpath $(libdir) $(libxml2_la_OBJECTS) $(libxml2_la_LIBADD) $(LIBS) testdso.la: $(testdso_la_OBJECTS) $(testdso_la_DEPENDENCIES) $(EXTRA_testdso_la_DEPENDENCIES) diff --git a/patches/libxml2/0008-use-glibc-strlen.patch b/patches/libxml2/0004-use-glibc-strlen.patch similarity index 100% rename from patches/libxml2/0008-use-glibc-strlen.patch rename to patches/libxml2/0004-use-glibc-strlen.patch diff --git a/patches/libxml2/0005-Fix-infinite-loop-in-xmlStringLenDecodeEntities.patch b/patches/libxml2/0005-Fix-infinite-loop-in-xmlStringLenDecodeEntities.patch deleted file mode 100644 index 56026fb569..0000000000 --- a/patches/libxml2/0005-Fix-infinite-loop-in-xmlStringLenDecodeEntities.patch +++ /dev/null @@ -1,32 +0,0 @@ -From 0e1a49c8907645d2e155f0d89d4d9895ac5112b5 Mon Sep 17 00:00:00 2001 -From: Zhipeng Xie <xiezhipeng1@huawei.com> -Date: Thu, 12 Dec 2019 17:30:55 +0800 -Subject: [PATCH] Fix infinite loop in xmlStringLenDecodeEntities - -When ctxt->instate == XML_PARSER_EOF,xmlParseStringEntityRef -return NULL which cause a infinite loop in xmlStringLenDecodeEntities - -Found with libFuzzer. - -Signed-off-by: Zhipeng Xie <xiezhipeng1@huawei.com> ---- - parser.c | 3 ++- - 1 file changed, 2 insertions(+), 1 deletion(-) - -diff --git a/parser.c b/parser.c -index d1c3196..a34bb6c 100644 ---- a/parser.c -+++ b/parser.c -@@ -2646,7 +2646,8 @@ xmlStringLenDecodeEntities(xmlParserCtxtPtr ctxt, const xmlChar *str, int len, - else - c = 0; - while ((c != 0) && (c != end) && /* non input consuming loop */ -- (c != end2) && (c != end3)) { -+ (c != end2) && (c != end3) && -+ (ctxt->instate != XML_PARSER_EOF)) { - - if (c == 0) break; - if ((c == '&') && (str[1] == '#')) { --- -2.17.1 - diff --git a/patches/libxml2/0009-avoid-isnan-isinf.patch b/patches/libxml2/0005-avoid-isnan-isinf.patch similarity index 94% rename from patches/libxml2/0009-avoid-isnan-isinf.patch rename to patches/libxml2/0005-avoid-isnan-isinf.patch index ffd82cba6c..38a867522b 100644 --- a/patches/libxml2/0009-avoid-isnan-isinf.patch +++ b/patches/libxml2/0005-avoid-isnan-isinf.patch @@ -51,7 +51,7 @@ diff --git a/xpath.c b/xpath.c index 9f64ab9..5b6d999 100644 --- a/xpath.c +++ b/xpath.c -@@ -509,11 +509,7 @@ xmlXPathInit(void) { +@@ -515,11 +515,7 @@ xmlXPathInit(void) { */ int xmlXPathIsNaN(double val) { @@ -63,16 +63,16 @@ index 9f64ab9..5b6d999 100644 } /** -@@ -524,15 +520,11 @@ xmlXPathIsNaN(double val) { +@@ -530,15 +530,11 @@ xmlXPathIsNaN(double val) { */ int xmlXPathIsInf(double val) { -#ifdef isinf - return isinf(val) ? (val > 0 ? 1 : -1) : 0; -#else - if (val >= INFINITY) + if (val >= xmlXPathPINF) return 1; - if (val <= -INFINITY) + if (val <= -xmlXPathPINF) return -1; return 0; -#endif diff --git a/patches/libxml2/0006-htmlParseComment-treat-as-if-it-closed-the-comment.patch b/patches/libxml2/0006-htmlParseComment-treat-as-if-it-closed-the-comment.patch deleted file mode 100644 index 5a687dfba1..0000000000 --- a/patches/libxml2/0006-htmlParseComment-treat-as-if-it-closed-the-comment.patch +++ /dev/null @@ -1,73 +0,0 @@ -From 4f51a6d2b1755ce5b36c524c215aad70d864ac1d Mon Sep 17 00:00:00 2001 -From: Mike Dalessio <mike.dalessio@gmail.com> -Date: Mon, 3 Aug 2020 17:36:05 -0400 -Subject: [PATCH 1/2] htmlParseComment: treat `--!>` as if it closed the - comment - -See guidance provided on incorrectly-closed comments here: - -https://html.spec.whatwg.org/multipage/parsing.html#parse-error-incorrectly-closed-comment ---- - HTMLparser.c | 28 ++++++++++++++++++++-------- - 1 file changed, 20 insertions(+), 8 deletions(-) - -diff --git a/HTMLparser.c b/HTMLparser.c -index 7b6d689..4d43479 100644 ---- a/HTMLparser.c -+++ b/HTMLparser.c -@@ -3300,6 +3300,7 @@ htmlParseComment(htmlParserCtxtPtr ctxt) { - int q, ql; - int r, rl; - int cur, l; -+ int next, nl; - xmlParserInputState state; - - /* -@@ -3332,6 +3333,21 @@ htmlParseComment(htmlParserCtxtPtr ctxt) { - while (IS_CHAR(cur) && - ((cur != '>') || - (r != '-') || (q != '-'))) { -+ NEXTL(l); -+ next = CUR_CHAR(nl); -+ if (next == 0) { -+ SHRINK; -+ GROW; -+ next = CUR_CHAR(nl); -+ } -+ -+ if ((q == '-') && (r == '-') && (cur == '!') && (next == '>')) { -+ htmlParseErr(ctxt, XML_ERR_COMMENT_NOT_FINISHED, -+ "Comment incorrectly closed by '--!>'", NULL, NULL); -+ cur = '>'; -+ break; -+ } -+ - if (len + 5 >= size) { - xmlChar *tmp; - -@@ -3345,18 +3361,14 @@ htmlParseComment(htmlParserCtxtPtr ctxt) { - } - buf = tmp; - } -- COPY_BUF(ql,buf,len,q); -+ COPY_BUF(ql,buf,len,q); -+ - q = r; - ql = rl; - r = cur; - rl = l; -- NEXTL(l); -- cur = CUR_CHAR(l); -- if (cur == 0) { -- SHRINK; -- GROW; -- cur = CUR_CHAR(l); -- } -+ cur = next; -+ l = nl; - } - buf[len] = 0; - if (IS_CHAR(cur)) { --- -2.25.1 - diff --git a/patches/libxml2/0011-update-automake-files-for-arm64.patch b/patches/libxml2/0006-update-automake-files-for-arm64.patch similarity index 100% rename from patches/libxml2/0011-update-automake-files-for-arm64.patch rename to patches/libxml2/0006-update-automake-files-for-arm64.patch diff --git a/patches/libxml2/0007-Fix-XPath-recursion-limit.patch b/patches/libxml2/0007-Fix-XPath-recursion-limit.patch new file mode 100644 index 0000000000..11342b33d8 --- /dev/null +++ b/patches/libxml2/0007-Fix-XPath-recursion-limit.patch @@ -0,0 +1,31 @@ +From 3e1aad4fe584747fd7d17cc7b2863a78e2d21a77 Mon Sep 17 00:00:00 2001 +From: Nick Wellnhofer <wellnhofer@aevum.de> +Date: Wed, 2 Jun 2021 17:31:49 +0200 +Subject: [PATCH] Fix XPath recursion limit + +Fix accounting of recursion depth when parsing XPath expressions. + +This silly bug introduced in commit 804c5297 could lead to spurious +errors when parsing larger expressions or XSLT documents. + +Should fix #264. +--- + xpath.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/xpath.c b/xpath.c +index 7497ba0..1aa2f1a 100644 +--- a/xpath.c ++++ b/xpath.c +@@ -10983,7 +10983,7 @@ xmlXPathCompileExpr(xmlXPathParserContextPtr ctxt, int sort) { + } + + if (xpctxt != NULL) +- xpctxt->depth -= 1; ++ xpctxt->depth -= 10; + } + + /** +-- +2.31.0 + diff --git a/patches/libxml2/0007-use-new-htmlParseLookupCommentEnd-to-find-comment-en.patch b/patches/libxml2/0007-use-new-htmlParseLookupCommentEnd-to-find-comment-en.patch deleted file mode 100644 index 3ba50bff8e..0000000000 --- a/patches/libxml2/0007-use-new-htmlParseLookupCommentEnd-to-find-comment-en.patch +++ /dev/null @@ -1,103 +0,0 @@ -From b20d746fa7cbb74716171bc49d836af99927e41e Mon Sep 17 00:00:00 2001 -From: Mike Dalessio <mike.dalessio@gmail.com> -Date: Sun, 11 Oct 2020 14:15:37 -0400 -Subject: [PATCH 2/2] use new htmlParseLookupCommentEnd to find comment ends - -Note that the caret in error messages generated during comment parsing -may have moved by one byte. - -See guidance provided on incorrectly-closed comments here: - -https://html.spec.whatwg.org/multipage/parsing.html#parse-error-incorrectly-closed-comment ---- - HTMLparser.c | 46 +++++++++++++++++++++++++++++++++++++--------- - 1 file changed, 37 insertions(+), 9 deletions(-) - -diff --git a/HTMLparser.c b/HTMLparser.c -index 4d43479..000dc3d 100644 ---- a/HTMLparser.c -+++ b/HTMLparser.c -@@ -5331,6 +5331,39 @@ htmlParseLookupChars(htmlParserCtxtPtr ctxt, const xmlChar * stop, - return (-1); - } - -+/** -+ * htmlParseLookupCommentEnd: -+ * @ctxt: an HTML parser context -+ * -+ * Try to find a comment end tag in the input stream -+ * The search includes "-->" as well as WHATWG-recommended incorrectly-closed tags. -+ * (See https://html.spec.whatwg.org/multipage/parsing.html#parse-error-incorrectly-closed-comment) -+ * This function has a side effect of (possibly) incrementing ctxt->checkIndex -+ * to avoid rescanning sequences of bytes, it DOES change the state of the -+ * parser, do not use liberally. -+ * This wraps to htmlParseLookupSequence() -+ * -+ * Returns the index to the current parsing point if the full sequence is available, -1 otherwise. -+ */ -+static int -+htmlParseLookupCommentEnd(htmlParserCtxtPtr ctxt) -+{ -+ int mark = 0; -+ int cur = CUR_PTR - BASE_PTR; -+ -+ while (mark >= 0) { -+ mark = htmlParseLookupSequence(ctxt, '-', '-', 0, 1, 1); -+ if ((mark < 0) || -+ (NXT(mark+2) == '>') || -+ ((NXT(mark+2) == '!') && (NXT(mark+3) == '>'))) { -+ return mark; -+ } -+ ctxt->checkIndex = cur + mark + 1; -+ } -+ return mark; -+} -+ -+ - /** - * htmlParseTryOrFinish: - * @ctxt: an HTML parser context -@@ -5507,8 +5540,7 @@ htmlParseTryOrFinish(htmlParserCtxtPtr ctxt, int terminate) { - cur = in->cur[0]; - if ((cur == '<') && (next == '!') && - (in->cur[2] == '-') && (in->cur[3] == '-')) { -- if ((!terminate) && -- (htmlParseLookupSequence(ctxt, '-', '-', '>', 1, 1) < 0)) -+ if ((!terminate) && (htmlParseLookupCommentEnd(ctxt) < 0)) - goto done; - #ifdef DEBUG_PUSH - xmlGenericError(xmlGenericErrorContext, -@@ -5567,8 +5599,7 @@ htmlParseTryOrFinish(htmlParserCtxtPtr ctxt, int terminate) { - next = in->cur[1]; - if ((cur == '<') && (next == '!') && - (in->cur[2] == '-') && (in->cur[3] == '-')) { -- if ((!terminate) && -- (htmlParseLookupSequence(ctxt, '-', '-', '>', 1, 1) < 0)) -+ if ((!terminate) && (htmlParseLookupCommentEnd(ctxt) < 0)) - goto done; - #ifdef DEBUG_PUSH - xmlGenericError(xmlGenericErrorContext, -@@ -5614,8 +5645,7 @@ htmlParseTryOrFinish(htmlParserCtxtPtr ctxt, int terminate) { - next = in->cur[1]; - if ((cur == '<') && (next == '!') && - (in->cur[2] == '-') && (in->cur[3] == '-')) { -- if ((!terminate) && -- (htmlParseLookupSequence(ctxt, '-', '-', '>', 1, 1) < 0)) -+ if ((!terminate) && (htmlParseLookupCommentEnd(ctxt) < 0)) - goto done; - #ifdef DEBUG_PUSH - xmlGenericError(xmlGenericErrorContext, -@@ -5871,9 +5901,7 @@ htmlParseTryOrFinish(htmlParserCtxtPtr ctxt, int terminate) { - htmlParseDocTypeDecl(ctxt); - } else if ((cur == '<') && (next == '!') && - (in->cur[2] == '-') && (in->cur[3] == '-')) { -- if ((!terminate) && -- (htmlParseLookupSequence( -- ctxt, '-', '-', '>', 1, 1) < 0)) -+ if ((!terminate) && (htmlParseLookupCommentEnd(ctxt) < 0)) - goto done; - #ifdef DEBUG_PUSH - xmlGenericError(xmlGenericErrorContext, --- -2.25.1 - diff --git a/patches/libxml2/0010-parser.c-shrink-the-input-buffer-when-appropriate.patch b/patches/libxml2/0010-parser.c-shrink-the-input-buffer-when-appropriate.patch deleted file mode 100644 index d3d9ad46a4..0000000000 --- a/patches/libxml2/0010-parser.c-shrink-the-input-buffer-when-appropriate.patch +++ /dev/null @@ -1,70 +0,0 @@ -From ca565c1edef9a455453fa8564270cc9c5813e1b9 Mon Sep 17 00:00:00 2001 -From: Mike Dalessio <mike.dalessio@gmail.com> -Date: Sun, 31 Jan 2021 09:53:56 -0500 -Subject: [PATCH] parser.c: shrink the input buffer when appropriate - -Fixes GNOME/libxml2#200 - -Also see discussions at: -- GNOME/libxml2#192 -- https://gitlab.gnome.org/nwellnhof/libxml2/-/commit/99bda1e -- https://github.com/sparklemotion/nokogiri/issues/2132 ---- - parser.c | 6 ++++++ - 1 file changed, 6 insertions(+) - -diff --git a/parser.c b/parser.c -index a7bdc7f..efde672 100644 ---- a/parser.c -+++ b/parser.c -@@ -4204,6 +4204,7 @@ xmlParseSystemLiteral(xmlParserCtxtPtr ctxt) { - } - count++; - if (count > 50) { -+ SHRINK; - GROW; - count = 0; - if (ctxt->instate == XML_PARSER_EOF) { -@@ -4291,6 +4292,7 @@ xmlParsePubidLiteral(xmlParserCtxtPtr ctxt) { - buf[len++] = cur; - count++; - if (count > 50) { -+ SHRINK; - GROW; - count = 0; - if (ctxt->instate == XML_PARSER_EOF) { -@@ -4571,6 +4573,7 @@ xmlParseCharDataComplex(xmlParserCtxtPtr ctxt, int cdata) { - } - count++; - if (count > 50) { -+ SHRINK; - GROW; - count = 0; - if (ctxt->instate == XML_PARSER_EOF) -@@ -4776,6 +4779,7 @@ xmlParseCommentComplex(xmlParserCtxtPtr ctxt, xmlChar *buf, - - count++; - if (count > 50) { -+ SHRINK; - GROW; - count = 0; - if (ctxt->instate == XML_PARSER_EOF) { -@@ -5186,6 +5190,7 @@ xmlParsePI(xmlParserCtxtPtr ctxt) { - } - count++; - if (count > 50) { -+ SHRINK; - GROW; - if (ctxt->instate == XML_PARSER_EOF) { - xmlFree(buf); -@@ -9783,6 +9788,7 @@ xmlParseCDSect(xmlParserCtxtPtr ctxt) { - sl = l; - count++; - if (count > 50) { -+ SHRINK; - GROW; - if (ctxt->instate == XML_PARSER_EOF) { - xmlFree(buf); --- -2.25.1 - diff --git a/patches/libxslt/0002-Fix-xml2-config-check-in-configure-script.patch b/patches/libxslt/0002-Fix-xml2-config-check-in-configure-script.patch new file mode 100644 index 0000000000..a882149aa3 --- /dev/null +++ b/patches/libxslt/0002-Fix-xml2-config-check-in-configure-script.patch @@ -0,0 +1,19 @@ +From: Nick Wellnhofer <wellnhofer@aevum.de> +Date: Fri, 15 Nov 2019 11:53:11 +0100 +Subject: [PATCH] Fix xml2-config check in configure script + +A 'print' option has never been supported. After a recent change to +libxml2, invalid options cause xml2-config to fail. +diff --git a/configure b/configure +index c63adc5..6061227 100755 +--- a/configure ++++ b/configure +@@ -14860,7 +14860,7 @@ PKG_CONFIG=$_save_PKG_CONFIG + fi + + +-if test "x$LIBXML_LIBS" = "x" && ${XML_CONFIG} --libs print > /dev/null 2>&1 ++if test "x$LIBXML_LIBS" = "x" && ${XML_CONFIG} --libs > /dev/null 2>&1 + then + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for libxml libraries >= $LIBXML_REQUIRED_VERSION" >&5 + $as_echo_n "checking for libxml libraries >= $LIBXML_REQUIRED_VERSION... " >&6; } diff --git a/rakelib/cext-depend.rake b/rakelib/cext-depend.rake index d8247a8c6f..da02510f18 100644 --- a/rakelib/cext-depend.rake +++ b/rakelib/cext-depend.rake @@ -1,3 +1,4 @@ +# frozen_string_literal: true require "set" namespace "cext" do @@ -8,8 +9,11 @@ namespace "cext" do ext_dir = File.dirname(extconf) Dir.chdir(ext_dir) do puts "(in #{ext_dir})" - File.exist?("depend") or FileUtils.touch("depend") + File.exist?("depend") || FileUtils.touch("depend") sh "makedepend -f depend -Y -I. *.c 2> /dev/null" + dep = File.read("depend") + dep.gsub!(%r{ \./}, " $(srcdir)/") + File.open("depend", "w") { |f| f.write(dep) } end end end diff --git a/rakelib/check-manifest.rake b/rakelib/check-manifest.rake index d5eda72380..ff4c7e7afb 100644 --- a/rakelib/check-manifest.rake +++ b/rakelib/check-manifest.rake @@ -12,10 +12,11 @@ task :check_manifest do .github .vagrant .yardoc - concourse coverage doc gems + nokogumbo-import + oci-images patches pkg ports @@ -32,9 +33,8 @@ task :check_manifest do .cross_rubies .editorconfig .gitignore + .gitmodules .yardopts - appveyor.yml - nokogiri.gemspec CHANGELOG.md CODE_OF_CONDUCT.md CONTRIBUTING.md @@ -44,8 +44,13 @@ task :check_manifest do SECURITY.md STANDARD_RESPONSES.md Vagrantfile - [0-9]+-.* - [a-z.]+.(log|out) + [a-z]*.{log,out} + [0-9]* + appveyor.yml + gumbo-parser/test/* + lib/nokogiri/**/nokogiri.{jar,so} + lib/nokogiri/nokogiri.{jar,so} + nokogiri.gemspec ] intended_directories = Dir.children(".") @@ -54,10 +59,11 @@ task :check_manifest do intended_files = Dir.children(".") .select { |filename| File.file?(filename) } - .reject { |filename| ignore_files.any? { |ig| File.fnmatch?(ig, filename) } } + .reject { |filename| ignore_files.any? { |ig| File.fnmatch?(ig, filename, File::FNM_EXTGLOB) } } intended_files += Dir.glob(intended_directories.map { |d| File.join(d, "/**/*") }) .select { |filename| File.file?(filename) } + .reject { |filename| ignore_files.any? { |ig| File.fnmatch?(ig, filename, File::FNM_EXTGLOB) } } .sort spec_files = raw_gemspec.files.sort diff --git a/rakelib/clean.rake b/rakelib/clean.rake index 2ea8b962a0..098acde375 100644 --- a/rakelib/clean.rake +++ b/rakelib/clean.rake @@ -1,6 +1,5 @@ require "rake/clean" CLEAN.add( - "concourse/images/*.generated", "coverage", "ext/nokogiri/include", "lib/nokogiri/[0-9].[0-9]", diff --git a/rakelib/concourse.rake b/rakelib/concourse.rake deleted file mode 100644 index ae2ec124f2..0000000000 --- a/rakelib/concourse.rake +++ /dev/null @@ -1,7 +0,0 @@ -require "concourse" - -Concourse.new("nokogiri", fly_target: "ci", format: true) do |c| - c.add_pipeline "nokogiri", "nokogiri.yml", ytt: true - c.add_pipeline "nokogiri-pr", "nokogiri-pr.yml", ytt: true - c.add_pipeline "nokogiri-truffleruby", "nokogiri-truffleruby.yml", ytt: true -end diff --git a/rakelib/css-generate.rake b/rakelib/css-generate.rake index 80f3d67c95..0f436bcfea 100644 --- a/rakelib/css-generate.rake +++ b/rakelib/css-generate.rake @@ -22,5 +22,3 @@ end task "compile" => "css:generate" task "check_manifest" => "css:generate" - -task "generate" => "css:generate" # backwards compatibility, see concourse/tasks/rake-test/run.sh diff --git a/rakelib/docker.rake b/rakelib/docker.rake index 054558c04a..da4c8711a9 100644 --- a/rakelib/docker.rake +++ b/rakelib/docker.rake @@ -2,7 +2,14 @@ # docker docker docker # namespace "docker" do - IMAGE_DIR = "concourse/images" + IMAGE_DIR = "oci-images/nokogiri-test" + RUBIES = { + :mri=>["2.5", "2.6", "2.7", "3.0"], + :jruby=>["9.2"], + :rbx=>["latest"], + :windows=>["2.3", "2.4", "2.5", "2.6"], + :truffle=>["stable", "nightly"], + } def docker_tag_for(engine, version = nil) [engine, version].compact.join("-") @@ -19,9 +26,9 @@ namespace "docker" do def docker_files_each Dir[File.join(IMAGE_DIR, "Dockerfile.*.erb")].each do |template_path| tag_or_engine = File.basename(template_path).gsub(/Dockerfile\.(.*)\.erb/, '\1').to_sym - if Concourse::RUBIES.keys.include?(tag_or_engine) + if RUBIES.keys.include?(tag_or_engine) # engine - Concourse::RUBIES[tag_or_engine].each do |version| + RUBIES[tag_or_engine].each do |version| dockerfile_path = docker_file_for(tag_or_engine, version) yield File.read(template_path), dockerfile_path, version, docker_image_for(tag_or_engine, version) end @@ -35,6 +42,7 @@ namespace "docker" do desc "Generate Dockerfiles" task "generate" do + require "erb" docker_files_each do |template, dockerfile_path, version, _| puts "writing #{dockerfile_path} ..." File.open(dockerfile_path, "w") do |dockerfile| @@ -76,3 +84,5 @@ end desc "Build and push a docker image for testing" task "docker" => ["docker:generate", "docker:pull", "docker:build", "docker:push"] + +CLEAN.add("oci-images/nokogiri-test/*.generated") diff --git a/rakelib/extensions.rake b/rakelib/extensions.rake index bc5067e3f6..1fd24d4973 100644 --- a/rakelib/extensions.rake +++ b/rakelib/extensions.rake @@ -307,7 +307,9 @@ if java? jruby_home = RbConfig::CONFIG['prefix'] jars = ["#{jruby_home}/lib/jruby.jar"] + FileList['lib/*.jar'] + # Keep the extension C files because they have docstrings (and Java files don't) ext.gem_spec.files.reject! { |path| File.fnmatch?("ext/nokogiri/*.h", path) } + ext.gem_spec.files.reject! { |path| File.fnmatch?("gumbo-parser/**/*", path) } ext.ext_dir = 'ext/java' ext.lib_dir = 'lib/nokogiri' @@ -326,7 +328,7 @@ else dependencies = YAML.load_file("dependencies.yml") task gem_build_path do - NOKOGIRI_SPEC.files.reject! { |f| f =~ %r{\.(java|jar)$} } + NOKOGIRI_SPEC.files.reject! { |path| File.fnmatch?("**/*.{java,jar}", path, File::FNM_EXTGLOB) } ["libxml2", "libxslt"].each do |lib| version = dependencies[lib]["version"] @@ -345,7 +347,8 @@ else end Rake::ExtensionTask.new("nokogiri", NOKOGIRI_SPEC) do |ext| - ext.gem_spec.files.reject! { |f| f =~ %r{\.(java|jar)$} } + ext.source_pattern = "*.{c,cc,cpp,h}" + ext.gem_spec.files.reject! { |path| File.fnmatch?("**/*.{java,jar}", path, File::FNM_EXTGLOB) } ext.lib_dir = File.join(*['lib', 'nokogiri', ENV['FAT_DIR']].compact) ext.config_options << ENV['EXTOPTS'] @@ -354,6 +357,7 @@ else ext.cross_config_options << "--enable-cross-build" ext.cross_compiling do |spec| spec.files.reject! { |path| File.fnmatch?('ports/*', path) } + spec.files.reject! { |path| File.fnmatch?("gumbo-parser/**/*", path) } spec.dependencies.reject! { |dep| dep.name=='mini_portile2' } # when pre-compiling a native gem, package all the C headers sitting in ext/nokogiri/include diff --git a/rakelib/format.rake b/rakelib/format.rake index 9ed8542bef..306227c5a5 100644 --- a/rakelib/format.rake +++ b/rakelib/format.rake @@ -3,7 +3,7 @@ namespace "format" do def assert_astyle require "mkmf" - find_executable("astyle") || raise("Could not find command 'astyle'") + find_executable0("astyle") || raise("Could not find command 'astyle'") end def astyle_args @@ -13,6 +13,7 @@ namespace "format" do [ # indentation "--indent=spaces=2", + "--indent-switches", # brackets "--style=1tbs", @@ -37,11 +38,11 @@ namespace "format" do end def astyle_c_files - FileList.new("ext/nokogiri/*.[ch]") + FileList.new("ext/**/*.[ch]") end def astyle_java_files - FileList.new("ext/java/nokogiri/**/*.java") + FileList.new("ext/**/*.java") end desc "Format Nokogiri's C code" @@ -60,7 +61,6 @@ namespace "format" do CLEAN.add(astyle_c_files.map { |f| "#{f}.orig" }) CLEAN.add(astyle_java_files.map { |f| "#{f}.orig" }) - CLEAN.add("mkmf.log") # because of find_executable end task "format" => ["format:c", "format:java"] diff --git a/rakelib/gumbo.rake b/rakelib/gumbo.rake new file mode 100644 index 0000000000..b48e5f574b --- /dev/null +++ b/rakelib/gumbo.rake @@ -0,0 +1,29 @@ + +namespace "gumbo" do + gtest_pkg = "gumbo-parser/googletest" + gtest_lib = File.join(gtest_pkg, "make/gtest_main.a") + + file gtest_lib => gtest_pkg do + sh("make -C gumbo-parser/googletest/make gtest_main.a") + end + + file gtest_pkg do + sh(<<~EOF) + curl -L https://github.com/google/googletest/archive/release-1.8.0.tar.gz | \ + tar zxf - --strip-components 1 -C gumbo-parser googletest-release-1.8.0/googletest + EOF + end + + task "test" => gtest_lib do + sh("make -j2 -C gumbo-parser") + end + + task "clean" do + sh("make -j2 -C gumbo-parser clean") + end + + CLOBBER.add(gtest_pkg) +end + +task "gumbo" => "gumbo:test" +task "clean" => "gumbo:clean" diff --git a/rakelib/set-version-to-timestamp.rake b/rakelib/set-version-to-timestamp.rake index 5a028d3d40..a396636458 100644 --- a/rakelib/set-version-to-timestamp.rake +++ b/rakelib/set-version-to-timestamp.rake @@ -1,7 +1,7 @@ # frozen_string_literal: true desc "Temporarily set Nokogiri::VERSION to a unique timestamp" task "set-version-to-timestamp" do - # this task is used by concourse/tasks/gem-test/gem-build.sh + # this task is used by scripts/test-gem-build # to test building, packaging, and installing a Nokogiri gem version_constant_re = /^\s*VERSION\s*=\s*["'](.*)["']$/ diff --git a/rakelib/test.rake b/rakelib/test.rake index b9a64a9743..dcbabec066 100644 --- a/rakelib/test.rake +++ b/rakelib/test.rake @@ -75,6 +75,7 @@ def nokogiri_test_task_configuration(t) t.libs << "test" t.test_files = FileList["test/**/test_*.rb"] t.verbose = true + t.options = "-v" if ENV["CI"] end Rake::TestTask.new do |t| diff --git a/scripts/build-gems b/scripts/build-gems index b5a0f63d32..37b448d26c 100755 --- a/scripts/build-gems +++ b/scripts/build-gems @@ -39,3 +39,7 @@ cp -v pkg/nokogiri-*darwin*.gem gems # test those gem files! $(dirname $0)/test-gem-set gems/*.gem + +pushd gems + ls *.gem | sort | xargs sha256sum +popd diff --git a/scripts/generate-html5-entities b/scripts/generate-html5-entities new file mode 100755 index 0000000000..1d71784583 --- /dev/null +++ b/scripts/generate-html5-entities @@ -0,0 +1,96 @@ +#!/usr/bin/env ruby + +# Copyright 2018 Stephen Checkoway +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The output of this script should be compiled with ragel 6.10 which is the +# current stable version. Version 7 changes some stuff but what, precisely, +# isn't documented yet. + +require 'psych' +require 'open-uri' + +ENTITIES_URL = 'https://html.spec.whatwg.org/entities.json' + +# Modify the json to remove the characters item because it contains surrogate +# pairs that Psych errors on. +entities = URI.parse(ENTITIES_URL).read +entities.gsub!(/, "characters"[^}]*/, '') +entities = Psych.safe_load(entities) + +without_semicolon = {} +with_semicolon = {} +entities.each do |key, val| + if key.end_with?(';') + with_semicolon[key[1..-1]] = val['codepoints'] + else + without_semicolon[key[1..-1]] = val['codepoints'] + end +end + +without_semicolon.each do |entity, codepoints| + puts "#{entity} has the wrong number of codepoints" unless [1,2].include?(codepoints.length) + with = "#{entity};" + unless with_semicolon.key?(with) + puts("#{entity} but not #{with}") + next + end + if codepoints != with_semicolon[with] + puts("#{entity} and #{with} have different codepoints!") + end +end + +puts(<<-EOF) +// Generated from <#{ENTITIES_URL}> by entities.rb +#include "char_ref.h" +%%{ +machine named_char_ref; +named_char_ref := |* +EOF +with_semicolon.each do |entity, codepoints| + without = entity.chop + if without_semicolon.key?(without) + pattern = "'#{without}' . ';'?" + else + pattern = "'#{entity}'" + end + code = codepoints.map.with_index { |cp, idx| sprintf("output[%d]=%#04x", idx, cp) } + puts(" #{pattern} => {#{code.join('; ')}; fbreak;};") +end +puts(<<-EOF) +*|; +write data noerror nofinal noentry noprefix; +}%% + +size_t match_named_char_ref ( + const char *str, + size_t size, + int output[2] +) { + int cs; + int act; + const char *p = str; + const char *pe = str + size; + const char *const eof = pe; + const char *ts; + const char *te; + output[0] = output[1] = kGumboNoChar; + %% write init; + %% write exec; + (void)ts; + (void)act; + size = p - str; + return cs >= %%{ write first_final; }%%? size:0; +} +EOF diff --git a/scripts/test-exported-symbols b/scripts/test-exported-symbols index 7a0ef44552..27c2c97e88 100755 --- a/scripts/test-exported-symbols +++ b/scripts/test-exported-symbols @@ -31,11 +31,13 @@ noko_so = find_that_file("lib/**/nokogiri.so") libxml2_archive = find_that_file("ports/**/libxml2.a") libxslt_archive = find_that_file("ports/**/libxslt.a") libexslt_archive = find_that_file("ports/**/libexslt.a") +libgumbo_archive = find_that_file("tmp/**/libgumbo.a") symbols = external_symbols(noko_so) symbols -= external_symbols(libxml2_archive) symbols -= external_symbols(libxslt_archive) symbols -= external_symbols(libexslt_archive) +symbols -= external_symbols(libgumbo_archive) puts "#{noko_so} exports the following surprising symbols:" symbols.to_a.sort.each do |symbol| diff --git a/scripts/test-gem-build b/scripts/test-gem-build new file mode 100755 index 0000000000..243253ad7b --- /dev/null +++ b/scripts/test-gem-build @@ -0,0 +1,43 @@ +#! /usr/bin/env bash +# +# run as part of CI, see gem-install.yml +# +if [[ $# -lt 2 ]] ; then + echo "usage: $(basename $0) <output_dir> <platform>" + exit 1 +fi + +set -e -u + +OUTPUT_DIR=$1 +BUILD_NATIVE_GEM=$2 + +test -e /etc/os-release && cat /etc/os-release + +if [[ "${XRUBIES:-}" != "" ]] ; then + # normally part of rake-compiler-dock "runas" + . /etc/rubybashrc + ln -s /usr/local/rake-compiler "$HOME"/.rake-compiler + export RAKE_EXTENSION_TASK_NO_NATIVE=true +fi + +set -x + +bundle install --local || bundle install +bundle exec rake set-version-to-timestamp + +if [[ "${BUILD_NATIVE_GEM}" == "ruby" ]] ; then + # TODO we're only compiling so that we retrieve libxml2/libxslt tarballs, we can do better. + bundle exec rake clean compile + bundle exec rake gem +elif [[ "${BUILD_NATIVE_GEM}" == "java" ]] ; then + bundle exec rake java gem +else + bundle exec rake gem:${BUILD_NATIVE_GEM}:builder FORCE_CROSS_COMPILING=true +fi + +./scripts/test-gem-file-contents pkg/nokogiri*.gem + +mkdir -p ${OUTPUT_DIR} +cp -v pkg/nokogiri*.gem ${OUTPUT_DIR} +ls -l ${OUTPUT_DIR}/* diff --git a/scripts/test-gem-file-contents b/scripts/test-gem-file-contents index 08bee5ff73..c567bf40d5 100755 --- a/scripts/test-gem-file-contents +++ b/scripts/test-gem-file-contents @@ -78,7 +78,7 @@ describe File.basename(gemfile) do assert_operator(actual, :>, 60, "expected gemfile to contain more than #{actual} files") end - it "gemspec is a Gem::Specfication" do + it "gemspec is a Gem::Specification" do assert_equal(Gem::Specification, gemspec.class) end end @@ -93,6 +93,19 @@ describe File.basename(gemfile) do end describe "ruby platform" do + it "depends on mini_portile2" do + assert(gemspec.dependencies.find { |d| d.name == "mini_portile2" }) + end + + it "contains ext/nokogiri C and header files" do + assert_operator(gemfile_contents.grep(%r{^ext/nokogiri/.*\.c}).length, :>, 20) + assert_operator(gemfile_contents.grep(%r{^ext/nokogiri/.*\.h}).length, :>, 0) + end + + it "includes C files in extra_rdoc_files" do + assert_operator(gemspec.extra_rdoc_files.grep(%r{ext/nokogiri/.*\.c$}).length, :>, 10) + end + it "contains the port files" do actual_ports = gemfile_contents.grep(%r{^ports/}) assert_equal(1, actual_ports.grep(/libxml2-\d+\.\d+\.\d+\.tar\.gz/).length, @@ -106,31 +119,37 @@ describe File.basename(gemfile) do assert_operator(gemfile_contents.grep(%r{^patches/}).length, :>, 0) end - it "contains ext/nokogiri C and header files" do - assert_operator(gemfile_contents.grep(%r{^ext/nokogiri/.*\.c}).length, :>, 20) - assert_operator(gemfile_contents.grep(%r{^ext/nokogiri/.*\.h}).length, :>, 0) - end - it "does not contain packaged libraries' header files" do # these files are present after installation if the packaged libraries are used assert_empty(gemfile_contents.grep(%r{^ext/nokogiri/include/})) end + it "contains the gumbo parser source code" do + assert_includes(gemfile_contents, "gumbo-parser/src/Makefile") + assert_operator(gemfile_contents.grep(%r{^gumbo-parser/src/.*\.c}).length, :>, 10) + assert_operator(gemfile_contents.grep(%r{^gumbo-parser/src/.*\.h}).length, :>, 10) + end + it "does not contain java files" do assert_empty(gemfile_contents.grep(%r{^ext/java/})) assert_empty(gemfile_contents.grep(/.*\.jar$/)) end + end if gemspec.platform == Gem::Platform::RUBY - it "depends on mini_portile2" do - assert(gemspec.dependencies.find { |d| d.name == "mini_portile2" }) + describe "native platform" do + it "does not depend on mini_portile2" do + refute(gemspec.dependencies.find { |d| d.name == "mini_portile2" }) + end + + it "contains ext/nokogiri C and header files" do + assert_operator(gemfile_contents.grep(%r{^ext/nokogiri/.*\.c}).length, :>, 20) + assert_operator(gemfile_contents.grep(%r{^ext/nokogiri/.*\.h}).length, :>, 20) end it "includes C files in extra_rdoc_files" do assert_operator(gemspec.extra_rdoc_files.grep(%r{ext/nokogiri/.*\.c$}).length, :>, 10) end - end if gemspec.platform == Gem::Platform::RUBY - describe "native platform" do it "does not contain the port files" do assert_empty(gemfile_contents.grep(%r{^ports/})) end @@ -139,30 +158,21 @@ describe File.basename(gemfile) do assert_empty(gemfile_contents.grep(%r{^patches/})) end - it "contains ext/nokogiri C and header files" do - assert_operator(gemfile_contents.grep(%r{^ext/nokogiri/.*\.c}).length, :>, 20) - assert_operator(gemfile_contents.grep(%r{^ext/nokogiri/.*\.h}).length, :>, 20) - end - it "contains packaged libraries' header files" do assert_includes(gemfile_contents, "ext/nokogiri/include/libxml2/libxml/tree.h") assert_includes(gemfile_contents, "ext/nokogiri/include/libxslt/xslt.h") assert_includes(gemfile_contents, "ext/nokogiri/include/libexslt/exslt.h") end + it "does not contain the gumbo parser source code" do + assert_empty(gemfile_contents.grep(%r{^gumbo-parser/src/})) + end + it "does not contain java files" do assert_empty(gemfile_contents.grep(%r{^ext/java/})) assert_empty(gemfile_contents.grep(/.*\.jar$/)) end - it "does not depend on mini_portile2" do - refute(gemspec.dependencies.find { |d| d.name == "mini_portile2" }) - end - - it "includes C files in extra_rdoc_files" do - assert_operator(gemspec.extra_rdoc_files.grep(%r{ext/nokogiri/.*\.c$}).length, :>, 10) - end - it "contains expected shared library files " do native_ruby_versions.each do |version| actual = gemfile_contents.find do |p| @@ -184,15 +194,12 @@ describe File.basename(gemfile) do end if gemspec.platform.is_a?(Gem::Platform) && gemspec.platform.cpu describe "java platform" do - it "does not contain the port files" do - assert_empty(gemfile_contents.grep(%r{^ports/})) - end - - it "does not contain the patch files" do - assert_empty(gemfile_contents.grep(%r{^patches/})) + it "does not depend on mini_portile2" do + refute(gemspec.dependencies.find { |d| d.name == "mini_portile2" }) end it "contains ext/nokogiri C files" do + # Note: we keep the C files because they have docstrings and Java files don't assert_operator(gemfile_contents.grep(%r{^ext/nokogiri/.*\.c}).length, :>, 20) end @@ -200,10 +207,30 @@ describe File.basename(gemfile) do assert_empty(gemfile_contents.grep(%r{^ext/nokogiri/.*\.h})) end + it "includes C files in extra_rdoc_files" do + assert_operator(gemspec.extra_rdoc_files.grep(%r{ext/nokogiri/.*\.c$}).length, :>, 10) + end + + it "does not contain the port files" do + assert_empty(gemfile_contents.grep(%r{^ports/})) + end + + it "does not contain the patch files" do + assert_empty(gemfile_contents.grep(%r{^patches/})) + end + it "does not contain packaged libraries' header files" do assert_empty(gemfile_contents.grep(%r{^ext/nokogiri/include/})) end + it "does not contain the gumbo parser source code" do + assert_empty(gemfile_contents.grep(%r{^gumbo-parser/src/})) + end + + it "contains java source files" do + assert_operator(gemfile_contents.grep(%r{^ext/java/.*\.java}).length, :>, 20) + end + it "contains the java jar files" do actual_jars = gemfile_contents.grep(/.*\.jar$/) expected_jars = [ @@ -221,9 +248,5 @@ describe File.basename(gemfile) do assert_equal(1, actual_jars.grep(%r{/#{jar}\.jar$}).length, "expected to contain #{jar}.jar") end end - - it "does not depend on mini_portile2" do - refute(gemspec.dependencies.find { |d| d.name == "mini_portile2" }) - end end if gemspec.platform == Gem::Platform.new("java") end diff --git a/scripts/test-gem-install b/scripts/test-gem-install new file mode 100755 index 0000000000..c650ef5eeb --- /dev/null +++ b/scripts/test-gem-install @@ -0,0 +1,42 @@ +#! /usr/bin/env bash +# +# run as part of CI, see gem-install.yml +# +if [[ $# -lt 1 ]] ; then + echo "usage: $(basename $0) <gems_dir> [install_flags]" + exit 1 +fi + +GEMS_DIR=$1 +shift +INSTALL_FLAGS=$* + +test -e /etc/os-release && cat /etc/os-release + +set -e -x -u + +pushd $GEMS_DIR + + gemfile=$(ls *.gem | head -n1) + ls -l ${gemfile} + gem install --no-document ${gemfile} -- ${INSTALL_FLAGS} + gem list -d nokogiri + nokogiri -v + +popd + +if [ -n "${BUNDLE_APP_CONFIG:-}" ] ; then + export BUNDLE_CACHE_PATH="${BUNDLE_APP_CONFIG}/cache" +fi + +bundle install --local || bundle install + +rm -rf lib ext # ensure we don't use the local files +rake test + +./scripts/test-gem-installation + +# delete the Gemfile because that's confusing to older versions of rubygems (e.g., bionic32) +rm -f Gemfile Gemfile.lock +./scripts/test-nokogumbo-compatibility 2.0.4 +./scripts/test-nokogumbo-compatibility 2.0.5 diff --git a/scripts/test-gem-installation b/scripts/test-gem-installation index d2b64bdcc5..2e3745577e 100755 --- a/scripts/test-gem-installation +++ b/scripts/test-gem-installation @@ -13,7 +13,7 @@ # this line needs to come before the bundler bit, to assert that we're running against an # already-installed version (and not some other version that bundler/inline might install if it came # first) -gemspec = Gem::Specification.find_by_name('nokogiri') +gemspec = Gem::Specification.find_all_by_name("nokogiri").sort_by(&:version).last raise "could not find installed gem" unless gemspec require "bundler/inline" diff --git a/scripts/test-gem-set b/scripts/test-gem-set index 253a792d44..523cb36aa8 100755 --- a/scripts/test-gem-set +++ b/scripts/test-gem-set @@ -4,7 +4,7 @@ # - test-gem-file-contents # - conditionally, if the local system can do it, test-gem-installation # -source "$HOME/.rvm/scripts/rvm" +eval "$(rbenv init - bash)" set -o errexit set -o pipefail @@ -18,9 +18,10 @@ function remove_all_nokogiris { function install_and_test { gem=$1 if [[ $gem =~ "java" ]] ; then - rvm use jruby + jruby_version=$(rbenv versions --bare | fgrep jruby | sort | tail -1) + rbenv shell ${jruby_version} else - rvm use default + rbenv shell --unset fi remove_all_nokogiris gem install --local $gem @@ -35,7 +36,7 @@ function install_and_test { gems=$* -rvm use default +rbenv shell --unset for gem in $gems ; do ./scripts/test-gem-file-contents $gem diff --git a/scripts/test-nokogumbo-compatibility b/scripts/test-nokogumbo-compatibility index 3c57b31df7..6ccd53f5d4 100755 --- a/scripts/test-nokogumbo-compatibility +++ b/scripts/test-nokogumbo-compatibility @@ -17,23 +17,31 @@ end # this line needs to come before the bundler bit, to assert that we're running against an # already-installed version (and not some other version that bundler/inline might install if it came # first) -gemspec = Gem::Specification.find_by_name('nokogiri') +gemspec = Gem::Specification.find_all_by_name("nokogiri").sort_by(&:version).last raise "could not find installed gem" unless gemspec require "bundler/inline" +nokogumbo_version = if ARGV[0] =~ /\d+\.\d+\.\d+/ + ARGV[0] +end + gemfile(true) do source "https://rubygems.org" gem "minitest" gem "minitest-reporters" gem "nokogiri", "=#{gemspec.version}" - gem "nokogumbo" + if nokogumbo_version + gem "nokogumbo", "=#{nokogumbo_version}" + else + gem "nokogumbo" + end end -nokogumbo_gemspec = Gem::Specification.find_by_name('nokogumbo') +nokogumbo_gemspec = Gem::Specification.find_by_name("nokogumbo") -require 'nokogumbo' -require 'yaml' +require "nokogumbo" +require "yaml" if ARGV.include?("-v") puts "---------- Nokogiri version info ----------" @@ -54,6 +62,8 @@ Minitest::Reporters.use!([Minitest::Reporters::SpecReporter.new]) puts "Testing #{gemspec.full_name} installed in #{gemspec.base_dir}" describe gemspec.full_name do it "works" do - assert(Nokogiri::HTML5::Document.parse("<div>ahoy</div>")) + doc = Nokogiri::HTML5::Document.parse("<div>ahoy</div>") + assert(doc) + assert_equal("ahoy", doc.at_css("/html/body/div").text) end end diff --git a/suppressions/nokogiri_ruby-3.1.supp b/suppressions/nokogiri_ruby-3.1.supp new file mode 100644 index 0000000000..351a358f69 --- /dev/null +++ b/suppressions/nokogiri_ruby-3.1.supp @@ -0,0 +1,69 @@ +{ + ruby 3.1 startup + Memcheck:Addr4 + fun:iseq_peephole_optimize + fun:iseq_optimize* + fun:iseq_setup_insn + fun:iseq_setup_insn + fun:rb_iseq_compile_node + fun:rb_iseq_new_with_opt + fun:new_child_iseq + ... + # fun:iseq_compile_each0 + # fun:iseq_compile_each + # fun:iseq_compile_each0 + # fun:iseq_compile_each.constprop.1 + # fun:rb_iseq_compile_node + # fun:rb_iseq_new_with_opt + # fun:new_child_iseq + # fun:iseq_compile_each0 + # fun:iseq_compile_each + # fun:iseq_compile_each0 + # fun:iseq_compile_each.constprop.1 + # fun:rb_iseq_compile_node + # fun:rb_iseq_new_with_opt + # fun:rb_iseq_new_top + # fun:load_iseq_eval + # fun:require_internal + # fun:rb_require_string + # fun:vm_call_cfunc_with_frame + # fun:vm_sendish + # fun:vm_exec_core + # fun:rb_vm_exec + # fun:load_iseq_eval + # fun:require_internal + # fun:rb_require_string + # fun:vm_call_cfunc_with_frame + # fun:vm_call_method_each_type + # fun:vm_call_alias + # fun:vm_sendish + # fun:vm_exec_core + # fun:rb_vm_exec + # fun:load_iseq_eval + # fun:require_internal + # fun:rb_require_string + # fun:vm_call_cfunc_with_frame + # fun:vm_call_method_each_type + # fun:vm_call_alias + # fun:vm_sendish + # fun:vm_exec_core + # fun:rb_vm_exec + # fun:load_iseq_eval + # fun:require_internal + # fun:rb_require_string + # fun:vm_call_cfunc_with_frame + # fun:vm_call_method_each_type + # fun:vm_call_alias + # fun:vm_sendish + # fun:vm_exec_core + # fun:rb_vm_exec + # fun:load_iseq_eval + # fun:require_internal + # fun:rb_require_string + # fun:vm_call_cfunc_with_frame + # fun:vm_call_method_each_type + # fun:vm_call_alias + # fun:vm_sendish + # fun:vm_exec_core + # fun:rb_vm_exec +} diff --git a/test/files/huge-xpath-query.txt b/test/files/huge-xpath-query.txt new file mode 100644 index 0000000000..edcdf0dd43 --- /dev/null +++ b/test/files/huge-xpath-query.txt @@ -0,0 +1 @@ +( (((sum(ct:AIACapitalAllowancesInc) + sum(../ct:NotIncluded/ct:AIACapitalAllowancesNotInc))) - (( ( ceiling( ( number((round(date:seconds(normalize-space('2015-04-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2016-01-01')) div 86400))) * ( ( number((round(date:seconds(normalize-space('2015-04-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400))) * number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400) < round(date:seconds(normalize-space('2016-01-01')) div 86400))) * (round(date:seconds(normalize-space(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D')))) div 86400) - round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400)) ) + ( number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2015-04-01')) div 86400))) * number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400) < round(date:seconds(normalize-space('2016-01-01')) div 86400))) * (round(date:seconds(normalize-space(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D')))) div 86400) - round(date:seconds(normalize-space('2015-04-01')) div 86400)) ) + ( number((round(date:seconds(normalize-space('2015-04-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400))) * number((round(date:seconds(normalize-space('2016-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * (round(date:seconds(normalize-space('2016-01-01')) div 86400) - round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400)) ) + ( number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2015-04-01')) div 86400))) * number((round(date:seconds(normalize-space('2016-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * (round(date:seconds(normalize-space('2016-01-01')) div 86400) - round(date:seconds(normalize-space('2015-04-01')) div 86400)) ) ) * 500000 div (366 - number( ( ( (round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2015-04-01')) div 86400)) and (round(date:seconds(normalize-space('2015-04-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400)) ) or ( (round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2016-01-01')) div 86400)) and (round(date:seconds(normalize-space('2016-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400)) ) or ( (round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2019-01-01')) div 86400)) and (round(date:seconds(normalize-space('2019-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400)) ) or ( (round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2022-01-01')) div 86400)) and (round(date:seconds(normalize-space('2022-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400)) ) ) or ((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400) - round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400)) + 1 < 366) )) ) ) + ceiling( ( number((round(date:seconds(normalize-space('2016-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2019-01-01')) div 86400))) * ( ( number((round(date:seconds(normalize-space('2016-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400))) * number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400) < round(date:seconds(normalize-space('2019-01-01')) div 86400))) * (round(date:seconds(normalize-space(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D')))) div 86400) - round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400)) ) + ( number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2016-01-01')) div 86400))) * number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400) < round(date:seconds(normalize-space('2019-01-01')) div 86400))) * (round(date:seconds(normalize-space(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D')))) div 86400) - round(date:seconds(normalize-space('2016-01-01')) div 86400)) ) + ( number((round(date:seconds(normalize-space('2016-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400))) * number((round(date:seconds(normalize-space('2019-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * (round(date:seconds(normalize-space('2019-01-01')) div 86400) - round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400)) ) + ( number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2016-01-01')) div 86400))) * number((round(date:seconds(normalize-space('2019-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * (round(date:seconds(normalize-space('2019-01-01')) div 86400) - round(date:seconds(normalize-space('2016-01-01')) div 86400)) ) ) * 200000 div (366 - number( ( ( (round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2015-04-01')) div 86400)) and (round(date:seconds(normalize-space('2015-04-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400)) ) or ( (round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2016-01-01')) div 86400)) and (round(date:seconds(normalize-space('2016-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400)) ) or ( (round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2019-01-01')) div 86400)) and (round(date:seconds(normalize-space('2019-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400)) ) or ( (round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2022-01-01')) div 86400)) and (round(date:seconds(normalize-space('2022-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400)) ) ) or ((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400) - round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400)) + 1 < 366) )) ) ) + ceiling( ( number((round(date:seconds(normalize-space('2019-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2022-01-01')) div 86400))) * ( ( number((round(date:seconds(normalize-space('2019-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400))) * number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400) < round(date:seconds(normalize-space('2022-01-01')) div 86400))) * (round(date:seconds(normalize-space(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D')))) div 86400) - round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400)) ) + ( number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2019-01-01')) div 86400))) * number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400) < round(date:seconds(normalize-space('2022-01-01')) div 86400))) * (round(date:seconds(normalize-space(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D')))) div 86400) - round(date:seconds(normalize-space('2019-01-01')) div 86400)) ) + ( number((round(date:seconds(normalize-space('2019-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400))) * number((round(date:seconds(normalize-space('2022-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * (round(date:seconds(normalize-space('2022-01-01')) div 86400) - round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400)) ) + ( number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2019-01-01')) div 86400))) * number((round(date:seconds(normalize-space('2022-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * (round(date:seconds(normalize-space('2022-01-01')) div 86400) - round(date:seconds(normalize-space('2019-01-01')) div 86400)) ) ) * 1000000 div (366 - number( ( ( (round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2015-04-01')) div 86400)) and (round(date:seconds(normalize-space('2015-04-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400)) ) or ( (round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2016-01-01')) div 86400)) and (round(date:seconds(normalize-space('2016-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400)) ) or ( (round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2019-01-01')) div 86400)) and (round(date:seconds(normalize-space('2019-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400)) ) or ( (round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2022-01-01')) div 86400)) and (round(date:seconds(normalize-space('2022-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400)) ) ) or ((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400) - round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400)) + 1 < 366) )) ) ) + ceiling( ( number((round(date:seconds(normalize-space('2022-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * ( ( number((round(date:seconds(normalize-space('2022-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400))) * (round(date:seconds(normalize-space(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D')))) div 86400) - round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400)) ) + ( number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2022-01-01')) div 86400))) * (round(date:seconds(normalize-space(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D')))) div 86400) - round(date:seconds(normalize-space('2022-01-01')) div 86400)) ) ) * 200000 div (366 - number( ( ( (round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2015-04-01')) div 86400)) and (round(date:seconds(normalize-space('2015-04-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400)) ) or ( (round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2016-01-01')) div 86400)) and (round(date:seconds(normalize-space('2016-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400)) ) or ( (round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2019-01-01')) div 86400)) and (round(date:seconds(normalize-space('2019-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400)) ) or ( (round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2022-01-01')) div 86400)) and (round(date:seconds(normalize-space('2022-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400)) ) ) or ((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400) - round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400)) + 1 < 366) )) ) ) ) )) < 0.001) ) or ( ( date:day-in-month(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) = 1 and date:day-in-month(normalize-space(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1DT12H')))) = 1 ) and ( (((sum(ct:AIACapitalAllowancesInc) + sum(../ct:NotIncluded/ct:AIACapitalAllowancesNotInc))) - ((( ceiling( ( number((round(date:seconds(normalize-space('2015-04-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2016-01-01')) div 86400))) * ( ( number((round(date:seconds(normalize-space('2015-04-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400))) * number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400) < round(date:seconds(normalize-space('2016-01-01')) div 86400))) * (12 * (date:year(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D'))) - date:year(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) + date:month-in-year(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D'))) - date:month-in-year(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) ) + ( number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2015-04-01')) div 86400))) * number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400) < round(date:seconds(normalize-space('2016-01-01')) div 86400))) * (12 * (date:year(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D'))) - date:year('2015-04-01')) + date:month-in-year(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D'))) - date:month-in-year('2015-04-01')) ) + ( number((round(date:seconds(normalize-space('2015-04-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400))) * number((round(date:seconds(normalize-space('2016-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * (12 * (date:year('2016-01-01') - date:year(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) + date:month-in-year('2016-01-01') - date:month-in-year(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) ) + ( number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2015-04-01')) div 86400))) * number((round(date:seconds(normalize-space('2016-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * (12 * (date:year('2016-01-01') - date:year('2015-04-01')) + date:month-in-year('2016-01-01') - date:month-in-year('2015-04-01')) ) ) * 500000 div 12 ) ) + ceiling( ( number((round(date:seconds(normalize-space('2016-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2019-01-01')) div 86400))) * ( ( number((round(date:seconds(normalize-space('2016-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400))) * number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400) < round(date:seconds(normalize-space('2019-01-01')) div 86400))) * (12 * (date:year(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D'))) - date:year(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) + date:month-in-year(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D'))) - date:month-in-year(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) ) + ( number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2016-01-01')) div 86400))) * number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400) < round(date:seconds(normalize-space('2019-01-01')) div 86400))) * (12 * (date:year(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D'))) - date:year('2016-01-01')) + date:month-in-year(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D'))) - date:month-in-year('2016-01-01')) ) + ( number((round(date:seconds(normalize-space('2016-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400))) * number((round(date:seconds(normalize-space('2019-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * (12 * (date:year('2019-01-01') - date:year(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) + date:month-in-year('2019-01-01') - date:month-in-year(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) ) + ( number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2016-01-01')) div 86400))) * number((round(date:seconds(normalize-space('2019-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * (12 * (date:year('2019-01-01') - date:year('2016-01-01')) + date:month-in-year('2019-01-01') - date:month-in-year('2016-01-01')) ) ) * 200000 div 12 ) ) + ceiling( ( number((round(date:seconds(normalize-space('2019-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2022-01-01')) div 86400))) * ( ( number((round(date:seconds(normalize-space('2019-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400))) * number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400) < round(date:seconds(normalize-space('2022-01-01')) div 86400))) * (12 * (date:year(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D'))) - date:year(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) + date:month-in-year(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D'))) - date:month-in-year(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) ) + ( number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2019-01-01')) div 86400))) * number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400) < round(date:seconds(normalize-space('2022-01-01')) div 86400))) * (12 * (date:year(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D'))) - date:year('2019-01-01')) + date:month-in-year(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D'))) - date:month-in-year('2019-01-01')) ) + ( number((round(date:seconds(normalize-space('2019-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400))) * number((round(date:seconds(normalize-space('2022-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * (12 * (date:year('2022-01-01') - date:year(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) + date:month-in-year('2022-01-01') - date:month-in-year(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) ) + ( number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2019-01-01')) div 86400))) * number((round(date:seconds(normalize-space('2022-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * (12 * (date:year('2022-01-01') - date:year('2019-01-01')) + date:month-in-year('2022-01-01') - date:month-in-year('2019-01-01')) ) ) * 1000000 div 12 ) ) + ceiling( ( number((round(date:seconds(normalize-space('2022-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To)) div 86400))) * ( ( number((round(date:seconds(normalize-space('2022-01-01')) div 86400) <= round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400))) * (12 * (date:year(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D'))) - date:year(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) + date:month-in-year(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D'))) - date:month-in-year(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) ) + ( number((round(date:seconds(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:From)) div 86400) < round(date:seconds(normalize-space('2022-01-01')) div 86400))) * (12 * (date:year(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D'))) - date:year('2022-01-01')) + date:month-in-year(date:add(normalize-space(../ct:CompanyInformation/ct:PeriodCovered/ct:To),normalize-space('P1D'))) - date:month-in-year('2022-01-01')) ) ) * 200000 div 12 ) ) ) )) < 0.001) ) ) diff --git a/test/helper.rb b/test/helper.rb index eed661705c..ed96d3baaf 100644 --- a/test/helper.rb +++ b/test/helper.rb @@ -6,8 +6,8 @@ # - "normal" - normal GC functionality # - "minor" - force a minor GC cycle after each test # - "major" (default for Rubies without compaction) - force a major GC cycle after each test -# - "compact" (default for Rubies with compaction) - force a major GC and a compaction after each test -# - "verify" - force a major GC, a compaction, and verify references after each test +# - "compact" (default for Rubies with compaction) - force a major GC after each test and GC compaction after every 20 tests +# - "verify" - force a major GC after each test and verify references-after-compaction after every 20 tests # - "stress" - run tests with GC.stress set to true # - NOKOGIRI_GC: read more in test/test_memory_leak.rb # @@ -87,7 +87,11 @@ class TestCase < MiniTest::Spec warn "#{__FILE__}:#{__LINE__}: NOKOGIRI_TEST_GC_LEVEL: #{GC_LEVEL}" end + COMPACT_EVERY = 20 + @@test_count = 0 + def setup + @@test_count += 1 if Nokogiri.uses_libxml? @fake_error_handler_called = false Nokogiri::Test.__foreign_error_handler do @@ -110,10 +114,16 @@ def teardown when "major" GC.start(full_mark: true) when "compact" - GC.compact + if @@test_count % COMPACT_EVERY == 0 + GC.compact + else + GC.start(full_mark: true) + end when "verify" - # https://alanwu.space/post/check-compaction/ - GC.verify_compaction_references(double_heap: true, toward: :empty) + if @@test_count % COMPACT_EVERY == 0 + # https://alanwu.space/post/check-compaction/ + GC.verify_compaction_references(double_heap: true, toward: :empty) + end GC.start(full_mark: true) when "stress" GC.stress = false @@ -201,6 +211,15 @@ def i_am_running_in_valgrind # https://stackoverflow.com/questions/365458/how-can-i-detect-if-a-program-is-running-from-within-valgrind/62364698#62364698 ENV["LD_PRELOAD"] =~ /valgrind|vgpreload/ end + + def i_am_running_with_asan + # https://stackoverflow.com/questions/35012059/check-whether-sanitizer-like-addresssanitizer-is-active + begin + %x"ldd #{Gem.ruby}" =~ /libasan\.so/ + rescue + false + end + end end module SAX diff --git a/test/html/sax/test_parser.rb b/test/html4/sax/test_parser.rb similarity index 100% rename from test/html/sax/test_parser.rb rename to test/html4/sax/test_parser.rb diff --git a/test/html/sax/test_parser_context.rb b/test/html4/sax/test_parser_context.rb similarity index 100% rename from test/html/sax/test_parser_context.rb rename to test/html4/sax/test_parser_context.rb diff --git a/test/html/sax/test_parser_text.rb b/test/html4/sax/test_parser_text.rb similarity index 100% rename from test/html/sax/test_parser_text.rb rename to test/html4/sax/test_parser_text.rb diff --git a/test/html/sax/test_push_parser.rb b/test/html4/sax/test_push_parser.rb similarity index 80% rename from test/html/sax/test_push_parser.rb rename to test/html4/sax/test_push_parser.rb index dd64253907..f83440d272 100644 --- a/test/html/sax/test_push_parser.rb +++ b/test/html4/sax/test_push_parser.rb @@ -70,17 +70,6 @@ def test_chevron def test_default_options assert_equal(0, @parser.options) end - - def test_broken_encoding - skip_unless_libxml2("ultra hard to fix for pure Java version") - @parser.options |= XML::ParseOptions::RECOVER - # This is ISO_8859-1: - @parser.<< "<?xml version='1.0' encoding='UTF-8'?><r>Gau\337</r>" - @parser.finish - assert(@parser.document.errors.size >= 1) - assert_equal("Gau\337", @parser.document.data.join) - assert_equal([["r"], ["body"], ["html"]], @parser.document.end_elements) - end end end end diff --git a/test/html/test_attributes.rb b/test/html4/test_attributes.rb similarity index 100% rename from test/html/test_attributes.rb rename to test/html4/test_attributes.rb diff --git a/test/html/test_attributes_properly_escaped.rb b/test/html4/test_attributes_properly_escaped.rb similarity index 58% rename from test/html/test_attributes_properly_escaped.rb rename to test/html4/test_attributes_properly_escaped.rb index 91da4ccbe2..0debd26a4f 100755 --- a/test/html/test_attributes_properly_escaped.rb +++ b/test/html4/test_attributes_properly_escaped.rb @@ -5,7 +5,9 @@ module HTML class TestAttributesProperlyEscaped < Nokogiri::TestCase def test_attribute_macros_are_escaped - skip if Nokogiri::VersionInfo.instance.libxml2? && Nokogiri::VersionInfo.instance.libxml2_using_system? + if Nokogiri.uses_libxml? && !Nokogiri::VERSION_INFO["libxml"]["patches"]&.include?("0001-Remove-script-macro-support.patch") + skip("libxml2 has not been patched to be safe against attribute macros") + end html = "<p><i for=\"&{<test>}\"></i></p>" document = Nokogiri::HTML::Document.new @@ -15,7 +17,9 @@ def test_attribute_macros_are_escaped end def test_libxml_escapes_server_side_includes - skip if Nokogiri::VersionInfo.instance.libxml2? && Nokogiri::VersionInfo.instance.libxml2_using_system? + if Nokogiri.uses_libxml? && !Nokogiri::VERSION_INFO["libxml"]["patches"]&.include?("0002-Update-entities-to-remove-handling-of-ssi.patch") + skip("libxml2 has not been patched to be safe against SSI") + end original_html = %(<p><a href='<!--"><test>-->'></a></p>) document = Nokogiri::HTML::Document.new @@ -25,7 +29,9 @@ def test_libxml_escapes_server_side_includes end def test_libxml_escapes_server_side_includes_without_nested_quotes - skip if Nokogiri::VersionInfo.instance.libxml2? && Nokogiri::VersionInfo.instance.libxml2_using_system? + if Nokogiri.uses_libxml? && !Nokogiri::VERSION_INFO["libxml"]["patches"]&.include?("0002-Update-entities-to-remove-handling-of-ssi.patch") + skip("libxml2 has not been patched to be safe against SSI") + end original_html = %(<p><i for="<!--<test>-->"></i></p>) document = Nokogiri::HTML::Document.new diff --git a/test/html/test_builder.rb b/test/html4/test_builder.rb similarity index 100% rename from test/html/test_builder.rb rename to test/html4/test_builder.rb diff --git a/test/html/test_comments.rb b/test/html4/test_comments.rb similarity index 95% rename from test/html/test_comments.rb rename to test/html4/test_comments.rb index 476fc4b473..7d207815f5 100644 --- a/test/html/test_comments.rb +++ b/test/html4/test_comments.rb @@ -113,8 +113,7 @@ class TestComment < Nokogiri::TestCase let(:subject) { doc.at_css("div#under-test") } let(:inner_div) { doc.at_css("div#do-i-exist") } - if Nokogiri.uses_libxml? && Nokogiri::VersionInfo.instance.libxml2_using_packaged? - # see patches/libxml2/0006-htmlParseComment-treat-as-if-it-closed-the-comment.patch + if Nokogiri::VersionInfo.instance.libxml2_using_packaged? || (Nokogiri::VersionInfo.instance.libxml2_using_system? && Nokogiri.uses_libxml?(">=2.9.11")) it "behaves as if the comment is normally closed" do # COMPLIANT assert_equal 3, subject.children.length assert subject.children[0].comment? @@ -128,9 +127,7 @@ class TestComment < Nokogiri::TestCase end end - if Nokogiri.jruby? || Nokogiri::VersionInfo.instance.libxml2_using_system? - # this behavior may change to the above in libxml v2.9.11 depending on whether - # https://gitlab.gnome.org/GNOME/libxml2/-/merge_requests/82 is merged + if Nokogiri.jruby? || (Nokogiri::VersionInfo.instance.libxml2_using_system? && Nokogiri.uses_libxml?("<2.9.11")) it "behaves as if the comment encompasses the inner div" do # NON-COMPLIANT assert_equal 1, subject.children.length assert subject.children.first.comment? diff --git a/test/html/test_document.rb b/test/html4/test_document.rb similarity index 98% rename from test/html/test_document.rb rename to test/html4/test_document.rb index 29d87cf6d9..c6f0bfe862 100644 --- a/test/html/test_document.rb +++ b/test/html4/test_document.rb @@ -370,6 +370,15 @@ def test_to_xhtml assert_match("UTF-8", html.to_xhtml(encoding: "UTF-8")) end + def test_to_xhtml_self_closing_tags + # https://github.com/sparklemotion/nokogiri/issues/2324 + html = "<html><body><br><table><colgroup><col>" + doc = Nokogiri::HTML::Document.parse(html) + xhtml = doc.to_xhtml + assert_match(%r(<br ?/>), xhtml) + assert_match(%r(<col ?/>), xhtml) + end + def test_no_xml_header html = Nokogiri::HTML(<<~EOHTML) <html> diff --git a/test/html/test_document_encoding.rb b/test/html4/test_document_encoding.rb similarity index 100% rename from test/html/test_document_encoding.rb rename to test/html4/test_document_encoding.rb diff --git a/test/html/test_document_fragment.rb b/test/html4/test_document_fragment.rb similarity index 100% rename from test/html/test_document_fragment.rb rename to test/html4/test_document_fragment.rb diff --git a/test/html/test_element_description.rb b/test/html4/test_element_description.rb similarity index 100% rename from test/html/test_element_description.rb rename to test/html4/test_element_description.rb diff --git a/test/html4/test_html_module.rb b/test/html4/test_html_module.rb new file mode 100644 index 0000000000..6d142f016b --- /dev/null +++ b/test/html4/test_html_module.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true +require "helper" + +class Nokogiri::TestCase + describe Nokogiri::HTML do + it "is the same as Nokogiri::HTML4" do + assert_same(Nokogiri::HTML, Nokogiri::HTML4) + end + end + + describe "Nokogiri.HTML()" do + it "is the same as Nokogiri.HTML4()" do + assert_equal(Nokogiri.method(:HTML), Nokogiri.method(:HTML4)) + end + end +end diff --git a/test/html/test_named_characters.rb b/test/html4/test_named_characters.rb similarity index 100% rename from test/html/test_named_characters.rb rename to test/html4/test_named_characters.rb diff --git a/test/html/test_node.rb b/test/html4/test_node.rb similarity index 90% rename from test/html/test_node.rb rename to test/html4/test_node.rb index a55e2db13f..57165d8cdd 100644 --- a/test/html/test_node.rb +++ b/test/html4/test_node.rb @@ -187,21 +187,19 @@ def test_to_html_does_not_contain_entities nokogiri.at("p").to_html.gsub(/ /, '') end end - end - def test_GH_1042 - file = File.join(ASSETS_DIR, 'GH_1042.html'); - html = Nokogiri::HTML(File.read(file)) - table = html.xpath("//table")[1] - trs = table.xpath("tr").drop(1) - - # the jruby inplementation of drop uses dup() on the IRubyObject (which - # is NOT the same dup() method on the ruby Object) which produces a - # shallow clone. a shallow of valid XMLNode triggers several - # NullPointerException on inspect() since loads of invariants - # are not set. the fix for GH1042 ensures a proper working clone. - assert_nothing_raised do - trs.inspect + def test_GH_1042 + file = File.join(ASSETS_DIR, 'GH_1042.html'); + html = Nokogiri::HTML(File.read(file)) + table = html.xpath("//table")[1] + trs = table.xpath("tr").drop(1) + + # the jruby inplementation of drop uses dup() on the IRubyObject (which + # is NOT the same dup() method on the ruby Object) which produces a + # shallow clone. a shallow of valid XMLNode triggers several + # NullPointerException on inspect() since loads of invariants + # are not set. the fix for GH1042 ensures a proper working clone. + trs.inspect # assert_nothing_raised end end end diff --git a/test/html/test_node_encoding.rb b/test/html4/test_node_encoding.rb similarity index 100% rename from test/html/test_node_encoding.rb rename to test/html4/test_node_encoding.rb diff --git a/test/html5/test_api.rb b/test/html5/test_api.rb new file mode 100644 index 0000000000..c1c03652c4 --- /dev/null +++ b/test/html5/test_api.rb @@ -0,0 +1,178 @@ +# coding: utf-8 +# frozen_string_literal: true +require "helper" + +class TestHtml5API < Nokogiri::TestCase + def test_parse_convenience_methods + html = "<!DOCTYPE html><p>hi" + base = Nokogiri::HTML5::Document.parse(html) + html5_parse = Nokogiri::HTML5.parse(html) + html5 = Nokogiri::HTML5(html) + str = base.to_html + assert_equal(str, html5_parse.to_html) + assert_equal(str, html5.to_html) + end + + def test_fragment_convenience_methods + frag = "<div><p>hi</div>" + base = Nokogiri::HTML5::DocumentFragment.parse(frag) + html5_fragment = Nokogiri::HTML5.fragment(frag) + assert_equal(base.to_html, html5_fragment.to_html) + end + + def test_url + html = "<p>hi" + url = "http://example.com" + doc = Nokogiri::HTML5::Document.parse(html, url, max_errors: 1) + assert_equal(url, doc.errors[0].file) + + doc = Nokogiri::HTML5.parse(html, url, max_errors: 1) + assert_equal(url, doc.errors[0].file) + + doc = Nokogiri::HTML5(html, url, max_errors: 1) + assert_equal(url, doc.errors[0].file) + end + + def test_parse_encoding + utf8 = "<!DOCTYPE html><body><p>おはようございます" + shift_jis = utf8.encode(Encoding::SHIFT_JIS) + raw = shift_jis.dup + raw.force_encoding(Encoding::ASCII_8BIT) + + assert_match(/おはようございます/, Nokogiri::HTML5(utf8).to_s) + assert_match(/おはようございます/, Nokogiri::HTML5(shift_jis).to_s) + refute_match(/おはようございます/, Nokogiri::HTML5(raw).to_s) + + assert_match(/おはようございます/, Nokogiri::HTML5(raw, nil, Encoding::SHIFT_JIS).to_s) + assert_match(/おはようございます/, Nokogiri::HTML5.parse(raw, nil, Encoding::SHIFT_JIS).to_s) + assert_match(/おはようございます/, Nokogiri::HTML5::Document.parse(raw, nil, Encoding::SHIFT_JIS).to_s) + end + + def test_fragment_encoding + utf8 = "<div><p>おはようございます</div>" + shift_jis = utf8.encode(Encoding::SHIFT_JIS) + raw = shift_jis.dup + raw.force_encoding(Encoding::ASCII_8BIT) + + assert_match(/おはようございます/, Nokogiri::HTML5.fragment(utf8).to_s) + assert_match(/おはようございます/, Nokogiri::HTML5.fragment(shift_jis).to_s) + refute_match(/おはようございます/, Nokogiri::HTML5.fragment(raw).to_s) + + assert_match(/おはようございます/, Nokogiri::HTML5.fragment(raw, Encoding::SHIFT_JIS).to_s) + assert_match(/おはようございます/, Nokogiri::HTML5::DocumentFragment.parse(raw, Encoding::SHIFT_JIS).to_s) + end + + def test_fragment_serialization_encoding + frag = Nokogiri::HTML5.fragment("<span>아는 길도 물어가라</span>") + html = frag.serialize(encoding: "US-ASCII") + assert_equal("<span>아는 길도 물어가라</span>", html) + frag = Nokogiri::HTML5.fragment(html) + assert_equal("<span>아는 길도 물어가라</span>", frag.serialize) + end + + def test_serialization_encoding + html = "<!DOCUMENT html><span>ฉันไม่พูดภาษาไทย</span>" + doc = Nokogiri::HTML5(html) + span = doc.at("/html/body/span") + serialized = span.inner_html(encoding: "US-ASCII") + assert_match(/^(?:&#(?:\d+|x\h+);)*$/, serialized) + assert_equal("ฉันไม่พูดภาษาไทย".each_char.map(&:ord), + serialized.scan(/&#(\d+|x\h+);/).map do |s| + s = s.first + if s.start_with?("x") + s[1..-1].to_i(16) + else + s.to_i + end + end) + + doc2 = Nokogiri::HTML5(doc.serialize(encoding: "Big5")) + html2 = doc2.serialize(encoding: "UTF-8") + assert_match("ฉันไม่พูดภาษาไทย", html2) + end + + %w[pre listing textarea].each do |tag| + define_method("test_serialize_preserve_newline_#{tag}".to_sym) do + doc = Nokogiri::HTML5("<!DOCTYPE html><#{tag}>\n\nContent</#{tag}>") + html = doc.at("/html/body/#{tag}").serialize(preserve_newline: true) + assert_equal "<#{tag}>\n\nContent</#{tag}>", html + end + + define_method("test_inner_html_preserve_newline_#{tag}".to_sym) do + doc = Nokogiri::HTML5("<!DOCTYPE html><#{tag}>\n\nContent</#{tag}>") + html = doc.at("/html/body/#{tag}").inner_html(preserve_newline: true) + assert_equal "\n\nContent", html + end + end + + def test_document_io + html = StringIO.new("<!DOCTYPE html><span>test</span>", "r") + doc = Nokogiri::HTML5::Document.read_io(html) + refute_nil(doc.at_xpath("/html/body/span")) + end + + def test_document_memory + html = "<!DOCTYPE html><span>test</span>" + doc = Nokogiri::HTML5::Document.read_memory(html) + refute_nil(doc) + refute_nil(doc.at_xpath("/html/body/span")) + end + + def test_document_io_failure + html = "<!DOCTYPE html><span>test</span>" + assert_raises(ArgumentError) { Nokogiri::HTML5::Document.read_io(html) } + end + + def test_document_memory_failure + html = StringIO.new("<!DOCTYPE html><span>test</span>", "r") + assert_raises(ArgumentError) { Nokogiri::HTML5::Document.read_memory(html) } + end + + def test_document_parse_failure + html = ["Neither a string, nor I/O"] + assert_raises(ArgumentError) { Nokogiri::HTML5::Document.parse(html) } + end + + def test_ownership + # Test that we don't change the passed in string, even if we need to + # re-encode it. + html = "<!DOCTYPE html><html></html>" + refute_nil(Nokogiri::HTML5.parse(html)) + + iso8859_1 = html.encode(Encoding::ISO_8859_1).freeze + refute_nil(Nokogiri::HTML5.parse(iso8859_1)) + + ascii_8bit = html.encode(Encoding::ASCII_8BIT).freeze + refute_nil(Nokogiri::HTML5.parse(ascii_8bit)) + end + + def test_fragment_from_node + doc = Nokogiri.HTML5("<!DOCTYPE html><form><span></span></form>") + span = doc.at_xpath("/html/body/form/span") + refute_nil(span) + frag = span.fragment("<form>Nested forms should be ignored</form>") + assert(frag.is_a?(Nokogiri::HTML5::DocumentFragment)) + assert_equal(1, frag.children.length) + nested_form = frag.at_xpath("form") + assert_nil(nested_form) + assert(frag.children[0].text?) + end + + def test_fragment_from_node_no_form + doc = Nokogiri.HTML5("<!DOCTYPE html><span></span></form>") + span = doc.at_xpath("/html/body/span") + refute_nil(span) + frag = span.fragment("<form><span>Form should not be ignored</span></form>") + assert(frag.is_a?(Nokogiri::HTML5::DocumentFragment)) + assert_equal(1, frag.children.length) + form = frag.at_xpath("form") + refute_nil(form) + end + + def test_empty_fragment + doc = Nokogiri.HTML5("<!DOCTYPE html><body>") + frag = doc.fragment + assert(frag.is_a?(Nokogiri::HTML5::DocumentFragment)) + assert(frag.children.empty?) + end +end if Nokogiri.uses_gumbo? diff --git a/test/html5/test_encoding.rb b/test/html5/test_encoding.rb new file mode 100644 index 0000000000..bbe24771ba --- /dev/null +++ b/test/html5/test_encoding.rb @@ -0,0 +1,209 @@ +# coding: utf-8 +# frozen_string_literal: true +require "helper" + +class TestHtml5Encoding < Nokogiri::TestCase + if "".respond_to?("encoding") + def test_macroman_encoding + mac = String.new("<span>\xCA</span>").force_encoding("macroman") + doc = Nokogiri::HTML5(mac) + assert_equal("<span> </span>", doc.at("span").to_xml) + end + + def test_iso8859_encoding + iso8859 = String.new("<span>Se\xF1or</span>").force_encoding(Encoding::ASCII_8BIT) + doc = Nokogiri::HTML5(iso8859) + assert_equal("<span>Señor</span>", doc.at("span").to_xml) + end + + def test_charset_encoding + utf8 = String.new("<meta charset='utf-8'><span>Se\xC3\xB1or</span>") + .force_encoding(Encoding::ASCII_8BIT) + doc = Nokogiri::HTML5(utf8) + assert_equal("<span>Señor</span>", doc.at("span").to_xml) + end + + def test_bogus_encoding + bogus = String.new("<meta charset='bogus'><span>Se\xF1or</span>") + .force_encoding(Encoding::ASCII_8BIT) + doc = Nokogiri::HTML5(bogus) + assert_equal("<span>Señor</span>", doc.at("span").to_xml) + end + + def test_utf8_bom + utf8 = "\uFEFF<!DOCTYPE html><html></html>".encode("UTF-8") + doc = Nokogiri::HTML5(utf8, max_errors: 10) + assert_equal([], doc.errors) + end + + def test_utf16le_bom + utf16le = "\uFEFF<!DOCTYPE html><html></html>".encode("UTF-16LE") + doc = Nokogiri::HTML5(utf16le, max_errors: 10) + assert_equal([], doc.errors) + end + + def test_utf16be_bom + utf16be = "\uFEFF<!DOCTYPE html><html></html>".encode("UTF-16BE") + doc = Nokogiri::HTML5(utf16be, max_errors: 10) + assert_equal([], doc.errors) + end + + def test_utf8_bom_ascii + utf8 = "\uFEFF<!DOCTYPE html><html></html>".encode("UTF-8") + utf8.force_encoding(Encoding::ASCII_8BIT) + doc = Nokogiri::HTML5(utf8, max_errors: 10) + doc.errors.each { |err| puts(err) } + assert_equal([], doc.errors) + end + + def test_utf16le_bom_ascii + utf16le = "\uFEFF<!DOCTYPE html><html></html>".encode("UTF-16LE") + utf16le.force_encoding(Encoding::ASCII_8BIT) + doc = Nokogiri::HTML5(utf16le, max_errors: 10) + assert_equal([], doc.errors) + doc.errors.each { |err| puts(err) } + end + + def test_utf16be_bom_ascii + utf16be = "\uFEFF<!DOCTYPE html><html></html>".encode("UTF-16BE") + utf16be.force_encoding(Encoding::ASCII_8BIT) + doc = Nokogiri::HTML5(utf16be, max_errors: 10) + assert_equal([], doc.errors) + doc.errors.each { |err| puts(err) } + end + + def test_tag_after_utf8_bom + utf8 = "\uFEFF<b></b>".encode("UTF-8") + doc = Nokogiri::HTML5.fragment(utf8, max_errors: 10) + assert_equal([], doc.errors) + end + end + + # https://github.com/rubys/nokogumbo/issues/68 + def test_charset_sniff_to_html + html = <<-EOF.gsub(/^ /, "") + <!DOCTYPE html> + <html> + <head> + <meta http-equiv="Content-Type" content="text/html; charset=utf-8; width=device-width"> + <style> .commit-tease, .user-profile-mini-avatar, .avatar, .vcard-details, .signup-prompt-bg { display: none !IMPORTANT; } </style> <script> document.addEventListener('DOMContentLoaded', function() { this.querySelectorAll('a').forEach(anchor => { anchor.addEventListener('click', e => { e.preventDefault(); const redact = new URLSearchParams(window.location.search).get('redact'); const hasExistingParams = anchor.href.includes('?'); window.location.href = anchor.href + (hasExistingParams ? `&redact=${redact}` : `?redact=${redact}`); }); }); }); </script> </head> + <body> + Hello! + </body> + </html> + EOF + doc = Nokogiri::HTML5(html, max_errors: 10) + assert_equal(0, doc.errors.length) + refute_equal("", doc.to_html) + end + + # https://encoding.spec.whatwg.org/#names-and-labels + # I chose these by looking at the Wikipedia page for each encoding, picked + # one of the languages it was supposed to encode, and then Googled for a + # proverb in the language. Apologies if these are ill-chosen or nonsensical. + # I'm happy to change them. I'm just pasting them in here so I'm pretty sure + # the right-to-left languages are backward. Corrections welcome. + ENCODINGS = [ + ["UTF-8", "Let's concatentate all of these for UTF-8"], # English + ["IBM866", "А дело бывало -- и коза волка съедала"], # Russian + ["ISO-8859-2", "Co můžeš udělat dnes, neodkládej na zítřek."], # Czech + ["ISO-8859-3", "Yukarda mavi gök, asağıda yağız yer yaratıldıkta"], # Turkish + ["ISO-8859-4", "Ceļš uz elli ir bruģēts ar labiem nodomiem."], # Latvian + ["ISO-8859-5", "Каде има сила, нема правдина."], # Macedonian + ["ISO-8859-6", "أباد الله خضراءهم ابذل لصديقك دمك ومالك"], # Arabic + ["ISO-8859-7", "Η καλύτερη άμυνα είναι η επίθεση."], # Greek + ["ISO-8859-8", "אין הנחתום מעיד על עיסתו"], # Hebrew + ["ISO-8859-8-I", "אל תסתכל בקנקן, אלא במה שבתוכו"], # Hebrew + ["ISO-8859-10", "Alla känner apan, men apan känner ingen."], # Swedish + ["ISO-8859-13", "Lašas po lašo ir akmenį pratašo."], # Lithuanian + ["ISO-8859-14", "ha bhòrd bòrd gun aran ach 's bòrd aran leis fhèin."], # Scottish Gaelic + ["ISO-8859-15", "This is essentially ISO 8859-1 but with € Š š Ž ž Œ œ Ÿ"], # English + ["ISO-8859-16", "Kiedy wszedłeś między wrony, musisz krakać jak i one."], # Polish + ["KOI8-R", "А дело бывало -- и коза волка съедала"], # Russian + ["KOI8-U", "Яблуко від яблуньки не далеко. Ґ, Є, І, Ї"], # Ukrainian + ["macroman", "Some good old Mac Roman œ∑´®†¥¨ˆøπå߃©"], # English + ["windows-874", "กระต่ายหมายจันทร์"], # Thai + ["windows-1250", "Addig nyújtózkodj, amíg a takaród ér."], # Hungarian + ["windows-1251", "Бързата работа - срам за майстора."], # Bulgarian + ["windows-1252", "Basically ISO 8859-1 with ‘differences’™ •"], # English + ["windows-1253", "Και οι τοίχοι έχουν αυτιά."], # Greek + ["windows-1254", "Baban nasılsa oğlu da öyledir."], # Turkish + ["windows-1255", "אל תקנה חתול בשק; ₪"], # Hebrew + ["windows-1256", "أبطأ من سلحفاة"], # Arabic + ["windows-1257", "Hommikune töö kuld, õhtune muld."], # Estonian + ["windows-1258", "Ăn theo thuở, ở theo thời."], # Vietnamese + ["macCyrillic", "А дело бывало -- и коза волка съедала"], # Russian + ["GBK", "不闻不若闻之，闻之不若见之，见之不若知之，知之不若行之；学至于行之而止矣"], # Simplified Chinese + ["gb18030", "不聞不若聞之，聞之不若見之，見之不若知之，知之不若行之；學至於行之而止矣"], # Traditional Chinese + ["Big5", "有其父必有其子"], # Traditional Chinese + ["EUC-JP", "猿も木から落ちる"], # Japanese + ["ISO-2022-JP", "井の中の蛙大海を知らず"], # Japanese + ["Shift_JIS", "鳥なき里の蝙蝠"], # Japanese + ["EUC-KR", "아는 길도 물어가라"], # Korean + ["replacement", "콩 심은데 콩나고, 팥 심은데 팥난다"], # Korean + ["UTF-16BE", "Everything had better be representable!"], # English + ["UTF-16LE", "Same as with UTF-16BE"], # English + ["US-ASCII", "Surprisingly not one of the required encodings"], # English + ].freeze + + def encodings_html + @encodings_html ||= + "<!DOCTYPE html><html><head> <style> .commit-tease, .user-profile-mini-avatar, .avatar, .vcard-details, .signup-prompt-bg { display: none !IMPORTANT; } </style> <script> document.addEventListener('DOMContentLoaded', function() { this.querySelectorAll('a').forEach(anchor => { anchor.addEventListener('click', e => { e.preventDefault(); const redact = new URLSearchParams(window.location.search).get('redact'); const hasExistingParams = anchor.href.includes('?'); window.location.href = anchor.href + (hasExistingParams ? `&redact=${redact}` : `?redact=${redact}`); }); }); }); </script> </head><body>" + + ENCODINGS.map { |enc| %(<span id="#{enc[0]}">#{enc[1]}</span>) }.join + + "</body></html>" + end + + def encodings_doc + @encodings_doc ||= Nokogiri::HTML5(encodings_html) + end + + def round_trip_through(str, enc) + begin + encoding = Encoding.find(enc) + rescue ArgumentError + skip("#{enc} not supported") + end + begin + encoded = str.encode(encoding) + rescue Encoding::ConverterNotFoundError + skip("Converting UTF-8 to #{enc} not supported") + end + begin + decoded = encoded.encode("UTF-8") + rescue Encoding::ConverterNotFoundError + skip("Converting #{enc} to UTF-8 not supported") + end + assert_equal(str, decoded, "'#{str}' did not round trip through #{enc[0]}") + encoded + end + + ENCODINGS.each do |enc| + define_method("test_parse_encoded_#{enc[0]}".to_sym) do + html = "<!DOCTYPE html><span>#{enc[1]}</span>" + encoded_html = round_trip_through(html, enc[0]) + doc = Nokogiri::HTML5(encoded_html, encoding: enc[0]) + span = doc.at("/html/body/span") + refute_nil span + assert_equal enc[1], span.content + end + + define_method("test_inner_html_encoded_#{enc[0]}".to_sym) do + encoded = round_trip_through(enc[1], enc[0]) + span = encodings_doc.at(%(/html/body/span[@id="#{enc[0]}"])) + refute_nil span + assert_equal encoded, span.inner_html(encoding: enc[0]) + end + + define_method("test_roundtrip_through_#{enc[0]}".to_sym) do + # https://bugs.ruby-lang.org/issues/15033 + # Ruby has a bug with the `:fallback` parameter passed to `#encode` when + # multiple conversions have to happen. I'm not sure it's worth working + # around. It impacts this test though. + skip "https://bugs.ruby-lang.org/issues/15033" if enc[0] == "ISO-2022-JP" + round_trip_through(enc[1], enc[0]) + encoded = encodings_doc.serialize(encoding: enc[0]) + doc = Nokogiri::HTML5(encoded, encoding: enc[0]) + assert_equal encodings_html, doc.serialize + end + end +end if Nokogiri.uses_gumbo? diff --git a/test/html5/test_monkey_patch.rb b/test/html5/test_monkey_patch.rb new file mode 100644 index 0000000000..42ff0ac6ab --- /dev/null +++ b/test/html5/test_monkey_patch.rb @@ -0,0 +1,15 @@ +# encoding: utf-8 +require 'helper' + +class TestHtml5SerializationMonkeyPatch < Nokogiri::TestCase + def test_to_xml + xml = Nokogiri.HTML5('<!DOCTYPE html><source>').to_xml + assert_match(/\A<\?xml version/, xml) + assert_match(/<source\s*\/>/, xml) + end + + def test_html4_fragment + frag = Nokogiri::HTML.fragment('<span></span>') + assert frag.is_a?(Nokogiri::HTML::DocumentFragment) + end +end if Nokogiri.uses_gumbo? diff --git a/test/html5/test_nokogumbo.rb b/test/html5/test_nokogumbo.rb new file mode 100644 index 0000000000..a49b9e9848 --- /dev/null +++ b/test/html5/test_nokogumbo.rb @@ -0,0 +1,353 @@ +# encoding: utf-8 +# frozen_string_literal: true +require "helper" + +class TestHtml5Nokogumbo < Nokogiri::TestCase + def test_element_text + doc = Nokogiri::HTML5(buffer) + assert_equal("content", doc.at("span").text) + end + + def test_element_cdata_textarea + doc = Nokogiri::HTML5(buffer) + assert_equal("foo<x>bar", doc.at("textarea").text.strip) + end + + def test_element_cdata_script + doc = Nokogiri::HTML5.fragment(buffer) + assert_equal(true, doc.document.html?) + assert_equal("<script> if (a < b) alert(1) </script>", doc.at("script").to_s) + end + + def test_attr_value + doc = Nokogiri::HTML5(buffer) + assert_equal("utf-8", doc.at("meta")["charset"]) + end + + def test_comment + doc = Nokogiri::HTML5(buffer) + assert_equal(" test comment ", doc.xpath("//comment()").text) + end + + def test_unknown_element + doc = Nokogiri::HTML5(buffer) + assert_equal("main", doc.at("main").name) + end + + def test_IO + require "stringio" + doc = Nokogiri::HTML5(StringIO.new(buffer)) + assert_equal("textarea", doc.at("form").element_children.first.name) + end + + def test_nil + doc = Nokogiri::HTML5(nil) + assert_equal(1, doc.search("body").count) + + fragment = Nokogiri::HTML5.fragment(nil) + assert_equal(0, fragment.errors.length) + end + + def test_html5_doctype + doc = Nokogiri::HTML5.parse("<!DOCTYPE html><html></html>") + assert_match(/<!DOCTYPE html>/, doc.to_html) + end + + def test_fragment_no_errors + doc = Nokogiri::HTML5.fragment("no missing DOCTYPE errors", max_errors: 10) + assert_equal(0, doc.errors.length) + end + + # This should be deleted when `:max_parse_errors` is removed. + def test_fragment_max_parse_errors + doc = Nokogiri::HTML5.fragment("testing deprecated :max_parse_errors", max_parse_errors: 10) + assert_equal(0, doc.errors.length) + end + + def test_fragment_head + doc = Nokogiri::HTML5.fragment(buffer[%r{<head>(.*?) <style> .commit-tease, .user-profile-mini-avatar, .avatar, .vcard-details, .signup-prompt-bg { display: none !IMPORTANT; } </style> <script> document.addEventListener('DOMContentLoaded', function() { this.querySelectorAll('a').forEach(anchor => { anchor.addEventListener('click', e => { e.preventDefault(); const redact = new URLSearchParams(window.location.search).get('redact'); const hasExistingParams = anchor.href.includes('?'); window.location.href = anchor.href + (hasExistingParams ? `&redact=${redact}` : `?redact=${redact}`); }); }); }); </script> </head>}m, 1]) + assert_equal("hello world", doc.xpath("title").text) + assert_equal("utf-8", doc.xpath("meta").first["charset"]) + end + + def test_fragment_body + doc = Nokogiri::HTML5.fragment(buffer[%r{<body>(.*?)</body>}m, 1]) + assert_equal("<span>content</span>", doc.xpath("main/span").to_xml) + assert_equal(" test comment ", doc.xpath("comment()").text) + end + + def test_xlink_attribute + source = <<-EOF.gsub(/^ {6}/, "") + <!DOCTYPE html> + <svg xmlns="http://www.w3.org/2000/svg"> + <a xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="#s1"/> + </svg> + EOF + doc = Nokogiri::HTML5.parse(source) + a = doc.at_xpath("/html/body/svg:svg/svg:a") + refute_nil(a) + refute_nil(a["xlink:href"]) + refute_nil(a["xmlns:xlink"]) + end + + def test_xlink_attribute_fragment + source = <<-EOF.gsub(/^ {6}/, "") + <svg xmlns="http://www.w3.org/2000/svg"> + <a xmlns:xlink="http://www.w3.org/1999/xlink" xlink:href="#s1"/> + </svg> + EOF + doc = Nokogiri::HTML5.fragment(source) + a = doc.at_xpath("svg:svg/svg:a") + refute_nil(a) + refute_nil(a["xlink:href"]) + refute_nil(a["xmlns:xlink"]) + end + + def test_template + source = <<-EOF.gsub(/^ {6}/, "") + <template id="productrow"> + <tr> + <td class="record"></td> + <td></td> + </tr> + </template> + EOF + doc = Nokogiri::HTML5.fragment(source) + template = doc.at("template") + assert_equal("productrow", template["id"]) + assert_equal("record", template.at("td")["class"]) + end + + def test_root_comments + doc = Nokogiri::HTML5("<!DOCTYPE html><!-- start --><html></html><!-- -->") + assert_equal(["html", "comment", "html", "comment"], doc.children.map(&:name)) + end + + def test_max_attributes + html = '<div id="i" class="c" title="t"><img src="s" alt="a"></div>' + + assert_raises(ArgumentError) { Nokogiri::HTML5(html, max_attributes: 2) } + assert_raises(ArgumentError) { Nokogiri::HTML5(html, max_attributes: 1) } + assert_raises(ArgumentError) { Nokogiri::HTML5(html, max_attributes: 0) } + + # -1 disables limit + doc = Nokogiri::HTML5(html, max_attributes: -1) + assert_equal({ "id" => "i", "class" => "c", "title" => "t" }, attributes(doc.at_css("div"))) + assert_equal({ "src" => "s", "alt" => "a" }, attributes(doc.at_css("img"))) + end + + def test_max_attributes_boolean + html = '<label><input checked type="checkbox" disabled name="cheese"> Cheese</label>' + + doc = Nokogiri::HTML5(html, max_attributes: 4) + assert_equal({ "checked" => "", "type" => "checkbox", "disabled" => "", "name" => "cheese" }, + attributes(doc.at_css("input"))) + + assert_raises(ArgumentError) { Nokogiri::HTML5(html, max_attributes: 3) } + assert_raises(ArgumentError) { Nokogiri::HTML5(html, max_attributes: 2) } + assert_raises(ArgumentError) { Nokogiri::HTML5(html, max_attributes: 1) } + assert_raises(ArgumentError) { Nokogiri::HTML5(html, max_attributes: 0) } + end + + def test_default_max_attributes + a = String.new("a") + attrs = 50_000.times.map { x = a.dup; a.succ!; x } + + # <div> contains 50,000 attributes, but default limit is 400. Parsing this would take ages if + # we were not enforcing any limit on attributes. All attributes are duplicated to make sure + # this doesn’t alter performance or end result. + html = <<~EOF + <div #{attrs.map.with_index { |x, i| "data-#{x}=#{i} data-#{x}=#{i}" }.join(" ")}> + hello + </div> + EOF + + assert_raises(ArgumentError) { Nokogiri::HTML5(html) } + end + + def test_fragment_max_attributes + html = '<div id="i" class="c" title="t"><img src="s" alt="a"></div>' + + assert_raises(ArgumentError) { Nokogiri::HTML5.fragment(html, max_attributes: 2) } + assert_raises(ArgumentError) { Nokogiri::HTML5.fragment(html, max_attributes: 1) } + assert_raises(ArgumentError) { Nokogiri::HTML5.fragment(html, max_attributes: 0) } + + # -1 disables limit + doc = Nokogiri::HTML5.fragment(html, max_attributes: -1) + assert_equal({ "id" => "i", "class" => "c", "title" => "t" }, attributes(doc.at_css("div"))) + assert_equal({ "src" => "s", "alt" => "a" }, attributes(doc.at_css("img"))) + end + + def test_fragment_default_max_attributes + a = String.new("a") + attrs = 50_000.times.map { x = a.dup; a.succ!; x } + + # <div> contains 50,000 attributes, but default limit is 400. Parsing this would take ages if + # we were not enforcing any limit on attributes. All attributes are duplicated to make sure + # this doesn’t alter performance or end result. + html = <<~EOF + <div #{attrs.map.with_index { |x, i| "data-#{x}=#{i} data-#{x}=#{i}" }.join(" ")}> + hello + </div> + EOF + + assert_raises(ArgumentError) { Nokogiri::HTML5.fragment(html) } + end + + def test_parse_errors + doc = Nokogiri::HTML5("<!DOCTYPE html><html><!-- <!-- --></a>", max_errors: 10) + assert_equal(doc.errors.length, 2) + doc = Nokogiri::HTML5("<!DOCTYPE html><html>", max_errors: 10) + assert_empty(doc.errors) + end + + def test_max_errors + # This document contains 2 parse errors, but we force limit to 1. + doc = Nokogiri::HTML5("<!DOCTYPE html><html><!-- -- --></a>", max_errors: 1) + assert_equal(1, doc.errors.length) + doc = Nokogiri::HTML5("<!DOCTYPE html><html>", max_errors: 1) + assert_empty(doc.errors) + end + + def test_default_max_errors + # This document contains 200 parse errors, but default limit is 0. + doc = Nokogiri::HTML5("<!DOCTYPE html><html>" + "</p>" * 200) + assert_equal(0, doc.errors.length) + end + + def test_parse_fragment_errors + doc = Nokogiri::HTML5.fragment("<\r\n", max_errors: 10) + refute_empty(doc.errors) + end + + def test_fragment_max_errors + # This fragment contains 2 parse errors, but we force limit to 1. + doc = Nokogiri::HTML5.fragment("<!-- <!-- --></span>", max_errors: 1) + assert_equal(1, doc.errors.length) + doc = Nokogiri::HTML5.fragment("<!-- <!-- --></span>", max_errors: 10) + assert_equal(2, doc.errors.length) + end + + def test_fragment_default_max_errors + # This fragment contains 200 parse errors, but default limit is 0. + doc = Nokogiri::HTML5.fragment("</p>" * 200) + assert_equal(0, Nokogiri::Gumbo::DEFAULT_MAX_ERRORS) + assert_equal(0, doc.errors.length) + end + + def test_default_max_depth_parse + assert_raises(ArgumentError) do + depth = Nokogiri::Gumbo::DEFAULT_MAX_TREE_DEPTH + 1 + Nokogiri::HTML5("<!DOCTYPE html><html><body>" + "<div>" * (depth - 2)) + end + end + + def test_default_max_depth_fragment + assert_raises(ArgumentError) do + depth = Nokogiri::Gumbo::DEFAULT_MAX_TREE_DEPTH + 1 + Nokogiri::HTML5.fragment("<div>" * depth) + end + end + + def test_max_depth_parse + depth = 10 + html = "<!DOCTYPE html><html><body>" + "<div>" * (depth - 2) + assert_raises(ArgumentError) do + Nokogiri::HTML5(html, max_tree_depth: depth - 1) + end + + begin + Nokogiri::HTML5(html, max_tree_depth: depth) + pass + rescue ArgumentError + flunk("Expected document parse to succeed") + end + end + + def test_max_depth_fragment + depth = 10 + html = "<div>" * depth + assert_raises(ArgumentError) do + Nokogiri::HTML5.fragment(html, max_tree_depth: depth - 1) + end + + begin + Nokogiri::HTML5.fragment(html, max_tree_depth: depth) + pass + rescue ArgumentError + flunk("Expected fragment parse to succeed") + end + end + + def test_document_encoding + html = <<-TEXT + <html> + <head> + <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"> + <style> .commit-tease, .user-profile-mini-avatar, .avatar, .vcard-details, .signup-prompt-bg { display: none !IMPORTANT; } </style> <script> document.addEventListener('DOMContentLoaded', function() { this.querySelectorAll('a').forEach(anchor => { anchor.addEventListener('click', e => { e.preventDefault(); const redact = new URLSearchParams(window.location.search).get('redact'); const hasExistingParams = anchor.href.includes('?'); window.location.href = anchor.href + (hasExistingParams ? `&redact=${redact}` : `?redact=${redact}`); }); }); }); </script> </head> + <body> + Кирилические символы + </body> + </html> + TEXT + doc = Nokogiri::HTML5.parse(html) + assert_equal("UTF-8", doc.encoding) + assert_equal("Кирилические символы", doc.at("body").text.gsub(/\n\s+/, "")) + end + + def test_line_text + doc = Nokogiri.HTML5("<!DOCTYPE html>\ntext node") + assert_equal(2, doc.at_xpath("/html/body/text()").line) + end + + def test_line_comment + doc = Nokogiri.HTML5("<!DOCTYPE html>\n\n<!-- comment -->") + assert_equal(3, doc.at_xpath("/comment()").line) + end + + def test_line_element + doc = Nokogiri.HTML5("<!DOCTYPE html>\n<p>") + assert_equal(2, doc.at_xpath("/html/body/p").line) + end + + def test_line_template + doc = Nokogiri.HTML5("<!DOCTYPE html>\n\n<template></template>") + assert_equal(3, doc.at_xpath("/html/head/template").line) + end + + def test_line_cdata + html = "<!DOCTYPE html>\n<svg>\n<script><![CDATA[ ]]></script></svg>" + doc = Nokogiri.HTML5(html) + node = doc.at_xpath("/html/body/svg:svg/svg:script/text()") + assert(node.cdata?) + assert_equal(3, node.line) + end + + private + + def buffer + <<-EOF.gsub(/^ /, "") + <html> + <head> + <meta charset="utf-8"/> + <title>hello world</title> + <script> if (a < b) alert(1) </script> + <style> .commit-tease, .user-profile-mini-avatar, .avatar, .vcard-details, .signup-prompt-bg { display: none !IMPORTANT; } </style> <script> document.addEventListener('DOMContentLoaded', function() { this.querySelectorAll('a').forEach(anchor => { anchor.addEventListener('click', e => { e.preventDefault(); const redact = new URLSearchParams(window.location.search).get('redact'); const hasExistingParams = anchor.href.includes('?'); window.location.href = anchor.href + (hasExistingParams ? `&redact=${redact}` : `?redact=${redact}`); }); }); }); </script> </head> + <body> + <h1>hello world</h1> + <main> + <span>content</span> + </main> + <!-- test comment --> + <form> + <textarea>foo<x>bar</textarea> + </form> + </body> + </html> + EOF + end + + def attributes(element) + element.attributes.map { |name, attribute| [name, attribute.value] }.to_h + end +end if Nokogiri.uses_gumbo? diff --git a/test/html5/test_null.rb b/test/html5/test_null.rb new file mode 100644 index 0000000000..d0a326c291 --- /dev/null +++ b/test/html5/test_null.rb @@ -0,0 +1,112 @@ +# encoding: utf-8 +# frozen_string_literal: true +require "helper" + +class TestHtml5Null < Nokogiri::TestCase + def fragment(s) + Nokogiri::HTML5.fragment(s, max_errors: 10) + end + + def test_null_char_ref + frag = fragment("�") + assert_equal(1, frag.errors.length) + end + + def test_data_state + frag = fragment("\u0000") + # 12.2.5.1 Data state: unexpected-null-character parse error + # 12.2.6.4.7 The "in body" insertion mode: Parse error + assert_equal(2, frag.errors.length) + end + + def test_data_rcdata_state + # 12.2.6.4.7 The "in body" insertion mode: textarea swiches to RCDATA + # state + frag = fragment("<textarea>\u0000</textarea>") + # 12.2.5.2 RCDATA state: unexpected-null-character parse error + assert_equal(1, frag.errors.length) + end + + def test_data_scriptdata_state + # 12.2.6.4.7 The "in body" insertion mode: Process "script" using rules + # for "in head" insertion mode + # 12.2.6.4.4 The "in head" insertion mode: "script" switches to script + # data state + frag = fragment("<script>\u0000</script>") + # 12.2.5.4 Script data state: unexpected-null-character parse error + assert_equal(1, frag.errors.length) + end + + def test_data_plaintext_state + frag = fragment("<plaintext>\u0000</plaintext>") + # 12.2.5.5 PLAINTEXT state: unexpected-null-character parse error + # EOF parse error because there's no way to switch out of plaintext! + assert_equal(2, frag.errors.length) + end + + def test_data_tag_name_state + frag = fragment("<x\u0000></x\ufffd>") + # 12.2.5.8 Tag name state: unexpected-null-character parse error + assert_equal(1, frag.errors.length) + end + + # XXX: There are 6 script states to test. + + def test_attribute_name_state + frag = fragment("<p \u0000>") + # 12.2.5.33 Attribute name state + assert_equal(1, frag.errors.length) + end + + def test_attribute_value_states + frag = fragment("<p x=\"\u0000\"><p x='\u0000'><p x=\u0000>") + # 12.2.5.36 Attribute value (double-quoted) state + # 12.2.5.37 Attribute value (single-quoted) state + # 12.2.5.38 Attribute value (unquoted) state + assert_equal(3, frag.errors.length) + end + + def test_bogus_comment_state + frag = fragment("<!\u0000>") + # 12.2.5.42 Markup declaration open state: incorrectly-opened-comment + # parse error + # 12.2.5.41 Bogus comment state: unexpected-null-character parse error + assert_equal(2, frag.errors.length) + end + + def test_comment_state + frag = fragment("<!-- \u0000 -->") + # 12.2.5.45 Comment state: unexpected-null-character parse error + assert_equal(1, frag.errors.length) + end + + def test_doctype_name_states + # There are two missing here for double quoted PUBLIC and SYSTEM values. + doc = Nokogiri::HTML5.parse("<!DOCTYPE \u0000\u0000 PUBLIC '\u0000' '\u0000' \u0000>", + max_errors: 10) + # 12.2.5.54 Before DOCTYPE name state: unexpected-null-character parse + # error + # 12.2.5.55 DOCTYPE name state: unexpected-null-character parse error + # 12.2.5.60 DOCTYPE public identifier (single-quoted) state: + # unexpected-null-character parse error + # 12.2.5.66 DOCTYPE system identifier (single-quoted) state: + # unexpected-null-character parse error + # 12.2.5.67 After DOCTYPE system identifier state: + # unexpected-character-after-doctype-system-identifier parse error + # 12.2.5.68 Bogus DOCTYPE state: unexpected-null-character parse error + # 12.2.6.4.1 The "initial" insertion mode: parse error + assert_equal(7, doc.errors.length) + end + + def test_cdata_section_state + frag = fragment("<script>//<![CDATA[\n\u0000\n//]]></script>") + # 12.2.6.5 The rules for parsing tokens in foreign content: parse error + assert_equal(1, frag.errors.length) + end + + def test_error_api_with_null + frag = fragment("<p \u0000>") + assert(frag.errors.any?) + assert_includes(frag.errors[0].to_s, "<p \u0000>") + end +end if Nokogiri.uses_gumbo? diff --git a/test/html5/test_serialize.rb b/test/html5/test_serialize.rb new file mode 100644 index 0000000000..32cd759061 --- /dev/null +++ b/test/html5/test_serialize.rb @@ -0,0 +1,506 @@ +# encoding: utf-8 +# frozen_string_literal: true +require "helper" + +class TestHtml5Serialize < Nokogiri::TestCase + # https://github.com/web-platform-tests/wpt/blob/master/html/syntax/serializing-html-fragments/initial-linefeed-pre.html + def initial_linefeed_pre + @initial_linefeed_pre ||= begin + html = <<~EOF.gsub(/^ /, "").freeze + <!DOCTYPE html> + <div id="outer"> + <div id="inner"> + <pre id="pre1"> + x</pre> + <pre id="pre2"> + + x</pre> + <textarea id="textarea1"> + x</textarea> + <textarea id="textarea2"> + + x</textarea> + <listing id="listing1"> + x</listing> + <listing id="listing2"> + + x</listing> + </div> + </div> + EOF + Nokogiri::HTML5(html) + end + @initial_linefeed_pre + end + + def test_initial_linefeed_pre_outer + expected = %{\n<div id="inner">\n<pre id="pre1">x</pre>\n<pre id="pre2">\nx</pre>\n<textarea id="textarea1">x</textarea>\n<textarea id="textarea2">\nx</textarea>\n<listing id="listing1">x</listing>\n<listing id="listing2">\nx</listing>\n</div>\n} + outer = initial_linefeed_pre.xpath('//div[@id="outer"]')[0] + refute_nil(outer) + assert_equal(expected, outer.inner_html) + end + + def test_initial_linefeed_pre_inner + expected = %{\n<pre id="pre1">x</pre>\n<pre id="pre2">\nx</pre>\n<textarea id="textarea1">x</textarea>\n<textarea id="textarea2">\nx</textarea>\n<listing id="listing1">x</listing>\n<listing id="listing2">\nx</listing>\n} + inner = initial_linefeed_pre.at('//div[@id="inner"]') + refute_nil(inner) + assert_equal(expected, inner.inner_html) + end + + %w[pre textarea listing].each do |tag| + define_method("test_initial_linefeed_#{tag}1".to_sym) do + elem = initial_linefeed_pre.at("//*[@id=\"#{tag}1\"]") + refute_nil elem + assert_equal "x", elem.inner_html + end + + define_method("test_initial_linefeed_#{tag}2".to_sym) do + elem = initial_linefeed_pre.at("//*[@id=\"#{tag}2\"]") + refute_nil elem + assert_equal "\nx", elem.inner_html + end + end + + # https://github.com/web-platform-tests/wpt/blob/master/html/syntax/serializing-html-fragments/outerHTML.html + ELEMENTS_WITH_END_TAG = [ + "a", + "abbr", + "address", + "article", + "aside", + "audio", + "b", + "bdi", + "bdo", + "blockquote", + "body", + "button", + "canvas", + "caption", + "cite", + "code", + "colgroup", + "command", + "datalist", + "dd", + "del", + "details", + "dfn", + "dialog", + "div", + "dl", + "dt", + "em", + "fieldset", + "figcaption", + "figure", + "footer", + "form", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "head", + "header", + "hgroup", + "html", + "i", + "iframe", + "ins", + "kbd", + "label", + "legend", + "li", + "map", + "mark", + "menu", + "meter", + "nav", + "noscript", + "object", + "ol", + "optgroup", + "option", + "output", + "p", + "pre", + "progress", + "q", + "rp", + "rt", + "ruby", + "s", + "samp", + "script", + "section", + "select", + "small", + "span", + "strong", + "style", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "textarea", + "tfoot", + "th", + "thead", + "time", + "title", + "tr", + "u", + "ul", + "var", + "video", + "data", + ].freeze + + ELEMENTS_WITHOUT_END_TAG = [ + "area", + "base", + "br", + "col", + "embed", + "hr", + "img", + "input", + "keygen", + "link", + "meta", + "param", + "source", + "track", + "wbr", + ] + + ELEMENTS_WITH_END_TAG.each do |tag| + define_method("test_outer_html_#{tag}".to_sym) do + doc = Nokogiri::HTML5::Document.new + child = Nokogiri::XML::Element.new(tag, doc) + assert_equal "<#{tag}></#{tag}>", child.serialize(save_with: 0) + end + end + + ELEMENTS_WITHOUT_END_TAG.each do |tag| + define_method("test_outer_html_#{tag}".to_sym) do + doc = Nokogiri::HTML5::Document.new + child = Nokogiri::XML::Element.new(tag, doc) + assert_equal "<#{tag}>", child.serialize(save_with: 0) + end + end + + # https://github.com/web-platform-tests/wpt/blob/master/html/syntax/serializing-html-fragments/serializing.html + def serializing_test_data + @serializing_test_data ||= begin + html = <<~EOF.gsub(/ /, "") + <!DOCTYPE html> + <div id="test" style="display:none"> + <span></span> + <span><a></a></span> + <span><a b=c></a></span> + <span><a b='c'></a></span> + <span><a b='&'></a></span> + <span><a b=' '></a></span> + <span><a b='"'></a></span> + <span><a b="<"></a></span> + <span><a b=">"></a></span> + <span><svg xlink:href="a"></svg></span> + <span><svg xmlns:svg="test"></svg></span> + <span>a</span> + <span>&</span> + <span> </span> + <span><</span> + <span>></span> + <span>"</span> + <span><style><&></style></span> + <span><script type="test"><&></script></span> + <script type="test"><&></script> + <span><xmp><&></xmp></span> + <span><iframe><&></iframe></span> + <span><noembed><&></noembed></span> + <span><noframes><&></noframes></span> + <span><noscript><&></noscript></span> + <span><!--data--></span> + <span><a><b><c></c></b><d>e</d><f><g>h</g></f></a></span> + <span b=c></span> + </div> + EOF + Nokogiri::HTML5(html).xpath("/html/body/div/*") + end + @serializing_test_data + end + + EXPECTED = [ + ["", "<span></span>"], + ["<a></a>", "<span><a></a></span>"], + ["<a b=\"c\"></a>", "<span><a b=\"c\"></a></span>"], + ["<a b=\"c\"></a>", "<span><a b=\"c\"></a></span>"], + ["<a b=\"&\"></a>", "<span><a b=\"&\"></a></span>"], + ["<a b=\" \"></a>", "<span><a b=\" \"></a></span>"], + ["<a b=\""\"></a>", "<span><a b=\""\"></a></span>"], + ["<a b=\"<\"></a>", "<span><a b=\"<\"></a></span>"], + ["<a b=\">\"></a>", "<span><a b=\">\"></a></span>"], + ["<svg xlink:href=\"a\"></svg>", "<span><svg xlink:href=\"a\"></svg></span>"], + ["<svg xmlns:svg=\"test\"></svg>", "<span><svg xmlns:svg=\"test\"></svg></span>"], + ["a", "<span>a</span>"], + ["&", "<span>&</span>"], + [" ", "<span> </span>"], + ["<", "<span><</span>"], + [">", "<span>></span>"], + ["\"", "<span>\"</span>"], + ["<style><&></style>", "<span><style><&></style></span>"], + ["<script type=\"test\"><&><\/script>", "<span><script type=\"test\"><&><\/script></span>"], + ["<&>", "<script type=\"test\"><&><\/script>"], + ["<xmp><&></xmp>", "<span><xmp><&></xmp></span>"], + ["<iframe><&></iframe>", "<span><iframe><&></iframe></span>"], + ["<noembed><&></noembed>", "<span><noembed><&></noembed></span>"], + ["<noframes><&></noframes>", "<span><noframes><&></noframes></span>"], + ["<noscript><&></noscript>", "<span><noscript><&></noscript></span>"], + ["<!--data-->", "<span><!--data--></span>"], + ["<a><b><c></c></b><d>e</d><f><g>h</g></f></a>", "<span><a><b><c></c></b><d>e</d><f><g>h</g></f></a></span>"], + ["", "<span b=\"c\"></span>"], + ].freeze + + DOM_TESTS = [ + ["Attribute in the XML namespace", + lambda do + doc = Nokogiri::HTML5::Document.new + span = Nokogiri::XML::Element.new("span", doc) + svg = Nokogiri::XML::Element.new("svg", doc) + span.add_child(svg) + svg.add_namespace("xml", "http://www.w3.org/XML/1998/namespace") + svg["xml:foo"] = "test" + span + end, + '<svg xml:foo="test"></svg>', + '<span><svg xml:foo="test"></svg></span>'], + + ["Attribute in the XML namespace with the prefix not set to xml:", + lambda do + doc = Nokogiri::HTML5::Document.new + span = Nokogiri::XML::Element.new("span", doc) + svg = Nokogiri::XML::Element.new("svg", doc) + span.add_child(svg) + svg["abc:foo"] = "test" + ns = svg.add_namespace("xml", "http://www.w3.org/XML/1998/namespace") + svg.attribute("abc:foo").namespace = ns + span + end, + '<svg xml:foo="test"></svg>', + '<span><svg xml:foo="test"></svg></span>'], + + ["Non-'xmlns' attribute in the xmlns namespace", + lambda do + doc = Nokogiri::HTML5::Document.new + span = Nokogiri::XML::Element.new("span", doc) + svg = Nokogiri::XML::Element.new("svg", doc) + span.add_child(svg) + svg.add_namespace("xmlns", "http://www.w3.org/2000/xmlns/") + svg["xmlns:foo"] = "test" + span + end, + '<svg xmlns:foo="test"></svg>', + '<span><svg xmlns:foo="test"></svg></span>'], + + ["'xmlns' attribute in the xmlns namespace", + lambda do + doc = Nokogiri::HTML5::Document.new + span = Nokogiri::XML::Element.new("span", doc) + svg = Nokogiri::XML::Element.new("svg", doc) + span.add_child(svg) + svg.add_namespace("xmlns", "http://www.w3.org/2000/xmlns/") + svg["xmlns"] = "test" + span + end, + '<svg xmlns="test"></svg>', + '<span><svg xmlns="test"></svg></span>'], + + ["Attribute in non-standard namespace", + lambda do + doc = Nokogiri::HTML5::Document.new + span = Nokogiri::XML::Element.new("span", doc) + svg = Nokogiri::XML::Element.new("svg", doc) + span.add_child(svg) + svg.add_namespace("abc", "fake_ns") + svg["abc:def"] = "test" + span + end, + '<svg abc:def="test"></svg>', + '<span><svg abc:def="test"></svg></span>'], + + ["<span> starting with U+000A", + lambda do + doc = Nokogiri::HTML5::Document.new + span = Nokogiri::XML::Element.new("span", doc) + text = Nokogiri::XML::Text.new("\x0A", doc) + span.add_child(text) + span + end, + "\x0A", + "<span>\x0A</span>"], + # TODO: Processing instructions + ] + + TEXT_ELEMENTS = %w[pre textarea listing] + TEXT_TESTS = [ + ["<%text> context starting with U+000A", + lambda do |tag| + doc = Nokogiri::HTML5::Document.new + elem = Nokogiri::XML::Element.new(tag, doc) + text = Nokogiri::XML::Text.new("\x0A", doc) + elem.add_child(text) + elem + end, + "\x0A", + "<%text>\x0A</%text>"], + + ["<%text> context not starting with U+000A", + lambda do |tag| + doc = Nokogiri::HTML5::Document.new + elem = Nokogiri::XML::Element.new(tag, doc) + text = Nokogiri::XML::Text.new("a\x0A", doc) + elem.add_child(text) + elem + end, + "a\x0A", + "<%text>a\x0A</%text>"], + + ["<%text> non-context starting with U+000A", + lambda do |tag| + doc = Nokogiri::HTML5::Document.new + elem = Nokogiri::XML::Element.new(tag, doc) + span = Nokogiri::XML::Element.new("span", doc) + text = Nokogiri::XML::Text.new("\x0A", doc) + elem.add_child(text) + span.add_child(elem) + span + end, + "<%text>\x0A</%text>", + "<span><%text>\x0A</%text></span>"], + + ["<%text> non-context not starting with U+000A", + lambda do |tag| + doc = Nokogiri::HTML5::Document.new + elem = Nokogiri::XML::Element.new(tag, doc) + span = Nokogiri::XML::Element.new("span", doc) + text = Nokogiri::XML::Text.new("a\x0A", doc) + elem.add_child(text) + span.add_child(elem) + span + end, + "<%text>a\x0A</%text>", + "<span><%text>a\x0A</%text></span>"], + ] + + VOID_ELEMENTS = [ + "area", "base", "basefont", "bgsound", "br", "col", "embed", + "frame", "hr", "img", "input", "keygen", "link", + "meta", "param", "source", "track", "wbr" + ] + VOID_TESTS = [ + ["Void context node", + lambda do |tag| + doc = Nokogiri::HTML5::Document.new + Nokogiri::XML::Element.new(tag, doc) + end, + "", + "<%void>"], + + ["void as first child with following siblings", + lambda do |tag| + doc = Nokogiri::HTML5::Document.new + span = Nokogiri::XML::Element.new("span", doc) + span.add_child(Nokogiri::XML::Element.new(tag, doc)) + span.add_child(Nokogiri::XML::Element.new("a", doc)) + .add_child(Nokogiri::XML::Text.new("test", doc)) + span.add_child(Nokogiri::XML::Element.new("b", doc)) + span + end, + "<%void><a>test</a><b></b>", + "<span><%void><a>test</a><b></b></span>"], + + ["void as second child with following siblings", + lambda do |tag| + doc = Nokogiri::HTML5::Document.new + span = Nokogiri::XML::Element.new("span", doc) + span.add_child(Nokogiri::XML::Element.new("a", doc)) + .add_child(Nokogiri::XML::Text.new("test", doc)) + span.add_child(Nokogiri::XML::Element.new(tag, doc)) + span.add_child(Nokogiri::XML::Element.new("b", doc)) + span + end, + "<a>test</a><%void><b></b>", + "<span><a>test</a><%void><b></b></span>"], + ["void as last child with preceding siblings", + lambda do |tag| + doc = Nokogiri::HTML5::Document.new + span = Nokogiri::XML::Element.new("span", doc) + span.add_child(Nokogiri::XML::Element.new("a", doc)) + .add_child(Nokogiri::XML::Text.new("test", doc)) + span.add_child(Nokogiri::XML::Element.new("b", doc)) + span.add_child(Nokogiri::XML::Element.new(tag, doc)) + span + end, + "<a>test</a><b></b><%void>", + "<span><a>test</a><b></b><%void></span>"], + ] + + # Generate tests + def self.cross_map(a1, a2) + rv = [] + a1.each do |a1_elem| + a2.each do |a2_elem| + rv << yield(a1_elem, a2_elem) + end + end + end + + EXPECTED.each_with_index do |item, i| + define_method("test_serializing_html_innerHTML_#{i}".to_sym) do + assert_equal item[0], serializing_test_data[i].inner_html + end + + define_method("test_serializing_html_outerHTML_#{i}".to_sym) do + assert_equal item[1], serializing_test_data[i].serialize + end + end + + DOM_TESTS.each do |test| + define_method("test_serializing_dom_innerHTML_#{test[0]}".to_sym) do + elem = test[1].call + refute_nil elem + assert_equal test[2], elem.inner_html + end + + define_method("test_serializing_dom_outerHTML_#{test[0]}".to_sym) do + elem = test[1].call + refute_nil elem + assert_equal test[3], elem.serialize + end + end + + cross_map(TEXT_TESTS, TEXT_ELEMENTS) do |test_data, tag| + define_method("test_serializing_text_innerHTML_#{test_data[0].gsub("%text", tag)}".to_sym) do + assert_equal test_data[2].gsub("%text", tag), test_data[1].call(tag).inner_html + end + + define_method("test_serialization_text_outerHTML_#{test_data[0].gsub("%text", tag)}".to_sym) do + assert_equal test_data[3].gsub("%text", tag), test_data[1].call(tag).serialize + end + end + + cross_map(VOID_TESTS, VOID_ELEMENTS) do |test_data, tag| + define_method("test_serializing_void_innerHTML_#{test_data[0]}_#{tag}".to_sym) do + assert_equal test_data[2].gsub("%void", tag), test_data[1].call(tag).inner_html + end + + define_method("test_serialization_void_outerHTML_#{test_data[0]}_#{tag}".to_sym) do + assert_equal test_data[3].gsub("%void", tag), test_data[1].call(tag).serialize + end + end +end if Nokogiri.uses_gumbo? diff --git a/test/html5/test_tree-construction.rb b/test/html5/test_tree-construction.rb new file mode 100644 index 0000000000..37fdee20d4 --- /dev/null +++ b/test/html5/test_tree-construction.rb @@ -0,0 +1,282 @@ +# encoding: utf-8 +# frozen_string_literal: true +require "helper" + +if Nokogiri.uses_gumbo? + def parse_test(test_data) + test = { script: :both } + index = /(?:^#errors\n|\n#errors\n)/ =~ test_data + abort("Expected #errors in\n#{test_data}") if index.nil? + skip_amount = $~[0].length + # Omit the final new line + test[:data] = test_data[0...index] + + # Process the rest line by line + lines = test_data[index + skip_amount..-1].split("\n") + index = lines.find_index do |line| + line == "#document-fragment" || + line == "#document" || + line == "#script-off" || + line == "#script-on" || + line == "#new-errors" + end + abort("Expected #document") if index.nil? + test[:errors] = lines[0...index] + test[:new_errors] = [] + if lines[index] == "#new-errors" + index += 1 + until %w[#document-fragment #document #script-off #script-on].include?(lines[index]) + test[:new_errors] << lines[index] + index += 1 + end + end + + if lines[index] == "#document-fragment" + test[:context] = lines[index + 1].chomp.split(" ", 2) + index += 2 + end + abort("failed to find fragment: #{index}: #{lines[index]}") if test_data.include?("#document-fragment") && test[:context].nil? + + if lines[index] =~ /#script-(on|off)/ + test[:script] = $~[1].to_sym + index += 1 + end + + abort("Expected #document, got #{lines[index]}") unless lines[index] == "#document" + index += 1 + + document = { + type: test[:context] ? :fragment : :document, + children: [], + } + open_nodes = [document] + while index < lines.length + abort("Expected '| ' but got #{lines[index]}") unless /^\| ( *)([^ ].*$)/ =~ lines[index] + depth = $~[1].length + if depth.odd? + abort("Invalid nesting depth") + else + depth /= 2 + end + abort("Too deep") if depth >= open_nodes.length + + node = {} + node_text = $~[2] + if node_text[0] == '"' + if node_text == '"' || node_text[-1] != '"' + loop do + index += 1 + node_text << "\n" + lines[index] + break if node_text[-1] == '"' + end + end + node[:type] = :text + node[:contents] = node_text[1..-2] + elsif /^<!DOCTYPE ([^ >]*)(?: "([^"]*)" "(.*)")?>$/ =~ node_text + node[:type] = :doctype + node[:name] = $~[1] + node[:public_id] = $~[2].nil? || $~[2].empty? ? nil : $~[2] + node[:system_id] = $~[3].nil? || $~[3].empty? ? nil : $~[3] + elsif node_text.start_with?('<!-- ') + loop do + break if lines[index].end_with?(' -->') + index += 1 + node_text << "\n" + lines[index] + end + node[:type] = :comment + node[:contents] = node_text[5..-5] + elsif /^<(svg |math )?(.+)>$/ =~ node_text + node[:type] = :element + node[:ns] = $~[1].nil? ? nil : $~[1].rstrip + node[:tag] = $~[2] + node[:attributes] = [] + node[:children] = [] + elsif /^([^ ]+ )?([^=]+)="(.*)"$/ =~ node_text + node[:type] = :attribute + node[:ns] = $~[1].nil? ? nil : $~[1].rstrip + node[:name] = $~[2] + node[:value] = $~[3] + elsif node_text == "content" + node[:type] = :template + else + abort("Unexpected node_text: #{node_text}") + end + + if node[:type] == :attribute + abort("depth #{depth} != #{open_nodes.length}") unless depth == open_nodes.length - 1 + abort("type :#{open_nodes[-1][:type]} != :element") unless open_nodes[-1][:type] == :element + abort("element has children") unless open_nodes[-1][:children].empty? + open_nodes[-1][:attributes] << node + elsif node[:type] == :template + abort("depth #{depth} != #{open_nodes.length}") unless depth == open_nodes.length - 1 + abort("type :#{open_nodes[-1][:type]} != :element") unless open_nodes[-1][:type] == :element + abort("tag :#{open_nodes[-1][:tag]} != template") unless open_nodes[-1][:tag] == "template" + abort("template has children before the 'content'") unless open_nodes[-1][:children].empty? + # Hack. We want the children of this template node to be reparented as + # children of the template element. + # XXX: Template contents are _not_ supposed to be children of the + # template, but we currently mishandle this. + open_nodes << open_nodes[-1] + else + open_nodes[depth][:children] << node + open_nodes[depth + 1..-1] = [] + if node[:type] == :element + open_nodes << node + end + end + index += 1 + end + test[:document] = document + test + end + + class TestHtml5TreeConstructionBase < Nokogiri::TestCase + def assert_equal_or_nil(exp, act) + if exp.nil? + assert_nil(act) + else + assert_equal(exp, act) + end + end + + def compare_nodes(node, ng_node) + case ng_node.type + when Nokogiri::XML::Node::ELEMENT_NODE + assert_equal(node[:type], :element) + if node[:ns] + refute_nil(ng_node.namespace) + assert_equal(node[:ns], ng_node.namespace.prefix) + end + assert_equal(node[:tag], ng_node.name) + attributes = ng_node.attributes + assert_equal(node[:attributes].length, attributes.length) + node[:attributes].each do |attr| + value = if attr[:ns] + ng_node["#{attr[:ns]}:#{attr[:name]}"] + else + attributes[attr[:name]].value + end + assert_equal(attr[:value], value) + end + assert_equal(node[:children].length, ng_node.children.length, + "Element <#{node[:tag]}> has wrong number of children #{ng_node.children.map { |c| c.name }} in #{@test[:data]}") + when Nokogiri::XML::Node::TEXT_NODE, Nokogiri::XML::Node::CDATA_SECTION_NODE + # We preserve the CDATA in the tree, but the tests represent it as text. + assert_equal(node[:type], :text) + assert_equal(node[:contents], ng_node.content) + when Nokogiri::XML::Node::COMMENT_NODE + assert_equal(node[:type], :comment) + assert_equal(node[:contents], ng_node.content) + when Nokogiri::XML::Node::HTML_DOCUMENT_NODE + assert_equal(node[:type], :document) + assert_equal(node[:children].length, ng_node.children.length) + when Nokogiri::XML::Node::DOCUMENT_FRAG_NODE + assert_equal(node[:type], :fragment) + assert_equal(node[:children].length, ng_node.children.length, + "Fragment node has wrong number of children #{ng_node.children.map { |c| c.name }} in #{@test[:data]}") + when Nokogiri::XML::Node::DTD_NODE + assert_equal(node[:type], :doctype) + assert_equal(node[:name], ng_node.name) + assert_equal_or_nil(node[:public_id], ng_node.external_id) + assert_equal_or_nil(node[:system_id], ng_node.system_id) + else + flunk("Unknown node type #{ng_node.type} (expected #{node[:type]})") + end + end + + def run_test + if @test[:context] + ctx = @test[:context].join(":") + doc = Nokogiri::HTML5::Document.new + doc = Nokogiri::HTML5::DocumentFragment.new(doc, @test[:data], ctx, max_errors: @test[:errors].length + 10) + else + doc = Nokogiri::HTML5.parse(@test[:data], max_errors: @test[:errors].length + 10) + end + # Walk the tree. + exp_nodes = [@test[:document]] + act_nodes = [doc] + children = [0] + compare_nodes(exp_nodes[0], doc) + while children.any? + child_index = children[-1] + exp = exp_nodes[-1] + act = act_nodes[-1] + if child_index == exp[:children].length + exp_nodes.pop + act_nodes.pop + children.pop + next + end + exp_child = exp[:children][child_index] + act_child = act.children[child_index] + compare_nodes(exp_child, act_child) + children[-1] = child_index + 1 + next unless exp_child.has_key?(:children) + exp_nodes << exp_child + act_nodes << act_child + children << 0 + end + + # Test the errors. + assert_equal(@test[:errors].length, doc.errors.length, "Wrong number of errors for #{@test[:data]}") + + # The new, standardized tokenizer errors live in @test[:new_errors]. Let's + # match each one to exactly one error in doc.errors. Unfortunately, the + # tests specify the column the error is detected, _not_ the column of the + # start of the problematic HTML (e.g., the start of a character reference + # or <![CDATA[) the way gumbo does. So check that Gumbo's column is no + # later than the error's column. + errors = doc.errors.map { |err| { line: err.line, column: err.column, code: err.str1 } } + errors.reject! { |err| err[:code] == "generic-parser" } + error_regex = /^\((?<line>\d+):(?<column>\d+)(?:-\d+:\d+)?\) (?<code>.*)$/ + @test[:new_errors].each do |err| + assert_match(error_regex, err, "New error format does not match: #{mu_pp(err)}") + m = err.match(error_regex) + line = m[:line].to_i + column = m[:column].to_i + code = m[:code] + idx = errors.index do |e| + e[:line] == line && + e[:code] == code && + e[:column] <= column + end + # This error should be the first error in the list. + # refute_nil(idx, "Expected to find error #{code} at #{line}:#{column}") + assert_equal(0, idx, "Expected to find error #{code} at #{line}:#{column} in #{@test[:data]}") + errors.delete_at(idx) + end + end + end + + tc_path = File.expand_path("../../html5lib-tests/tree-construction", __FILE__) + Dir[File.join(tc_path, "*.dat")].each do |path| + test_name = "TestHtml5TreeConstruction" + File.basename(path, ".dat") + .split(/[_-]/) + .map { |s| s.capitalize } + .join("") + tests = [] + File.open(path, "r", encoding: "UTF-8") do |f| + f.each("\n\n#data\n") do |test_data| + if test_data.start_with?("#data\n") + test_data = test_data[6..-1] + end + if test_data.end_with?("\n\n#data\n") + test_data = test_data[0..-9] + end + tests << parse_test(test_data) + end + end + + klass = Class.new(TestHtml5TreeConstructionBase) do + tests.each_with_index do |test, index| + next if test[:script] == :on + define_method "test_#{index}".to_sym do + @test = test + @index = index + run_test + end + end + end + Object.const_set(test_name, klass) + end +end diff --git a/test/html5lib-tests b/test/html5lib-tests new file mode 160000 index 0000000000..6030cb6e40 --- /dev/null +++ b/test/html5lib-tests @@ -0,0 +1 @@ +Subproject commit 6030cb6e40a0cf68ae38bf0001bb85b727b80a26 diff --git a/test/namespaces/test_namespaces_in_created_doc.rb b/test/namespaces/test_namespaces_in_created_doc.rb index 43dd7a9b8b..a1cc701ad1 100644 --- a/test/namespaces/test_namespaces_in_created_doc.rb +++ b/test/namespaces/test_namespaces_in_created_doc.rb @@ -7,7 +7,7 @@ def setup super @doc = Nokogiri::XML('<fruit xmlns="ns:fruit" xmlns:veg="ns:veg" xmlns:xlink="http://www.w3.org/1999/xlink"/>') pear = @doc.create_element('pear') - bosc = @doc.create_element('bosc') + bosc = @doc.create_element('bosc', {"veg:any" => "none"}) pear.add_child(bosc) @doc.root << pear @doc.root.add_child('<orange/>') @@ -38,6 +38,9 @@ def test_created_parent_default_ns def test_created_grandparent_default_ns assert_equal 'ns:fruit', check_namespace(@doc.root.elements[0].elements[0]) end + def test_created_attribute_with_ns + assert_equal 'ns:veg', check_namespace(@doc.root.elements[0].elements[0].attribute_nodes.first) + end def test_created_parent_nondefault_ns assert_equal 'ns:veg', check_namespace(@doc.root.elements[2]) end diff --git a/test/test_nokogiri.rb b/test/test_nokogiri.rb index e1a90b43f1..d44a6d5252 100644 --- a/test/test_nokogiri.rb +++ b/test/test_nokogiri.rb @@ -1,54 +1,55 @@ +# frozen_string_literal: true require "helper" -class TestNokogiri < Nokogiri::TestCase - def test_libxml_iconv - skip_unless_libxml2("this constant is only set in the C extension when libxml2 is used") - assert Nokogiri.const_defined?(:LIBXML_ICONV_ENABLED) - end - - def test_parse_with_io - doc = Nokogiri.parse( - StringIO.new("<html><head><title></title><body></body></html>") - ) - assert_instance_of Nokogiri::HTML::Document, doc - end - - def test_xml? - doc = Nokogiri.parse(File.read(XML_FILE)) - assert doc.xml? - assert !doc.html? - end - - def test_atom_is_xml? - doc = Nokogiri.parse(File.read(XML_ATOM_FILE)) - assert doc.xml? - assert !doc.html? - end - - def test_html? - doc = Nokogiri.parse(File.read(HTML_FILE)) - assert !doc.xml? - assert doc.html? - end - - def test_nokogiri_method_with_html - doc1 = Nokogiri(File.read(HTML_FILE)) - doc2 = Nokogiri.parse(File.read(HTML_FILE)) - assert_equal doc1.serialize, doc2.serialize - end - - def test_nokogiri_method_with_block - doc = Nokogiri { b "bold tag" } - assert_equal("<b>bold tag</b>", doc.to_html.chomp) - end - - def test_make_with_html - doc = Nokogiri.make("<b>bold tag</b>") - assert_equal("<b>bold tag</b>", doc.to_html.chomp) - end - - def test_make_with_block - doc = Nokogiri.make { b "bold tag" } - assert_equal("<b>bold tag</b>", doc.to_html.chomp) +class Nokogiri::TestCase + describe Nokogiri do + def test_libxml_iconv + skip_unless_libxml2("this constant is only set in the C extension when libxml2 is used") + assert(Nokogiri.const_defined?(:LIBXML_ICONV_ENABLED)) + end + + def test_parse_with_io + doc = Nokogiri.parse(StringIO.new("<html><head><title></title><body></body></html>")) + assert_instance_of(Nokogiri::HTML::Document, doc) + end + + def test_xml? + doc = Nokogiri.parse(File.read(XML_FILE)) + assert(doc.xml?) + assert(!doc.html?) + end + + def test_atom_is_xml? + doc = Nokogiri.parse(File.read(XML_ATOM_FILE)) + assert(doc.xml?) + assert(!doc.html?) + end + + def test_html? + doc = Nokogiri.parse(File.read(HTML_FILE)) + assert(!doc.xml?) + assert(doc.html?) + end + + def test_nokogiri_method_with_html + doc1 = Nokogiri(File.read(HTML_FILE)) + doc2 = Nokogiri.parse(File.read(HTML_FILE)) + assert_equal(doc1.serialize, doc2.serialize) + end + + def test_nokogiri_method_with_block + doc = Nokogiri { b("bold tag") } + assert_equal("<b>bold tag</b>", doc.to_html.chomp) + end + + def test_make_with_html + doc = Nokogiri.make("<b>bold tag</b>") + assert_equal("<b>bold tag</b>", doc.to_html.chomp) + end + + def test_make_with_block + doc = Nokogiri.make { b("bold tag") } + assert_equal("<b>bold tag</b>", doc.to_html.chomp) + end end end diff --git a/test/test_nokogumbo_contract.rb b/test/test_nokogumbo_contract.rb new file mode 100644 index 0000000000..83df5837ea --- /dev/null +++ b/test/test_nokogumbo_contract.rb @@ -0,0 +1,26 @@ +require "helper" + +describe "Nokogumbo contract expectations" do + # per https://github.com/rubys/nokogumbo/pull/171 + it "includes the HTML5 public interface" do + skip("Gumbo is not supported on this platform") unless Nokogiri.uses_gumbo? + + assert_includes(::Nokogiri.singleton_methods, :HTML5) + + assert_equal(defined?(::Nokogiri::HTML5), "constant") + assert_includes(::Nokogiri::HTML5.singleton_methods, :parse) + assert_includes(::Nokogiri::HTML5.singleton_methods, :fragment) + + assert_equal(defined?(::Nokogiri::HTML5::Node), "constant") + assert_equal(defined?(::Nokogiri::HTML5::Document), "constant") + assert_equal(defined?(::Nokogiri::HTML5::DocumentFragment), "constant") + end + + it "includes a replacement for the Nokogumbo private interface" do + skip("Gumbo is not supported on this platform") unless Nokogiri.uses_gumbo? + + assert_equal(defined?(::Nokogiri::Gumbo), "constant") + assert_includes(::Nokogiri::Gumbo.singleton_methods, :parse) + assert_includes(::Nokogiri::Gumbo.singleton_methods, :fragment) + end +end diff --git a/test/test_version.rb b/test/test_version.rb index dda8f392ba..e8f7468b6f 100644 --- a/test/test_version.rb +++ b/test/test_version.rb @@ -59,6 +59,7 @@ def test_version_info_for_libxml assert_equal("#{major}.#{minor}.#{bug}", Nokogiri::VERSION_INFO["libxml"]["loaded"]) assert(version_info["libxml"].key?("iconv_enabled")) + assert(version_info["libxslt"].key?("datetime_enabled")) end def test_version_info_for_libxslt diff --git a/test/test_xslt_transforms.rb b/test/test_xslt_transforms.rb index 708215dc80..1f34290b6a 100644 --- a/test/test_xslt_transforms.rb +++ b/test/test_xslt_transforms.rb @@ -1,372 +1,406 @@ # frozen_string_literal: true require "helper" -class TestXsltTransforms < Nokogiri::TestCase - def setup - super - @doc = Nokogiri::XML(File.open(XML_FILE)) - end +class Nokogiri::TestCase + describe Nokogiri::XSLT::Stylesheet do + let(:doc) { Nokogiri::XML(File.open(XML_FILE)) } - def test_class_methods - style = Nokogiri::XSLT(File.read(XSLT_FILE)) + def test_class_methods + style = Nokogiri::XSLT(File.read(XSLT_FILE)) - assert(result = style.apply_to(@doc, ['title', '"Grandma"'])) - assert_match(%r{<h1>Grandma</h1>}, result) - end + assert(result = style.apply_to(doc, ["title", '"Grandma"'])) + assert_match(%r{<h1>Grandma</h1>}, result) + end - def test_transform - assert(style = Nokogiri::XSLT.parse(File.read(XSLT_FILE))) - - assert(result = style.apply_to(@doc, ['title', '"Booyah"'])) - assert_match(%r{<h1>Booyah</h1>}, result) - assert_match(%r{<th.*Employee ID</th>}, result) - assert_match(%r{<th.*Name</th>}, result) - assert_match(%r{<th.*Position</th>}, result) - assert_match(%r{<th.*Salary</th>}, result) - assert_match(%r{<td>EMP0003</td>}, result) - assert_match(%r{<td>Margaret Martin</td>}, result) - assert_match(%r{<td>Computer Specialist</td>}, result) - assert_match(%r{<td>100,000</td>}, result) - assert_no_match(/Dallas|Texas/, result) - assert_no_match(/Female/, result) - - assert(result = style.apply_to(@doc, ['title', '"Grandma"'])) - assert_match(%r{<h1>Grandma</h1>}, result) - - assert(result = style.apply_to(@doc)) - assert_match(%r{<h1></h1>|<h1/>}, result) - end + def test_transform + assert(style = Nokogiri::XSLT.parse(File.read(XSLT_FILE))) + + assert(result = style.apply_to(doc, ["title", '"Booyah"'])) + assert_match(%r{<h1>Booyah</h1>}, result) + assert_match(%r{<th.*Employee ID</th>}, result) + assert_match(%r{<th.*Name</th>}, result) + assert_match(%r{<th.*Position</th>}, result) + assert_match(%r{<th.*Salary</th>}, result) + assert_match(%r{<td>EMP0003</td>}, result) + assert_match(%r{<td>Margaret Martin</td>}, result) + assert_match(%r{<td>Computer Specialist</td>}, result) + assert_match(%r{<td>100,000</td>}, result) + assert_no_match(/Dallas|Texas/, result) + assert_no_match(/Female/, result) + + assert(result = style.apply_to(doc, ["title", '"Grandma"'])) + assert_match(%r{<h1>Grandma</h1>}, result) + + assert(result = style.apply_to(doc)) + assert_match(%r{<h1></h1>|<h1/>}, result) + end - def test_xml_declaration - input_xml = <<~EOS - <?xml version="1.0" encoding="utf-8"?> - <report> - <title>My Report</title> - </report> - EOS - - input_xsl = <<~EOS - <?xml version="1.0" encoding="utf-8"?> - <xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> - <xsl:output method="xml" version="1.0" encoding="utf-8" indent="yes"/> - <xsl:template match="/"> - <html> - <head> - <title><xsl:value-of select="report/title"/></title> - <style> .commit-tease, .user-profile-mini-avatar, .avatar, .vcard-details, .signup-prompt-bg { display: none !IMPORTANT; } </style> <script> document.addEventListener('DOMContentLoaded', function() { this.querySelectorAll('a').forEach(anchor => { anchor.addEventListener('click', e => { e.preventDefault(); const redact = new URLSearchParams(window.location.search).get('redact'); const hasExistingParams = anchor.href.includes('?'); window.location.href = anchor.href + (hasExistingParams ? `&redact=${redact}` : `?redact=${redact}`); }); }); }); </script> </head> - <body> - <h1><xsl:value-of select="report/title"/></h1> - </body> - </html> - </xsl:template> - </xsl:stylesheet> - EOS + def test_xml_declaration + input_xml = <<~EOS + <?xml version="1.0" encoding="utf-8"?> + <report> + <title>My Report</title> + </report> + EOS + + input_xsl = <<~EOS + <?xml version="1.0" encoding="utf-8"?> + <xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> + <xsl:output method="xml" version="1.0" encoding="utf-8" indent="yes"/> + <xsl:template match="/"> + <html> + <head> + <title><xsl:value-of select="report/title"/></title> + <style> .commit-tease, .user-profile-mini-avatar, .avatar, .vcard-details, .signup-prompt-bg { display: none !IMPORTANT; } </style> <script> document.addEventListener('DOMContentLoaded', function() { this.querySelectorAll('a').forEach(anchor => { anchor.addEventListener('click', e => { e.preventDefault(); const redact = new URLSearchParams(window.location.search).get('redact'); const hasExistingParams = anchor.href.includes('?'); window.location.href = anchor.href + (hasExistingParams ? `&redact=${redact}` : `?redact=${redact}`); }); }); }); </script> </head> + <body> + <h1><xsl:value-of select="report/title"/></h1> + </body> + </html> + </xsl:template> + </xsl:stylesheet> + EOS - require 'nokogiri' + require "nokogiri" - xml = ::Nokogiri::XML(input_xml) - xsl = ::Nokogiri::XSLT(input_xsl) + xml = ::Nokogiri::XML(input_xml) + xsl = ::Nokogiri::XSLT(input_xsl) - assert_includes(xsl.apply_to(xml), '<?xml version="1.0" encoding="utf-8"?>') - end + assert_includes(xsl.apply_to(xml), '<?xml version="1.0" encoding="utf-8"?>') + end - def test_transform_with_output_style - xslt = "" - xslt = if Nokogiri.jruby? - Nokogiri::XSLT(<<~eoxslt) - <?xml version="1.0" encoding="ISO-8859-1"?> + def test_transform_with_output_style + xslt = "" + xslt = if Nokogiri.jruby? + Nokogiri::XSLT(<<~eoxslt) + <?xml version="1.0" encoding="ISO-8859-1"?> - <xsl:stylesheet version="1.0" - xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> - <xsl:output method="text" version="1.0" - encoding="iso-8859-1" indent="yes"/> - - <xsl:param name="title"/> - - <xsl:template match="/"> - <html> - <body> - <xsl:for-each select="staff/employee"> - <tr> - <td><xsl:value-of select="employeeId"/></td> - <td><xsl:value-of select="name"/></td> - <td><xsl:value-of select="position"/></td> - <td><xsl:value-of select="salary"/></td> - </tr> - </xsl:for-each> - </body> - </html> - </xsl:template> + <xsl:stylesheet version="1.0" + xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> + <xsl:output method="text" version="1.0" + encoding="iso-8859-1" indent="yes"/> - </xsl:stylesheet> - eoxslt - else - Nokogiri::XSLT(<<~eoxslt) - <?xml version="1.0" encoding="ISO-8859-1"?> + <xsl:param name="title"/> - <xsl:stylesheet version="1.0" - xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> - <xsl:output method="text" version="1.0" - encoding="iso-8859-1" indent="yes"/> - - <xsl:param name="title"/> - - <xsl:template match="/"> - <html> - <body> - <xsl:for-each select="staff/employee"> - <tr> - <td><xsl:value-of select="employeeId"/></td> - <td><xsl:value-of select="name"/></td> - <td><xsl:value-of select="position"/></td> - <td><xsl:value-of select="salary"/></td> - </tr> - </xsl:for-each> - </table> - </body> - </html> - </xsl:template> + <xsl:template match="/"> + <html> + <body> + <xsl:for-each select="staff/employee"> + <tr> + <td><xsl:value-of select="employeeId"/></td> + <td><xsl:value-of select="name"/></td> + <td><xsl:value-of select="position"/></td> + <td><xsl:value-of select="salary"/></td> + </tr> + </xsl:for-each> + </body> + </html> + </xsl:template> - </xsl:stylesheet> - eoxslt + </xsl:stylesheet> + eoxslt + else + Nokogiri::XSLT(<<~eoxslt) + <?xml version="1.0" encoding="ISO-8859-1"?> + + <xsl:stylesheet version="1.0" + xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> + <xsl:output method="text" version="1.0" + encoding="iso-8859-1" indent="yes"/> + + <xsl:param name="title"/> + + <xsl:template match="/"> + <html> + <body> + <xsl:for-each select="staff/employee"> + <tr> + <td><xsl:value-of select="employeeId"/></td> + <td><xsl:value-of select="name"/></td> + <td><xsl:value-of select="position"/></td> + <td><xsl:value-of select="salary"/></td> + </tr> + </xsl:for-each> + </table> + </body> + </html> + </xsl:template> + + </xsl:stylesheet> + eoxslt + end + result = xslt.apply_to(doc, ["title", "foo"]) + assert_no_match(/<td>/, result) + + # the entity-form is for systems with this bug with Encodings.properties + # https://issues.apache.org/jira/browse/XALANJ-2618 + # a.k.a. "Attempt to output character of integral value 48 that is not represented in specified output encoding of iso-8859-1." + assert_match( + /This is an adjacent|This is an adjacent/, result + ) end - result = xslt.apply_to(@doc, ['title', 'foo']) - assert_no_match(/<td>/, result) - - # the entity-form is for systems with this bug with Encodings.properties - # https://issues.apache.org/jira/browse/XALANJ-2618 - # a.k.a. "Attempt to output character of integral value 48 that is not represented in specified output encoding of iso-8859-1." - assert_match( - /This is an adjacent|This is an adjacent/, result - ) - end - def test_transform_arg_error - assert(style = Nokogiri::XSLT(File.read(XSLT_FILE))) - assert_raises(TypeError) do - style.transform(@doc, :foo) + def test_transform_arg_error + assert(style = Nokogiri::XSLT(File.read(XSLT_FILE))) + assert_raises(TypeError) do + style.transform(doc, :foo) + end end - end - def test_transform_with_hash - assert(style = Nokogiri::XSLT(File.read(XSLT_FILE))) - result = style.transform(@doc, { 'title' => '"Booyah"' }) - assert(result.html?) - assert_equal("Booyah", result.at_css("h1").content) - end + def test_transform_with_hash + assert(style = Nokogiri::XSLT(File.read(XSLT_FILE))) + result = style.transform(doc, { "title" => '"Booyah"' }) + assert(result.html?) + assert_equal("Booyah", result.at_css("h1").content) + end - def test_transform2 - assert(style = Nokogiri::XSLT(File.open(XSLT_FILE))) - assert(result_doc = style.transform(@doc)) - assert(result_doc.html?) - assert_equal("", result_doc.at_css("h1").content) + def test_transform2 + assert(style = Nokogiri::XSLT(File.open(XSLT_FILE))) + assert(result_doc = style.transform(doc)) + assert(result_doc.html?) + assert_equal("", result_doc.at_css("h1").content) - assert(style = Nokogiri::XSLT(File.read(XSLT_FILE))) - assert(result_doc = style.transform(@doc, ['title', '"Booyah"'])) - assert(result_doc.html?) - assert_equal("Booyah", result_doc.at_css("h1").content) + assert(style = Nokogiri::XSLT(File.read(XSLT_FILE))) + assert(result_doc = style.transform(doc, ["title", '"Booyah"'])) + assert(result_doc.html?) + assert_equal("Booyah", result_doc.at_css("h1").content) - assert(result_string = style.apply_to(@doc, ['title', '"Booyah"'])) - assert_equal(result_string, style.serialize(result_doc)) - end + assert(result_string = style.apply_to(doc, ["title", '"Booyah"'])) + assert_equal(result_string, style.serialize(result_doc)) + end - def test_transform_with_quote_params - assert(style = Nokogiri::XSLT(File.open(XSLT_FILE))) - assert(result_doc = style.transform(@doc, Nokogiri::XSLT.quote_params(['title', 'Booyah']))) - assert(result_doc.html?) - assert_equal("Booyah", result_doc.at_css("h1").content) + def test_transform_with_quote_params + assert(style = Nokogiri::XSLT(File.open(XSLT_FILE))) + assert(result_doc = style.transform(doc, Nokogiri::XSLT.quote_params(["title", "Booyah"]))) + assert(result_doc.html?) + assert_equal("Booyah", result_doc.at_css("h1").content) - assert(style = Nokogiri::XSLT.parse(File.read(XSLT_FILE))) - assert(result_doc = style.transform(@doc, Nokogiri::XSLT.quote_params({ 'title' => 'Booyah' }))) - assert(result_doc.html?) - assert_equal("Booyah", result_doc.at_css("h1").content) - end + assert(style = Nokogiri::XSLT.parse(File.read(XSLT_FILE))) + assert(result_doc = style.transform(doc, Nokogiri::XSLT.quote_params({ "title" => "Booyah" }))) + assert(result_doc.html?) + assert_equal("Booyah", result_doc.at_css("h1").content) + end - def test_quote_params - h = { - :sym => %{xxx}, - 'str' => %{"xxx"}, - :sym2 => %{'xxx'}, - 'str2' => %{x'x'x}, - :sym3 => %{x"x"x}, - } - hh = h.dup - result_hash = Nokogiri::XSLT.quote_params(h) - assert_equal(hh, h) # non-destructive - - a = h.to_a.flatten - result_array = Nokogiri::XSLT.quote_params(a) - assert_equal(h.to_a.flatten, a) # non-destructive - - assert_equal(result_array, result_hash) - end + def test_quote_params + h = { + :sym => %{xxx}, + "str" => %{"xxx"}, + :sym2 => %{'xxx'}, + "str2" => %{x'x'x}, + :sym3 => %{x"x"x}, + } + hh = h.dup + result_hash = Nokogiri::XSLT.quote_params(h) + assert_equal(hh, h) # non-destructive + + a = h.to_a.flatten + result_array = Nokogiri::XSLT.quote_params(a) + assert_equal(h.to_a.flatten, a) # non-destructive + + assert_equal(result_array, result_hash) + end - def test_exslt - # see http://yokolet.blogspot.com/2010/10/pure-java-nokogiri-xslt-extension.html") - skip_unless_libxml2("cannot get it working on JRuby") - - assert(doc = Nokogiri::XML.parse(File.read(EXML_FILE))) - assert(doc.xml?) - - assert(style = Nokogiri::XSLT.parse(File.read(EXSLT_FILE))) - params = { - p1: 'xxx', - p2: "x'x'x", - p3: 'x"x"x', - p4: '"xxx"', - } - result_doc = Nokogiri::XML.parse(style.apply_to(doc, + def test_exslt + # see http://yokolet.blogspot.com/2010/10/pure-java-nokogiri-xslt-extension.html") + skip_unless_libxml2("cannot get it working on JRuby") + + assert(doc = Nokogiri::XML.parse(File.read(EXML_FILE))) + assert(doc.xml?) + + assert(style = Nokogiri::XSLT.parse(File.read(EXSLT_FILE))) + params = { + p1: "xxx", + p2: "x'x'x", + p3: 'x"x"x', + p4: '"xxx"', + } + result_doc = Nokogiri::XML.parse(style.apply_to(doc, Nokogiri::XSLT.quote_params(params))) - assert_equal('func-result', result_doc.at('/root/function').content) - assert_equal(3, result_doc.at('/root/max').content.to_i) - assert_match( - /\d{4}-\d\d-\d\d([-|+]\d\d:\d\d)?/, - result_doc.at('/root/date').content - ) - result_doc.xpath('/root/params/*').each do |p| - assert_equal(p.content, params[p.name.intern]) - end - check_params(result_doc, params) - result_doc = Nokogiri::XML.parse(style.apply_to(doc, + assert_equal("func-result", result_doc.at("/root/function").content) + assert_equal(3, result_doc.at("/root/max").content.to_i) + if Nokogiri::VersionInfo.instance.libxslt_has_datetime? + assert_match( + /\d{4}-\d\d-\d\d([-|+]\d\d:\d\d)?/, + result_doc.at("/root/date").content + ) + end + result_doc.xpath("/root/params/*").each do |p| + assert_equal(p.content, params[p.name.intern]) + end + check_params(result_doc, params) + result_doc = Nokogiri::XML.parse(style.apply_to(doc, Nokogiri::XSLT.quote_params(params.to_a.flatten))) - check_params(result_doc, params) - end + check_params(result_doc, params) + end - def test_xslt_paramaters - # see http://yokolet.blogspot.com/2010/10/pure-java-nokogiri-xslt-extension.html") - skip_unless_libxml2("cannot get it working on JRuby") + def test_xslt_paramaters + # see http://yokolet.blogspot.com/2010/10/pure-java-nokogiri-xslt-extension.html") + skip_unless_libxml2("cannot get it working on JRuby") - xslt_str = <<~EOX - <xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" > - <xsl:template match="/"> - <xsl:value-of select="$foo" /> - </xsl:template> - </xsl:stylesheet> + xslt_str = <<~EOX + <xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" > + <xsl:template match="/"> + <xsl:value-of select="$foo" /> + </xsl:template> + </xsl:stylesheet> EOX - xslt = Nokogiri::XSLT(xslt_str) - doc = Nokogiri::XML("<root />") - assert_match(/bar/, xslt.transform(doc, Nokogiri::XSLT.quote_params('foo' => 'bar')).to_s) - end + xslt = Nokogiri::XSLT(xslt_str) + doc = Nokogiri::XML("<root />") + assert_match(/bar/, xslt.transform(doc, Nokogiri::XSLT.quote_params("foo" => "bar")).to_s) + end - def test_xslt_transform_error - # see http://yokolet.blogspot.com/2010/10/pure-java-nokogiri-xslt-extension.html") - skip_unless_libxml2("cannot get it working on JRuby") + def test_xslt_transform_error + # see http://yokolet.blogspot.com/2010/10/pure-java-nokogiri-xslt-extension.html") + skip_unless_libxml2("cannot get it working on JRuby") - xslt_str = <<~EOX - <xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" > - <xsl:template match="/"> - <xsl:value-of select="$foo" /> - </xsl:template> - </xsl:stylesheet> + xslt_str = <<~EOX + <xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" > + <xsl:template match="/"> + <xsl:value-of select="$foo" /> + </xsl:template> + </xsl:stylesheet> EOX - xslt = Nokogiri::XSLT(xslt_str) - doc = Nokogiri::XML("<root />") - assert_raises(RuntimeError) { xslt.transform(doc) } - end + xslt = Nokogiri::XSLT(xslt_str) + doc = Nokogiri::XML("<root />") + assert_raises(RuntimeError) { xslt.transform(doc) } + end - def test_xslt_parse_error - xslt_str = <<~EOX - <xsl:stylesheet version="1.0" - xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> - <!-- Not well-formed: --> - <xsl:template match="/"/> - <values> - <xsl:for-each select="//*"> - <value> - <xsl:value-of select="@id"/> - </value> - </xsl:for-each> - </values> - </xsl:template> - </xsl:stylesheet>} - EOX - assert_raises(RuntimeError) { Nokogiri::XSLT.parse(xslt_str) } - end + def test_xslt_parse_error + xslt_str = <<~EOX + <xsl:stylesheet version="1.0" + xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> + <!-- Not well-formed: --> + <xsl:template match="/"/> + <values> + <xsl:for-each select="//*"> + <value> + <xsl:value-of select="@id"/> + </value> + </xsl:for-each> + </values> + </xsl:template> + </xsl:stylesheet>} + EOX + assert_raises(RuntimeError) { Nokogiri::XSLT.parse(xslt_str) } + end - def test_passing_a_non_document_to_transform - xsl = Nokogiri::XSLT('<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform"></xsl:stylesheet>') - assert_raises(ArgumentError) { xsl.transform("<div></div>") } - assert_raises(ArgumentError) { xsl.transform(Nokogiri::HTML("").css("body")) } - end + def test_passing_a_non_document_to_transform + xsl = Nokogiri::XSLT('<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform"></xsl:stylesheet>') + assert_raises(ArgumentError) { xsl.transform("<div></div>") } + assert_raises(ArgumentError) { xsl.transform(Nokogiri::HTML("").css("body")) } + end - def check_params(result_doc, params) - result_doc.xpath('/root/params/*').each do |p| - assert_equal(p.content, params[p.name.intern]) + def check_params(result_doc, params) + result_doc.xpath("/root/params/*").each do |p| + assert_equal(p.content, params[p.name.intern]) + end end - end - def test_non_html_xslt_transform - xml = Nokogiri.XML(<<~EOXML) - <a> - <b> - <c>123</c> - </b> - </a> - EOXML - - xsl = Nokogiri.XSLT(<<~EOXSL) - <xsl:stylesheet version="1.0" - xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> - - <xsl:output encoding="UTF-8" indent="yes" method="xml" /> - - <xsl:template match="/"> - <a><xsl:value-of select="/a" /></a> - </xsl:template> - </xsl:stylesheet> - EOXSL - - result = xsl.transform(xml) - assert(!result.html?) - end + def test_non_html_xslt_transform + xml = Nokogiri.XML(<<~EOXML) + <a> + <b> + <c>123</c> + </b> + </a> + EOXML - it "should not crash when given XPath 2.0 features" do - # - # https://github.com/sparklemotion/nokogiri/issues/1802 - # - # note that here the XPath 2.0 feature is `decimal`. - # this test case is taken from the example provided in the original issue. - # - xml = <<~EOXML - <?xml version="1.0" encoding="UTF-8"?> - <Invoice xmlns="urn:oasis:names:specification:ubl:schema:xsd:Invoice-2" - xmlns:cac="urn:oasis:names:specification:ubl:schema:xsd:CommonAggregateComponents-2" - xmlns:cbc="urn:oasis:names:specification:ubl:schema:xsd:CommonBasicComponents-2" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" - xsi:schemaLocation="urn:oasis:names:specification:ubl:schema:xsd:Invoice-2"> - <cac:TaxTotal> - <cbc:TaxAmount currencyID="EUR">48.00</cbc:TaxAmount> - </cac:TaxTotal> - </Invoice> + xsl = Nokogiri.XSLT(<<~EOXSL) + <xsl:stylesheet version="1.0" + xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> + + <xsl:output encoding="UTF-8" indent="yes" method="xml" /> + + <xsl:template match="/"> + <a><xsl:value-of select="/a" /></a> + </xsl:template> + </xsl:stylesheet> + EOXSL + + result = xsl.transform(xml) + assert(!result.html?) + end + + it "should not crash when given XPath 2.0 features" do + # + # https://github.com/sparklemotion/nokogiri/issues/1802 + # + # note that here the XPath 2.0 feature is `decimal`. + # this test case is taken from the example provided in the original issue. + # + xml = <<~EOXML + <?xml version="1.0" encoding="UTF-8"?> + <Invoice xmlns="urn:oasis:names:specification:ubl:schema:xsd:Invoice-2" + xmlns:cac="urn:oasis:names:specification:ubl:schema:xsd:CommonAggregateComponents-2" + xmlns:cbc="urn:oasis:names:specification:ubl:schema:xsd:CommonBasicComponents-2" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="urn:oasis:names:specification:ubl:schema:xsd:Invoice-2"> + <cac:TaxTotal> + <cbc:TaxAmount currencyID="EUR">48.00</cbc:TaxAmount> + </cac:TaxTotal> + </Invoice> EOXML - xsl = <<~EOXSL - <?xml version="1.0" encoding="UTF-8" standalone="yes"?> - <xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" - xmlns:cbc="urn:oasis:names:specification:ubl:schema:xsd:CommonBasicComponents-2" - xmlns:cac="urn:oasis:names:specification:ubl:schema:xsd:CommonAggregateComponents-2" - xmlns:ubl="urn:oasis:names:specification:ubl:schema:xsd:Invoice-2" - xmlns:xs="http://www.w3.org/2001/XMLSchema" - version="1.0"> - <xsl:template match="/"> - <xsl:apply-templates select="/" mode="qwerty"/> - </xsl:template> - <xsl:template match="/ubl:Invoice/cac:TaxTotal" priority="1001" mode="qwerty"> - <xsl:choose> - <xsl:when test="(round(xs:decimal(child::cbc:TaxAmount)))"/> - </xsl:choose> - </xsl:template> - </xsl:stylesheet> + xsl = <<~EOXSL + <?xml version="1.0" encoding="UTF-8" standalone="yes"?> + <xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" + xmlns:cbc="urn:oasis:names:specification:ubl:schema:xsd:CommonBasicComponents-2" + xmlns:cac="urn:oasis:names:specification:ubl:schema:xsd:CommonAggregateComponents-2" + xmlns:ubl="urn:oasis:names:specification:ubl:schema:xsd:Invoice-2" + xmlns:xs="http://www.w3.org/2001/XMLSchema" + version="1.0"> + <xsl:template match="/"> + <xsl:apply-templates select="/" mode="qwerty"/> + </xsl:template> + <xsl:template match="/ubl:Invoice/cac:TaxTotal" priority="1001" mode="qwerty"> + <xsl:choose> + <xsl:when test="(round(xs:decimal(child::cbc:TaxAmount)))"/> + </xsl:choose> + </xsl:template> + </xsl:stylesheet> EOXSL - doc = Nokogiri::XML(xml) - xslt = Nokogiri::XSLT(xsl) - exception = assert_raise(RuntimeError) do - xslt.transform(doc) + doc = Nokogiri::XML(xml) + xslt = Nokogiri::XSLT(xsl) + exception = assert_raise(RuntimeError) do + xslt.transform(doc) + end + assert_match(/decimal/, exception.message) + end + + describe "DEFAULT_XSLT parse options" do + it "is the union of DEFAULT_XML and libxslt's XSLT_PARSE_OPTIONS" do + xslt_parse_options = Nokogiri::XML::ParseOptions.new.noent.dtdload.dtdattr.nocdata + expected = Nokogiri::XML::ParseOptions::DEFAULT_XML | xslt_parse_options.options + assert_equal(expected, Nokogiri::XML::ParseOptions::DEFAULT_XSLT) + end + + it "parses docs the same as xsltproc" do + skip_unless_libxml2("JRuby implementation disallows this edge case XSLT") + + # see https://github.com/sparklemotion/nokogiri/issues/1940 + xml = "<t></t>" + xsl = <<~EOF + <?xml version="1.0"?> + <xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform"> + <xsl:output method="text" omit-xml-declaration="no" /> + <xsl:template match="/"> + <xsl:text disable-output-escaping="yes"><![CDATA[<>]]></xsl:text> + </xsl:template> + </xsl:stylesheet> + EOF + + doc = Nokogiri::XML(xml) + stylesheet = Nokogiri::XSLT(xsl) + + # TODO: ideally I'd like to be able to access the parse options in the final object + # assert_equal(Nokogiri::XML::ParseOptions::DEFAULT_XSLT, stylesheet.document.parse_options) + + result = stylesheet.transform(doc) + assert_equal("<>", result.children.to_xml) + end end - assert_match(/decimal/, exception.message) end end diff --git a/test/xml/sax/test_parser.rb b/test/xml/sax/test_parser.rb index de078d6115..1a54ff638b 100644 --- a/test/xml/sax/test_parser.rb +++ b/test/xml/sax/test_parser.rb @@ -2,425 +2,462 @@ require "helper" -module Nokogiri - module XML - module SAX - class TestParser < Nokogiri::SAX::TestCase - def setup - super - @parser = XML::SAX::Parser.new(Doc.new) - end - - def test_parser_context_yielded_io - doc = Doc.new - parser = XML::SAX::Parser.new(doc) - xml = "<foo a='&b'/>" - - block_called = false - parser.parse(StringIO.new(xml)) do |ctx| - block_called = true - ctx.replace_entities = true - end - - assert(block_called) - - assert_equal([["foo", [["a", "&b"]]]], doc.start_elements) - end - - def test_parser_context_yielded_in_memory - doc = Doc.new - parser = XML::SAX::Parser.new(doc) - xml = "<foo a='&b'/>" - - block_called = false - parser.parse(xml) do |ctx| - block_called = true - ctx.replace_entities = true - end - - assert(block_called) - - assert_equal([["foo", [["a", "&b"]]]], doc.start_elements) - end - - def test_empty_decl - parser = XML::SAX::Parser.new(Doc.new) - - xml = "<root />" - parser.parse(xml) - assert(parser.document.start_document_called, xml) - assert_nil(parser.document.xmldecls, xml) - end - - def test_xml_decl - [ - ['<?xml version="1.0" ?>', ["1.0"]], - ['<?xml version="1.0" encoding="UTF-8" ?>', ["1.0", "UTF-8"]], - ['<?xml version="1.0" standalone="yes"?>', ["1.0", "yes"]], - ['<?xml version="1.0" standalone="no"?>', ["1.0", "no"]], - ['<?xml version="1.0" encoding="UTF-8" standalone="no"?>', ["1.0", "UTF-8", "no"]], - ['<?xml version="1.0" encoding="ISO-8859-1" standalone="yes"?>', ["1.0", "ISO-8859-1", "yes"]], - ].each do |decl, value| - parser = XML::SAX::Parser.new(Doc.new) - - xml = "#{decl}\n<root />" - parser.parse(xml) - assert(parser.document.start_document_called, xml) - assert_equal(value, parser.document.xmldecls, xml) - end - end - - def test_parse_empty - assert_raises(RuntimeError) do - @parser.parse("") - end - end - - def test_namespace_declaration_order_is_saved - @parser.parse(<<~eoxml) - <root xmlns:foo='http://foo.example.com/' xmlns='http://example.com/'> - <a foo:bar='hello' /> - </root> - eoxml - assert_equal(2, @parser.document.start_elements_namespace.length) - el = @parser.document.start_elements_namespace.first - namespaces = el.last - assert_equal(["foo", "http://foo.example.com/"], namespaces.first) - assert_equal([nil, "http://example.com/"], namespaces.last) - end - - def test_bad_document_calls_error_handler - @parser.parse("<foo><bar></foo>") - assert(@parser.document.errors) - assert(@parser.document.errors.length > 0) - end - - def test_namespace_are_super_fun_to_parse - @parser.parse(<<~eoxml) - <root xmlns:foo='http://foo.example.com/'> - <a foo:bar='hello' /> - <b xmlns:foo='http://bar.example.com/'> - <a foo:bar='hello' /> - </b> - <foo:bar>hello world</foo:bar> - </root> - eoxml - - assert(@parser.document.start_elements_namespace.length > 0) - el = @parser.document.start_elements_namespace[1] - assert_equal("a", el.first) - assert_equal(1, el[1].length) - - attribute = el[1].first - assert_equal("bar", attribute.localname) - assert_equal("foo", attribute.prefix) - assert_equal("hello", attribute.value) - assert_equal("http://foo.example.com/", attribute.uri) - end - - def test_sax_v1_namespace_attribute_declarations - @parser.parse(<<~eoxml) - <root xmlns:foo='http://foo.example.com/' xmlns='http://example.com/'> - <a foo:bar='hello' /> - <b xmlns:foo='http://bar.example.com/'> - <a foo:bar='hello' /> - </b> - <foo:bar>hello world</foo:bar> - </root> - eoxml - assert(@parser.document.start_elements.length > 0) - elm = @parser.document.start_elements.first - assert_equal("root", elm.first) - assert(elm[1].include?(["xmlns:foo", "http://foo.example.com/"])) - assert(elm[1].include?(["xmlns", "http://example.com/"])) - end - - def test_sax_v1_namespace_nodes - @parser.parse(<<~eoxml) - <root xmlns:foo='http://foo.example.com/' xmlns='http://example.com/'> - <a foo:bar='hello' /> - <b xmlns:foo='http://bar.example.com/'> - <a foo:bar='hello' /> - </b> - <foo:bar>hello world</foo:bar> - </root> - eoxml - assert_equal(5, @parser.document.start_elements.length) - assert(@parser.document.start_elements.map(&:first).include?("foo:bar")) - assert(@parser.document.end_elements.map(&:first).include?("foo:bar")) - end - - def test_start_is_called_without_namespace - @parser.parse(<<~eoxml) - <root xmlns:foo='http://foo.example.com/' xmlns='http://example.com/'> - <foo:f><bar></foo:f> - </root> - eoxml - assert_equal(["root", "foo:f", "bar"], - @parser.document.start_elements.map(&:first)) - end - - def test_parser_sets_encoding - parser = XML::SAX::Parser.new(Doc.new, "UTF-8") - assert_equal("UTF-8", parser.encoding) - end - - def test_errors_set_after_parsing_bad_dom - doc = Nokogiri::XML("<foo><bar></foo>") - assert(doc.errors) - - @parser.parse("<foo><bar></foo>") - assert(@parser.document.errors) - assert(@parser.document.errors.length > 0) - - doc.errors.each do |error| - assert_equal("UTF-8", error.message.encoding.name) - end - - # when using JRuby Nokogiri, more errors will be generated as the DOM - # parser continue to parse an ill formed document, while the sax parser - # will stop at the first error - unless Nokogiri.jruby? - assert_equal(doc.errors.length, @parser.document.errors.length) - end - end - - def test_parse_with_memory_argument - @parser.parse(File.read(XML_FILE)) - assert(@parser.document.cdata_blocks.length > 0) - end - - def test_parse_with_io_argument - File.open(XML_FILE, "rb") do |f| - @parser.parse(f) - end - assert(@parser.document.cdata_blocks.length > 0) - end - - def test_parse_io - call_parse_io_with_encoding("UTF-8") - end - - # issue #828 - def test_parse_io_lower_case_encoding - call_parse_io_with_encoding("utf-8") - end - - def call_parse_io_with_encoding(encoding) - File.open(XML_FILE, "rb") do |f| - @parser.parse_io(f, encoding) - end - assert(@parser.document.cdata_blocks.length > 0) - - called = false - @parser.document.start_elements.flatten.each do |thing| - assert_equal("UTF-8", thing.encoding.name) - called = true - end - assert(called) - - called = false - @parser.document.end_elements.flatten.each do |thing| - assert_equal("UTF-8", thing.encoding.name) - called = true - end - assert(called) - - called = false - @parser.document.data.each do |thing| - assert_equal("UTF-8", thing.encoding.name) - called = true - end - assert(called) - - called = false - @parser.document.comments.flatten.each do |thing| - assert_equal("UTF-8", thing.encoding.name) - called = true - end - assert(called) - - called = false - @parser.document.cdata_blocks.flatten.each do |thing| - assert_equal("UTF-8", thing.encoding.name) - called = true - end - assert(called) - end - - def test_parse_file - @parser.parse_file(XML_FILE) - - assert_raises(ArgumentError) do - @parser.parse_file(nil) - end - - assert_raises(Errno::ENOENT) do - @parser.parse_file("") - end - assert_raises(Errno::EISDIR) do - @parser.parse_file(File.expand_path(File.dirname(__FILE__))) - end - end - - def test_render_parse_nil_param - assert_raises(ArgumentError) { @parser.parse_memory(nil) } - end - - def test_bad_encoding_args - assert_raises(ArgumentError) { XML::SAX::Parser.new(Doc.new, "not an encoding") } - assert_raises(ArgumentError) { @parser.parse_io(StringIO.new("<root/>"), "not an encoding") } - end - - def test_ctag - @parser.parse_memory(<<~eoxml) - <p id="asdfasdf"> - <![CDATA[ This is a comment ]]> - Paragraph 1 - </p> - eoxml - assert_equal([" This is a comment "], @parser.document.cdata_blocks) - end - - def test_comment - @parser.parse_memory(<<~eoxml) - <p id="asdfasdf"> - <!-- This is a comment --> - Paragraph 1 - </p> - eoxml - assert_equal([" This is a comment "], @parser.document.comments) - end - - def test_characters - @parser.parse_memory(<<~eoxml) - <p id="asdfasdf">Paragraph 1</p> - eoxml - assert_equal(["Paragraph 1"], @parser.document.data) - end - - def test_end_document - @parser.parse_memory(<<~eoxml) - <p id="asdfasdf">Paragraph 1</p> - eoxml - assert(@parser.document.end_document_called) - end - - def test_end_element - @parser.parse_memory(<<~eoxml) - <p id="asdfasdf">Paragraph 1</p> - eoxml - assert_equal([["p"]], @parser.document.end_elements) - end - - def test_start_element_attrs - @parser.parse_memory(<<~eoxml) - <p id="asdfasdf">Paragraph 1</p> - eoxml - assert_equal([["p", [["id", "asdfasdf"]]]], @parser.document.start_elements) - end - - def test_start_element_attrs_include_namespaces - @parser.parse_memory(<<~eoxml) - <p xmlns:foo='http://foo.example.com/'>Paragraph 1</p> - eoxml - assert_equal([["p", [["xmlns:foo", "http://foo.example.com/"]]]], - @parser.document.start_elements) - end - - def test_processing_instruction - @parser.parse_memory(<<~eoxml) - <?xml-stylesheet href="a.xsl" type="text/xsl"?> - <?xml version="1.0"?> - eoxml - assert_equal([["xml-stylesheet", 'href="a.xsl" type="text/xsl"']], - @parser.document.processing_instructions) - end - - def test_parse_document - skip_unless_libxml2("JRuby SAXParser only parses well-formed XML documents") - @parser.parse_memory(<<~eoxml) - <p>Paragraph 1</p> - <p>Paragraph 2</p> - eoxml - end - - def test_parser_attributes - xml = <<~eoxml - <?xml version="1.0" ?><root><foo a="&b" c=">d" /></root> - eoxml - - block_called = false - @parser.parse(xml) do |ctx| - block_called = true - ctx.replace_entities = true - end - - assert(block_called) - - assert_equal([["root", []], ["foo", [["a", "&b"], ["c", ">d"]]]], @parser.document.start_elements) - end - - def test_recovery_from_incorrect_xml - xml = <<~eoxml - <?xml version="1.0" ?><Root><Data><?xml version='1.0'?><Item>hey</Item></Data><Data><Item>hey yourself</Item></Data></Root> - eoxml - - block_called = false - @parser.parse(xml) do |ctx| - block_called = true - ctx.recovery = true - end - - assert(block_called) - - assert_equal([["Root", []], ["Data", []], ["Item", []], ["Data", []], ["Item", []]], - @parser.document.start_elements) - end - - def test_square_bracket_in_text # issue 1261 - xml = <<~eoxml - <tu tuid="87dea04cf60af103ff09d1dba36ae820" segtype="block"> - <prop type="x-smartling-string-variant">en:#:home_page:#:stories:#:[6]:#:name</prop> - <tuv xml:lang="en-US"><seg>Sandy S.</seg></tuv> - </tu> - eoxml - @parser.parse(xml) - assert_includes(@parser.document.data, "en:#:home_page:#:stories:#:[6]:#:name") - end - - def test_large_cdata_is_handled - # see #2132 and https://gitlab.gnome.org/GNOME/libxml2/-/issues/200 - skip("Upstream libxml2 <= 2.9.10 needs to be patched") if Nokogiri::VersionInfo.instance.libxml2_using_system? - - template = <<~EOF - <?xml version="1.0" encoding="UTF-8"?> - <soapenv:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:ns="http://example.com"> - <soapenv:Header> - <AuthHeader xsi:type="ns:vAuthHeader"> - <userName xsi:type="xsd:string">gorilla</userName> - <password xsi:type="xsd:string">secret</password> - </AuthHeader> - </soapenv:Header> - <soapenv:Body> - <ns:checkToken soapenv:encodingStyle="http://schemas.xmlsoap.org/soap/encoding/"> - <checkToken xsi:type="xsd:string"><![CDATA[%s]]></checkToken> - </ns:checkToken> - </soapenv:Body> - </soapenv:Envelope> - EOF - - factor = 10 - huge_data = "a" * (1024 * 1024 * factor) - xml = StringIO.new(template % (huge_data)) - - handler = Nokogiri::SAX::TestCase::Doc.new - parser = Nokogiri::XML::SAX::Parser.new(handler) - parser.parse(xml) - - assert_predicate(handler.errors, :empty?) - end +class Nokogiri::SAX::TestCase + describe Nokogiri::XML::SAX::Parser do + let(:parser) { Nokogiri::XML::SAX::Parser.new(Doc.new) } + + it :test_parser_context_yielded_io do + doc = Doc.new + parser = Nokogiri::XML::SAX::Parser.new(doc) + xml = "<foo a='&b'/>" + + block_called = false + parser.parse(StringIO.new(xml)) do |ctx| + block_called = true + ctx.replace_entities = true end + + assert(block_called) + + assert_equal([["foo", [["a", "&b"]]]], doc.start_elements) + end + + it :test_parser_context_yielded_in_memory do + doc = Doc.new + parser = Nokogiri::XML::SAX::Parser.new(doc) + xml = "<foo a='&b'/>" + + block_called = false + parser.parse(xml) do |ctx| + block_called = true + ctx.replace_entities = true + end + + assert(block_called) + + assert_equal([["foo", [["a", "&b"]]]], doc.start_elements) + end + + it :test_empty_decl do + parser = Nokogiri::XML::SAX::Parser.new(Doc.new) + + xml = "<root />" + parser.parse(xml) + assert(parser.document.start_document_called, xml) + assert_nil(parser.document.xmldecls, xml) + end + + it :test_xml_decl do + [ + ['<?xml version="1.0" ?>', ["1.0"]], + ['<?xml version="1.0" encoding="UTF-8" ?>', ["1.0", "UTF-8"]], + ['<?xml version="1.0" standalone="yes"?>', ["1.0", "yes"]], + ['<?xml version="1.0" standalone="no"?>', ["1.0", "no"]], + ['<?xml version="1.0" encoding="UTF-8" standalone="no"?>', ["1.0", "UTF-8", "no"]], + ['<?xml version="1.0" encoding="ISO-8859-1" standalone="yes"?>', ["1.0", "ISO-8859-1", "yes"]], + ].each do |decl, value| + parser = Nokogiri::XML::SAX::Parser.new(Doc.new) + + xml = "#{decl}\n<root />" + parser.parse(xml) + assert(parser.document.start_document_called, xml) + assert_equal(value, parser.document.xmldecls, xml) + end + end + + it :test_parse_empty do + assert_raises(RuntimeError) do + parser.parse("") + end + end + + it :test_namespace_declaration_order_is_saved do + parser.parse(<<~EOF) + <root xmlns:foo='http://foo.example.com/' xmlns='http://example.com/'> + <a foo:bar='hello' /> + </root> + EOF + assert_equal(2, parser.document.start_elements_namespace.length) + el = parser.document.start_elements_namespace.first + namespaces = el.last + assert_equal(["foo", "http://foo.example.com/"], namespaces.first) + assert_equal([nil, "http://example.com/"], namespaces.last) + end + + it :test_bad_document_calls_error_handler do + parser.parse("<foo><bar></foo>") + assert(parser.document.errors) + assert(parser.document.errors.length > 0) + end + + it :test_namespace_are_super_fun_to_parse do + parser.parse(<<~EOF) + <root xmlns:foo='http://foo.example.com/'> + <a foo:bar='hello' /> + <b xmlns:foo='http://bar.example.com/'> + <a foo:bar='hello' /> + </b> + <foo:bar>hello world</foo:bar> + </root> + EOF + + assert(parser.document.start_elements_namespace.length > 0) + el = parser.document.start_elements_namespace[1] + assert_equal("a", el.first) + assert_equal(1, el[1].length) + + attribute = el[1].first + assert_equal("bar", attribute.localname) + assert_equal("foo", attribute.prefix) + assert_equal("hello", attribute.value) + assert_equal("http://foo.example.com/", attribute.uri) + end + + it :test_sax_v1_namespace_attribute_declarations do + parser.parse(<<~EOF) + <root xmlns:foo='http://foo.example.com/' xmlns='http://example.com/'> + <a foo:bar='hello' /> + <b xmlns:foo='http://bar.example.com/'> + <a foo:bar='hello' /> + </b> + <foo:bar>hello world</foo:bar> + </root> + EOF + assert(parser.document.start_elements.length > 0) + elm = parser.document.start_elements.first + assert_equal("root", elm.first) + assert(elm[1].include?(["xmlns:foo", "http://foo.example.com/"])) + assert(elm[1].include?(["xmlns", "http://example.com/"])) + end + + it :test_sax_v1_namespace_nodes do + parser.parse(<<~EOF) + <root xmlns:foo='http://foo.example.com/' xmlns='http://example.com/'> + <a foo:bar='hello' /> + <b xmlns:foo='http://bar.example.com/'> + <a foo:bar='hello' /> + </b> + <foo:bar>hello world</foo:bar> + </root> + EOF + assert_equal(5, parser.document.start_elements.length) + assert(parser.document.start_elements.map(&:first).include?("foo:bar")) + assert(parser.document.end_elements.map(&:first).include?("foo:bar")) + end + + it :test_start_is_called_without_namespace do + parser.parse(<<~EOF) + <root xmlns:foo='http://foo.example.com/' xmlns='http://example.com/'> + <foo:f><bar></foo:f> + </root> + EOF + assert_equal( + ["root", "foo:f", "bar"], + parser.document.start_elements.map(&:first) + ) + end + + it :test_parser_sets_encoding do + parser = Nokogiri::XML::SAX::Parser.new(Doc.new, "UTF-8") + assert_equal("UTF-8", parser.encoding) + end + + it :test_errors_set_after_parsing_bad_dom do + doc = Nokogiri::XML("<foo><bar></foo>") + assert(doc.errors) + + parser.parse("<foo><bar></foo>") + assert(parser.document.errors) + assert(parser.document.errors.length > 0) + + doc.errors.each do |error| + assert_equal("UTF-8", error.message.encoding.name) + end + + # when using JRuby Nokogiri, more errors will be generated as the DOM + # parser continue to parse an ill formed document, while the sax parser + # will stop at the first error + unless Nokogiri.jruby? + assert_equal(doc.errors.length, parser.document.errors.length) + end + end + + it :test_parse_with_memory_argument do + parser.parse(File.read(XML_FILE)) + assert(parser.document.cdata_blocks.length > 0) + end + + it :test_parse_with_io_argument do + File.open(XML_FILE, "rb") do |f| + parser.parse(f) + end + assert(parser.document.cdata_blocks.length > 0) + end + + it :test_parse_io do + call_parse_io_with_encoding("UTF-8") + end + + # issue #828 + it :test_parse_io_lower_case_encoding do + call_parse_io_with_encoding("utf-8") + end + + def call_parse_io_with_encoding(encoding) + File.open(XML_FILE, "rb") do |f| + parser.parse_io(f, encoding) + end + assert(parser.document.cdata_blocks.length > 0) + + called = false + parser.document.start_elements.flatten.each do |thing| + assert_equal("UTF-8", thing.encoding.name) + called = true + end + assert(called) + + called = false + parser.document.end_elements.flatten.each do |thing| + assert_equal("UTF-8", thing.encoding.name) + called = true + end + assert(called) + + called = false + parser.document.data.each do |thing| + assert_equal("UTF-8", thing.encoding.name) + called = true + end + assert(called) + + called = false + parser.document.comments.flatten.each do |thing| + assert_equal("UTF-8", thing.encoding.name) + called = true + end + assert(called) + + called = false + parser.document.cdata_blocks.flatten.each do |thing| + assert_equal("UTF-8", thing.encoding.name) + called = true + end + assert(called) + end + + it :test_parse_file do + parser.parse_file(XML_FILE) + + assert_raises(ArgumentError) do + parser.parse_file(nil) + end + + assert_raises(Errno::ENOENT) do + parser.parse_file("") + end + assert_raises(Errno::EISDIR) do + parser.parse_file(File.expand_path(File.dirname(__FILE__))) + end + end + + it :test_render_parse_nil_param do + assert_raises(ArgumentError) { parser.parse_memory(nil) } + end + + it :test_bad_encoding_args do + assert_raises(ArgumentError) { Nokogiri::XML::SAX::Parser.new(Doc.new, "not an encoding") } + assert_raises(ArgumentError) { parser.parse_io(StringIO.new("<root/>"), "not an encoding") } + end + + it :test_ctag do + parser.parse_memory(<<~EOF) + <p id="asdfasdf"> + <![CDATA[ This is a comment ]]> + Paragraph 1 + </p> + EOF + assert_equal([" This is a comment "], parser.document.cdata_blocks) + end + + it :test_comment do + parser.parse_memory(<<~EOF) + <p id="asdfasdf"> + <!-- This is a comment --> + Paragraph 1 + </p> + EOF + assert_equal([" This is a comment "], parser.document.comments) + end + + it :test_characters do + parser.parse_memory(<<~EOF) + <p id="asdfasdf">Paragraph 1</p> + EOF + assert_equal(["Paragraph 1"], parser.document.data) + end + + it :test_end_document do + parser.parse_memory(<<~EOF) + <p id="asdfasdf">Paragraph 1</p> + EOF + assert(parser.document.end_document_called) + end + + it :test_end_element do + parser.parse_memory(<<~EOF) + <p id="asdfasdf">Paragraph 1</p> + EOF + assert_equal([["p"]], parser.document.end_elements) + end + + it :test_start_element_attrs do + parser.parse_memory(<<~EOF) + <p id="asdfasdf">Paragraph 1</p> + EOF + assert_equal([["p", [["id", "asdfasdf"]]]], parser.document.start_elements) + end + + it :test_start_element_attrs_include_namespaces do + parser.parse_memory(<<~EOF) + <p xmlns:foo='http://foo.example.com/'>Paragraph 1</p> + EOF + assert_equal( + [["p", [["xmlns:foo", "http://foo.example.com/"]]]], + parser.document.start_elements + ) + end + + it :test_processing_instruction do + parser.parse_memory(<<~EOF) + <?xml-stylesheet href="a.xsl" type="text/xsl"?> + <?xml version="1.0"?> + EOF + assert_equal( + [["xml-stylesheet", 'href="a.xsl" type="text/xsl"']], + parser.document.processing_instructions + ) + end + + it :test_parse_document do + skip_unless_libxml2("JRuby SAXParser only parses well-formed XML documents") + parser.parse_memory(<<~EOF) + <p>Paragraph 1</p> + <p>Paragraph 2</p> + EOF + end + + it :test_parser_attributes do + xml = <<~EOF + <?xml version="1.0" ?><root><foo a="&b" c=">d" /></root> + EOF + + block_called = false + parser.parse(xml) do |ctx| + block_called = true + ctx.replace_entities = true + end + + assert(block_called) + + assert_equal( + [["root", []], ["foo", [["a", "&b"], ["c", ">d"]]]], parser.document.start_elements + ) + end + + it :test_recovery_from_incorrect_xml do + xml = <<~EOF + <?xml version="1.0" ?><Root><Data><?xml version='1.0'?><Item>hey</Item></Data><Data><Item>hey yourself</Item></Data></Root> + EOF + + block_called = false + parser.parse(xml) do |ctx| + block_called = true + ctx.recovery = true + end + + assert(block_called) + + assert_equal( + [["Root", []], ["Data", []], ["Item", []], ["Data", []], ["Item", []]], + parser.document.start_elements + ) + end + + it :test_square_bracket_in_text do + # issue 1261 + xml = <<~EOF + <tu tuid="87dea04cf60af103ff09d1dba36ae820" segtype="block"> + <prop type="x-smartling-string-variant">en:#:home_page:#:stories:#:[6]:#:name</prop> + <tuv xml:lang="en-US"><seg>Sandy S.</seg></tuv> + </tu> + EOF + parser.parse(xml) + assert_includes(parser.document.data, "en:#:home_page:#:stories:#:[6]:#:name") + end + + it :test_large_cdata_is_handled do + # see #2132 and https://gitlab.gnome.org/GNOME/libxml2/-/issues/200 + skip("Upstream libxml2 <= 2.9.10 needs to be patched") if Nokogiri::VersionInfo.instance.libxml2_using_system? + + template = <<~EOF + <?xml version="1.0" encoding="UTF-8"?> + <soapenv:Envelope xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:ns="http://example.com"> + <soapenv:Header> + <AuthHeader xsi:type="ns:vAuthHeader"> + <userName xsi:type="xsd:string">gorilla</userName> + <password xsi:type="xsd:string">secret</password> + </AuthHeader> + </soapenv:Header> + <soapenv:Body> + <ns:checkToken soapenv:encodingStyle="http://schemas.xmlsoap.org/soap/encoding/"> + <checkToken xsi:type="xsd:string"><![CDATA[%s]]></checkToken> + </ns:checkToken> + </soapenv:Body> + </soapenv:Envelope> + EOF + + factor = 10 + huge_data = "a" * (1024 * 1024 * factor) + xml = StringIO.new(template % huge_data) + + handler = Nokogiri::SAX::TestCase::Doc.new + parser = Nokogiri::XML::SAX::Parser.new(handler) + parser.parse(xml) + + assert_predicate(handler.errors, :empty?) + end + + it "does not resolve entities by default" do + xml = <<~EOF + <?xml version="1.0" encoding="UTF-8"?> + <!DOCTYPE doc [ + <!ENTITY local SYSTEM "file:///#{File.expand_path(__FILE__)}"> + <!ENTITY custom "resolved> + ]> + <doc><foo>&local;</foo><foo>&custom;</foo></doc> + EOF + + doc = Doc.new + parser = Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(xml) + + assert_nil(doc.data) + end + + it "does not resolve network external entities by default" do + xml = <<~EOF + <?xml version="1.0" encoding="UTF-8"?> + <!DOCTYPE doc [ + <!ENTITY remote SYSTEM "http://0.0.0.0:8080/evil.dtd"> + ]> + <doc><foo>&remote;</foo></doc> + EOF + + doc = Doc.new + parser = Nokogiri::XML::SAX::Parser.new(doc) + parser.parse(xml) + + assert_nil(doc.data) end end end diff --git a/test/xml/sax/test_push_parser.rb b/test/xml/sax/test_push_parser.rb index 587bd54a1a..dfab1793fa 100644 --- a/test/xml/sax/test_push_parser.rb +++ b/test/xml/sax/test_push_parser.rb @@ -1,266 +1,243 @@ # -*- coding: utf-8 -*- +# frozen_string_literal: true require "helper" -module Nokogiri - module XML - module SAX - class TestPushParser < Nokogiri::SAX::TestCase - def setup - super - @parser = XML::SAX::PushParser.new(Doc.new) - end - - def test_exception - assert_raises(SyntaxError) do - @parser << "<foo /><foo />" - end - - assert_raises(SyntaxError) do - @parser << nil - end - end +class Nokogiri::SAX::TestCase + describe Nokogiri::XML::SAX::PushParser do + let(:parser) { Nokogiri::XML::SAX::PushParser.new(Doc.new) } - def test_early_finish - @parser << "<foo>" - assert_raises(SyntaxError) do - @parser.finish - end - end + it :test_exception do + assert_raises(Nokogiri::XML::SyntaxError) do + parser << "<foo /><foo />" + end - def test_write_last_chunk - @parser << "<foo>" - @parser.write "</foo>", true - assert_equal [["foo", []]], @parser.document.start_elements - assert_equal [["foo"]], @parser.document.end_elements - end + assert_raises(Nokogiri::XML::SyntaxError) do + parser << nil + end + end - def test_empty_doc - @parser.options |= XML::ParseOptions::RECOVER - @parser.write "", true - assert_nil @parser.document.start_elements - assert_nil @parser.document.end_elements - end + it :test_early_finish do + parser << "<foo>" + assert_raises(Nokogiri::XML::SyntaxError) do + parser.finish + end + end + it :test_write_last_chunk do + parser << "<foo>" + parser.write("</foo>", true) + assert_equal [["foo", []]], parser.document.start_elements + assert_equal [["foo"]], parser.document.end_elements + end - def test_finish_should_rethrow_last_error - begin - @parser << "</foo>" - rescue => e - expected = e - end + it :test_empty_doc do + parser.options |= Nokogiri::XML::ParseOptions::RECOVER + parser.write("", true) + assert_nil parser.document.start_elements + assert_nil parser.document.end_elements + end - begin - @parser.finish - rescue => e - actual = e - end + it :test_finish_should_rethrow_last_error do + expected = assert_raise(Nokogiri::XML::SyntaxError) { parser << "</foo>" } + actual = assert_raise(Nokogiri::XML::SyntaxError) { parser.finish } + assert_equal actual.message, expected.message + end - assert_equal actual.message, expected.message + it :test_should_throw_error_returned_by_document do + doc = Doc.new + class << doc + def error(msg) + raise "parse error" end + end + parser = Nokogiri::XML::SAX::PushParser.new(doc) - def test_should_throw_error_returned_by_document - doc = Doc.new - class << doc - def error msg - raise "parse error" - end - end - - @parser = XML::SAX::PushParser.new(doc) - begin - @parser << "</foo>" - rescue => e - actual = e - end - fail 'PushParser should throw error when fed ill-formed data' if actual.nil? - - assert_equal actual.message, "parse error" - end + exception = assert_raise(RuntimeError) { parser << "</foo>" } + assert_equal exception.message, "parse error" + end - def test_writing_nil - assert_equal @parser.write(nil), @parser - end + it :test_writing_nil do + assert_equal parser.write(nil), parser + end - def test_end_document_called - @parser.<<(<<-eoxml) - <p id="asdfasdf"> - <!-- This is a comment --> - Paragraph 1 - </p> - eoxml - assert ! @parser.document.end_document_called - @parser.finish - assert @parser.document.end_document_called - end + it :test_end_document_called do + parser.<<(<<~EOF) + <p id="asdfasdf"> + <!-- This is a comment --> + Paragraph 1 + </p> + EOF + assert !parser.document.end_document_called + parser.finish + assert parser.document.end_document_called + end - def test_start_element - @parser.<<(<<-eoxml) - <p id="asdfasdf"> - eoxml - - assert_equal [["p", [["id", "asdfasdf"]]]], - @parser.document.start_elements - - @parser.<<(<<-eoxml) - <!-- This is a comment --> - Paragraph 1 - </p> - eoxml - assert_equal [' This is a comment '], @parser.document.comments - @parser.finish - end + it :test_start_element do + parser.<<(<<~EOF) + <p id="asdfasdf"> + EOF + + assert_equal [["p", [["id", "asdfasdf"]]]], + parser.document.start_elements + + parser.<<(<<~EOF) + <!-- This is a comment --> + Paragraph 1 + </p> + EOF + assert_equal [" This is a comment "], parser.document.comments + parser.finish + end - def test_start_element_with_namespaces - @parser.<<(<<-eoxml) - <p xmlns:foo="http://foo.example.com/"> - eoxml - - assert_equal [["p", [["xmlns:foo", "http://foo.example.com/"]]]], - @parser.document.start_elements - - @parser.<<(<<-eoxml) - <!-- This is a comment --> - Paragraph 1 - </p> - eoxml - assert_equal [' This is a comment '], @parser.document.comments - @parser.finish - end + it :test_start_element_with_namespaces do + parser.<<(<<~EOF) + <p xmlns:foo="http://foo.example.com/"> + EOF + + assert_equal [["p", [["xmlns:foo", "http://foo.example.com/"]]]], + parser.document.start_elements + + parser.<<(<<~EOF) + <!-- This is a comment --> + Paragraph 1 + </p> + EOF + assert_equal [" This is a comment "], parser.document.comments + parser.finish + end - def test_start_element_ns - @parser.<<(<<-eoxml) - <stream:stream xmlns='jabber:client' xmlns:stream='http://etherx.jabber.org/streams' version='1.0' size='large'></stream:stream> - eoxml + it :test_start_element_ns do + parser.<<(<<~EOF) + <stream:stream xmlns='jabber:client' xmlns:stream='http://etherx.jabber.org/streams' version='1.0' size='large'></stream:stream> + EOF - assert_equal 1, @parser.document.start_elements_namespace.length - el = @parser.document.start_elements_namespace.first + assert_equal 1, parser.document.start_elements_namespace.length + el = parser.document.start_elements_namespace.first - assert_equal 'stream', el.first - assert_equal 2, el[1].length - assert_equal [['version', '1.0'], ['size', 'large']], - el[1].map { |x| [x.localname, x.value] } + assert_equal "stream", el.first + assert_equal 2, el[1].length + assert_equal [["version", "1.0"], ["size", "large"]], + el[1].map { |x| [x.localname, x.value] } - assert_equal 'stream', el[2] - assert_equal 'http://etherx.jabber.org/streams', el[3] - @parser.finish - end + assert_equal "stream", el[2] + assert_equal "http://etherx.jabber.org/streams", el[3] + parser.finish + end - def test_end_element_ns - @parser.<<(<<-eoxml) - <stream:stream xmlns='jabber:client' xmlns:stream='http://etherx.jabber.org/streams' version='1.0'></stream:stream> - eoxml + it :test_end_element_ns do + parser.<<(<<~EOF) + <stream:stream xmlns='jabber:client' xmlns:stream='http://etherx.jabber.org/streams' version='1.0'></stream:stream> + EOF - assert_equal [['stream', 'stream', 'http://etherx.jabber.org/streams']], - @parser.document.end_elements_namespace - @parser.finish - end + assert_equal [["stream", "stream", "http://etherx.jabber.org/streams"]], + parser.document.end_elements_namespace + parser.finish + end - def test_chevron_partial_xml - @parser.<<(<<-eoxml) - <p id="asdfasdf"> - eoxml - - @parser.<<(<<-eoxml) - <!-- This is a comment --> - Paragraph 1 - </p> - eoxml - assert_equal [' This is a comment '], @parser.document.comments - @parser.finish - end + it :test_chevron_partial_xml do + parser.<<(<<~EOF) + <p id="asdfasdf"> + EOF + + parser.<<(<<~EOF) + <!-- This is a comment --> + Paragraph 1 + </p> + EOF + assert_equal [" This is a comment "], parser.document.comments + parser.finish + end - def test_chevron - @parser.<<(<<-eoxml) - <p id="asdfasdf"> - <!-- This is a comment --> - Paragraph 1 - </p> - eoxml - @parser.finish - assert_equal [' This is a comment '], @parser.document.comments - end + it :test_chevron do + parser.<<(<<~EOF) + <p id="asdfasdf"> + <!-- This is a comment --> + Paragraph 1 + </p> + EOF + parser.finish + assert_equal [" This is a comment "], parser.document.comments + end - def test_default_options - assert_equal 0, @parser.options - end + it :test_default_options do + assert_equal 0, parser.options + end - def test_recover - @parser.options |= XML::ParseOptions::RECOVER - @parser.<<(<<-eoxml) - <p> - Foo - <bar> - Bar - </p> - eoxml - @parser.finish - assert(@parser.document.errors.size >= 1) - assert_equal [["p", []], ["bar", []]], @parser.document.start_elements - assert_equal "FooBar", @parser.document.data.map { |x| - x.gsub(/\s/, '') - }.join - end + it :test_recover do + parser.options |= Nokogiri::XML::ParseOptions::RECOVER + parser.<<(<<~EOF) + <p> + Foo + <bar> + Bar + </p> + EOF + parser.finish + assert(parser.document.errors.size >= 1) + assert_equal [["p", []], ["bar", []]], parser.document.start_elements + assert_equal "FooBar", parser.document.data.map { |x| + x.gsub(/\s/, "") + }.join + end - def test_broken_encoding - skip_unless_libxml2("ultra hard to fix for pure Java version") - @parser.options |= XML::ParseOptions::RECOVER - # This is ISO_8859-1: - @parser.<< "<?xml version='1.0' encoding='UTF-8'?><r>Gau\337</r>" - @parser.finish - assert(@parser.document.errors.size >= 1) - assert_equal "Gau\337", @parser.document.data.join - assert_equal [["r"]], @parser.document.end_elements - end + it :test_broken_encoding do + skip_unless_libxml2("ultra hard to fix for pure Java version") + parser.options |= Nokogiri::XML::ParseOptions::RECOVER + # This is ISO_8859-1: + parser.<< "<?xml version='1.0' encoding='UTF-8'?><r>Gau\337</r>" + parser.finish + assert(parser.document.errors.size >= 1) + assert_equal "Gau\337", parser.document.data.join + assert_equal [["r"]], parser.document.end_elements + end - def test_replace_entities_attribute_behavior - if Nokogiri.uses_libxml? - # initially false - assert_equal false, @parser.replace_entities - - # can be set to true - @parser.replace_entities = true - assert_equal true, @parser.replace_entities - - # can be set to false - @parser.replace_entities = false - assert_equal false, @parser.replace_entities - else - # initially true - assert_equal true, @parser.replace_entities - - # ignore attempts to set to false - @parser.replace_entities = false # TODO: should we raise an exception here? - assert_equal true, @parser.replace_entities - end - end + it :test_replace_entities_attribute_behavior do + if Nokogiri.uses_libxml? + # initially false + assert_equal false, parser.replace_entities + + # can be set to true + parser.replace_entities = true + assert_equal true, parser.replace_entities + + # can be set to false + parser.replace_entities = false + assert_equal false, parser.replace_entities + else + # initially true + assert_equal true, parser.replace_entities + + # ignore attempts to set to false + parser.replace_entities = false # TODO: should we raise an exception here? + assert_equal true, parser.replace_entities + end + end - def test_untouched_entities - skip_unless_libxml2("entities are always replaced in pure Java version") - @parser.<<(<<-eoxml) - <p id="asdf&asdf"> - <!-- This is a comment --> - Paragraph 1 & 2 - </p> - eoxml - @parser.finish - assert_equal [["p", [["id", "asdf&asdf"]]]], @parser.document.start_elements - assert_equal "Paragraph 1 & 2", @parser.document.data.join.strip - end + it :test_untouched_entities do + skip_unless_libxml2("entities are always replaced in pure Java version") + parser.<<(<<~EOF) + <p id="asdf&asdf"> + <!-- This is a comment --> + Paragraph 1 & 2 + </p> + EOF + parser.finish + assert_equal [["p", [["id", "asdf&asdf"]]]], parser.document.start_elements + assert_equal "Paragraph 1 & 2", parser.document.data.join.strip + end - def test_replaced_entities - @parser.replace_entities = true - @parser.<<(<<-eoxml) - <p id="asdf&asdf"> - <!-- This is a comment --> - Paragraph 1 & 2 - </p> - eoxml - @parser.finish - assert_equal [["p", [["id", "asdf&asdf"]]]], @parser.document.start_elements - assert_equal "Paragraph 1 & 2", @parser.document.data.join.strip - end - end + it :test_replaced_entities do + parser.replace_entities = true + parser.<<(<<~EOF) + <p id="asdf&asdf"> + <!-- This is a comment --> + Paragraph 1 & 2 + </p> + EOF + parser.finish + assert_equal [["p", [["id", "asdf&asdf"]]]], parser.document.start_elements + assert_equal "Paragraph 1 & 2", parser.document.data.join.strip end end end diff --git a/test/xml/test_builder.rb b/test/xml/test_builder.rb index a17c0b5b9e..022a200970 100644 --- a/test/xml/test_builder.rb +++ b/test/xml/test_builder.rb @@ -107,6 +107,54 @@ def test_non_root_namespace assert_equal "one", b.doc.at("hello", "xmlns" => "one").namespace.href end + def test_builder_namespace_inheritance_true + # see https://github.com/sparklemotion/nokogiri/issues/2317 + result = Nokogiri::XML::Builder.new(encoding: 'utf-8') do |xml| + xml["soapenv"].Envelope("xmlns:soapenv" => "http://schemas.xmlsoap.org/soap/envelope/") do + xml.Header + end + end + assert(result.doc.namespace_inheritance) + assert( + result.doc.at_xpath("//soapenv:Header", "soapenv" => "http://schemas.xmlsoap.org/soap/envelope/"), + "header element should have a namespace" + ) + end + + def test_builder_namespace_inheritance_false + # see https://github.com/sparklemotion/nokogiri/issues/2317 + result = Nokogiri::XML::Builder.new(encoding: 'utf-8', namespace_inheritance: false) do |xml| + xml["soapenv"].Envelope("xmlns:soapenv" => "http://schemas.xmlsoap.org/soap/envelope/") do + xml.Header + end + end + refute(result.doc.namespace_inheritance) + assert( + result.doc.at_xpath("//Header"), + "header element should not have a namespace" + ) + end + + def test_builder_namespace_inheritance_false_part_deux + # see https://github.com/sparklemotion/nokogiri/issues/1712 + result = Nokogiri::XML::Builder.new(encoding: 'utf-8', namespace_inheritance: false) do |xml| + xml['soapenv'].Envelope('xmlns:soapenv' => 'http://schemas.xmlsoap.org/soap/envelope/', 'xmlns:emer' => 'http://dashcs.com/api/v1/emergency') do + xml['soapenv'].Header + xml['soapenv'].Body do + xml['emer'].validateLocation do + # these should not have a namespace + xml.location do + xml.address 'Some place over the rainbow' + end + end + end + end + end + assert(result.doc.at_xpath("//emer:validateLocation", {"emer" => "http://dashcs.com/api/v1/emergency"}), + "expected validateLocation node to have a namespace") + assert(result.doc.at_xpath("//location"), "expected location node to not have a namespace") + end + def test_specify_namespace b = Nokogiri::XML::Builder.new { |xml| xml.root("xmlns:foo" => "bar") do @@ -177,12 +225,18 @@ def test_specified_namespace_undeclared } end + def test_set_namespace_inheritance + assert(Nokogiri::XML::Builder.new.doc.namespace_inheritance) + refute(Nokogiri::XML::Builder.new(namespace_inheritance: false).doc.namespace_inheritance) + end + def test_set_encoding builder = Nokogiri::XML::Builder.new(:encoding => "UTF-8") do |xml| xml.root do xml.bar "blah" end end + assert_equal "UTF-8", builder.doc.encoding assert_match "UTF-8", builder.to_xml end diff --git a/test/xml/test_document.rb b/test/xml/test_document.rb index 2462ebab8b..48eed1d10e 100644 --- a/test/xml/test_document.rb +++ b/test/xml/test_document.rb @@ -15,6 +15,13 @@ class TestDocument < Nokogiri::TestCase let(:xml) { Nokogiri::XML.parse(File.read(XML_FILE), XML_FILE) } + def test_default_namespace_inheritance + doc = Nokogiri::XML::Document.new + refute(doc.namespace_inheritance) + doc.namespace_inheritance = true + assert(doc.namespace_inheritance) + end + def test_dtd_with_empty_internal_subset doc = Nokogiri::XML(<<~eoxml) <?xml version="1.0"?> @@ -518,7 +525,7 @@ def test_non_existant_function def test_xpath_syntax_error assert_raises(Nokogiri::XML::XPath::SyntaxError) do - xml.xpath('\\') + xml.xpath("\\") end end @@ -704,7 +711,7 @@ def test_parse_can_take_pathnames # an arbitrary assertion on the structure of the document assert_equal(20, doc.xpath("/xmlns:feed/xmlns:entry/xmlns:author", - "xmlns" => "http://www.w3.org/2005/Atom").length) + "xmlns" => "http://www.w3.org/2005/Atom").length) assert_equal(XML_ATOM_FILE, doc.url) end @@ -742,7 +749,7 @@ def test_document_search_with_multiple_queries def test_bad_xpath_raises_syntax_error assert_raises(XML::XPath::SyntaxError) do - xml.xpath('\\') + xml.xpath("\\") end end @@ -957,16 +964,16 @@ def test_can_be_closed rb_doc = Nokogiri::XML::Document.wrap(java_doc) assert_equal(0, rb_doc.root.children.length) - java_doc.getFirstChild().appendChild(java_doc.createElement("bar")) + java_doc.getFirstChild.appendChild(java_doc.createElement("bar")) assert_equal(1, rb_doc.root.children.length) end it "shares the same data structure as shown when modified via Nokogiri" do rb_doc = Nokogiri::XML::Document.wrap(java_doc) - assert_equal(0, java_doc.getFirstChild().getChildNodes().getLength()) + assert_equal(0, java_doc.getFirstChild.getChildNodes.getLength) rb_doc.root.add_child("<bar />") - assert_equal(1, java_doc.getFirstChild().getChildNodes().getLength()) + assert_equal(1, java_doc.getFirstChild.getChildNodes.getLength) end else it "does not have the method" do @@ -984,22 +991,22 @@ def test_can_be_closed it "returns the underlying java object" do java_doc = rb_doc.to_java assert_kind_of(org.w3c.dom.Document, java_doc) - assert_equal("foo", java_doc.getDocumentElement().getTagName()) + assert_equal("foo", java_doc.getDocumentElement.getTagName) end it "shares the same data structure as shown when modified via Nokogiri" do java_doc = rb_doc.to_java - assert_equal(1, java_doc.getFirstChild().getChildNodes().getLength()) + assert_equal(1, java_doc.getFirstChild.getChildNodes.getLength) rb_doc.root.add_child("<baz />") - assert_equal(2, java_doc.getFirstChild().getChildNodes().getLength()) + assert_equal(2, java_doc.getFirstChild.getChildNodes.getLength) end it "shares the same data structure as shown when modified via Java interop" do java_doc = rb_doc.to_java assert_equal(1, rb_doc.root.children.length) - java_doc.getFirstChild().appendChild(java_doc.createElement("baz")) + java_doc.getFirstChild.appendChild(java_doc.createElement("baz")) assert_equal(2, rb_doc.root.children.length) end else @@ -1145,9 +1152,9 @@ def initialize(*args) end it "doesn't leak the replaced node" do - skip("only run if NOKOGIRI_GC is set") unless ENV['NOKOGIRI_GC'] + skip("only run if NOKOGIRI_GC is set") unless ENV["NOKOGIRI_GC"] doc = Nokogiri::XML("<root>test</root>") - doc2 = Nokogiri::XML("<root>#{'x' * 5000000}</root>") + doc2 = Nokogiri::XML("<root>#{"x" * 5000000}</root>") doc2.root = doc.root end @@ -1164,7 +1171,19 @@ def initialize(*args) e = assert_raises(ArgumentError) do doc.root = node_set end - assert_equal("expected Nokogiri::XML::Node but received Nokogiri::XML::NodeSet", e.message); + assert_equal("expected Nokogiri::XML::Node but received Nokogiri::XML::NodeSet", e.message) + end + end + + describe "#path" do + it "should return '/'" do + xml = <<~EOF + <root></root> + EOF + + doc = Nokogiri::XML::Document.parse(xml) + assert_equal("/", doc.path) + assert_equal(doc, doc.at_xpath(doc.path)) # make sure we can round-trip end end end diff --git a/test/xml/test_document_fragment.rb b/test/xml/test_document_fragment.rb index b7de6f3444..ce5947a4a2 100644 --- a/test/xml/test_document_fragment.rb +++ b/test/xml/test_document_fragment.rb @@ -11,26 +11,26 @@ def test_replace_text_node html = "foo" doc = Nokogiri::XML::DocumentFragment.parse(html) doc.children[0].replace("bar") - assert_equal('bar', doc.children[0].content) + assert_equal("bar", doc.children[0].content) end def test_fragment_is_relative doc = Nokogiri::XML('<root><a xmlns="blah" /></root>') ctx = doc.root.child - fragment = Nokogiri::XML::DocumentFragment.new(doc, '<hello />', ctx) + fragment = Nokogiri::XML::DocumentFragment.new(doc, "<hello />", ctx) hello = fragment.child - assert_equal('hello', hello.name) + assert_equal("hello", hello.name) assert_equal(doc.root.child.namespace, hello.namespace) end def test_node_fragment_is_relative doc = Nokogiri::XML('<root><a xmlns="blah" /></root>') assert(doc.root.child) - fragment = doc.root.child.fragment('<hello />') + fragment = doc.root.child.fragment("<hello />") hello = fragment.child - assert_equal('hello', hello.name) + assert_equal("hello", hello.name) assert_equal(doc.root.child.namespace, hello.namespace) end @@ -45,7 +45,7 @@ def test_fragment_should_have_document def test_name fragment = Nokogiri::XML::DocumentFragment.new(xml) - assert_equal('#document-fragment', fragment.name) + assert_equal("#document-fragment", fragment.name) end def test_static_method @@ -112,12 +112,12 @@ def test_fragment_children_search fragment = Nokogiri::XML::Document.new.fragment( '<div><p id="content">hi</p></div>' ) - expected = fragment.children.xpath('.//p') + expected = fragment.children.xpath(".//p") assert_equal(1, expected.length) - css = fragment.children.css('p') - search_css = fragment.children.search('p') - search_xpath = fragment.children.search('.//p') + css = fragment.children.css("p") + search_css = fragment.children.search("p") + search_xpath = fragment.children.search(".//p") assert_equal(expected, css) assert_equal(expected, search_css) assert_equal(expected, search_xpath) @@ -130,14 +130,14 @@ def test_fragment_css_search_with_whitespace_and_node_removal fragment = Nokogiri::XML::DocumentFragment.parse(<<~EOXML) <p id="content">hi</p> x <!--y--> <p>another paragraph</p> EOXML - children = fragment.css('p') + children = fragment.css("p") assert_equal(2, children.length) # removing the last node instead does not yield the error. Probably the # node removal leaves around two consecutive text nodes which make the # css search crash? children.first.remove - assert_equal(1, fragment.xpath('.//p | self::p').length) - assert_equal(1, fragment.css('p').length) + assert_equal(1, fragment.xpath(".//p | self::p").length) + assert_equal(1, fragment.css("p").length) end def test_fragment_search_three_ways @@ -146,13 +146,13 @@ def test_fragment_search_three_ways assert_equal(2, expected.length) [ - [:css, '#content'], - [:search, '#content'], - [:search, './*[@id = \'content\']'], + [:css, "#content"], + [:search, "#content"], + [:search, "./*[@id = 'content']"], ].each do |method, query| result = frag.send(method, query) assert_equal(expected, result, - "fragment search with :#{method} using '#{query}' expected '#{expected}' got '#{result}'") + "fragment search with :#{method} using '#{query}' expected '#{expected}' got '#{result}'") end end @@ -169,9 +169,9 @@ def test_fragment_search_with_multiple_queries fragment = Nokogiri::XML.fragment(xml) assert_kind_of(Nokogiri::XML::DocumentFragment, fragment) - assert_equal(3, fragment.xpath('.//div', './/p').length) - assert_equal(3, fragment.css('.title', '.content', 'p').length) - assert_equal(3, fragment.search('.//div', 'p.blah').length) + assert_equal(3, fragment.xpath(".//div", ".//p").length) + assert_equal(3, fragment.css(".title", ".content", "p").length) + assert_equal(3, fragment.search(".//div", "p.blah").length) end def test_fragment_without_a_namespace_does_not_get_a_namespace @@ -215,7 +215,7 @@ def awesome! util_decorate(xml, x) fragment = Nokogiri::XML::DocumentFragment.new(xml, "<div>a</div><div>b</div>") - assert(node_set = fragment.css('div')) + assert(node_set = fragment.css("div")) assert(node_set.respond_to?(:awesome!)) node_set.each do |node| assert(node.respond_to?(:awesome!), node.class) @@ -234,8 +234,8 @@ def awesome! end def test_add_node_to_doc_fragment_segfault - frag = Nokogiri::XML::DocumentFragment.new(xml, '<p>hello world</p>') - Nokogiri::XML::Comment.new(frag, 'moo') + frag = Nokogiri::XML::DocumentFragment.new(xml, "<p>hello world</p>") + Nokogiri::XML::Comment.new(frag, "moo") end def test_issue_1077_parsing_of_frozen_strings @@ -363,6 +363,25 @@ def initialize(*args) end end end + + describe "#path" do + it "should return '?'" do + # see https://github.com/sparklemotion/nokogiri/issues/2250 + # this behavior is clearly undesirable, but is what libxml <= 2.9.10 returned, and so we + # do this for now to preserve the behavior across libxml2 versions. + xml = <<~EOF + <root1></root1> + <root2></root2> + EOF + + frag = Nokogiri::XML::DocumentFragment.parse(xml) + assert_equal "?", frag.path + + # # TODO: we should circle back and fix both the `#path` behavior and the `#xpath` + # # behavior so we can round-trip and get the DocumentFragment back again. + # assert_equal(frag, frag.at_xpath(doc.path)) # make sure we can round-trip + end + end end end end diff --git a/test/xml/test_node.rb b/test/xml/test_node.rb index 5995055cb7..f125d41d54 100644 --- a/test/xml/test_node.rb +++ b/test/xml/test_node.rb @@ -334,7 +334,7 @@ def test_namespace_definitions_when_no_exist assert_equal(0, namespace_definitions.length) end - def test_namespace_goes_to_children + def test_default_namespace_goes_to_children fruits = Nokogiri::XML(<<~eoxml) <Fruit xmlns='www.fruits.org'> </Fruit> @@ -347,6 +347,22 @@ def test_namespace_goes_to_children assert(fruits.at('//fruit:Apple', { 'fruit' => 'www.fruits.org' })) end + def test_parent_namespace_is_not_inherited + fruits = Nokogiri::XML(<<-eoxml) + <fruit xmlns:fruit="http://fruits.org"> + <fruit:apple /> + </fruit> + eoxml + + apple = fruits.at_xpath('//fruit:apple', {"fruit" => "http://fruits.org"}) + assert(apple) + + orange = Nokogiri::XML::Node.new('orange', fruits) + apple.add_child(orange) + + assert_equal(orange, fruits.at_xpath('//orange')) + end + def test_description assert_nil(xml.at('employee').description) end @@ -1049,18 +1065,24 @@ def test_namespace_without_an_href_on_html_node # describe how we handle microsoft word's HTML formatting. # this test is descriptive, not prescriptive. # - skip_unless_libxml2("Xerces handles this edge case completely differently") - - xml = Nokogiri::HTML.parse(<<~EOF) + html = Nokogiri::HTML.parse(<<~EOF) <div><o:p>foo</o:p></div> EOF - - node = xml.at("p") + node = html.at("div").children.first assert_not_nil(node) - assert_equal(1, node.namespaces.keys.size) - assert(node.namespaces.has_key?('xmlns:o')) - assert_nil(node.namespaces['xmlns:o']) + if Nokogiri.uses_libxml?(">= 2.9.12") + assert_empty(node.namespaces.keys) + assert_equal("<p>foo</p>", node.to_html) + elsif Nokogiri.uses_libxml? + assert_equal(1, node.namespaces.keys.size) + assert(node.namespaces.has_key?('xmlns:o')) + assert_nil(node.namespaces['xmlns:o']) + assert_equal("<p>foo</p>", node.to_html) + else # jruby + assert_empty(node.namespaces.keys) + assert_equal("<o:p>foo</o:p>", node.to_html) + end end def test_xpath_results_have_document_and_are_decorated diff --git a/test/xml/test_node_reparenting.rb b/test/xml/test_node_reparenting.rb index 2bd80cf9f0..4bb9f43bd8 100644 --- a/test/xml/test_node_reparenting.rb +++ b/test/xml/test_node_reparenting.rb @@ -360,6 +360,30 @@ def coerce(data) assert_match(%r{<first xmlns=\"http://tenderlovemaking.com/\">}, @doc.to_xml) end end + + describe "and a child node has a namespaced attribute" do + # https://github.com/sparklemotion/nokogiri/issues/2228 + it "should not lose attribute namespace" do + source_doc = Nokogiri::XML::Document.parse(<<~EOXML) + <pre1:root xmlns:pre1="ns1" xmlns:pre2="ns2"> + <pre1:child pre2:attr="attrval"> + </pre1:root> + EOXML + assert(source_node = source_doc.at_xpath("//pre1:child", {"pre1" => "ns1"})) + assert_equal("attrval", source_node.attribute_with_ns("attr", "ns2")&.value) + + dest_doc = Nokogiri::XML::Document.parse(<<~EOXML) + <pre1:root xmlns:pre1="ns1" xmlns:pre2="ns2"> + </pre1:root> + EOXML + assert(dest_node = dest_doc.at_xpath("//pre1:root", {"pre1" => "ns1"})) + + inserted = dest_node.add_child(source_node) + + assert_equal("attrval", inserted.attribute_with_ns("attr", "ns2")&.value, + "inserted node attribute should be namespaced") + end + end end describe "given a parent node with a default and non-default namespace" do @@ -533,6 +557,35 @@ def coerce(data) end end end + + describe "given a parent node with a non-default namespace" do + let(:doc) do + Nokogiri::XML(<<~EOF) + <root xmlns:foo="http://nokogiri.org/default_ns/test/foo"> + <foo:parent> + </foo:parent> + </root> + EOF + end + let(:parent) { doc.at_xpath("//foo:parent", "foo" => "http://nokogiri.org/default_ns/test/foo") } + + describe "and namespace_inheritance is off" do + it "inserts a child node that does not inherit the parent's namespace" do + refute(doc.namespace_inheritance) + child = parent.add_child("<child></child>").first + assert_nil(child.namespace) + end + end + + describe "and namespace_inheritance is on" do + it "inserts a child node that inherits the parent's namespace" do + doc.namespace_inheritance = true + child = parent.add_child("<child></child>").first + assert_not_nil(child.namespace) + assert_equal("http://nokogiri.org/default_ns/test/foo", child.namespace.href) + end + end + end end describe "#add_previous_sibling" do diff --git a/test/xml/test_node_set.rb b/test/xml/test_node_set.rb index be5cdc8eb0..c386bc8278 100644 --- a/test/xml/test_node_set.rb +++ b/test/xml/test_node_set.rb @@ -740,6 +740,7 @@ def awesome(ns) skip("cannot use Ruby 2.5 Enumerator#each in a systemd container") end skip("enumerators confuse valgrind") if i_am_running_in_valgrind + skip("enumerators confuse ASan") if i_am_running_with_asan employees = xml.search("//employee") enum = employees.each diff --git a/test/xml/test_schema.rb b/test/xml/test_schema.rb index 6cba39abe7..f09678b579 100644 --- a/test/xml/test_schema.rb +++ b/test/xml/test_schema.rb @@ -214,6 +214,21 @@ def test_xsd_with_dtd end end + def test_xsd_import_with_no_systemid + # https://github.com/sparklemotion/nokogiri/pull/2296 + xsd = <<~EOF + <?xml version="1.0" encoding="UTF-8"?> + <xs:schema + xmlns:xs="http://www.w3.org/2001/XMLSchema" + xmlns="http://www.w3.org/1998/Math/MathML" + targetNamespace="http://www.w3.org/1998/Math/MathML" + > + <xs:import/> + </xs:schema> + EOF + Nokogiri::XML::Schema(xsd) # assert_nothing_raised + end + describe "CVE-2020-26247" do # https://github.com/sparklemotion/nokogiri/security/advisories/GHSA-vr8q-g5c7-m54m let(:schema) do diff --git a/test/xml/test_xpath.rb b/test/xml/test_xpath.rb index 271412a70f..504a30caa8 100644 --- a/test/xml/test_xpath.rb +++ b/test/xml/test_xpath.rb @@ -340,7 +340,7 @@ def name_equals(nodeset, name, *args) # long list of long arguments, to apply GC pressure during # ruby_funcall argument marshalling xpath = ["//tool[name_equals(.,'hammer'"] - 1000.times { xpath << "'unused argument #{'x' * 1000}'" } + 500.times { xpath << "'unused argument #{'x' * 1000}'" } xpath << "'unused argument')]" xpath = xpath.join(',') @@ -470,6 +470,27 @@ def test_xpath_syntax_error end end + def test_huge_xpath_query + if Nokogiri.uses_libxml?("~>2.9.11") && !Nokogiri::VERSION_INFO["libxml"]["patches"]&.include?("0007-Fix-XPath-recursion-limit.patch") + skip("libxml2 under test is broken with respect to xpath query recusion depth") + end + + # real world example + # from https://github.com/sparklemotion/nokogiri/issues/2257 + query = File.read(File.join(ASSETS_DIR, 'huge-xpath-query.txt')) + + doc = Nokogiri::XML::Document.parse("<root></root>") + handler = Class.new do + def seconds(context) + 42 + end + def add(context, rhs) + 42 + end + end + doc.xpath(query, {"ct" => "https://test.nokogiri.org/ct", "date" => "https://test.nokogiri.org/date"}, handler.new) + end + describe "nokogiri-builtin:css-class xpath function" do before do @doc = Nokogiri::HTML::Document.parse("<html></html>") diff --git a/test/xslt/test_custom_functions.rb b/test/xslt/test_custom_functions.rb index 01daba28dd..34f509e56c 100644 --- a/test/xslt/test_custom_functions.rb +++ b/test/xslt/test_custom_functions.rb @@ -41,15 +41,15 @@ def capitalize(nodes) xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:f="http://e.org/functions" extension-element-prefixes="f"> - <xsl:template match="text()"> - <xsl:copy-of select="f:capitalize(.)"/> - </xsl:template> <xsl:template match="@*|node()"> <xsl:copy> <xsl:apply-templates select="@*|node()"/> <xsl:apply-imports/> </xsl:copy> </xsl:template> + <xsl:template match="text()"> + <xsl:copy-of select="f:capitalize(.)"/> + </xsl:template> </xsl:stylesheet> EOXSL result = xsl.transform(@xml) @@ -108,15 +108,15 @@ def america(nodes) xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:f="http://e.org/functions" extension-element-prefixes="f"> - <xsl:template match="text()"> - <xsl:copy-of select="f:america(.)"/> - </xsl:template> <xsl:template match="@*|node()"> <xsl:copy> <xsl:apply-templates select="@*|node()"/> <xsl:apply-imports/> </xsl:copy> </xsl:template> + <xsl:template match="text()"> + <xsl:copy-of select="f:america(.)"/> + </xsl:template> </xsl:stylesheet> EOXSL result = xsl.transform(@xml)