Merge pull request #26 from tech-engine/dev

Feature: Added css & xpath node selectors

Author: tech-engine

go.mod

@@ -15,6 +15,8 @@ require (
 	google.golang.org/api v0.136.0
 )
 
+require github.com/antchfx/xpath v1.3.1 // indirect
+
 require (
 	cloud.google.com/go v0.110.6 // indirect
 	cloud.google.com/go/compute v1.23.0 // indirect
@@ -23,6 +25,8 @@ require (
 	cloud.google.com/go/iam v1.1.1 // indirect
 	cloud.google.com/go/longrunning v0.5.1 // indirect
 	cloud.google.com/go/storage v1.33.0 // indirect
+	github.com/andybalholm/cascadia v1.3.2
+	github.com/antchfx/htmlquery v1.3.2
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect

go.sum

@@ -17,6 +17,12 @@ cloud.google.com/go/storage v1.33.0/go.mod h1:Hhh/dogNRGca7IWv1RC2YqEn0c0G77ctA/
 firebase.google.com/go v3.13.0+incompatible h1:3TdYC3DDi6aHn20qoRkxwGqNgdjtblwVAyRLQwGn/+4=
 firebase.google.com/go v3.13.0+incompatible/go.mod h1:xlah6XbEyW6tbfSklcfe5FHJIwjt8toICdV5Wh9ptHs=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
+github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
+github.com/antchfx/htmlquery v1.3.2 h1:85YdttVkR1rAY+Oiv/nKI4FCimID+NXhDn82kz3mEvs=
+github.com/antchfx/htmlquery v1.3.2/go.mod h1:1mbkcEgEarAokJiWhTfr4hR06w/q2ZZjnYLrDt6CTUk=
+github.com/antchfx/xpath v1.3.1 h1:PNbFuUqHwWl0xRjvUPjJ95Agbmdj2uzzIwmQKgu4oCk=
+github.com/antchfx/xpath v1.3.1/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@@ -139,6 +145,7 @@ golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTk
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -152,6 +159,9 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
 golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
 golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -163,6 +173,7 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
 golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -175,17 +186,22 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
 golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
@@ -197,6 +213,7 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

pkg/core/ports.go

@@ -7,6 +7,7 @@ import (
 	"net/url"
 
 	"github.com/tech-engine/goscrapy/internal/fsm"
+	"golang.org/x/net/html"
 )
 
 type IEngine[OUT any] interface {
@@ -50,6 +51,7 @@ type IResponseReader interface {
 	Cookies() []*http.Cookie
 	Request() *http.Request
 	Meta(string) (any, bool)
+	ISelector
 }
 
 type IJob interface {
@@ -64,3 +66,15 @@ type IOutput[OUT any] interface {
 }
 
 type ResponseCallback func(context.Context, IResponseReader)
+type ISelectorGetter interface {
+	Get() *html.Node
+	GetAll() []*html.Node
+	Text(...string) []string
+	Attr(string) []string
+}
+
+type ISelector interface {
+	Css(string) ISelector
+	Xpath(string) ISelector
+	ISelectorGetter
+}

pkg/scheduler/response.go

@@ -6,6 +6,8 @@ import (
 	"net/http"
 
 	"github.com/tech-engine/goscrapy/internal/fsm"
+	"github.com/tech-engine/goscrapy/pkg/core"
+	"golang.org/x/net/html"
 )
 
 func NewResponse() *response {
@@ -19,6 +21,7 @@ type response struct {
 	cookies    []*http.Cookie
 	request    *http.Request
 	meta       *fsm.FixedSizeMap[string, any]
+	nodes      Selectors
 }
 
 // response implementing core.ResponseReader
@@ -61,6 +64,7 @@ func (r *response) Reset() {
 	// because we there isn't guarantee that we will have the same pair for req-res from the pools,
 	// we must set it meta=nil upon releasing req-res to their respective pools, otherwise we will have corrupt data.
 	r.meta = nil
+	r.nodes = nil
 }
 
 // response implementing engine.ResponseWriter
@@ -87,3 +91,40 @@ func (r *response) WriteCookies(cookies []*http.Cookie) {
 func (r *response) WriteMeta(meta *fsm.FixedSizeMap[string, any]) {
 	r.meta = meta
 }
+
+func (r *response) Css(selector string) core.ISelector {
+
+	if r.nodes == nil {
+		if nodes, err := NewSelector(r.body); err == nil {
+			r.nodes = nodes
+		}
+	}
+
+	return r.nodes.Css(selector)
+}
+
+func (r *response) Xpath(xpath string) core.ISelector {
+
+	if r.nodes == nil {
+		if nodes, err := NewSelector(r.body); err == nil {
+			r.nodes = nodes
+		}
+	}
+	return r.nodes.Xpath(xpath)
+}
+
+func (r *response) Text(def ...string) []string {
+	return r.nodes.Text(def...)
+}
+
+func (r *response) Attr(attrName string) []string {
+	return r.nodes.Attr(attrName)
+}
+
+func (r *response) Get() *html.Node {
+	return r.nodes.Get()
+}
+
+func (r *response) GetAll() []*html.Node {
+	return r.nodes.GetAll()
+}

pkg/scheduler/selectors.go

@@ -0,0 +1,89 @@
+package scheduler
+
+import (
+	"io"
+	"strings"
+
+	"github.com/andybalholm/cascadia"
+	"github.com/antchfx/htmlquery"
+	"github.com/tech-engine/goscrapy/pkg/core"
+	"golang.org/x/net/html"
+)
+
+type Selectors []*html.Node
+
+func NewSelector(r io.Reader) (Selectors, error) {
+	root, err := html.Parse(r)
+	if err != nil {
+		return nil, err
+	}
+	return Selectors([]*html.Node{root}), nil
+}
+
+// Css selector - select element by id, class, nodename etc.
+func (nodes Selectors) Css(selector string) core.ISelector {
+	sel, err := cascadia.ParseWithPseudoElement(selector)
+	if err != nil {
+		return Selectors([]*html.Node{})
+	}
+
+	selected := make(Selectors, 0, len(nodes))
+	for _, node := range nodes {
+		selected = append(selected, cascadia.QueryAll(node, sel)...)
+	}
+
+	return selected
+}
+
+// Xpath selector - select element using an xpath expression.
+func (nodes Selectors) Xpath(xpath string) core.ISelector {
+	selected := make(Selectors, 0, len(nodes))
+	for _, node := range nodes {
+		matches, err := htmlquery.QueryAll(node, xpath)
+		if err != nil {
+			continue
+		}
+		selected = append(selected, matches...)
+	}
+	return selected
+}
+
+// Extracts all the text of a node and it's descendents.
+func (nodes Selectors) Text(def ...string) []string {
+	texts := make([]string, 0, len(nodes))
+	for _, node := range nodes {
+		text := strings.TrimSpace(htmlquery.InnerText(node))
+		if text == "" && len(def) > 0 {
+			texts = append(texts, def[0])
+			continue
+		}
+		texts = append(texts, text)
+	}
+	return texts
+}
+
+// Extracts attribute values
+func (nodes Selectors) Attr(attrName string) []string {
+	attrs := make([]string, 0, len(nodes))
+	for _, node := range nodes {
+		for _, attr := range node.Attr {
+			if attr.Key == attrName {
+				attrs = append(attrs, attr.Val)
+			}
+		}
+	}
+	return attrs
+}
+
+// Get the first matched node
+func (nodes Selectors) Get() *html.Node {
+	if len(nodes) <= 0 {
+		return nil
+	}
+	return nodes[0]
+}
+
+// Gets all the matched nodes
+func (nodes Selectors) GetAll() []*html.Node {
+	return nodes
+}

pkg/scheduler/selectors_test.go

@@ -0,0 +1,56 @@
+package scheduler
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSelectors(t *testing.T) {
+	html := `
+	<html>
+		<body>
+			<div id="main" class="content">
+				<h1>Title</h1>
+				<p class="intro">Introduction paragraph 1</p>
+				<a href="http://example.com">Example Link</a>
+				<p>This is test paragraph</p>
+				<p class="intro" data-mg="test">Introduction paragraph 3</p>
+			</div>
+		</body>
+	</html>
+	`
+
+	selector, err := NewSelector(strings.NewReader(html))
+
+	assert.NoError(t, err)
+
+	cssSelector := selector.Css("p.intro")
+
+	cssNodes := cssSelector.GetAll()
+	assert.Len(t, cssNodes, 2, "expected nodes=2, got=%s", len(cssNodes))
+
+	cssNodesTexts := cssSelector.Text()
+	assert.Equal(t, "Introduction paragraph 1", cssNodesTexts[0], "expected paragraph text=Introduction paragraph 1, got=%s", cssNodesTexts[0])
+
+	xpathSelector := selector.Xpath("//p[@data-mg='test']")
+
+	xpathNodes := xpathSelector.GetAll()
+	assert.Len(t, xpathNodes, 1, "expected xpath nodes=1, got=%s", len(xpathNodes))
+
+	xpathNodesTexts := xpathSelector.Text()
+	assert.Len(t, xpathNodesTexts, 1, "expected xpathNodesTexts=1, got=%s", len(xpathNodesTexts))
+	assert.Equal(t, "Introduction paragraph 3", xpathNodesTexts[0], "expected paragraph text=Introduction paragraph 3, got=%s", xpathNodesTexts[0])
+
+	attrValues := selector.Css("a").Attr("href")
+	assert.Len(t, xpathNodesTexts, 1, "expected attrValues=1, got=%s", len(attrValues))
+	assert.Equal(t, "http://example.com", attrValues[0], "expected href=http://example.com, got=%s", attrValues[0])
+
+	noCssElements := selector.Css("p.box").GetAll()
+	assert.Empty(t, noCssElements, "expected element=0, got=%s", len(noCssElements))
+
+	noXpathElements := selector.Xpath("//p[@class='test']").GetAll()
+	assert.Empty(t, noXpathElements, "expected element=0, got=%s", len(noXpathElements))
+
+}