Skip to content

Commit 1f0d78d

Browse files
waybackarchiverwaybackarchiver
committed
Add support for tag retrieval
1 parent 567017e commit 1f0d78d

25 files changed

+415
-60
lines changed

‎README.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -215,7 +215,8 @@ You can also specify configuration options either via command flags or via envir
215215
| - | `WAYBACK_POOLING_SIZE` | `3` | Number of worker pool for wayback at once |
216216
| - | `WAYBACK_BOLT_PATH` | `./wayback.db` | File path of bolt database |
217217
| - | `WAYBACK_STORAGE_DIR` | - | Directory to store binary file, e.g. PDF, html file |
218-
| - | `WAYBACK_MAX_MEDIA_SIZE` | `512MB` | Max size to limit download stream media |
218+
| - | `WAYBACK_MAX_MEDIA_SIZE` | `512MB` | Maximum size to limit download stream media |
219+
| - | `WAYBACK_MAX_TAG_SIZE` | `3` | Maximum size of tags can be extracted from a webpage |
219220
| - | `WAYBACK_TIMEOUT` | `300` | Timeout for single wayback request, defaults to 300 second |
220221
| - | `WAYBACK_MAX_RETRIES` | `2` | Max retries for single wayback request, defaults to 2 |
221222
| - | `WAYBACK_USERAGENT` | `WaybackArchiver/1.0` | User-Agent for a wayback request |

‎config/config_test.go‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1421,3 +1421,43 @@ func TestWaybackFallback(t *testing.T) {
14211421
})
14221422
}
14231423
}
1424+
1425+
func TestMaxTagSize(t *testing.T) {
1426+
t.Parallel()
1427+
1428+
var tests = []struct {
1429+
size string
1430+
expected int
1431+
}{
1432+
{
1433+
size: "0",
1434+
expected: 0,
1435+
},
1436+
{
1437+
size: "-1",
1438+
expected: 0,
1439+
},
1440+
{
1441+
size: "5",
1442+
expected: 5,
1443+
},
1444+
}
1445+
1446+
for _, test := range tests {
1447+
t.Run(test.size, func(t *testing.T) {
1448+
os.Clearenv()
1449+
os.Setenv("WAYBACK_MAX_TAG_SIZE", test.size)
1450+
1451+
parser := NewParser()
1452+
opts, err := parser.ParseEnvironmentVariables()
1453+
if err != nil {
1454+
t.Fatalf(`Parsing environment variables failed: %v`, err)
1455+
}
1456+
1457+
got := opts.MaxTagSize()
1458+
if got != test.expected {
1459+
t.Errorf(`Unexpected set max tag size got %d instead of %d`, got, test.expected)
1460+
}
1461+
})
1462+
}
1463+
}

‎config/options.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ const (
7676
defPoolingSize = 3
7777
defStorageDir = ""
7878
defMaxMediaSize = "512MB"
79+
defMaxTagSize = 3
7980
defWaybackTimeout = 300
8081
defWaybackMaxRetries = 2
8182
defWaybackUserAgent = "WaybackArchiver/1.0"
@@ -113,6 +114,7 @@ type Options struct {
113114
poolingSize int
114115
storageDir string
115116
maxMediaSize string
117+
maxTagSize int
116118
waybackTimeout int
117119
waybackMaxRetries int
118120
waybackUserAgent string
@@ -206,6 +208,7 @@ func NewOptions() *Options {
206208
poolingSize: defPoolingSize,
207209
storageDir: defStorageDir,
208210
maxMediaSize: defMaxMediaSize,
211+
maxTagSize: defMaxTagSize,
209212
waybackTimeout: defWaybackTimeout,
210213
waybackMaxRetries: defWaybackMaxRetries,
211214
waybackUserAgent: defWaybackUserAgent,
@@ -681,3 +684,11 @@ func (o *Options) WaybackUserAgent() string {
681684
func (o *Options) WaybackFallback() bool {
682685
return o.waybackFallback
683686
}
687+
688+
// MaxTagSize returns the maximum number of tags can be extracted from a webpage.
689+
func (o *Options) MaxTagSize() int {
690+
if o.maxTagSize < 0 {
691+
return 0
692+
}
693+
return o.maxTagSize
694+
}

‎config/parser.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,8 @@ func (p *Parser) parseLines(lines []string) (err error) {
189189
p.opts.storageDir = parseString(val, defStorageDir)
190190
case "WAYBACK_MAX_MEDIA_SIZE":
191191
p.opts.maxMediaSize = parseString(val, defMaxMediaSize)
192+
case "WAYBACK_MAX_TAG_SIZE":
193+
p.opts.maxTagSize = parseInt(val, defMaxTagSize)
192194
case "WAYBACK_TIMEOUT":
193195
p.opts.waybackTimeout = parseInt(val, defWaybackTimeout)
194196
case "WAYBACK_MAX_RETRIES":

‎go.mod‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
module github.com/wabarc/wayback
22

3-
// +heroku goVersion go1.17
3+
// +heroku goVersion go1.18
44

5-
go 1.17
5+
go 1.18
66

77
require (
88
github.com/bwmarrin/discordgo v0.23.3-0.20210627161652-421e14965030
@@ -39,8 +39,8 @@ require (
3939
github.com/wabarc/screenshot v1.5.1-0.20220318140348-632a135d50db
4040
github.com/wabarc/telegra.ph v0.0.0-20220304132636-fce723bd6eae
4141
github.com/wabarc/warcraft v0.2.2-0.20211107142816-7beea5a75ab5
42+
github.com/zoomio/tagify v0.53.0
4243
go.etcd.io/bbolt v1.3.6
43-
golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd // indirect
4444
golang.org/x/net v0.0.0-20220225172249-27dd8689420f
4545
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
4646
gopkg.in/telebot.v3 v3.0.0-20220130115853-f0291132d3c3
@@ -51,6 +51,7 @@ require (
5151
github.com/MercuryEngineering/CookieMonster v0.0.0-20180304172713-1584578b3403 // indirect
5252
github.com/PuerkitoBio/goquery v1.8.0 // indirect
5353
github.com/VividCortex/ewma v1.2.0 // indirect
54+
github.com/abadojack/whatlanggo v1.0.1 // indirect
5455
github.com/andybalholm/cascadia v1.3.1 // indirect
5556
github.com/beorn7/perks v1.0.1 // indirect
5657
github.com/bitly/go-simplejson v0.5.0 // indirect
@@ -82,6 +83,7 @@ require (
8283
github.com/ipfs/go-ipfs-files v0.1.1 // indirect
8384
github.com/itchyny/gojq v0.12.7 // indirect
8485
github.com/itchyny/timefmt-go v0.1.3 // indirect
86+
github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a // indirect
8587
github.com/josharian/intern v1.0.0 // indirect
8688
github.com/json-iterator/go v1.1.12 // indirect
8789
github.com/kallydev/telegraph-go v1.0.0 // indirect
@@ -125,8 +127,11 @@ require (
125127
github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect
126128
github.com/wabarc/imgbb v1.0.0 // indirect
127129
github.com/wabarc/memento v0.0.0-20210703205719-adc2f8ab8bae // indirect
128-
github.com/whyrusleeping/tar-utils v0.0.0-20201201191210-20a61371de5b // indirect
130+
github.com/whyrusleeping/tar-utils v0.0.0-20180509141711-8c6c8ba81d5c // indirect
129131
github.com/ybbus/httpretry v1.0.1 // indirect
132+
github.com/zoomio/inout v0.11.0 // indirect
133+
github.com/zoomio/stopwords v0.7.0 // indirect
134+
golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd // indirect
130135
golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 // indirect
131136
golang.org/x/text v0.3.7 // indirect
132137
google.golang.org/protobuf v1.27.1 // indirect

‎go.sum‎

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,8 @@ github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejn
6262
github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
6363
github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
6464
github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
65+
github.com/abadojack/whatlanggo v1.0.1 h1:19N6YogDnf71CTHm3Mp2qhYfkRdyvbgwWdd2EPxJRG4=
66+
github.com/abadojack/whatlanggo v1.0.1/go.mod h1:66WiQbSbJBIlOZMsvbKe5m6pzQovxCH9B/K8tQB2uoc=
6567
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
6668
github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
6769
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -113,7 +115,6 @@ github.com/cenkalti/backoff/v4 v4.1.2 h1:6Yo7N8UP2K6LWZnW94DLVSSrbobcWdVzAYOisuD
113115
github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
114116
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
115117
github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
116-
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
117118
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
118119
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
119120
github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
@@ -122,8 +123,10 @@ github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyV
122123
github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U=
123124
github.com/cheggaaa/pb/v3 v3.0.8 h1:bC8oemdChbke2FHIIGy9mn4DPJ2caZYQnfbRqwmdCoA=
124125
github.com/cheggaaa/pb/v3 v3.0.8/go.mod h1:UICbiLec/XO6Hw6k+BHEtHeQFzzBH4i2/qk/ow1EJTA=
126+
github.com/chromedp/cdproto v0.0.0-20210713064928-7d28b402946a/go.mod h1:At5TxYYdxkbQL0TSefRjhLE3Q0lgvqKKMSFUglJ7i1U=
125127
github.com/chromedp/cdproto v0.0.0-20220217222649-d8c14a5c6edf h1:1omDWNUsWxn2HpiMiMuyRmzjl9uG7RP3IE6GTlpgJWU=
126128
github.com/chromedp/cdproto v0.0.0-20220217222649-d8c14a5c6edf/go.mod h1:At5TxYYdxkbQL0TSefRjhLE3Q0lgvqKKMSFUglJ7i1U=
129+
github.com/chromedp/chromedp v0.7.4/go.mod h1:dBj+SXuQHznp6ZPwZeDDEBZKwclUwDLbZ0hjMialMYs=
127130
github.com/chromedp/chromedp v0.7.8 h1:JFPIFb28LPjcx6l6mUUzLOTD/TgswcTtg7KrDn8S/2I=
128131
github.com/chromedp/chromedp v0.7.8/go.mod h1:HcIUFBa5vA+u2QI3+xljiU59llUQ8lgGoLzYSCBfmUA=
129132
github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic=
@@ -391,7 +394,6 @@ github.com/ipfs/go-cid v0.1.0/go.mod h1:rH5/Xv83Rfy8Rw6xG+id3DYAMUVmem1MowoKwdXm
391394
github.com/ipfs/go-ipfs-api v0.3.0 h1:ZzVrsTV31Z53ZlUare6a5UJ46lC7lW93q/s1/fXyATk=
392395
github.com/ipfs/go-ipfs-api v0.3.0/go.mod h1:A1naQGm0Jg01GxDq7oDyVSZxt20SuRTNIBFNZJgPDmg=
393396
github.com/ipfs/go-ipfs-files v0.0.9/go.mod h1:aFv2uQ/qxWpL/6lidWvnSQmaVqCrf0TBGoUr+C1Fo84=
394-
github.com/ipfs/go-ipfs-files v0.1.1-0.20220118065129-90b5617c775d/go.mod h1:8xkIrMWH+Y5P7HvJ4Yc5XWwIW2e52dyXUiC0tZyjDbM=
395397
github.com/ipfs/go-ipfs-files v0.1.1 h1:/MbEowmpLo9PJTEQk16m9rKzUHjeP4KRU9nWJyJO324=
396398
github.com/ipfs/go-ipfs-files v0.1.1/go.mod h1:8xkIrMWH+Y5P7HvJ4Yc5XWwIW2e52dyXUiC0tZyjDbM=
397399
github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ=
@@ -403,6 +405,8 @@ github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5D
403405
github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4=
404406
github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
405407
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
408+
github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a h1:eeaG9XMUvRBYXJi4pg1ZKM7nxc5AfXfojeLLW7O5J3k=
409+
github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
406410
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
407411
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
408412
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
@@ -721,9 +725,8 @@ github.com/wabarc/telegra.ph v0.0.0-20220304132636-fce723bd6eae h1:qxDauBAL0F3P/
721725
github.com/wabarc/telegra.ph v0.0.0-20220304132636-fce723bd6eae/go.mod h1:s4RHlPLzyJ7nX4+3UHp+WfVZb0hPefHUl319kVBPLtg=
722726
github.com/wabarc/warcraft v0.2.2-0.20211107142816-7beea5a75ab5 h1:jY/jqIy/ddCMWWWuTIeAazE5F4QW8HAIvlI69XMJ1ew=
723727
github.com/wabarc/warcraft v0.2.2-0.20211107142816-7beea5a75ab5/go.mod h1:/BbCwReBjlqHRaw8Yh+7sfAicOesiMYNhiFpuL1x8Rc=
728+
github.com/whyrusleeping/tar-utils v0.0.0-20180509141711-8c6c8ba81d5c h1:GGsyl0dZ2jJgVT+VvWBf/cNijrHRhkrTjkmp5wg7li0=
724729
github.com/whyrusleeping/tar-utils v0.0.0-20180509141711-8c6c8ba81d5c/go.mod h1:xxcJeBb7SIUl/Wzkz1eVKJE/CB34YNrqX2TQI6jY9zs=
725-
github.com/whyrusleeping/tar-utils v0.0.0-20201201191210-20a61371de5b h1:wA3QeTsaAXybLL2kb2cKhCAQTHgYTMwuI8lBlJSv5V8=
726-
github.com/whyrusleeping/tar-utils v0.0.0-20201201191210-20a61371de5b/go.mod h1:xT1Y5p2JR2PfSZihE0s4mjdJaRGp1waCTf5JzhQLBck=
727730
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
728731
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
729732
github.com/ybbus/httpretry v1.0.1 h1:lpo0rx/qY4kvishWD6kDgEFW7uGzx4gB/9AgPO5rg9Q=
@@ -738,6 +741,12 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
738741
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
739742
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
740743
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
744+
github.com/zoomio/inout v0.11.0 h1:hnIdXa3J4tTXmemCJJqsHemyF5+T52GAsGGTnQW9Tq4=
745+
github.com/zoomio/inout v0.11.0/go.mod h1:kDGEuyxH0chOEslentwiv9TxGELUZyX5eqS9UtYj9gI=
746+
github.com/zoomio/stopwords v0.7.0 h1:8U+BpGVXN7xA4/ZqBgucdNuEVHOhrWsIiAP2EePNSHQ=
747+
github.com/zoomio/stopwords v0.7.0/go.mod h1:gaosgTBK3xJE1dRenc/N4f/RnaKxjB1gfIqFSqtck+k=
748+
github.com/zoomio/tagify v0.53.0 h1:coOOg5oa3grqo1+sdQhVxVwXrZbqZBZMbUYgmHozI/g=
749+
github.com/zoomio/tagify v0.53.0/go.mod h1:MOcXdwMWfcv/mtDdKhcIn7E9sao6djyTBHGA/9ZKMZc=
741750
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
742751
go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
743752
go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
@@ -871,11 +880,9 @@ golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qx
871880
golang.org/x/net v0.0.0-20210716203947-853a461950ff/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
872881
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
873882
golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
874-
golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
875883
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
876884
golang.org/x/net v0.0.0-20211215060638-4ddde0e984e9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
877885
golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
878-
golang.org/x/net v0.0.0-20220121210141-e204ce36a2ba/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
879886
golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc=
880887
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
881888
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -975,6 +982,7 @@ golang.org/x/sys v0.0.0-20210415045647-66c3f260301c/go.mod h1:h1NjWce9XRLGQEsW7w
975982
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
976983
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
977984
golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
985+
golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
978986
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
979987
golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
980988
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

‎publish/publish_test.go‎

Lines changed: 37 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -19,37 +19,44 @@ import (
1919
"github.com/wabarc/wayback"
2020
"github.com/wabarc/wayback/config"
2121
"github.com/wabarc/wayback/reduxer"
22+
2223
telegram "gopkg.in/telebot.v3"
2324
)
2425

25-
var collects = []wayback.Collect{
26-
{
27-
Arc: config.SLOT_IA,
28-
Dst: "https://web.archive.org/web/20211000000001/https://example.com/",
29-
Src: "https://example.com/",
30-
Ext: config.SLOT_IA,
31-
},
32-
{
33-
Arc: config.SLOT_IS,
34-
Dst: "http://archive.today/abcdE",
35-
Src: "https://example.com/",
36-
Ext: config.SLOT_IS,
37-
},
38-
{
39-
Arc: config.SLOT_IP,
40-
Dst: "https://ipfs.io/ipfs/QmTbDmpvQ3cPZG6TA5tnar4ZG6q9JMBYVmX2n3wypMQMtr",
41-
Src: "https://example.com/",
42-
Ext: config.SLOT_IP,
43-
},
44-
{
45-
Arc: config.SLOT_PH,
46-
Dst: "http://telegra.ph/title-01-01",
47-
Src: "https://example.com/",
48-
Ext: config.SLOT_PH,
49-
},
50-
}
26+
var (
27+
src = "https://example.com/"
28+
collects = []wayback.Collect{
29+
{
30+
Arc: config.SLOT_IA,
31+
Dst: "https://web.archive.org/web/20211000000001/https://example.com/",
32+
Src: src,
33+
Ext: config.SLOT_IA,
34+
},
35+
{
36+
Arc: config.SLOT_IS,
37+
Dst: "http://archive.today/abcdE",
38+
Src: src,
39+
Ext: config.SLOT_IS,
40+
},
41+
{
42+
Arc: config.SLOT_IP,
43+
Dst: "https://ipfs.io/ipfs/QmTbDmpvQ3cPZG6TA5tnar4ZG6q9JMBYVmX2n3wypMQMtr",
44+
Src: src,
45+
Ext: config.SLOT_IP,
46+
},
47+
{
48+
Arc: config.SLOT_PH,
49+
Dst: "http://telegra.ph/title-01-01",
50+
Src: src,
51+
Ext: config.SLOT_PH,
52+
},
53+
}
54+
textContent = `Example Domain
55+
This domain is for use in illustrative examples in documents. You may use this domain in literature without prior coordination or asking for permission.
5156
52-
var bundleExample = reduxer.BundleExample()
57+
More information...`
58+
bundleExample = reduxer.BundleExample()
59+
)
5360

5461
func unsetAllEnv() {
5562
lines := os.Environ()
@@ -142,6 +149,7 @@ func TestPublishTootFromMastodon(t *testing.T) {
142149
mstdn := NewMastodon(nil)
143150

144151
ctx := context.WithValue(context.Background(), FlagMastodon, mstdn.client)
152+
ctx = context.WithValue(ctx, PubBundle{}, bundleExample)
145153
To(ctx, collects, FlagMastodon.String())
146154
}
147155

@@ -170,6 +178,7 @@ func TestPublishTweetFromTwitter(t *testing.T) {
170178

171179
twi := NewTwitter(twitter.NewClient(httpClient))
172180
ctx := context.WithValue(context.Background(), FlagTwitter, twi.client)
181+
ctx = context.WithValue(ctx, PubBundle{}, bundleExample)
173182
To(ctx, collects, FlagTwitter.String())
174183
}
175184

@@ -204,5 +213,6 @@ func TestPublishToMatrixRoomFromMatrix(t *testing.T) {
204213

205214
mat := NewMatrix(nil)
206215
ctx := context.WithValue(context.Background(), "matrix", mat.client)
216+
ctx = context.WithValue(ctx, PubBundle{}, bundleExample)
207217
To(ctx, collects, "matrix")
208218
}

‎reduxer/example.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,20 @@
44

55
package reduxer // import "github.com/wabarc/wayback/reduxer"
66

7+
import (
8+
"strings"
9+
10+
"github.com/go-shiori/go-readability"
11+
"github.com/wabarc/wayback/tagging"
12+
)
13+
714
func BundleExample() Reduxer {
15+
textContent := `Example Domain
16+
This domain is for use in illustrative examples in documents. You may use this domain in literature without prior coordination or asking for permission.
17+
More information...`
18+
r := strings.NewReader(textContent)
19+
art, _ := readability.FromReader(r, nil)
20+
821
rdx := NewReduxer()
922
bnd := &bundle{
1023
artifact: Artifact{
@@ -58,6 +71,8 @@ func BundleExample() Reduxer {
5871
},
5972
},
6073
},
74+
article: art,
75+
tags: tagging.Annotation{"information", "more", "asking"},
6176
}
6277

6378
rdx.Store(Src("https://example.com/"), bnd)

0 commit comments

Comments
 (0)