NOT READY: warcio test

README.rst

@@ -368,6 +368,14 @@ of WARC records, if possible. An exit value of 1 indicates a failure.
 ``warcio check -v`` will print verbose output for each record in the
 WARC file.
 
+Test
+~~~~
+
+The ``warcio test`` command will check one or more WARC files against
+the WARC standard, giving commentary about standards violations,
+recommendations, and other issues.
+
+
 Recompress
 ~~~~~~~~~~
 

setup.py

@@ -4,6 +4,7 @@
 from setuptools import setup, find_packages
 from setuptools.command.test import test as TestCommand
 import glob
+import sys
 
 __version__ = '1.7.0.dev0'
 
@@ -21,6 +22,15 @@ def run_tests(self):
         errcode = pytest.main(['--doctest-module', './warcio', '--cov', 'warcio', '-v', 'test/'])
         sys.exit(errcode)
 
+tests_require = [
+    'pytest',
+    'pytest-cov',
+    'httpbin==0.5.0',
+    'requests',
+]
+if sys.version_info < (3, 3):
+    tests_require.append('ipaddress')
+
 setup(
     name='warcio',
     version=__version__,
@@ -44,12 +54,7 @@ def run_tests(self):
     """,
     cmdclass={'test': PyTest},
     test_suite='',
-    tests_require=[
-        'pytest',
-        'pytest-cov',
-        'httpbin==0.5.0',
-        'requests',
-    ],
+    tests_require=tests_require,
     classifiers=[
         'Development Status :: 4 - Beta',
         'Environment :: Web Environment',

test/data/standard-torture-missing.warc

@@ -0,0 +1,5 @@
+WARC/1.0
+WARC-Type: warcinfo
+Content-Length: 0
+
+

test/data/standard-torture-validate-field.warc

@@ -0,0 +1,53 @@
+WARC/1.0
+WARC-Target-URI: <http://example.com/>
+WARC-Target-URI: example.com
+WARC-Target-URI: ex ample.com
+WARC-Target-URI: h<>ttp://example.com/
+WARC-Type: does-not-exist
+WARC-Type: CAPITALIZED
+WARC-Concurrent-To: http://example.com/
+WARC-Record-ID: <foo:bar>
+WARC-Date: 2017-03-06T04:03:53Z
+WARC-Date: 2017-03-06T04:03:53.Z
+Content-Type: asdf
+Content-Type: has space/asdf
+Content-Type: asdf/has space
+Content-Type: asdf/has space;asdf
+WARC-Block-Digest: asdf
+WARC-Block-Digest: has space:asdf
+WARC-Block-Digest: sha1:&$*^&*^#*&^
+WARC-IP-Address: 1.2.3.4.5
+WARC-Truncated: invalid
+WARC-Warcinfo-ID: asdf:asdf
+WARC-Filename: not-yet-tested
+WARC-Profile: asdf
+WARC-Profile: http://netpreserve.org/warc/1.0/revisit/identical-payload-digest
+WARC-Identified-Payload-Type: asdf
+WARC-Segment-Origin-ID: http://example.com
+WARC-Segment-Number: not-an-integer
+WARC-Segment-Number: 0
+WARC-Segment-Number: 1
+WARC-Segment-Number: 2
+WARC-Segment-Total-Length: 0
+WARC-Segment-Total-Length: not-an-integer
+WARC-Refers-To-Target-URI: http://example.com
+WARC-Refers-To-Date: not-a-date
+WARC-Unknown-Field: asdf
+Content-Length: 0
+
+
+WARC/1.1
+WARC-Date: 2017-03-06T04:03:53Z
+WARC-Date: 2017-03-06T04:03:53.Z
+WARC-Date: 2017-03-06T04:03:53.0Z
+WARC-Type: invalid
+Content-Length: 0
+
+
+WARC/1.1
+WARC-Type: request
+WARC-Segment-Number: 1
+Content-Length: 0
+
+
+WARC/invalid

test/data/standard-torture-validate-record.warc

@@ -0,0 +1,111 @@
+WARC/1.0
+WARC-Type: warcinfo
+Content-Type: application/warc-fields
+WARC-Refers-To: probhibited
+Content-Length: 146
+
+ first line can't start with a space
+test: invalid utf8 �(
+test: lines should end with \r\n
+foo:
+ bar
+
+no colon
+token cannot have a space:
+
+
+WARC/1.0
+WARC-Type: warcinfo
+Content-Type: application/warc-fields
+Content-Length: 0
+
+
+WARC/1.0
+WARC-Type: response
+WARC-Target-URI: HtTp://example.com/
+Content-Type: text/plain
+Content-Length: 0
+
+
+WARC/1.0
+WARC-Type: resource
+WARC-Target-URI: DnS:asdfasdf
+Content-Type: text/plain
+Content-Length: 0
+
+
+WARC/1.0
+WARC-Type: resource
+WARC-Target-URI: DnS:asdfasdf
+Content-Type: text/dns
+Content-Length: 0
+
+
+WARC/1.0
+WARC-Type: resource
+WARC-Target-URI: foo:bar
+Content-Length: 0
+
+
+WARC/1.0
+WARC-Type: request
+WARC-Target-URI: hTtP://example.com/
+Content-Type: text/plain
+Content-Length: 0
+
+
+WARC/1.0
+WARC-Type: request
+WARC-Target-URI: hTtP://example.com/
+WARC-IP-Address: 1.2.3.4
+Content-Type: text/plain
+Content-Length: 0
+
+
+WARC/1.0
+WARC-Type: metadata
+Content-Type: application/warc-fields
+Content-Length: 0
+
+
+WARC/1.0
+WARC-Type: metadata
+Content-Type: not-application/warc-fields
+Content-Length: 0
+
+
+WARC/1.0
+WARC-Type: revisit
+WARC-Profile: none
+Content-Length: 0
+
+
+WARC/1.0
+WARC-Type: revisit
+WARC-Profile: http://netpreserve.org/warc/1.1/revisit/identical-payload-digest
+Content-Length: 0
+
+
+WARC/1.0
+WARC-Type: revisit
+WARC-Profile: http://netpreserve.org/warc/1.0/revisit/server-not-modified
+Content-Length: 0
+
+
+WARC/1.0
+WARC-Type: conversion
+Content-Length: 0
+
+
+WARC/1.0
+WARC-Type: continuation
+WARC-Segment-Number: 1
+Content-Length: 0
+
+
+WARC/1.0
+WARC-Type: continuation
+WARC-Segment-Number: 2
+Content-Length: 0
+
+

test/test_archiveiterator.py

@@ -185,6 +185,8 @@ def test_err_arc_iterator_on_warc(self):
     def test_corrects_wget_bug(self):
         with self._find_first_by_type('example-wget-bad-target-uri.warc.gz', 'response') as record:
             assert record.rec_headers.get('WARC-Target-URI') == 'http://example.com/'
+        with self._find_first_by_type('example-wget-bad-target-uri.warc.gz', 'response', fixup_bugs=False) as record:
+            assert record.rec_headers.get('WARC-Target-URI') == '<http://example.com/>'
 
     def _digests_mutilate_helper(self, contents, expected_t, expected_f, capsys, full_read=False):
         with pytest.raises(ArchiveLoadFailed):
@@ -243,9 +245,9 @@ def test_digests_file(self):
         expected_t = ['request', 'request', 'request']
 
         # record 1: invalid payload digest
-        assert self._load_archive('example-digest.warc', check_digests=True) == expected_t
-        assert self._load_archive('example-digest.warc', check_digests=False) == expected_f
+        assert self._load_archive('example-digest-bad.warc', check_digests=True) == expected_t
+        assert self._load_archive('example-digest-bad.warc', check_digests=False) == expected_f
 
         # record 2: b64 digest; record 3: b64 filename safe digest
-        assert self._load_archive('example-digest.warc', offset=922, check_digests=True) == expected_t
-        assert self._load_archive('example-digest.warc', offset=922, check_digests=False) == expected_t
+        assert self._load_archive('example-digest-bad.warc', offset=922, check_digests=True) == expected_t
+        assert self._load_archive('example-digest-bad.warc', offset=922, check_digests=False) == expected_t

test/test_cli.py

@@ -90,7 +90,7 @@ def test_check_valid():
 
 
 def test_check_invalid():
-    filenames = [get_test_file('example-digest.warc')]
+    filenames = [get_test_file('example-digest-bad.warc')]
 
     args = ['check'] + filenames
     value = check_helper(args, 1)