Updated docs. Updated sbxenc to use seqbox module. tweaks on the library.

Author: MarcoPon

notes.txt

@@ -1,37 +1,42 @@
 SeqBox - Sequenced Box container (SEQBOX/SBX)
 ===========================================
 
-Encode a file in a container that can be reconstructed even after total loss of file system structures.
-Use a blocksize equal or submultiple of a sector/cluster size, with a minimal header that include block
-sequence number, checksum and other info. 
-Additional, non critical info are contained in block 0 (like name, file size, other attributes, etc.).
+Encode a file in a container that can be reconstructed even after total
+loss of file system structures. Use a blocksize equal or submultiple of a
+sector/cluster size, with a minimal header that include block sequence
+number, checksum and other info. 
+Additional, non critical info/metadata are contained in block 0 (like name,
+file size, other attributes, etc.).
 
-Recovery can be performed simply scanning a disk / image, reading sector/cluster sized slices and checking
-block signature and then CRC to detect SeqBox blocks. Then blocks can be sorted by UID's and sequence numbers.
+Recovery can be performed simply scanning a disk / image, reading
+sector/cluster sized slices and checking block signature and then CRC to
+detect valid SeqBox blocks. Then blocks can be sorted by UID's and sequence
+numbers.
 
-Optionally blocks can be freely duplicated and/or stored in different media to enhance recoverability.
-Eventually even a simil-RAID system could be implemented between blocks.
+Optionally blocks can be freely duplicated and/or stored in different media
+to enhance recoverability.
 
 The UID can be anything, as long as is unique for the specific application. 
-It could be random generated, or a hash of the file content, or a simple sequence, etc.
+It could be random generated, or a hash of the file content, or a simple
+sequence, etc. For the tools is just a sequence of bytes.
 
 Overhead is minimal: from 16B/512B (+1 512B block) to 16B/32KB (+1 32KB block)
 
 Could become part of a File System.
 
-
 Command line tools to:
-- encode file to SBX
-- decode SBX to file
-- utility to get info and scramble / sort / test / damage SBX file
-- recovery/scan tool to find/list and recover series of RBX files from a file (image/device)
+- sbxenc: encode file to SBX
+- sbxdec: decode SBX to file (and also test or get info)
+- sbxscan: scan files to build an Sqlite db of blocks positions, num, uid
+           and a detailed log (in various formats, to enable other tools)
+- sbxrec: rebuild sbx files using previous scanned info
 
 
 Common blocks header:
 
   0-  2   3 Recoverable Block signature = 'SBx'
   3-  3   1 Version byte 
-  4-  5   2 Block CRC-16
+  4-  5   2 Block CRC-16 (Version is used as starting value)
   6- 11	  6 file UID (MD5 or other 32bit hash) 
  12- 15   4 Block sequence number
 
@@ -43,10 +48,22 @@ Block sequence = 0
  22- 29   8 file lenght
  30-284 255 file/dir name (UTF-8)
 285-316  32 file content crypto hash (optional)
-317-blksize padding
+317-nnn nnn encoded metadata
+nnn-blksize padding
 ------------------------
 
 Block sequence > 0:
 
  16-blocksize = data  
+------------------------
+
+metadata encoding:
+
+3 bytes str ID + 1 byte length + data
+
+ID:
 
+FNM filename (utf-8)
+SNM sbx filename (utf-8)
+FSZ filesize 8 bytes
+HSH SHA256 crypto hash

sbxenc.py

@@ -22,14 +22,7 @@
 
 import seqbox
 
-PROGRAM_VER = "0.03a"
-
-def errexit(errlev=1, mess=""):
-    """Display an error and exit."""
-    if mess != "":
-        print("%s: error: %s" % (os.path.split(sys.argv[0])[1], mess))
-    sys.exit(errlev)
-
+PROGRAM_VER = "0.04a"
 
 def get_cmdline():
     """Evaluate command line parameters, usage & help."""
@@ -58,12 +51,7 @@ def banner():
 def usage():
     print("""usage:
 
-seqbox e file.sbx file encode file in file.sbx
-seqbox d file.sbx file decode file from file.sbx
-seqbox i file.sbx show information on file.sbx
-seqbox t file.sbx test file.sbx for integrity
-seqbox r [-d path] filenames [filenames ...] recover sbx files from filenames
-         and store in path
+seqbox <file> [file.sbx] encode file in file.sbx
     """)
 
 def getcmdargs():
@@ -76,19 +64,23 @@ def getcmdargs():
         usage()
         errexit(0)
 
-    res["cmd"] = sys.argv[1].lower()
-
-    if res["cmd"] in ["e"]:
-        if len(sys.argv) == 4:
-            res["sbxfile"] = sys.argv[2]
-            res["file"] = sys.argv[3]
-        else:
-            usage()
-            errexit(1)
-    else:
-        errexit(1, "command %s not yet implemented." % res["cmd"])
-    
+    if len(sys.argv) > 1:
+        res["file"] = sys.argv[1]
+        res["sbxfile"] = res["file"] + ".sbx"
+    if len(sys.argv) > 2:
+        res["sbxfile"] = sys.argv[2]
+    if len(sys.argv) > 3:
+        usage()
+        errexit(1)
+
     return res
+
+
+def errexit(errlev=1, mess=""):
+    """Display an error and exit."""
+    if mess != "":
+        print("%s: error: %s" % (os.path.split(sys.argv[0])[1], mess))
+    sys.exit(errlev)
 
 
 def main():
@@ -99,18 +91,19 @@ def main():
 
     filename = cmdline["file"]
     sbxfilename = cmdline["sbxfile"]
-    
+
     print("reading %s..." % filename)
     filesize = os.path.getsize(filename)
     sha256 = getsha256(filename)
     fin = open(filename, "rb")
     fout = open(sbxfilename, "wb")
 
-    sbx = seqbox.sbxBlock()
+    sbx = seqbox.sbxBlock(uid=b'uiduid')
 
     #write block 0
     sbx.metadata = {"filesize":filesize,
                     "filename":filename,
+                    "sbxname":sbxfilename,
                     "hash":sha256}
     fout.write(sbx.encode())
 
@@ -131,43 +124,6 @@ def main():
 
     print("\nok!")
 
-    ######################
-
-    cmdline["sbxfile"] = r"c:\t\test.sbx"
-    cmdline["filename"] = r"c:\t\out.zip"
-
-    print("\nTesting...")
-
-    sbxfilename = cmdline["sbxfile"]
-    filename = cmdline["filename"]
-    fin = open(sbxfilename, "rb")
-    fout= open(filename, "wb")
-
-    sbx = seqbox.sbxBlock()
-    lastblocknum = 0
-    d = hashlib.sha256()
-    while True:
-        buffer = fin.read(sbx.blocksize)
-        if len(buffer) < sbx.blocksize:
-            break
-        if not sbx.decode(buffer):
-            errexit(errlev=1, mess="Invalid block.")
-        else:
-            print("Block #",sbx.blocknum)
-            if sbx.blocknum == 0:
-                #get metadata
-                pass
-            else:
-                fout.write(sbx.data)
-                d.update(sbx.data)
-    
-    fout.close()
-    fin.close()
-
-    print("File decoded.")
-    print("Hash:", d.hexdigest())
-    
-
 
 if __name__ == '__main__':
     main()

seqbox.py

@@ -35,9 +35,9 @@ def __init__(self, ver=0, uid="r"):
 
         if uid == "r":
             random.seed()
-            self.uid = random.getrandbits(48)
+            self.uid = random.getrandbits(6*8).to_bytes(6, byteorder='big')
         else:
-            self.uid = 0
+            self.uid = (b'\x00'*6 + uid)[-6:]
 
         self.parent_uid = 0
         self.metadata = {}
@@ -52,19 +52,22 @@ def encode(self):
             self.data = b""
             if "filename" in self.metadata:
                 bb = self.metadata["filename"].encode()
-                self.data += b"NM" + len(bb).to_bytes(1, byteorder='little') + bb
+                self.data += b"FNM" + len(bb).to_bytes(1, byteorder='big') + bb
+            if "sbxname" in self.metadata:
+                bb = self.metadata["sbxname"].encode()
+                self.data += b"SNM" + len(bb).to_bytes(1, byteorder='big') + bb
             if "filesize" in self.metadata:
-                bb = self.metadata["filesize"].to_bytes(8, byteorder='little', signed=True)
-                self.data += b"SZ" + len(bb).to_bytes(1, byteorder='little') + bb
+                bb = self.metadata["filesize"].to_bytes(8, byteorder='big')
+                self.data += b"FSZ" + len(bb).to_bytes(1, byteorder='big') + bb
             if "hash" in self.metadata:
                 bb = self.metadata["hash"]
-                self.data += b"HS" + len(bb).to_bytes(1, byteorder='little') + bb
+                self.data += b"HSH" + len(bb).to_bytes(1, byteorder='big') + bb
 
         data = self.data + b'\x1A' * (self.datasize - len(self.data))
-        buffer = (self.uid.to_bytes(6, byteorder='little') +
-                  self.blocknum.to_bytes(4, byteorder='little') +
+        buffer = (self.uid +
+                  self.blocknum.to_bytes(4, byteorder='big') +
                   data)
-        crc = binascii.crc_hqx(buffer,0).to_bytes(2,byteorder='little')
+        crc = binascii.crc_hqx(buffer, self.ver).to_bytes(2,byteorder='big')
         return (self.magic + crc + buffer)
 
     def decode(self, buffer):
@@ -79,13 +82,13 @@ def decode(self, buffer):
         print("Version:", buffer[3])
 
         #check CRC of rest of the block
-        crc = int.from_bytes(buffer[4:6], byteorder='little') 
-        if crc != binascii.crc_hqx(buffer[6:],0):
+        crc = int.from_bytes(buffer[4:6], byteorder='big') 
+        if crc != binascii.crc_hqx(buffer[6:], self.ver):
             return False
         print("CRC: OK!")
 
-        self.uid = int.from_bytes(buffer[6:12], byteorder='little') 
-        self.blocknum = int.from_bytes(buffer[12:16], byteorder='little') 
+        self.uid = buffer[6:12]
+        self.blocknum = int.from_bytes(buffer[12:16], byteorder='big') 
         self.data = buffer[16:]
 
         if self.blocknum == 0: