Implement HMAC nonce mode xchacha20-t3

Save/send ini at checkpoints only if cadence is lt 101 as this is useful only for stateful crypto counter

get_config: Resolve ts difference if only counters differ

send: benchmark fixes

Issues #158 #159

src/wyng

@@ -111,7 +111,8 @@ class ArchiveSet:
         # Format "magic" is "[WYNGvv]\n" where vv = format version: 9 bytes
         # Format mode is next as "ci = m0\n" where m = cipher mode: 8 bytes
         with open(self.confpath+ext, "rb") as f:
-            header0 = f.read(9)    ; header1 = f.read(8)    ; buf = f.read(self.max_conf_sz)
+            self.header = (header0 := f.read(9)) + (header1 := f.read(8))
+            buf = f.read(self.max_conf_sz)
         self.raw_hashval = hashlib.sha256(header0 + header1 + buf).hexdigest()
         if not (header0 == self.confprefix and header1.startswith(self.modeprefix)):
             if not header0.startswith(b"[var]\n"):
@@ -154,7 +155,7 @@ class ArchiveSet:
                                            arch_counter=self.mci_count,
                                            passphrase=passphrase_s1, agentkeys=agentkeys)
 
-        # initial decryption + auth
+        # initial decryption + auth of archive.ini (root body)
         try:
             if mcrypto:   buf = mcrypto.decrypt(buf)
         except ValueError as e:
@@ -185,8 +186,9 @@ class ArchiveSet:
             if not float(self.updated_at) < self.time_start:
                 raise ValueError("Current time is less than archive timestamp!")
 
+            cadence = 20 + (self.data_cipher.endswith(("-t1","-t2","-t3")) * 100)
             self.dataci_count = datacrypto.load(self.data_cipher, mcrypto.keyfile, slot=0,
-                                                cadence=20,  arch_counter=self.dataci_count,
+                                                cadence=cadence,  arch_counter=self.dataci_count,
                                                 passphrase=passphrase, agentkeys=agentkeys)
 
             # datacrypto may not have AE, so test data chunk against a volinfo hash
@@ -767,10 +769,10 @@ class DataCryptography:
     __slots__ = ("key","keyfile","ci_type","counter","ctstart","ctcadence","countsz","max_count",
                  "slot","slot_offset","key_sz","nonce_sz","tag_sz","randomsz","buf_start","mode",
                  "encrypt","decrypt","auth","AES_new","ChaCha20_new","ChaCha20_Poly1305_new",
-                 "time_start","monotonic_start","get_rnd","b2bhash")
+                 "time_start","monotonic_start","get_rnd","b2bhash","noncekey")
 
     def __init__(self):
-        self.key = self.keyfile = self.counter = self.ctstart = self.countsz \
+        self.key = self.noncekey = self.keyfile = self.counter = self.ctstart = self.countsz \
                  = self.slot_offset = None
 
     def common_xchacha20(self):
@@ -802,6 +804,7 @@ class DataCryptography:
             if not exists(keyfile) and init:   open(keyfile, "wb").close()
             keyfile =  open(keyfile, "r+b", buffering=0)
 
+        mknoncekey      = False
         self.keyfile    = keyfile          ; self.ci_type  = ci_type
         self.key        = None             ; kbits = self.crypto_key_bits
         self.slot       = slot             ; self.counter  = self.ctstart = None
@@ -833,6 +836,16 @@ class DataCryptography:
             self.common_xchacha20_poly1305()
             self.encrypt = self._enc_chacha20_poly1305_t2
 
+        elif ci_type == "xchacha20-t3":
+            self.common_xchacha20()
+            self.encrypt = self._enc_chacha20_t3
+            mknoncekey   = True
+
+        elif ci_type == "xchacha20-poly1305-t3":
+            self.common_xchacha20_poly1305()
+            self.encrypt = self._enc_chacha20_poly1305_t3
+            mknoncekey   = True
+
         elif ci_type == "aes-256-cbc":
             raise NotImplementedError()
 
@@ -858,43 +871,58 @@ class DataCryptography:
             self.key     = self.new_key(passphrase)
         else:
             keyfile.seek(self.slot_offset)    ; ctbytes = keyfile.read(self.countsz)
-            self.counter = self.ctstart = (cadence * 2) + int.from_bytes(ctbytes, "big")
+            # Advance counter with a safe margin 'cadence X2' if cadence is quick (<101)
+            self.counter = self.ctstart = (cadence * 2 * int(cadence < 101)) \
+                         + int.from_bytes(ctbytes, "big")
             salt         = keyfile.read(self.key_sz)   ; assert len(salt) == self.key_sz
 
             if agentkeys:
                 self.key = agentkeys[slot]
             else:
-                self.key = self.derive_key(salt, passphrase)
+                self.key = self.derive_key(salt, passphrase, self.key_sz)
             assert len(self.key) == self.key_sz and type(self.key) is bytearray
 
-        if arch_counter and arch_counter > self.counter:
-            self.counter = arch_counter    ; self.save_counter()
+        if mknoncekey:
+            self.noncekey = self.derive_noncekey(self.key, self.key_sz)
+            assert len(self.noncekey) == self.key_sz and type(self.noncekey) is bytearray
+
+        if arch_counter:   self.set_counter(arch_counter)
         return self.counter
 
 
     def __del__(self):
         if self.counter and self.counter > self.ctstart:   self.save_counter()
-        if self.key:   clear_array(self.key)
+        if self.key:        clear_array(self.key)
+        if self.noncekey:   clear_array(self.noncekey)
 
 
     # Key file binary format: counter=8B, key=key_sz
     def new_key(self, passphrase):
         salt = self.get_rnd(self.key_sz)      ; self.counter = self.ctstart = 0
         self.keyfile.seek(self.slot_offset)   ; self.keyfile.write(bytes(self.countsz) + salt)
-        return self.derive_key(salt, passphrase)
+        return self.derive_key(salt, passphrase, self.key_sz)
 
-    def derive_key(self, salt, passphrase):
+    def derive_key(self, salt, passphrase, size):
         key = bytearray(hashlib.scrypt(passphrase, salt=salt, n=2**19, r=8, p=1,
-                                        maxmem=640*1024*1024, dklen=self.key_sz))
+                                        maxmem=640*1024*1024, dklen=size))
         clear_array(passphrase)
         return key
 
+    def derive_noncekey(self, key, size):
+        noncekey = bytearray(hashlib.scrypt(key, salt=b'Wyng_Nonces', n=2**13, r=8, p=1,
+                                            maxmem=128*1024*1024, dklen=size))
+        return noncekey
+
     # Update key counter on disk; call directly at end of transaction if cadence > 1
     def save_counter(self):
         self.keyfile.seek(self.slot_offset)
         self.keyfile.write(self.counter.to_bytes(self.countsz, "big"))
         self.keyfile.flush()
 
+    def set_counter(self, ct):
+        if ct > self.counter:
+            self.counter = ct    ; self.save_counter()
+
     # Encrypt aes-256-siv:
     def _enc_aes_256_siv(self, buf):
         self.counter += 1
@@ -1010,6 +1038,35 @@ class DataCryptography:
         buf, ci_tag = cipher.encrypt_and_digest(buf)
         return  b''.join((nonce, ci_tag)), buf
 
+    # Encrypt [X]ChaCha20 (HMAC nonce)
+    def _enc_chacha20_t3(self, buf):
+        self.counter += 1
+        if self.counter % self.ctcadence == 0:   self.save_counter()
+        if self.counter > self.max_count:   raise ValueError("Key exhaustion.")
+
+        # Nonce from HMAC of rnd || buf
+        nonce_h = hmac.new(self.noncekey, msg=self.get_rnd(24), digestmod="sha256")
+        nonce_h.update(buf)
+        nonce   = nonce_h.digest()[:24]
+
+        cipher  = self.ChaCha20_new(key=self.key, nonce=nonce)
+        return  nonce, cipher.encrypt(buf)
+
+    # Encrypt [X]ChaCha20-Poly1305 (HMAC nonce)
+    def _enc_chacha20_poly1305_t3(self, buf):
+        self.counter += 1
+        if self.counter % self.ctcadence == 0:   self.save_counter()
+        if self.counter > self.max_count:   raise ValueError("Key exhaustion.")
+
+        # Nonce from HMAC of rnd || buf
+        nonce_h = hmac.new(self.noncekey, msg=self.get_rnd(24), digestmod="sha256")
+        nonce_h.update(buf)
+        nonce   = nonce_h.digest()[:24]
+
+        cipher = self.ChaCha20_Poly1305_new(key=self.key, nonce=nonce)
+        buf, ci_tag = cipher.encrypt_and_digest(buf)
+        return  b''.join((nonce, ci_tag)), buf
+
 
 # Define absolute paths of commands
 
@@ -1788,8 +1845,17 @@ def get_configs_remote(dest, base_dir):
             shutil.rmtree(arch_dir)
         else:
             if cache_aset.updated_at > aset.updated_at:
-                raise ValueError(f"Cached metadata is newer: "
-                                 f"{cache_aset.updated_at} vs. {aset.updated_at}")
+                # check if any non-crypto-counter vars differ
+                if cache_aset.header != aset.header \
+                or any((x != y for x, y in zip(dict(cache_aset.conf), dict(aset.conf))
+                                  if aset.mcrypto and x[0] not in ("mci_count","dataci_count"))):
+                    raise ValueError(f"Cached metadata is newer, from {cache_aset.path}\n"
+                                     f"{cache_aset.updated_at} vs. {aset.updated_at}\n")
+                elif aset.mcrypto:
+                    # difference was only in counters, so advance them and continue
+                    aset.mcrypto.set_counter(cache_aset.mci_count)
+                    aset.datacrypto.set_counter(cache_aset.dataci_count)
+
                 # Enh: Test-load the cache fully and/or fetch remote, but use largest counter#s
                 # Fix: Check for .tmp versions and use local copy if all other fields match;
                 # this allows recovery from interruption during send (archive.ini cadence).
@@ -2760,7 +2826,7 @@ def send_volume(storage, vol, curtime, ses_tags, send_all, benchmark=False):
     compress = compressors[aset.compression][2] ; compresslevel = int(aset.compr_level)
 
     if benchmark:   testtime = time.monotonic()
-    def tar_add_pass(**kwargs):   pass
+    def tar_add_pass(*args, **kwargs):   pass
 
     if len(vol.sessions):
         # Our chunks are usually smaller than LVM's, so generate a full manifest to detect
@@ -2792,11 +2858,11 @@ def send_volume(storage, vol, curtime, ses_tags, send_all, benchmark=False):
 
 
     if aset.datacrypto:
-        crypto  = True
+        crypto  = True    ; crypto_cadence = aset.datacrypto.ctcadence
         encrypt = aset.datacrypto.encrypt
     else:
         crypto  = False
-        etag    = b''
+        etag    = b''     ; crypto_cadence = 0
 
 
     # Use tar to stream files to destination
@@ -2858,7 +2924,9 @@ def send_volume(storage, vol, curtime, ses_tags, send_all, benchmark=False):
 
                 # Process checkpoint
                 if counter > checkpt:
-                    aset.save_conf()    ; tarf.add(aset.confname)
+                    # Keep updating aset counters if key has a low cadence
+                    if 0 < crypto_cadence < 101:
+                        aset.save_conf()    ; tarf_add(aset.confname)
                     # Show progress.
                     if verbose:
                         percent = int(addr/snap2size*1000)
@@ -2885,6 +2953,7 @@ def send_volume(storage, vol, curtime, ses_tags, send_all, benchmark=False):
                             stderr=SPr.DEVNULL)
                     tarf = tarfile.open(mode="w|", fileobj=untar.stdin)
                     tarf_addfile = tar_add_pass if benchmark else tarf.addfile
+                    tarf_add     = tar_add_pass if benchmark else tarf.add
                     TarInfo  = tarfile.TarInfo         ; LNKTYPE = tarfile.LNKTYPE
                     tar_info = TarInfo(sdir+"-tmp")    ; tar_info.type  = tarfile.DIRTYPE
                     tarf_addfile(tarinfo=tar_info)     ; stream_started = True
@@ -2940,7 +3009,7 @@ def send_volume(storage, vol, curtime, ses_tags, send_all, benchmark=False):
           ("\n  (reduced %0.1fM)" % (ddbytes/1000000)) if ddbytes and options.verbose else "",
           end="")
 
-    if benchmark:   print("\nTime:", time.monotonic() - timetest)
+    if benchmark:   print("\nTime:", time.monotonic() - testtime)
 
     # Send session info, end stream and cleanup
     if fullmanifest:   fullmanifest.close()
@@ -2949,8 +3018,8 @@ def send_volume(storage, vol, curtime, ses_tags, send_all, benchmark=False):
         if crypto:   aset.datacrypto.save_counter()
         ses.save_info(ext=".tmp")
         for f in ("manifest.z","info"):
-            fpath = sdir+"-tmp/"+f         ; tarf.add(fpath+".tmp", arcname=fpath)
-        tarf.add(vol.vid+"/volinfo.tmp")   ; tarf.add(aset.confname+".tmp")
+            fpath = sdir+"-tmp/"+f         ; tarf_add(fpath+".tmp", arcname=fpath)
+        tarf_add(vol.vid+"/volinfo.tmp")   ; tarf_add(aset.confname+".tmp")
 
         tarf.close()    ; untar.stdin.close()
         try:
@@ -4126,7 +4195,7 @@ def cleanup():
 
 # Constants / Globals
 prog_name             = "wyng"
-prog_version          = "0.4alpha3"     ; prog_date = "20230513"
+prog_version          = "0.4alpha3"     ; prog_date = "20230514"
 format_version        = 3               ; debug     = False       ; tmpdir = None
 admin_permission      = os.getuid() == 0
 time_start            = time.time()