From 6fbbcc401d7c7d76373f18ed6572274e8e883c70 Mon Sep 17 00:00:00 2001 From: fabiodl Date: Wed, 31 Jan 2024 10:59:39 +0900 Subject: [PATCH] add workflow --- .github/workflows/action.yml | 28 +++ SC3000/script/basicToWav.py | 33 ++++ SC3000/script/basicparse.py | 61 +++++++ SC3000/script/basparse.py | 115 +++++++++++++ SC3000/script/bitparse.py | 114 +++++++++++++ SC3000/script/command_table.py | 157 +++++++++++++++++ SC3000/script/requirements.txt | 1 + SC3000/script/sc3000decoder.py | 147 ++++++++++++++++ SC3000/script/sc3000encoder.py | 112 ++++++++++++ SC3000/script/section.py | 299 +++++++++++++++++++++++++++++++++ SC3000/script/util.py | 90 ++++++++++ SC3000/script/wavparse.py | 34 ++++ 12 files changed, 1191 insertions(+) create mode 100644 .github/workflows/action.yml create mode 100644 SC3000/script/basicToWav.py create mode 100644 SC3000/script/basicparse.py create mode 100644 SC3000/script/basparse.py create mode 100644 SC3000/script/bitparse.py create mode 100644 SC3000/script/command_table.py create mode 100644 SC3000/script/requirements.txt create mode 100644 SC3000/script/sc3000decoder.py create mode 100644 SC3000/script/sc3000encoder.py create mode 100644 SC3000/script/section.py create mode 100644 SC3000/script/util.py create mode 100644 SC3000/script/wavparse.py diff --git a/.github/workflows/action.yml b/.github/workflows/action.yml new file mode 100644 index 0000000..5cc93ad --- /dev/null +++ b/.github/workflows/action.yml @@ -0,0 +1,28 @@ +name: Convert Basic +on: + push: + branches: + - master +jobs: + convert: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: 'pip' + - run: pip install -r SC3000/script/requirements.txt + - run: | + CHANGED_FILES=$(git diff --name-only ${{ github.event.before }} ${{ github.event.after }} | grep '^SC3000/.*\.\(bas\|basic\)$') + for FILE in $CHANGED_FILES; do + python SC3000/script/basicToWav.py "$FILE" + done + CHANGED_FILES=`git diff --name-only ${{ github.event.before }} ${{ github.event.after }}` python script/convert.py + git config user.name "bot" + git config user.email "github-actions@users.noreply.github.com" + git add . + git commit -m "Add converted file" + git push origin master diff --git a/SC3000/script/basicToWav.py b/SC3000/script/basicToWav.py new file mode 100644 index 0000000..958a22c --- /dev/null +++ b/SC3000/script/basicToWav.py @@ -0,0 +1,33 @@ +import basicparse +import wavparse +import argparse +import pathlib +import section +import bitparse + + +def canonicalName(fname): + stem = pathlib.Path(fname).stem.upper() + return ''.join(char for char in stem if char.isalnum()).ljust(16)[:16] + + +if __name__ == "__main__": + remrate = 44100 + parser = argparse.ArgumentParser() + parser.add_argument("infile") + parser.add_argument("outfile", nargs="?", default=None) + parser.add_argument("--program_name") + args = parser.parse_args() + + opt = {k: v for k, v in vars(args).items() if v is not None} + + if "program_name" not in opt: + opt["program_name"] = canonicalName(args.infile) + + if args.outfile is None: + args.outfile = pathlib.Path(args.infile).with_suffix(".wav") + + d = basicparse.readBasic(args.infile, opt) + section.parseBytesSections(d["sections"], True, False) + d["signal"] = bitparse.genSignal(d, remrate, True) + wavparse.writeWav(str(args.outfile), d, opt) diff --git a/SC3000/script/basicparse.py b/SC3000/script/basicparse.py new file mode 100644 index 0000000..4f15b61 --- /dev/null +++ b/SC3000/script/basicparse.py @@ -0,0 +1,61 @@ +from sc3000decoder import read_bas_as_hex_string, decode_hex_string, print_decoded, save_decoded_to, escape_char +from sc3000encoder import encode_script_string +import binascii +from basparse import getBasicSections + +from section import KeyCode +from util import removeExtension +from pathlib import Path + + +def writeBasic(filename, d, opt): # already parsed + fname = removeExtension(filename) + codeChunks = [] + for s in d["sections"]: + if s["type"] == "bytes" and KeyCode.code[s["keycode"]] == KeyCode.BasicData: + codeChunks.append(s["Program"]) + + ext = Path(filename).suffix + if len(codeChunks) == 1: + decode(fname + ext, codeChunks[0]) + else: + for idx, c in enumerate(codeChunks): + decode(f"{fname}{idx}{ext}", c) + + +def writeFilename(filename, d, opt): + fname = removeExtension(filename) + names = [] + for s in d["sections"]: + if s["type"] == "bytes" and KeyCode.code[s["keycode"]] == KeyCode.BasicHeader: + names.append(s["Filename"]) + if len(names) == 1: + open(fname + ".filename", "w").write(names[0]) + else: + for idx, na in enumerate(names): + open(f"{fname}{idx}.filename", "w").write(na) + + +def readBasic(filename, opts): + if "basic_raw_encoding" in opts: + raw = open(filename, "rb").read() + # print("==RAW==", raw) + script_string = "".join([escape_char(ch, toPass=[0x0A]) + for ch in raw if ch not in [0x01, 0x0D]]) + else: + script_string = open(filename).read() + suppress_error = False + encoded = encode_script_string(script_string, suppress_error) + result = "" + for line in encoded: + result += line["encoded"] + resultb = list(binascii.a2b_hex(result)) + if "\\bin" not in encoded[-1]["raw"]: + resultb += [0x00, 0x00] + return getBasicSections(resultb, opts) + + +def decode(fname, chunk): + hex_string = bytes(chunk).hex().upper() + decoded = decode_hex_string(hex_string, suppress_error=True) + save_decoded_to(fname, decoded) diff --git a/SC3000/script/basparse.py b/SC3000/script/basparse.py new file mode 100644 index 0000000..cc771be --- /dev/null +++ b/SC3000/script/basparse.py @@ -0,0 +1,115 @@ +from section import KeyCode +from util import removeExtension, beint +import numpy as np + + +def writeFile(filename, l): + with open(filename, "wb") as f: + f.write(bytes(l)) + + +def writeBas(filename, d, opts): # already parsed + codeChunks = [] + for s in d["sections"]: + if s["type"] == "bytes" and KeyCode.code[s["keycode"]] == KeyCode.BasicData: + codeChunks.append(s["Program"]) + + if len(codeChunks)==1: + writeFile(filename,codeChunks[0]) + else: + fname=removeExtension(filename) + for idx,c in enumerate(codeChunks): + writeFile(f"{fname}{idx}.bas",c) + + + +def writeBin(filename,d,opt): #already parsed + fname=removeExtension(filename) + codeChunks=[] + idx=0 + for s in d["sections"]: + if s["type"]=="bytes": + writeFile(f"{fname}{idx}.bin",s["bytes"]) + idx+=1 + + +def parity(x): + return 0x100-(0xFF&int(np.sum(x))) + +def getBasicSections(program,opts): + d={ + "sections":[] + } + sections=d["sections"] + + if "program_type" in opts: + ptype=opts["program_type"] + if ptype=="machine": + isMachine=True + elif ptype=="basic": + isMachine=False + else: + raise Exception("Unknown code type "+ctype+" options are either 'basic' or 'machine'") + else: + isMachine=False + + + if isMachine: + if "program_start_addr" in opts: + startAddr=int(opts["program_start_addr"],16) + else: + print("\nWarning: start address not specified\n") + startAddr=0xC000 + + + if "program_name" not in opts: + print("\nWarning: program name not specified\n") + else: + if isMachine: + header=[KeyCode.MachineHeader] + else: + header=[KeyCode.BasicHeader] + filename=opts["program_name"].ljust(16)[:16] + filename=[ord(c) for c in filename] + programLength=beint(len(program),2) + + headerPayload=filename+programLength + if isMachine: + headerPayload+=beint(startAddr,2) + p=parity(headerPayload) + sections.append({ + "t":-1, + "type":"bytes", + "bytes": header+headerPayload+[p,0x00,0x00]}) + + if isMachine: + header=[KeyCode.MachineData] + else: + header=[KeyCode.BasicData] + p=parity(program) + + sections.append({ + "t":-1, + "type":"bytes", + "bytes":header+program+[p,0x00,0x00]}) + return d + + + +def readBas(filename,opts): + start,end=0,None + d=open(filename,"rb").read() + if "program_from" in opts: + start=int(opts["program_from"],16) + if "program_to" in opts: + end=int(opts["program_to"],16) + if "program_size" in opts: + end=start+int(opts["program_size"],16) + if "program_rstrip" in opts: + endchar=int(opts["program_rstrip"],16) + d=d.rstrip(bytes([endchar])) + if end is None: + end=len(d) + print(f"Reading {filename} range {start:04x} : {end:04x}") + program=list(d[start:end]) + return getBasicSections(program,opts) diff --git a/SC3000/script/bitparse.py b/SC3000/script/bitparse.py new file mode 100644 index 0000000..0574014 --- /dev/null +++ b/SC3000/script/bitparse.py @@ -0,0 +1,114 @@ +import numpy as np +from section import SectionList, KeyCode + + +def maybeByte(bs): + if len(bs) < 11: + return None + for b in bs[:11]: + if b not in ["0", "1"]: + return None + if bs[0] != "0" or bs[9] != "1" or bs[10] != "1": + return None + n = 0 + for i in range(8): + if bs[1+i] == "1": + n += (1 << i) + return n + + +def readBit(filename, opts): + ignore_section_errors = "ignore_section_errors" in opts + return getSections(open(filename).read(), ignore_section_errors) + + +def getSections(data, ignore_section_errors=False): + sl = SectionList() + offset = 0 + while offset < len(data): + n = maybeByte(data[offset:offset+11]) + if n is not None: + sl.pushByte(offset, n) + offset += 11 + elif data[offset] == "1": + sl.pushHeader(offset) + offset += 1 + elif data[offset] == " ": + sl.pushLevel(offset, 0, 1) + offset += 1 + else: + msg = f"Invalid char in bit file {data[offset]} at position {offset} out of {len(data)}, {data[offset:offset+11]}" + if ignore_section_errors: + print(msg) + offset += 1 + else: + raise Exception(msg) + sl.finalize() + d = {"bitrate": 1200, "sections": sl.sections} + return d + + +def encodeByte(b): + return "0" + "".join(["1" if ((b >> i) & 0x01) == 1 else "0" for i in range(8)])+"11" + + +def encodeBytes(x): + return "".join([encodeByte(b) for b in x]) + + +def toBitRaw(d): + data = "" + for s in d["sections"]: + stype = s["type"] + if stype == "level": + data += " "*int(np.round(s["length"]/d["bitrate"]*1200)) + elif stype == "header": + data += "1"*s["count"] + elif stype == "bytes": + data += encodeBytes(s["bytes"]) + + return data + + +def toBitRemaster(d, fastStart=True): + data = "" + for s in d["sections"]: + stype = s["type"] + # print("section", s) + if stype == "bytes": + code = KeyCode.code[s["keycode"]] + if fastStart: + n = 0 + elif code == KeyCode.BasicHeader or code == KeyCode.MachineHeader: + n = 10*1200 + elif code == KeyCode.BasicData or code == KeyCode.MachineData: + n = 1*1200 + data += " "*n+"1"*3600+encodeBytes(s["bytes"]) + fastStart = False + return data + + +def genSignal(d, sampleRate, sectionRemaster): + val = 1 + Space = np.zeros(int(sampleRate/1200)) + Zero = np.hstack([v*np.ones(int(sampleRate/1200/2)) for v in [val, -val]]) + One = np.hstack([v*np.ones(int(sampleRate/1200/4)) + for v in [val, -val, val, -val]]) + conv = {' ': Space, '0': Zero, '1': One} + + if sectionRemaster: + bits = toBitRemaster(d, True) + else: + bits = toBitRaw(d) + sig = np.hstack([conv[b] for b in bits]) + d["bitrate"] = sampleRate + return sig + + +def writeBit(filename, d, opt): + remaster = opt["remaster"] == "section" + with open(filename, "w") as f: + if remaster: + f.write(toBitRemaster(d)) + else: + f.write(toBitRaw(d)) diff --git a/SC3000/script/command_table.py b/SC3000/script/command_table.py new file mode 100644 index 0000000..4c71262 --- /dev/null +++ b/SC3000/script/command_table.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +"""SC3000 Basic Command Table +Reference: +http://www43.tok2.com/home/cmpslv/Sc3000/EnrSCbas.htm +""" + +COMMAND = { + "81": "INPUT$", + "82": "LIST", + "83": "LLIST", + "84": "AUTO", + "85": "DELETE", + "86": "RUN", + "87": "CONT", + "88": "LOAD", # CLOAD + "89": "SAVE", # CSAVE + "8A": "VERIFY", + "8B": "NEW", + "8C": "RENUM", + "8D": "FILES", + "8E": "LFILES", + "8F": "BOOT", + "90": "REM", + "91": "PRINT", # ? + "92": "LPRINT", # L? + "93": "DATA", + "94": "DEF", + "95": "INPUT", + "96": "READ", + "97": "STOP", + "98": "END", + "99": "LET", + "9A": "DIM", + "9B": "FOR", + "9C": "NEXT", + "9D": "GOTO", + "9E": "GOSUB", + "9F": "GO", + "A0": "ON", + "A1": "RETURN", + "A2": "ERASE", + "A3": "CURSOR", + "A4": "IF", + "A5": "RESTORE", + "A6": "SCREEN", + "A7": "COLOR", + "A8": "LINE", + "A9": "SOUND", + "AA": "BEEP", + "AB": "CONSOLE", + "AC": "CLS", + "AD": "OUT", + "AE": "CALL", + "AF": "POKE", + "B0": "PSET", + "B1": "PRESET", + "B2": "PAINT", + "B3": "BLINE", + "B4": "POSITION", + "B5": "HCOPY", + "B6": "SPRITE", + "B7": "PATTERN", + "B8": "CIRCLE", + "B9": "BCIRCLE", + "BA": "MAG", + "BB": "VPOKE", + "BC": "MOTOR", + "BD": "OPEN", + "BE": "CLOSE", + "BF": "COMSET", + "C0": "^", + "C1": "*", + "C2": "/", + "C3": "MOD", + "C4": "+", + "C5": "-", + "C6": "<>", # >< + "C7": ">=", # => + "C8": "<=", # =< + "C9": ">", + "CA": "<", + "CB": "=", + "CC": "NOT", + "CD": "AND", + "CE": "OR", + "CF": "XOR", + "D0": "CLOADM", + "D1": "CSAVEM", + "D2": "VERIFYM", + "D3": "SAVEM", + "D4": "LOADM", + "D5": "LIMIT", + "D6": "GET", + "D7": "PUT", + "D8": "DSKI$", + "D9": "DSKO$", + "DA": "KILL", + "DB": "SET", + "DC": "NAME", + "E0": "FN", + "E1": "TO", + "E2": "STEP", + "E3": "THEN", + "E4": "TAB", + "E5": "SPC", + # "F0": "SAVE", + "F5": "MERGE", + "F6": "COMSAVE", + "F7": "COMLOAD", + "F8": "UTILITY", + "F9": "MAXFILE", +} + +# When the byte is 80, the next byte will be of FUNC +FUNCTION = { + "80": "ABS", + "81": "RND", + "82": "SIN", + "83": "COS", + "84": "TAN", + "85": "ASN", + "86": "ACS", + "87": "ATN", + "88": "LOG", + "89": "LGT", + "8A": "LTW", + "8B": "EXP", + "8C": "RAD", + "8D": "DEG", + "8E": "PI", + "8F": "SQR", + "90": "INT", + "91": "SGN", + "92": "ASC", + "93": "LEN", + "94": "VAL", + "95": "PEEK", + "96": "INP", + "97": "FRE", + "98": "VPEEK", + "99": "STICK", + "9A": "STRIG", + "9B": "EOF", + "9C": "LOC", + "9D": "LOF", + "9E": "DSKF", + "A0": "CHR$", + "A1": "HEX$", + "A2": "INKEY$", + "A3": "LEFT$", + "A4": "RIGHT$", + "A5": "MID$", + "A6": "STR$", + "A7": "TIME$" +} diff --git a/SC3000/script/requirements.txt b/SC3000/script/requirements.txt new file mode 100644 index 0000000..af47163 --- /dev/null +++ b/SC3000/script/requirements.txt @@ -0,0 +1 @@ +numpy==1.26.3 diff --git a/SC3000/script/sc3000decoder.py b/SC3000/script/sc3000decoder.py new file mode 100644 index 0000000..3c8aac6 --- /dev/null +++ b/SC3000/script/sc3000decoder.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +"""SC3000 Basic Decoder +Reference: +http://www43.tok2.com/home/cmpslv/Sc3000/EnrSCbas.htm +""" + +from command_table import COMMAND, FUNCTION + + +class UnknownFunctionException(Exception): + pass + + +class UnknownCommandException(Exception): + pass + + +def read_bas_as_hex_string(filepath): + with open(filepath, "rb") as f: + content = f.read() + return content.hex().upper() + + +def print_decoded(decoded, pretty_format=True): + if pretty_format: + print(decoded["raw"]) + print("Script length: {}".format(len(decoded["raw"]))) + print_format = "({:04d}) {: >5} {}" + else: + print_format = "{1} {2}" + for line in decoded["result"]: + print(print_format.format(line["byte"], line["line"], line["cmd"])) + + +def decode_hex_string(hex_string, suppress_error=True): + result = {"raw": hex_string, "result": []} + i = 0 + while i < len(hex_string): + result_i = {"byte": i, "line": "", "cmd": "", "raw": ""} + if hex_string[i:i+4] == "0000": + bind = hex_string[i+4:] + if len(bind) > 0: + result_i["cmd"] = f"\\bin " + hex_string[i:] + di = len(hex_string) - i + else: + return result + elif len(hex_string) >= i+14: + try: + di, result_i["line"], result_i["cmd"] = decode_one_line( + hex_string[i:], suppress_error) + result_i["raw"] = " ".join( + [hex_string[j:j+2] for j in range(i, i+di, 2)]) + except UnknownFunctionException as e: + print("Error: {}".format(e)) + break + except UnknownCommandException as e: + print("Error: {}".format(e)) + break + else: + return result + + i += di + result["result"].append(result_i) + return result + + +# LEN LINE_L LINE_H 00 00 .... 0D +def decode_one_line(line, suppress_error=True): + result = "" + command_length = int(line[0:2], 16) + di = 10+command_length*2+2 + line_number = int(line[4:6]+line[2:4], 16) + blank = line[6:10] + result = decode_command(line[10:di-2], line_number, suppress_error) + return di, line_number, result + + +def decode_command(command, line_number, suppress_error=True): + result = "" + zipper = zip(command[0::2], command[1::2]) + inside_data = False + inside_rem = False + inside_quote = False + + for l, (i, j) in enumerate(zipper): + current_result = "" + if i+j < "80" or inside_data or inside_rem or inside_quote: + current_result = decode_ascii(i+j) + elif i+j == "80": + i, j = next(zipper) + try: + current_result = FUNCTION[i+j] + except KeyError: + msg = "Unknown Function {}{} on line {}".format( + i, j, line_number) + if suppress_error: + current_result = "\\f{}{}".format(i, j) + print("Warning: " + msg) + else: + raise UnknownFunctionException(msg) + else: + try: + current_result = COMMAND[i+j] + except KeyError: + msg = "Unknown Command {}{} on line {}".format( + i, j, line_number) + if suppress_error: + current_result = "\\c{}{}".format(i, j) + print("Warning: " + msg) + else: + raise UnknownCommandException(msg) + + # Characters between Double quote, or after DATA or REM, should be treated as ascii + if current_result == "DATA": + inside_data = True + elif current_result == "REM": + inside_rem = True + # Ignore the rest of the line + # https://www.c64-wiki.com/wiki/REM + elif current_result == '"': + inside_quote = not inside_quote + elif current_result == ":": + inside_data = False + result += current_result + + return result + + +def escape_char(ch, toPass=[]): + if ch in toPass or 0x20 <= ch <= 0x7E and ch != "\\": + return chr(ch) + else: + return f"\\x{ch:02X}" + + +def decode_ascii(byte): + ch = int(byte, 16) + return escape_char(ch) + + +def save_decoded_to(filepath, decoded): + with open(filepath, "w") as f: + for line in decoded["result"]: + f.write("{1} {2}\n".format( + line["byte"], line["line"], line["cmd"])) diff --git a/SC3000/script/sc3000encoder.py b/SC3000/script/sc3000encoder.py new file mode 100644 index 0000000..35884b5 --- /dev/null +++ b/SC3000/script/sc3000encoder.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import binascii + +from command_table import COMMAND, FUNCTION + +COMMAND_BY_WORD = {v: k for k, v in COMMAND.items()} +FUNCTION_BY_WORD = {v: k for k, v in FUNCTION.items()} + + +def encode_script_string(script_string, suppress_error=False): + result = [] + for line in script_string.split("\n"): + if not line: + continue + result.append({"raw": line, "encoded": encode_one_line(line)}) + return result + + +def encode_one_line(line): + try: + line_number, command = line.split(" ", 1) + encoded_line_number = encode_line_number(line_number) + encoded_command = encode_command(command) + encoded_command_length = encode_command_length(encoded_command) + encoded = (encoded_command_length + encoded_line_number + + "0000" + encoded_command + "0D").upper() + return encoded + + except Exception as e: + tok = line.split() + if len(tok) == 2: + line_number, command = tok + print("LINE NUMBER IS", line_number) + if line_number == "\\bin": + return command + + print(f"unable to convert LINE '{line}'") + raise e + + +def encode_line_number(line_number): + hex_number = f"{int(line_number):04x}" + return hex_number[2:4]+hex_number[0:2] + + +def encode_command(command): + result = "" + ascii_mode = False + in_comment = False + in_quotes = False + c = 0 + while c < len(command): + parsed = "" + if not ascii_mode: + result_c, parsed, dc = match_one_keyword(command[c:]) + if ascii_mode or not parsed: + result_c, parsed, dc = encode_one_ascii(command[c:]) + if parsed == '"': + in_quotes = not in_quotes + elif parsed in ['REM', 'DATA']: + in_comment = True + ascii_mode = in_comment or in_quotes + c += dc + result += result_c + return result + + +def match_one_keyword(command): + matching_commands = list( + filter(lambda cmd: command.startswith(cmd), COMMAND_BY_WORD.keys())) + matching_functions = list( + filter(lambda cmd: command.startswith(cmd), FUNCTION_BY_WORD.keys())) + if matching_commands: + result = max(matching_commands, key=len) + return COMMAND_BY_WORD[result], result, len(result) + elif matching_functions: + result = max(matching_functions, key=len) + return "80"+FUNCTION_BY_WORD[result], result, len(result) + else: + return "", "", 0 + + +def encode_one_ascii(command): + if command[0] == '\\' and command[1] == 'x': + return command[2]+command[3], command[0:4], 4 + else: + return f"{ord(command[0]):02x}", command[0], 1 + + +def encode_command_length(encoded_command): + return f"{int(len(encoded_command)/2):02x}" + + +def print_encoded(encoded, pretty_format=False): + result = "" + for line in encoded: + if pretty_format: + result += "{}\n {}\n".format(line["raw"], line["encoded"]) + else: + result += line["encoded"] + print(result) + + +def save_encoded_to(filepath, encoded): + with open(filepath, "wb") as f: + result = "" + for line in encoded: + result += line["encoded"] + resultb = binascii.a2b_hex(result) + f.write(resultb) diff --git a/SC3000/script/section.py b/SC3000/script/section.py new file mode 100644 index 0000000..a8b845b --- /dev/null +++ b/SC3000/script/section.py @@ -0,0 +1,299 @@ +from util import bigEndian, printable, lre, hexString +import numpy as np + + +class KeyCode: + BasicHeader, MachineHeader = 0x16, 0x26 + BasicData, MachineData = 0x17, 0x27 + MusicHeader, MusicData = 0x57, 0x58 + name = { + BasicHeader: "Basic header", + BasicData: "Basic data", + MachineHeader: "ML header", + MachineData: "ML data", + MusicHeader: "Music header", + MusicData: "Music data" + } + code = {v: k for k, v in name.items()} + + +class SectionList: + def __init__(self): + self.sections = [] + self.currSection = None + + def pushByte(self, t, val): + if self.currSection is None or self.currSection["type"] != "bytes": + self.finalize() + self.currSection = {"t": t, "type": "bytes", "bytes": []} + self.currSection["bytes"].append(val) + + def pushHeader(self, t): + if self.currSection is None or self.currSection["type"] != "header": + self.finalize() + self.currSection = {"t": t, "type": "header", "count": 0} + self.currSection["count"] += 1 + + def pushLevel(self, t, val, count): + if self.currSection is None or self.currSection["type"] != "level" or self.currSection["value"] != val: + self.finalize() + self.currSection = {"t": t, "type": "level", + "value": val, "length": 0} + self.currSection["length"] += count + + def finalize(self): + if self.currSection != None: + self.sections.append(self.currSection) + + +def splitChunks(data, chunkLens): + idx = 0 + chunks = [] + for l in chunkLens: + chunks.append(data[idx:idx+l]) + idx += l + return chunks + + +def parseBytes(si, so): + d = si["bytes"] + secType = d[0] + if secType in KeyCode.name: + d = d[1:] + if secType == KeyCode.BasicHeader: + cl = [16, 2, 1, 2] + if (len(d) < np.sum(cl)): + so["fail.short"] = np.sum(cl) + return False + filename, programLength, parity, dummyData = splitChunks(d, cl) + checkSum = np.sum(filename+programLength+parity) & 0xFF + if checkSum != 0: + so["fail.checksum"] = checkSum + print( + f"*checksum fail {0xFF&(0x100-np.sum(filename+programLength)):02x} vs {parity[0]:02x}") + return False + else: + print( + f"header checksum ok, prog len {bigEndian(programLength)}") + so["keycode"] = KeyCode.name[secType] + so["Filename"] = "".join([chr(c) for c in filename]) + so["ProgramLength"] = bigEndian(programLength) + so["Parity"] = parity + so["Dummy"] = dummyData + if secType == KeyCode.MachineHeader: + cl = [16, 2, 2, 1, 2] + if (len(d) < np.sum(cl)): + so["fail.short"] = np.sum(cl) + return False + filename, programLength, startAddr, parity, dummyData = splitChunks( + d, cl) + checkSum = np.sum(filename+programLength+startAddr+parity) & 0xFF + if checkSum != 0: + print("*checksum fail") + so["fail.checksum"] = checkSum + return False + so["keycode"] = KeyCode.name[secType] + so["Filename"] = "".join([chr(c) for c in filename]) + so["ProgramLength"] = bigEndian(programLength) + so["StartAddr"] = f"{bigEndian(startAddr):04x}" + so["Parity"] = parity + so["Dummy"] = dummyData + elif secType == KeyCode.BasicData or secType == KeyCode.MachineData: + program, parity, dummyData = d[:-3], d[-3:-2], d[-2:] + if not program or not parity: + so["fail.notenoughData"] = d + return False + checkSum = np.sum(program+parity) & 0xFF + if checkSum != 0: + print("*checksum fail") + so["fail.checksum"] = checkSum + so["fail.length"] = len(program) + return False + else: + print(KeyCode.name[secType], "checksum ok") + so["keycode"] = KeyCode.name[secType] + so["Program"] = program + so["Parity"] = parity + so["Dummy"] = dummyData + so["length"] = len(program) + else: + so["fail.keycode"] = hex(secType) + print("Unknown Keycode", secType) + return False + return True + + +def printSection(s): + print("{", end="") + for n, v in s.items(): + if n in ["fail.checksum", "checksum"]: + v = f"{v:02X}" + if n in ["bytes", "Program", "Dummy", "Parity"]: + v = hexString(v) + print(f"{n}: {v}", end=" ") + print(end="}") + + +def parseBytesSections(sl, exceptOnError, ignoreFFsections): + error = False + for s in sl: + if s["type"] == "bytes": + if ignoreFFsections and s["bytes"][0] == 0xFF: + s["type"] = "ignored_section" + print("ignoring section of length", len(s["bytes"])) + else: + if not parseBytes(s, s): + error = True + + if error: + printSection(s) + print("Section errors, decoded", len(sl), "sections") + if len(sl) < 10: + for sec in sl: + print(" Section at", sec["t"], sec["type"]) + if exceptOnError: + raise Exception("Error in parsing Section") + + +def printSummary(d, withSilence=True): + for s in d["sections"]: + if s["type"] == "header": + c = s["count"] + print(f"Header count={c}") + elif s["type"] == "level": + t = s["length"]/d["bitrate"] + if t > 1.0/1200 and withSilence: + print(f"Silence t={t:0.1f}s") + elif "keycode" in s: + print(s["keycode"], end=" ") + c = KeyCode.code[s["keycode"]] + if c == KeyCode.BasicHeader or c == KeyCode.MachineHeader: + fname = printable(s["Filename"]) + l = s["ProgramLength"] + print(f'filename ="{fname}" length={l}') + else: + l = s["length"] + print(f"length={l}") + + +def listContent(d): + filenames = [] + for s in d["sections"]: + if s["type"] == "bytes": + c = KeyCode.code[s["keycode"]] + if c in [KeyCode.BasicHeader, KeyCode.MachineHeader]: + filenames.append(s["Filename"]) + return filenames + + +def getStarts(pairs): + starts = [] + tl = 0 + for (v, l) in pairs: + starts.append(tl) + tl += l + return starts + + +def checkLengths(l, minv, maxv, ignoreBegin, ignoreEnd): + if l[0] < minv: + return False + if l[0] > maxv and not ignoreBegin: + return False + for d in l[1:-1]: + if d < minv or d > maxv: + return False + if l[-1] < minv: + return False + if l[-1] > maxv and not ignoreEnd: + return False + return True + + +def isZero(lop, lperiod, ignoreBegin, ignoreEnd): + if len(lop) < 2: + return False + l = [lop[i][1] for i in range(2)] + ok = checkLengths(l, 3/8*lperiod, 3/4*lperiod, ignoreBegin, ignoreEnd) + return ok and lop[0][0]*lop[1][0] == -1 + + +def isOne(lop, lperiod, ignoreBegin, ignoreEnd): + if len(lop) < 4: + return False + l = [lop[i][1] for i in range(4)] + ok = checkLengths(l, 1/8*lperiod, 3/8*lperiod, ignoreBegin, ignoreEnd) + for i in range(3): + if lop[i][0]*lop[i+1][0] != -1: + ok = False + return ok + + +def maybeByte(pairs, period, firstByte): + n = 0 + offset = 0 + if isZero(pairs[offset:offset+2], period, not firstByte, False): + offset += 2 + else: + return None + for i in range(8): + if isZero(pairs[offset:offset+2], period, False, False): + offset += 2 + elif isOne(pairs[offset:offset+4], period, False, False): + offset += 4 + n += (1 << i) + else: + return None + if isOne(pairs[offset:offset+4], period, False, False) and isOne(pairs[offset+4:offset+8], period, False, True): + offset += 8 + else: + return None + return offset, n + + +def getSections(d, pitch, removeSpikes=True): + # print("levels",levell,levelh,"period",period) + if "signal" not in d: + print("No signal analysis") + return + pairs = list(lre(d["signal"])) + period = d["bitrate"]*pitch/1200 + + starts = getStarts(pairs) + if removeSpikes: + # print("before removal",len(pairs)) + idx = [i for i, (v, l) in enumerate(pairs) if l > period/8] + pairs = [pairs[i] for i in idx] + starts = [starts[i] for i in idx] + # print("after removal",len(pairs)) + + offset = 0 + + t = 0 + sl = SectionList() + + def pushLongSpace(ps): + for p in ps: + if p[1] > period: + sl.pushLevel(t, p[0], p[1]) + + while offset < len(pairs): + t = starts[offset] + bi = maybeByte(pairs[offset:offset+4*11], period, offset == 0) + + if bi is not None: + off, val = bi + sl.pushByte(t, val) + pushLongSpace(pairs[offset:offset+4*11]) + offset += off + elif isOne(pairs[offset:offset+4], period, True, False): + pushLongSpace(pairs[offset:offset+4]) + sl.pushHeader(t) + offset += 4 + else: + sl.pushLevel(t, pairs[offset][0], pairs[offset][1]) + offset += 1 + + sl.finalize() + d["sections"] = sl.sections + return d diff --git a/SC3000/script/util.py b/SC3000/script/util.py new file mode 100644 index 0000000..79be4cf --- /dev/null +++ b/SC3000/script/util.py @@ -0,0 +1,90 @@ +import sys +import numpy as np +import itertools + + +def bigEndian(x): + return np.sum([v*(256**i) for i, v in enumerate(x[::-1])]) + + +def le(d): + return sum([c << (8*i) for i, c in enumerate(d)]) + + +def hexString(x): + return " ".join([f"{v:02X}" for v in x]) + + +def leint(x, n): + l = [] + for i in range(n): + l.append(x & 0xFF) + x = x >> 8 + return l + + +def beint(x, n): + l = [] + for i in range(n): + l.append((x >> ((n-1-i)*8)) & 0xFF) + return l + +# https://stackoverflow.com/questions/1066758/find-length-of-sequences-of-identical-values-in-a-numpy-array-run-length-encodi/32681075 + + +def lre(bits): + for bit, group in itertools.groupby(bits): + yield (bit, len(list(group))) + + +def printable(s): + return "".join([c for c in s if c.isprintable()]) + + +def getParam(index, default): + if len(sys.argv) > index: + return sys.argv[index] + return default + + +def removeExtension(filename): + return ".".join(filename.split(".")[:-1]) + + +def rhoSweep(func, filename, rho, opts): + if rho == "auto": + for div in range(1, 8): + s = 1.0/2**div + for rho in np.arange(s, 1, 2*s): + try: + print(f"Level at {rho:0.2f}") + d = func(filename, rho, rho, opts) + print(f"\nOK for {rho:0.2f}") + return d + except Exception as e: + print(f"Level at {rho:0.2f} failed:"+str(e)) + print("") + # raise + raise Exception("No level found") + else: + return func(filename, rho, rho, opts) + + +def rhoSweepMax(func, filename, opts): + best = -1 + bestd = None + for div in range(1, int(opts.get("search_precision", 8))): + s = 1.0/2**div + for rho in np.arange(s, 1, 2*s): + try: + print(f"Level at {rho:0.2f}", end=" ") + score, d = func(filename, rho, rho, opts) + print(f"score {score:.2f} max {best:.2f}") + if score > best: + best = score + bestd = d + except Exception as e: + print(f"Level at {rho:0.2f} failed:"+str(e)) + print("") + # raise + return bestd diff --git a/SC3000/script/wavparse.py b/SC3000/script/wavparse.py new file mode 100644 index 0000000..6022ae9 --- /dev/null +++ b/SC3000/script/wavparse.py @@ -0,0 +1,34 @@ +import numpy as np +import wave +import struct +import sys + +sampleRate = 48000 # hertz +val = 32767 + +Space = np.zeros(int(sampleRate/1200), dtype=np.int16).tobytes() +Zero = np.hstack([v*np.ones(int(sampleRate/1200/2), dtype=np.int16) + for v in [val, -val]]).tobytes() +One = np.hstack([v*np.ones(int(sampleRate/1200/4), dtype=np.int16) + for v in [val, -val, val, -val]]).tobytes() +conv = {' ': Space, '0': Zero, '1': One} + + +def writeWav(filename, d, opts): + obj = wave.open(filename, 'wb') + obj.setnchannels(1) # mono + obj.setsampwidth(2) + obj.setframerate(d["bitrate"]) + obj.writeframesraw(Space) + for d in d["signal"]: + obj.writeframesraw(np.int16(val*d)) + obj.writeframesraw(Space) + obj.close() + + +def bittowav(filename): + with open(filename) as f: + data = f.read() + outname = ".".join(filename.split(".")[:-1])+".wav" + writeWav(outname, data) + print("Wrote", outname)