RustNotMC/rsx-client/tools/struct_diff.py

"""Structural comparison between the original bytecode and the rebuilt fork.

javac proves a file compiles. It proves nothing about whether the compiler
emitted the same class the decompiler read, and a misdecompiled class can be
silently wrong: dropping an interface, losing a method, or emitting the wrong
superclass all produce clean output.

Both trees are on disk, so the honest check is to parse the class files
directly and compare shape. This is a structural pass only, not a bytecode
equivalence proof, but it catches the whole class of errors that compiling
cannot: missing classes, lost interfaces, absent methods and fields.

    python struct_diff.py <original-dir> <rebuilt-dir> [--json out.json]
"""

import json
import os
import struct
import sys

CONSTANT_UTF8 = 1
CONSTANT_CLASS = 7
CONSTANT_STRING = 8
CONSTANT_LONG = 5
CONSTANT_DOUBLE = 6
SKIP_TWO_SLOT = (CONSTANT_LONG, CONSTANT_DOUBLE)

INTERFACE = 0x0200
ABSTRACT = 0x0400
SYNTHETIC = 0x1000


class Reader:
    """Minimal big-endian cursor over a class file."""

    def __init__(self, data):
        self.data = data
        self.pos = 0

    def u1(self):
        value = self.data[self.pos]
        self.pos += 1
        return value

    def u2(self):
        value = struct.unpack_from('>H', self.data, self.pos)[0]
        self.pos += 2
        return value

    def u4(self):
        value = struct.unpack_from('>I', self.data, self.pos)[0]
        self.pos += 4
        return value

    def take(self, size):
        value = self.data[self.pos:self.pos + size]
        self.pos += size
        return value


def read_constant_pool(reader):
    """Returns a list indexed by pool index; entries are str or int."""
    count = reader.u2()
    pool = [None] * count
    index = 1
    while index < count:
        tag = reader.u1()
        if tag == CONSTANT_UTF8:
            length = reader.u2()
            pool[index] = reader.take(length).decode('utf-8', 'replace')
        elif tag in (CONSTANT_CLASS, CONSTANT_STRING):
            pool[index] = ('ref', reader.u2())
        elif tag in (3, 4):                      # Integer, Float
            pool[index] = reader.take(4)
        elif tag in SKIP_TWO_SLOT:               # Long, Double
            pool[index] = reader.take(8)
        elif tag in (9, 10, 11, 12, 17, 18):     # refs, NameAndType, Dynamic
            reader.take(4)
            pool[index] = None
        elif tag == 15:                          # MethodHandle
            reader.take(3)
            pool[index] = None
        elif tag in (16, 19, 20):                # MethodType, Module, Package
            reader.take(2)
            pool[index] = None
        else:
            raise ValueError('unknown constant pool tag %d' % tag)
        index += 2 if tag in SKIP_TWO_SLOT else 1
    return pool


def utf(pool, index):
    entry = pool[index]
    return entry if isinstance(entry, str) else '?'


def class_name(pool, index):
    entry = pool[index]
    return utf(pool, entry[1]) if isinstance(entry, tuple) else '?'


def skip_attributes(reader, pool):
    count = reader.u2()
    for _ in range(count):
        reader.u2()                              # name index
        length = reader.u4()
        reader.take(length)


def parse(path):
    """Returns (name, superName, interfaces, methods, fields, referenced)."""
    with open(path, 'rb') as handle:
        data = handle.read()
    reader = Reader(data)
    if reader.u4() != 0xCAFEBABE:
        raise ValueError('not a class file: %s' % path)
    reader.u2()                                  # minor
    reader.u2()                                  # major
    pool = read_constant_pool(reader)
    reader.u2()                                  # access flags
    this_class = reader.u2()
    super_class = reader.u2()
    referenced = set()
    for entry in pool:
        if isinstance(entry, tuple) and entry[0] == 'ref':
            target = pool[entry[1]]
            if isinstance(target, str):
                referenced.add(target)
    interfaces = []
    for _ in range(reader.u2()):
        interfaces.append(class_name(pool, reader.u2()))
    fields = set()
    for _ in range(reader.u2()):
        flags = reader.u2()
        name = utf(pool, reader.u2())
        descriptor = utf(pool, reader.u2())
        if not flags & SYNTHETIC:
            fields.add('%s %s' % (name, descriptor))
        skip_attributes(reader, pool)
    methods = set()
    for _ in range(reader.u2()):
        flags = reader.u2()
        name = utf(pool, reader.u2())
        descriptor = utf(pool, reader.u2())
        if not flags & SYNTHETIC and name not in ('<clinit>',):
            methods.add('%s %s' % (name, descriptor))
        skip_attributes(reader, pool)
    return (class_name(pool, this_class), class_name(pool, super_class),
            interfaces, methods, fields, referenced)


def collect(root):
    found = {}
    for base, _dirs, files in os.walk(root):
        for name in files:
            if not name.endswith('.class'):
                continue
            full = os.path.join(base, name)
            key = os.path.relpath(full, root).replace(os.sep, '/')
            try:
                found[key] = parse(full)
            except (ValueError, struct.error, IndexError) as error:
                found[key] = None
                sys.stderr.write('unreadable %s: %s\n' % (key, error))
    return found


def describe(kind, items):
    return sorted(items)


def main():
    original_root, rebuilt_root = sys.argv[1], sys.argv[2]
    report_path = None
    if '--json' in sys.argv:
        report_path = sys.argv[sys.argv.index('--json') + 1]

    original = collect(original_root)
    rebuilt = collect(rebuilt_root)

    only_original = sorted(set(original) - set(rebuilt))
    only_rebuilt = sorted(set(rebuilt) - set(original))

    lost_interfaces = []
    lost_methods = []
    lost_fields = []
    changed_super = []
    unreadable = []
    referenced_by_rebuild = set()

    for key in sorted(set(rebuilt)):
        entry = rebuilt[key]
        if entry is not None:
            referenced_by_rebuild |= entry[5]

    for key in sorted(set(original) & set(rebuilt)):
        left, right = original[key], rebuilt[key]
        if left is None or right is None:
            unreadable.append(key)
            continue
        if set(left[2]) != set(right[2]):
            lost_interfaces.append({
                'class': key,
                'missing': describe('i', set(left[2]) - set(right[2])),
                'added': describe('i', set(right[2]) - set(left[2])),
            })
        if left[1] != right[1]:
            changed_super.append({
                'class': key,
                'original': left[1],
                'rebuilt': right[1],
            })
        gone_methods = left[3] - right[3]
        if gone_methods:
            lost_methods.append({'class': key, 'missing': sorted(gone_methods)})
        gone_fields = left[4] - right[4]
        if gone_fields:
            lost_fields.append({'class': key, 'missing': sorted(gone_fields)})

    # a class the rebuild never mentions cannot be reached at runtime, so the
    # original bytecode on the classpath is never consulted for it
    orphans = [k for k in only_original
               if k[:-6] not in referenced_by_rebuild]
    still_needed = [k for k in only_original
                    if k[:-6] in referenced_by_rebuild]

    report = {
        'classes_original': len(original),
        'classes_rebuilt': len(rebuilt),
        'only_in_original': only_original,
        'only_in_rebuilt': only_rebuilt,
        'lost_interfaces': lost_interfaces,
        'changed_super': changed_super,
        'lost_methods': lost_methods,
        'lost_fields': lost_fields,
        'unreadable': unreadable,
        'orphans': orphans,
        'still_needed': still_needed,
    }

    print('classes: original %d, rebuilt %d' % (len(original), len(rebuilt)))
    print('classes only in original : %d' % len(only_original))
    print('classes only in rebuilt  : %d' % len(only_rebuilt))
    print('lost interfaces          : %d' % len(lost_interfaces))
    print('changed superclass       : %d' % len(changed_super))
    print('lost methods             : %d' % sum(len(r['missing']) for r in lost_methods))
    print('lost fields              : %d' % sum(len(r['missing']) for r in lost_fields))
    print('missing but unreferenced : %d (dead weight, never loaded)' % len(orphans))
    print('missing and still needed : %d (would fail without the fallback)' % len(still_needed))

    if lost_interfaces:
        print('\n--- LOST INTERFACES (fatal class of bug) ---')
        for row in lost_interfaces:
            print('  %s' % row['class'])
            print('      - %s' % ', '.join(row['missing']))
            if row['added']:
                print('      + %s' % ', '.join(row['added']))
    if changed_super:
        print('\n--- CHANGED SUPERCLASS ---')
        for row in changed_super:
            print('  %-60s %s -> %s' % (row['class'], row['original'], row['rebuilt']))
    if lost_methods:
        print('\n--- LOST METHODS ---')
        for row in lost_methods:
            print('  %s' % row['class'])
            for name in row['missing']:
                print('      - %s' % name)
    if lost_fields:
        print('\n--- LOST FIELDS ---')
        for row in lost_fields:
            print('  %s' % row['class'])
            for name in row['missing']:
                print('      - %s' % name)
    if only_original:
        print('\n--- ONLY IN ORIGINAL (first 60) ---')
        for key in only_original[:60]:
            print('  %s' % key)
        if len(only_original) > 60:
            print('  ... and %d more' % (len(only_original) - 60))

    if report_path:
        with open(report_path, 'w', encoding='utf-8') as handle:
            json.dump(report, handle, indent=2)
        print('\nreport written to %s' % report_path)


if __name__ == '__main__':
    main()