product-vamp/generator/solo/audition.py

"""Build-only snapshots and like-for-like listening comparisons; no engine presets."""
from __future__ import annotations

import argparse
import array
import collections
import copy
import fractions
import hashlib
import html
import importlib
import json
import pathlib
import sys
import time
import wave

import generator.arrange.generate
import generator.model
import generator.musicxml
import generator.musicxml_writer
import generator.parts
import generator.playback
import generator.rhythm
import generator.scales.library
import generator.solo.arrangement
import generator.solo.report
import generator.solo.sequences

STAGES = ('baseline', 'A', 'B', 'C', 'D')

def metrics(solo):
  notes = solo.notes()
  gaps, cursor, run, runs = [], fractions.Fraction(), 0, []
  for note in notes:
    if note.start > cursor:
      gaps.append((cursor, note.start))
    if note.end - note.start == fractions.Fraction(1, 2):
      run = run + 1 if note.start == cursor else 1
    else:
      run = 0
    runs.append(run)
    cursor = note.end
  if cursor < solo.end:
    gaps.append((cursor, solo.end))
  planned = [(p.start, p.end) for phrase in solo.phrases for p in getattr(phrase.plan, 'passages', ()) if p.intent == 'space']
  planned.extend((p.plan.end - p.plan.release, p.plan.end) for p in solo.phrases if p.plan.release)
  planned.extend((c.span.start, c.span.end) for c in solo.choices if c.scale is None)
  unplanned = []
  for start, end in gaps:
    fragments = [(start, end)]
    for a, b in planned:
      fragments = [(x, y) for low, high in fragments for x, y in ((low, min(high, a)), (max(low, b), high)) if y > x]
    unplanned.extend(b - a for a, b in fragments)
  quoted = sum(n.end - n.start for n in notes if n.role == 'quote')
  return {
    'phrases': len(solo.phrases), 'attacks': len(notes),
    'attacks_per_four_beats': float(4 * len(notes) / solo.end),
    'sounding_occupancy': float(sum(n.end - n.start for n in notes) / solo.end),
    'longest_unplanned_gap_beats': float(max(unplanned, default=0)),
    'longest_connected_eighth_run': max(runs, default=0),
    'quotation_coverage': float(quoted / solo.end),
    'treatments': dict(collections.Counter(d.treatment for p in solo.phrases for d in p.decisions)),
    'search': getattr(solo, 'diagnostics', {}),
  }

def fingerprint():
  digest = hashlib.sha256()
  for path in sorted(pathlib.Path('generator').rglob('*.py')):
    digest.update(path.as_posix().encode())
    digest.update(path.read_bytes())
  return digest.hexdigest()

def capture(root, stage):
  destination = root / stage
  if destination.exists():
    raise ValueError(f'{destination}: snapshot already exists; choose a new output root')
  rhythms = generator.rhythm.load_library(pathlib.Path('data/rhythms'))
  scales = generator.scales.library.load_library(pathlib.Path('data/scales'))
  sequences = generator.solo.sequences.load_library(pathlib.Path('data/sequences'))
  result = {'stage': stage, 'implementation': fingerprint(), 'songs': {}}
  try:
    result['policy'] = importlib.import_module('generator.solo.policy').describe()
  except ModuleNotFoundError:
    result['policy'] = {'baseline': True}
  for path in generator.parts.source_paths(pathlib.Path('data/songs')):
    started = time.monotonic()
    document = generator.parts.prepare_document(generator.musicxml.load_musicxml(path))
    plans = generator.arrange.generate.plan_arrangement(document, rhythms)
    parts = {part: generator.arrange.generate._arrange_document(document, part, plans) for part in generator.parts.PART_TYPES}
    solos = []
    generator.solo.arrangement.apply(document, parts, rhythms=rhythms, scales=scales, sequences=sequences, solos=solos)
    song = {'source_sha256': hashlib.sha256(path.read_bytes()).hexdigest(), 'parts': parts, 'solos': {}, 'elapsed_seconds': time.monotonic() - started}
    regions = generator.solo.arrangement.regions(document)
    for solo, index in zip(solos, (3, 4)):
      first, count, offset, length = regions[index]
      song['solos'][solo.part] = {'first': first, 'count': count, 'offset': str(offset), 'length': str(length), 'tempo': str(solo.tempo), 'metrics': metrics(solo), 'report': generator.solo.report.describe(solo, scales), 'structure': generator.solo.report.structure(solo)}
      print(path.stem, solo.part, json.dumps(song['solos'][solo.part]['metrics']), flush=True)
    result['songs'][path.stem] = song
  destination.mkdir(parents=True)
  (destination / 'snapshot.json').write_text(json.dumps(result, indent=2) + '\n')

def excerpt(document, first, count):
  result = copy.deepcopy(document)
  staff = result['staves'][0]
  staff['measures'] = staff['measures'][first:first + count]
  notes = [n for m in staff['measures'] for v in m['voices'] for n in v['notes']]
  identities = {p['id'] for n in notes for p in n['pitches']}
  events = {n['id'] for n in notes}
  for note in notes:
    for pitch in note['pitches']:
      for key in ('tie_from', 'tie_to'):
        if pitch.get(key) not in identities:
          pitch.pop(key, None)
  staff['tuplets'] = [t for t in staff.get('tuplets', []) if t['start'] in events and t['end'] in events]
  tuplets = {t['id'] for t in staff['tuplets']}
  for note in notes:
    if 'tuplet_ids' in note:
      note['tuplet_ids'] = [i for i in note['tuplet_ids'] if i in tuplets]
  offset = sum(generator.model.fraction(m['duration']) for m in document['staves'][0]['measures'][:first])
  end = offset + sum(generator.model.fraction(m['duration']) for m in staff['measures'])
  for key in ('scale_spans', 'octave_shifts'):
    staff[key] = [{**s, 'start': generator.model.rational(max(offset, generator.model.fraction(s['start'])) - offset), 'end': generator.model.rational(min(end, generator.model.fraction(s['end'])) - offset)} for s in staff.get(key, []) if generator.model.fraction(s['start']) < end and generator.model.fraction(s['end']) > offset]
  generator.model.validate_document(result)
  return result

def mix(documents, tempo, root, cache):
  timelines = [generator.playback.compile_timeline(document) for document in documents]
  rate = generator.playback.SAMPLE_RATE
  factor = fractions.Fraction(60 * rate, tempo)
  frames = round(max(generator.model.fraction(t['duration']) for t in timelines) * factor)
  output = array.array('f', [0]) * (frames + 2400)
  for index, timeline in enumerate(timelines):
    gain = 1.0 if index == 0 else 0.55
    for event in timeline['events']:
      pitch = generator.model.fraction(event['pitch'])
      if pitch not in cache:
        sample = generator.playback.generate_tone(root, pitch)
        with wave.open(str(root / sample['path'])) as audio:
          values = array.array('h', audio.readframes(audio.getnframes()))
        if sys.byteorder != 'little':
          values.byteswap()
        cache[pitch] = sample, values
      sample, values = cache[pitch]
      start = round(generator.model.fraction(event['onset']) * factor)
      end = round((generator.model.fraction(event['onset']) + generator.model.fraction(event['duration'])) * factor)
      for frame in range(start, min(len(output), end + 2400)):
        age = frame - start
        position = age if age < sample['loop_end'] else sample['loop_start'] + (age - sample['loop_start']) % (sample['loop_end'] - sample['loop_start'])
        release = 1 if frame < end else (1 - (frame - end) / 2399) ** 2
        output[frame] += gain * release * values[position] / 32767
  return output

def render(root):
  stages = ('before', 'after') if (root / 'before' / 'snapshot.json').exists() else STAGES
  snapshots = {stage: json.loads((root / stage / 'snapshot.json').read_text()) for stage in stages}
  cache, clips, manifest = {}, [], {'sample_rate': generator.playback.SAMPLE_RATE, 'stages': {}, 'clips': []}
  for stage, snapshot in snapshots.items():
    manifest['stages'][stage] = {k: v for k, v in snapshot.items() if k != 'songs'}
    for song_name, song in snapshot['songs'].items():
      baseline = snapshots[stages[0]]['songs'][song_name]
      if song['source_sha256'] != baseline['source_sha256']:
        raise ValueError(f'{song_name}: source changed between audition stages')
      for part, solo in song['solos'].items():
        other = 'electric' if part == 'acoustic' else 'acoustic'
        windows = [('full', 0, solo['count']), ('opening8', 0, min(8, solo['count'])), ('closing4', max(0, solo['count'] - 4), min(4, solo['count']))] if stages[0] == 'before' else [('', 0, min(8, solo['count']))]
        for label, offset, count in windows:
          first = solo['first'] + offset
          foreground = excerpt(song['parts'][part], first, count)
          prefix = root / stage / (f'{song_name}_{part}' + ('_' + label if label else ''))
          generator.musicxml_writer.write_musicxml(prefix.with_suffix('.musicxml'), foreground)
          prefix.with_suffix('.txt').write_text(solo['report'])
          if 'structure' in solo:
            prefix.with_suffix('.json').write_text(json.dumps(solo['structure'], indent=2) + '\n')
          for responsive in [False, True] if stage == stages[-1] else [False]:
            backing = excerpt((song if responsive else baseline)['parts'][other], first, count)
            vocal = excerpt(baseline['parts']['vocal'], first, count)
            path = prefix.with_name(prefix.name + ('_responsive' if responsive else '_fixed')).with_suffix('.wav')
            samples = mix((foreground, backing, vocal), fractions.Fraction(solo['tempo']), root, cache)
            clips.append((path, samples))
            manifest['clips'].append({'path': path.relative_to(root).as_posix(), 'window': label, 'first': offset, 'count': count, 'tempo': solo['tempo'], 'responsive_backing': responsive, 'source_sha256': song['source_sha256'], 'metrics': solo['metrics'], 'elapsed_seconds': song['elapsed_seconds']})
  # One common attenuation preserves relative loudness and identical backing.
  peak = max(abs(value) for _, samples in clips for value in samples)
  gain = min(1, 0.95 / peak) if peak else 1
  manifest['shared_gain'] = gain
  for path, samples in clips:
    generator.playback.write_pcm(path, (value * gain for value in samples))
  (root / 'manifest.json').write_text(json.dumps(manifest, indent=2) + '\n')
  write_passage_index(root, snapshots) if stages[0] == 'before' else write_index(root, snapshots)
  print(f'Rendered {len(clips)} comparisons under {root}', flush=True)

def write_index(root, snapshots):
  labels = {'baseline': 'Original engine', 'A': 'Scoring repair', 'B': 'Activity shapes', 'C': 'Motifs and retrieval', 'D': 'Phrase lookahead'}
  body = ['<!doctype html><html lang="en"><meta charset="utf-8"><title>Solo comparisons</title><style>body{font:16px system-ui;max-width:1200px;margin:3rem auto;padding:0 1rem;background:#faf9f5;color:#202820}table{border-collapse:collapse;width:100%;margin:1rem 0 3rem}td,th{text-align:left;padding:1rem;border-bottom:1px solid #ccd1c9}audio{width:280px}small{display:block;margin-top:.5rem}</style><h1>Solo comparisons</h1><p>Eight bars from the same solo entrance at the source tempo. All stages use the original backing and one shared mix gain. D also includes the normal backing that responds to the new solo.</p><p>A: scoring and register. B: activity and breathing. C: transformation contracts, contextual motifs and diverse paths. D: one-phrase lookahead. Expressive playback is deferred.</p><p>Metrics below describe the complete solo, not just the excerpt. These are diagnostics; listening and playing determine musical quality. <a href="manifest.json">Build fingerprints and diagnostics</a>.</p>']
  for name, song in snapshots['D']['songs'].items():
    for part in song['solos']:
      body.append('<h2>' + html.escape(name.replace('_', ' ').title() + ' — ' + part.title()) + '</h2><table><tr><th>Stage</th><th>Same backing</th><th>Complete solo</th></tr>')
      for stage in STAGES:
        prefix = stage + '/' + name + '_' + part
        metrics = snapshots[stage]['songs'][name]['solos'][part]['metrics']
        stats = f"{metrics['attacks_per_four_beats']:.2f} attacks / four beats<br>{metrics['sounding_occupancy']:.0%} sounding occupancy<br>{metrics['longest_connected_eighth_run']} connected eighths<br>{metrics['longest_unplanned_gap_beats']:g}-beat longest unplanned gap"
        body.append(f'<tr><td>{stage}: {labels[stage]}<small><a href="{prefix}.musicxml">Notation</a> · <a href="{prefix}.txt">Phrase report</a></small></td><td><audio controls preload="none" src="{prefix}_fixed.wav"></audio></td><td>{stats}</td></tr>')
      body.append(f'</table><p>D with responsive backing: <audio controls preload="none" src="D/{name}_{part}_responsive.wav"></audio></p>')
  (root / 'index.html').write_text(''.join(body) + '</html>\n')

def write_passage_index(root, snapshots):
  body = ['<!doctype html><html lang="en"><meta charset="utf-8"><title>Connected solo passages</title><style>body{font:16px system-ui;max-width:1250px;margin:3rem auto;padding:0 1rem;background:#faf9f5;color:#202820}table{border-collapse:collapse;width:100%;margin:1rem 0 3rem}td,th{text-align:left;padding:.75rem;border-bottom:1px solid #ccd1c9}audio{width:260px}small{display:block;margin-top:.5rem}</style><h1>Connected solo passages</h1><p>Before and after use the same source songs, scale-selection settings, activity targets, tempo and root seed inputs. Fixed comparisons retain the original backing. Responsive comparisons include accompaniment recomposed around the new foreground. The returning vocal pickup and one shared mix gain are preserved.</p><p>Listen for a convincing settlement on the current chord, an idea developing across several bars, intermediate arrivals that continue, and a prepared ending. Metrics alone do not establish musical improvement. <a href="manifest.json">Fingerprints and diagnostics</a>.</p>']
  for name, song in snapshots['after']['songs'].items():
    for part in song['solos']:
      body.append('<h2>' + html.escape(name.replace('_', ' ').title() + ' — ' + part.title()) + '</h2>')
      for stage in ('before', 'after'):
        metric = snapshots[stage]['songs'][name]['solos'][part]['metrics']
        body.append(f"<p>{stage.title()}: {metric['attacks_per_four_beats']:.2f} attacks / four beats; {metric['sounding_occupancy']:.1%} occupancy; longest unplanned gap {metric['longest_unplanned_gap_beats']:g} beats.</p>")
      body.append('<table><tr><th>Passage</th><th>Before, fixed backing</th><th>After, fixed backing</th><th>After, responsive backing</th></tr>')
      for window in ('full', 'opening8', 'closing4'):
        body.append(f'<tr><th>{window}</th>')
        for stage, mix_name in (('before', 'fixed'), ('after', 'fixed'), ('after', 'responsive')):
          prefix = stage + '/' + name + '_' + part + '_' + window
          body.append(f'<td><audio controls preload="none" src="{prefix}_{mix_name}.wav"></audio><small><a href="{prefix}.musicxml">MusicXML</a> · <a href="{prefix}.txt">Report</a>' + (f' · <a href="{prefix}.json">Passage data</a>' if stage == 'after' else '') + '</small></td>')
        body.append('</tr>')
      body.append('</table>')
  (root / 'index.html').write_text(''.join(body) + '</html>\n')

def main():
  parser = argparse.ArgumentParser(description=__doc__)
  parser.add_argument('--output', type=pathlib.Path, default=pathlib.Path('build/solo_auditions'))
  action = parser.add_mutually_exclusive_group(required=True)
  action.add_argument('--capture', choices=(*STAGES, 'before', 'after'))
  action.add_argument('--render', action='store_true')
  args = parser.parse_args()
  capture(args.output, args.capture) if args.capture else render(args.output)

if __name__ == '__main__':
  main()