Cleanup

generator/__main__.py

@@ -2,14 +2,14 @@ from __future__ import annotations
 
 import argparse
 import json
+import os
 import pathlib
+import platform
 import shutil
 import tempfile
 import time
 
-import modulus.filesystem
-
-import generator.arrange.generate
+import generator.audio_assets
 import generator.catalog
 import generator.engraving.cache
 import generator.engraving.fonts
@@ -17,12 +17,12 @@ import generator.engraving.pipeline
 import generator.engraving.types
 import generator.parts
 import generator.pico_artwork
-import generator.performance
 import generator.printing
+import generator.production
 import generator.score
+import generator.score_io
 import generator.summary
-import platform
-import os
+import modulus.filesystem
 
 RUNTIME_FONT_FILES = ("PetalumaScript.otf", "OFL.txt")
 
@@ -87,7 +87,7 @@ def generate_assets(args: argparse.Namespace) -> None:
     staged.mkdir(parents=True, exist_ok=True)
     copy_runtime_fonts(static_assets_dir, staged)
     report('generating performance parts')
-    performance_count = generator.arrange.generate.generate_performance_scores(data_dir / 'songs', staged / 'songs', composition_jobs=getattr(args, 'composition_jobs', 2), cache_dir=getattr(args, 'composition_cache', None), progress=progress, timings=timings)
+    performance_count = generator.production.generate_tree(data_dir / 'songs', staged / 'songs', jobs=getattr(args, 'composition_jobs', 2), cache_dir=getattr(args, 'composition_cache', None), progress=progress, timings=timings)
     if getattr(args, 'trace_level', 'summary') == 'summary':
       for path in (staged / 'analysis').glob('*.roles'):
         roles = json.loads(path.read_text())
@@ -95,13 +95,13 @@ def generate_assets(args: argparse.Namespace) -> None:
           trace = role.get('details', {}).get('selection_trace', {})
           trace.pop('decisions', None)
         path.write_text(json.dumps(roles, sort_keys=True, indent=2))
-    generator.score.validate_generated_tree(staged)
+    generator.score_io.validate_generated_tree(staged)
     if args.check:
       generator.summary.validate_expected_performance_summary(pathlib.Path(__file__).resolve().parents[1] / 'tests' / 'fixtures' / 'performance_summary.json', staged / 'songs')
     report('copying supplied recordings')
     song_ids = {generator.score.slugify(path.stem) for path in (data_dir / 'songs').glob('*.xml')}
-    audio_assets = generator.performance.copy_audio(static_assets_dir, staged, song_ids)
-    generator.performance.validate_recording_lengths(staged, audio_assets)
+    audio_assets = generator.audio_assets.copy_audio(static_assets_dir, staged, song_ids)
+    generator.audio_assets.validate_recording_lengths(staged, audio_assets)
     report('engraving pages')
     validated_pages = generator.engraving.pipeline.engrave_tree(staged, static_assets_dir / 'fonts' / 'petaluma', cache_dir=cache_dir, jobs=getattr(args, 'jobs', 1), progress=progress)
     report('generating printable parts')
@@ -109,7 +109,7 @@ def generate_assets(args: argparse.Namespace) -> None:
     report('rendering Pico titles')
     generator.pico_artwork.generate_titles(staged, static_assets_dir / 'fonts' / 'petaluma')
     report('validating score, page, and audio assets')
-    generator.score.validate_generated_tree(staged, validated_pages)
+    generator.score_io.validate_generated_tree(staged, validated_pages)
     report('building catalog database')
     generator.catalog.build_catalog(staged, audio_assets=audio_assets)
     with modulus.filesystem.publish_directories([(staged, output_dir)]):

generator/arrange/accompaniment.py

@@ -1,20 +1,13 @@
 """One learned accompaniment per instrument, reused at authored form positions."""
 from __future__ import annotations
 
-import copy
 import fractions
 
 import generator.accompaniment
-import generator.acoustic
-import generator.acoustic_voicing
-import generator.electric
-import generator.guitar
-import generator.fretboard
-import generator.model
-import generator.parts
-import generator.score
 import generator.arrange.form
 import generator.arrange.pickup
+import generator.model
+import generator.score
 
 def harmonic_key(measure, previous):
 

generator/arrange/assembly.py

@@ -0,0 +1,100 @@
+"""Assemble source-defined role windows without composing or respelling."""
+from __future__ import annotations
+import copy
+import fractions
+
+import generator.accompaniment
+import generator.arrange.pickup
+import generator.composition_types
+import generator.fretboard
+import generator.model
+import generator.parts
+import generator.validation
+
+def mask(document, allowed):
+  """Apply source-defined role exclusions, without observing another part."""
+  absolute = fractions.Fraction()
+  for measure in document['staves'][0]['measures']:
+    length = generator.model.fraction(measure['duration'])
+    for voice in measure['voices']:
+      selected = []
+      for n in voice['notes']:
+        start = absolute + generator.model.fraction(n['onset'])
+        end = start + generator.model.fraction(n['duration'])
+        intervals = [(max(start, a), min(end, b)) for a, b in allowed if a < end and b > start]
+        for a, b in intervals:
+          note = copy.deepcopy(n)
+          if a != start:
+            note['onset'] = generator.model.rational(a - absolute)
+            for pitch in note['pitches']:
+              pitch.pop('tie_from', None)
+          generator.arrange.pickup.truncate(note, b - absolute)
+          if a < b:
+            selected.append(note)
+      voice['notes'] = fill(selected, length, measure['id'] + '_' + voice['id'], voice['id'])
+    absolute += length
+  staff = document['staves'][0]
+  staff['tuplets'] = generator.arrange.pickup.retain_links(staff['measures'], staff.get('tuplets', []))
+
+def fill(notes, length, prefix, voice='voice_arranged'):
+  notes = sorted(notes, key=lambda n: generator.model.fraction(n['onset']))
+  result = []
+  cursor = fractions.Fraction()
+
+  def rests(end):
+    nonlocal cursor
+    for duration, written in generator.accompaniment.written_durations(end - cursor):
+      result.append(generator.model.make_event(prefix + '_rest_' + str(len(result)), cursor, duration, 'rest', written=written) | {'voice': voice})
+      cursor += duration
+
+  for n in notes:
+    start = generator.model.fraction(n['onset'])
+    if start < cursor:
+      raise ValueError(prefix + ': overlapping independently generated roles')
+    rests(start)
+    n['voice'] = voice
+    result.append(n)
+    cursor = start + generator.model.fraction(n['duration'])
+  rests(length)
+  return result
+
+def assemble(source, results):
+  """Union prevalidated role windows; no selection or pitch/fingering rewrite."""
+  documents = {}
+  for part in generator.parts.PART_TYPES:
+    selected = [results[r]['document'] for r in generator.composition_types.ROLES if r.startswith(part + '_')]
+    document = copy.deepcopy(selected[0])
+    staff = document['staves'][0]
+    if part == 'vocal':
+      documents[part] = document
+      continue
+    staff['tuplets'] = []
+    staff['octave_shifts'] = []
+    staff['scale_spans'] = []
+    for other in selected:
+      for field in ('tuplets', 'octave_shifts', 'scale_spans'):
+        staff[field].extend(copy.deepcopy(other['staves'][0].get(field, [])))
+    for i, m in enumerate(staff['measures']):
+      # Tuplet rests belong to the source/role rhythm and must survive union.
+      lanes = {}
+      for d in selected:
+        for voice in d['staves'][0]['measures'][i]['voices']:
+          lane = voice['id'] if part == 'acoustic' and voice['id'] in ('upper', 'bass') else 'voice_arranged'
+          sounding = [copy.deepcopy(n) for n in voice['notes'] if n['pitches'] or n.get('tuplet_ids')]
+          if sounding:
+            lanes.setdefault(lane, []).extend(sounding)
+      lanes = lanes or {'voice_arranged': []}
+      m['voices'] = [{'id': lane, 'notes': fill(notes, generator.model.fraction(m['duration']), m['id'] + '_' + lane, lane)} for lane, notes in lanes.items()]
+      # Attribute changes belong to source melody pickups and are assembly metadata.
+      for d in selected:
+        other = d['staves'][0]['measures'][i]
+        if other.get('attributes'):
+          m['attributes'] = copy.deepcopy(other['attributes'])
+    staff['tuplets'] = generator.arrange.pickup.retain_links(staff['measures'], staff['tuplets'])
+    owners = {n['id']: v['id'] for m in staff['measures'] for v in m['voices'] for n in v['notes']}
+    for span in staff['tuplets']:
+      span['voice'] = owners[span['start']]
+    generator.validation.validate_document(document)
+    generator.fretboard.validate_document(document)
+    documents[part] = document
+  return documents

generator/arrange/ending.py

@@ -4,44 +4,30 @@ import fractions
 
 import generator.accompaniment
 import generator.acoustic_voicing
+import generator.arrange.ending_plan
 import generator.comping
 import generator.fretboard
 import generator.model
 import generator.parts
 import generator.score
-
-BARS = 3
-
-def template(source):
-  result = copy.deepcopy(source[-1])
-  # An ending is three full bars in the final active meter, never a pickup.
-  for change in result.get('attributes', []):
-    for key in ('meter', 'key', 'clef', 'transpose', 'staff_details'):
-      if key in change:
-        result[key] = copy.deepcopy(change[key])
-  result.update(duration=generator.model.rational(generator.model.meter_length(result['meter'])), implicit=False,
-    attributes=[], harmonies=[], voices=[], tempos=[], section_label='')
-  result['time_signature'] = generator.model.meter_label(result['meter'])
-  return result
-
-def duration(source):
-  return BARS * generator.model.fraction(template(source)['duration']) if source else fractions.Fraction()
+import generator.vocabulary
 
 def chord(source):
   effective = [{**m, 'harmonies': [h for h in m['harmonies'] if not h.get('chord_optional')]} for m in source]
   prior = generator.accompaniment.carried_chords(effective)[-1]
   return generator.accompaniment.active_chord(effective[-1], generator.model.fraction(effective[-1]['duration']), prior)
 
-def append(source, output, part, *, sounding=False, center=55):
+def append(source, output, part, *, sounding=False, center=55, composition=None):
+  composition = composition or generator.vocabulary.current()
   if not source or not output:
     return
-  model = template(source)
+  model = generator.arrange.ending_plan.template(source)
   harmony = chord(source)
   grip = ()
   if sounding:
     if harmony is None:
       raise ValueError('Ending: final performed harmony is missing or N.C.; cannot invent a closing chord')
-    options = generator.comping.voicings(generator.comping.semantic_key(harmony), part)
+    options = generator.comping.voicings(generator.comping.semantic_key(harmony), part, composition=composition)
     recent = {}
     for m in reversed(output):
       for n in sorted(generator.score.measure_notes(m), key=lambda n: generator.model.fraction(n['onset']), reverse=True):
@@ -55,10 +41,10 @@ def append(source, output, part, *, sounding=False, center=55):
     if part == 'acoustic':
       bass = next((recent[s] for s in (5, 6) if s in recent), None)
       previous = generator.acoustic_voicing.Voicing(bass, *(recent[s] for s in (4, 3, 2))) if bass and all(s in recent for s in (4, 3, 2)) else ()
-    grip = min(options, key=lambda v: (generator.comping.local_cost(part, generator.comping.semantic_key(harmony), v, center).moved(generator.comping.movement(part, previous, v)), v))
+    grip = min(options, key=lambda v: (generator.comping.local_cost(part, generator.comping.semantic_key(harmony), v, center, composition=composition).moved(generator.comping.movement(part, previous, v)), v))
   length = generator.model.fraction(model['duration'])
   previous = None
-  for index in range(BARS):
+  for index in range(generator.arrange.ending_plan.BARS):
     measure = generator.parts.build_measure(model, f'measure_{len(output) + 1}', 'rest', part, None, 'Ending' if index == 0 else '')
     measure['ending_sustain'] = True
     if index == 0 and harmony:
@@ -77,4 +63,4 @@ def append(source, output, part, *, sounding=False, center=55):
         cursor += duration
       measure['voices'] = [{'id': 'voice_arranged', 'notes': notes}]
     output.append(measure)
-  generator.score.reidentify(output[-BARS:], [])
+  generator.score.reidentify(output[-generator.arrange.ending_plan.BARS:], [])

generator/arrange/ending_plan.py

@@ -0,0 +1,22 @@
+"""Source-derived ending geometry, independent of instrument composition."""
+import copy
+import fractions
+
+import generator.model
+
+BARS = 3
+
+def template(source):
+  result = copy.deepcopy(source[-1])
+  # An ending is three full bars in the final active meter, never a pickup.
+  for change in result.get('attributes', []):
+    for key in ('meter', 'key', 'clef', 'transpose', 'staff_details'):
+      if key in change:
+        result[key] = copy.deepcopy(change[key])
+  result.update(duration=generator.model.rational(generator.model.meter_length(result['meter'])), implicit=False,
+    attributes=[], harmonies=[], voices=[], tempos=[], section_label='')
+  result['time_signature'] = generator.model.meter_label(result['meter'])
+  return result
+
+def duration(source):
+  return BARS * generator.model.fraction(template(source)['duration']) if source else fractions.Fraction()

generator/arrange/form.py

@@ -1,8 +1,12 @@
 from __future__ import annotations
 
 import dataclasses
+import fractions
 import typing
 
+import generator.arrange.ending_plan
+import generator.arrange.pickup
+import generator.model
 import generator.score
 
 @dataclasses.dataclass(frozen=True)
@@ -55,3 +59,57 @@ def full_arrangement_sections(measures: list[dict], part_type: str) -> list[Arra
     for label, start, count, index in specs
     if count > 0
   ]
+
+@dataclasses.dataclass(frozen=True)
+class FormPlan:
+  sections: tuple[ArrangementSection, ...]
+  windows: tuple[tuple[fractions.Fraction, fractions.Fraction, str], ...]
+  stops: tuple[tuple[int, fractions.Fraction] | None, ...]
+  @classmethod
+  def read(cls, document, part):
+    source = generator.score.source_staff(document)['measures']
+    sections = tuple(full_arrangement_sections(source, part))
+    stops = tuple(section_stop(source, section, sections[i + 1] if i + 1 < len(sections) else None) for i, section in enumerate(sections))
+    return cls(sections, _windows(document, part, sections), stops)
+
+def section_stop(source, section, following):
+  if following is None or not generator.score.body_start_measure_index(source):
+    return None
+  last = section.source_measure_index + section.measure_count - 1
+  pad = generator.arrange.pickup.offset(source[last], source[0])
+  if following.role == 'slash':
+    return last, pad
+  if following.role == 'melody' and generator.arrange.pickup.enters(following, source):
+    attacks = [generator.model.fraction(note['onset']) for note in generator.score.measure_notes(source[0]) if note['pitches']]
+    if attacks:
+      return last, pad + min(attacks)
+  return None
+
+def windows(document, part):
+  return FormPlan.read(document, part).windows
+
+def _windows(document, part, sections):
+  source = generator.score.source_staff(document)['measures']
+  result, cursor = [], fractions.Fraction()
+  for section in sections:
+    duration = sum((generator.model.fraction(m['duration']) for m in source[section.source_measure_index:section.source_measure_index + section.measure_count]), fractions.Fraction())
+    role = {'slash': 'solo', 'vocal': 'melody'}.get(section.role, section.role)
+    if section.role in ('vocal', 'melody') and generator.arrange.pickup.enters(section, source) and result:
+      attacks = [generator.model.fraction(n['onset']) for n in generator.score.measure_notes(source[0]) if n['pitches']]
+      if attacks:
+        start = cursor - generator.model.fraction(source[0]['duration']) + min(attacks)
+        a, _, old = result[-1]
+        result[-1] = (a, start, old)
+        result.append((start, cursor, part + '_' + role))
+    end = cursor + duration
+    start = cursor
+    if section.role == 'slash' and generator.score.body_start_measure_index(source) and result:
+      # Allocate the source's complete pickup bar before composition. This
+      # does not inspect either generated solo; a role may leave it silent.
+      start = cursor - generator.model.fraction(source[0]['duration'])
+      a, _, old = result[-1]
+      result[-1] = (a, start, old)
+    result.append((start, end, part + '_' + role))
+    cursor = end
+  result.append((cursor, cursor + generator.arrange.ending_plan.duration(source), part + ('_accompaniment' if part != 'vocal' else '_rest')))
+  return tuple(result)

generator/arrange/generate.py

@@ -2,36 +2,31 @@ from __future__ import annotations
 
 import copy
 import fractions
-import pathlib
 
 import generator.accompaniment
-import generator.arrange.accompaniment
 import generator.acoustic
 import generator.acoustic_voicing
 import generator.arrange.ending
 import generator.arrange.form
 import generator.arrange.pickup
+import generator.comping
 import generator.electric
 import generator.fretboard
 import generator.guitar
 import generator.melody
 import generator.model
-import generator.musicxml
 import generator.parts
 import generator.score
-import generator.solo.arrangement
-import generator.comping
 import generator.vocabulary
-import generator.production
 
 FINAL_SECTION_INDEX = 5
 
-def arrange_document(document: dict, part_type: str) -> dict:
+def arrange_document(document: dict, part_type: str, *, composition=None) -> dict:
   """Canonical source-only template; solos are inserted by production roles."""
+  composition = composition or generator.vocabulary.CompositionContext.create(generator.vocabulary.load(), part_type + '_accompaniment')
   document = generator.parts.prepare_document(document)
-  with generator.vocabulary.using(generator.vocabulary.load(), part_type + '_accompaniment'):
-    backing = generator.comping.Canonical(document, part_type, 'canonical') if part_type != 'vocal' else None
-    return _arrange_document(document, part_type, accompaniment=backing)
+  backing = generator.comping.Canonical(document, part_type, 'canonical', composition=composition) if part_type != 'vocal' else None
+  return _arrange_document(document, part_type, accompaniment=backing, composition=composition)
 
 def section_source(source: list[dict], section, next_section=None) -> list[dict]:
   result = copy.deepcopy(source)
@@ -44,12 +39,14 @@ def section_source(source: list[dict], section, next_section=None) -> list[dict]
     generator.arrange.pickup.harmonies(destination, result[0])
   return result
 
-def _arrange_document(document: dict, part_type: str, plans=None, *, owned=None, accompaniment=None) -> dict:
+def _arrange_document(document: dict, part_type: str, plan=None, *, owned=None, accompaniment=None, composition=None) -> dict:
+  composition = composition or generator.vocabulary.current()
   if part_type not in generator.parts.PART_TYPES:
     raise ValueError(f'unknown arrangement part: {part_type}')
   staff = generator.score.source_staff(document)
   source = staff['measures']
-  sections = generator.arrange.form.full_arrangement_sections(source, part_type)
+  plan = plan or generator.arrange.form.FormPlan.read(document, part_type)
+  sections = plan.sections
   output, tuplets, octave_shifts, voicing_origins = [], [], [], []
   absolute = fractions.Fraction()
   source_starts = []
@@ -62,16 +59,8 @@ def _arrange_document(document: dict, part_type: str, plans=None, *, owned=None,
     section_measures = section_source(source, section, sections[index + 1] if index + 1 < len(sections) else None)
     previous = generator.accompaniment.carried_chords(section_measures)
     segment = []
-    stop = None
+    stop = plan.stops[index]
     next_section = sections[index + 1] if index + 1 < len(sections) else None
-    if next_section and generator.score.body_start_measure_index(source):
-      last = section.source_measure_index + section.measure_count - 1
-      if next_section.role == 'slash':
-        stop = (last, generator.model.fraction(section_measures[last]['duration']) - generator.model.fraction(source[0]['duration']))
-      elif next_section.role == 'melody' and generator.arrange.pickup.enters(next_section, source):
-        attacks = [generator.model.fraction(n['onset']) for n in generator.score.measure_notes(source[0]) if n['pitches']]
-        if attacks:
-          stop = (last, generator.model.fraction(section_measures[last]['duration']) - generator.model.fraction(source[0]['duration']) + min(attacks))
     for si in range(section.source_measure_index, section.source_measure_index + section.measure_count):
       m = section_measures[si]
       if role == 'accompaniment' and accompaniment is not None:
@@ -111,7 +100,7 @@ def _arrange_document(document: dict, part_type: str, plans=None, *, owned=None,
   if sections and sections[-1].section_index == FINAL_SECTION_INDEX:
     generator.arrange.ending.append(source, output, part_type,
       sounding=part_type != 'vocal' and (owned is None or 'accompaniment' in owned),
-      center=generator.vocabulary.role(part_type + '_accompaniment').get('center', 55))
+      center=composition.role(part_type + '_accompaniment').get('center', 55), composition=composition)
   tuplets = generator.arrange.pickup.retain_links(output, tuplets)
   result = generator.score.make_arranged_document(document, staff, staff_id='staff_arranged_' + part_type, staff_name=part_type.capitalize(), role='vocal' if part_type == 'vocal' else 'guitar', arrangement_part_type=part_type, measures=output, tuplets=tuplets, octave_shifts=octave_shifts)
   result['metadata'].setdefault('tempo', 124)
@@ -123,15 +112,12 @@ def _arrange_document(document: dict, part_type: str, plans=None, *, owned=None,
     generator.acoustic.combine(result)
   elif part_type == 'electric' and (owned is None or 'melody' in owned):
     generator.melody.harmonize(result)
-    generator.guitar.place_guitar(result)
+    generator.guitar.place_guitar(result, composition=composition)
     if generator.accompaniment.connect_continuations(result):
-      generator.guitar.place_guitar(result)
+      generator.guitar.place_guitar(result, composition=composition)
   # Source labels identify the head while arranging; published labels identify
   # the complete performance, including its lead-in and repeated choruses.
   for number, measure in enumerate(output, 1):
     measure['number'] = str(number)
   generator.fretboard.validate_document(result)
   return result
-
-def generate_performance_scores(source_dir: pathlib.Path, output_dir: pathlib.Path, *, composition_jobs=2, cache_dir=None, progress=False, timings=None) -> int:
-  return generator.production.generate_tree(source_dir, output_dir, jobs=composition_jobs, cache_dir=cache_dir, progress=progress, timings=timings)

generator/audio_assets.py

@@ -0,0 +1,68 @@
+"""Validate and copy supplied recordings without synthesis or transformation."""
+from __future__ import annotations
+
+import fractions
+import hashlib
+import json
+import re
+import shutil
+import wave
+
+import generator.model
+import generator.performance
+
+TEMPOS = tuple(range(60, 121, 10))
+
+def describe_wav(path, root, *, song_id='', bpm=0):
+  if path.is_symlink() or not path.is_file() or not path.resolve().is_relative_to(root.resolve()):
+    raise ValueError(f'{path}: expected a contained regular WAV')
+  try:
+    with wave.open(str(path), 'rb') as audio:
+      channels, rate, frames = audio.getnchannels(), audio.getframerate(), audio.getnframes()
+      if audio.getcomptype() != 'NONE' or audio.getsampwidth() != 2 or channels not in (1, 2) or not 8000 <= rate <= 192000 or frames <= 0:
+        raise ValueError('expected nonempty mono/stereo PCM16 WAV')
+      remaining = frames
+      while remaining:
+        count = min(remaining, 65536)
+        if len(audio.readframes(count)) != count * channels * 2:
+          raise ValueError('truncated WAV data')
+        remaining -= count
+  except (wave.Error, EOFError) as error:
+    raise ValueError(f'{path}: invalid WAV: {error}') from error
+  with path.open('rb') as source:
+    digest = hashlib.file_digest(source, 'sha256').hexdigest()
+  return dict(path=path.relative_to(root).as_posix(), sha256=digest, channels=channels, sample_rate=rate, frames=frames, song_id=song_id, bpm=bpm)
+
+def copy_audio(static, destination, song_ids):
+  entries = []
+  inputs = [(static / 'sounds' / name, '', 0) for name in ('beat1.wav', 'beat2.wav')]
+  for path in sorted((static / 'backing').glob('*.wav')):
+    match = re.fullmatch(r'([a-z0-9]+(?:_[a-z0-9]+)*)_(60|70|80|90|100|110|120)\.wav', path.name)
+    if not match:
+      raise ValueError(f'{path}: unsupported backing filename or tempo')
+    if match[1] not in song_ids:
+      continue
+    inputs.append((path, match[1], int(match[2])))
+  for source, song_id, bpm in inputs:
+    description = describe_wav(source, static, song_id=song_id, bpm=bpm)
+    target = destination / description['path']
+    target.parent.mkdir(parents=True, exist_ok=True)
+    shutil.copyfile(source, target)
+    entries.append(description)
+  return entries
+
+def validate_recording_lengths(root, entries):
+  durations = {}
+  for path in sorted((root / 'songs').glob('*.json')):
+    document = json.loads(path.read_text())
+    performance = generator.performance.timing(document)
+    song_id = performance['song_id']
+    duration = generator.model.fraction(performance['duration']) + performance['count_in_beats']
+    if song_id in durations and durations[song_id] != duration:
+      raise ValueError(f'{song_id}: parts disagree about performance duration')
+    durations[song_id] = duration
+  for entry in entries:
+    if entry['song_id']:
+      seconds = fractions.Fraction(entry['frames'], entry['sample_rate'])
+      if seconds < durations[entry['song_id']] * 60 / entry['bpm']:
+        raise ValueError(f"{entry['path']}: backing recording is shorter than count-in and score")

generator/cache.py

@@ -0,0 +1,82 @@
+"""Content identities and atomic storage shared by generator cache adapters."""
+from __future__ import annotations
+import ast
+import contextlib
+import functools
+import hashlib
+import json
+import os
+import pathlib
+import shutil
+import tempfile
+
+ROOT = pathlib.Path(__file__).resolve().parent
+
+@functools.lru_cache(maxsize=256)
+def imports(content):
+  modules = set()
+  for node in ast.walk(ast.parse(content)):
+    if isinstance(node, ast.Import):
+      modules.update(alias.name for alias in node.names if alias.name.startswith('generator.'))
+    elif isinstance(node, ast.ImportFrom) and node.module and node.module.startswith('generator'):
+      modules.add(node.module)
+      modules.update(node.module + '.' + alias.name for alias in node.names)
+  return tuple(sorted(modules))
+
+def source_identity(entrypoints):
+  """Conservatively include the transitive imports of explicit stage roots.
+
+  Read bytes on every call, so edits in a long-lived build process invalidate
+  both code content and newly introduced dependencies. Only parsing is cached.
+  """
+  pending, sources = list(entrypoints), {}
+  while pending:
+    module = pending.pop()
+    relative = module.split('.')[1:]
+    path = ROOT.joinpath(*relative).with_suffix('.py') if relative else ROOT / '__init__.py'
+    if not path.is_file():
+      path = ROOT.joinpath(*relative, '__init__.py')
+    if path in sources or not path.is_file():
+      continue
+    content = path.read_bytes()
+    sources[path] = hashlib.sha256(content).hexdigest()
+    pending.extend(imports(content))
+    pending.extend('.'.join(module.split('.')[:i]) for i in range(1, len(module.split('.'))))
+  return {path.relative_to(ROOT).as_posix(): sources[path] for path in sorted(sources)}
+
+def fingerprint(value):
+  return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(',', ':')).encode()).hexdigest()
+
+def write_json(path, value):
+  path.parent.mkdir(parents=True, exist_ok=True)
+  temporary = None
+  try:
+    with tempfile.NamedTemporaryFile('w', dir=path.parent, delete=False) as output:
+      temporary = pathlib.Path(output.name)
+      json.dump(value, output, sort_keys=True)
+    os.replace(temporary, path)
+  finally:
+    if temporary is not None:
+      temporary.unlink(missing_ok=True)
+
+def has_symlink(path):
+  return any(parent.is_symlink() for parent in (path, *path.parents))
+
+@contextlib.contextmanager
+def stage_entry(entry, prefix):
+  if has_symlink(entry.parent):
+    raise ValueError('cache directories must not be symlinks')
+  entry.parent.mkdir(parents=True, exist_ok=True)
+  if entry.is_symlink() or entry.is_file():
+    entry.unlink(missing_ok=True)
+  entry.mkdir(exist_ok=True)
+  with tempfile.TemporaryDirectory(prefix=prefix, dir=entry.parent) as temporary:
+    yield pathlib.Path(temporary)
+
+def publish_files(entry, staged, names):
+  """Publish complete files in order, with the artifact manifest last."""
+  for name in names:
+    target = entry / name
+    if target.is_dir() and not target.is_symlink():
+      shutil.rmtree(target)
+    (staged / name).replace(target)

generator/catalog.py

@@ -10,7 +10,7 @@ import sys
 if __name__ == '__main__' and __package__ is None:
   sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1]))
 
-import generator.performance
+import generator.audio_assets
 
 APPLICATION_ID = 0x56414D50
 SONG_COLUMNS = """
@@ -58,12 +58,12 @@ def populate(connection: sqlite3.Connection, root: pathlib.Path, audio_assets=No
     for path in sorted(root.glob('sounds/*.wav')):
       if path.name not in ('beat1.wav', 'beat2.wav'):
         raise ValueError(f'{path}: unknown metronome sample')
-      audio_assets.append(generator.performance.describe_wav(path, root))
+      audio_assets.append(generator.audio_assets.describe_wav(path, root))
     for path in sorted(root.glob('backing/*.wav')):
       match = re.fullmatch(r'([a-z0-9]+(?:_[a-z0-9]+)*)_(60|70|80|90|100|110|120)\.wav', path.name)
       if not match:
         raise ValueError(f'{path}: invalid backing filename')
-      audio_assets.append(generator.performance.describe_wav(path, root, song_id=match[1], bpm=int(match[2])))
+      audio_assets.append(generator.audio_assets.describe_wav(path, root, song_id=match[1], bpm=int(match[2])))
   folder = root / 'songs'
   if (folder.exists() or folder.is_symlink()) and not folder.is_dir():
     raise ValueError(f'{folder}: score asset directory not found')

generator/chord_families.py

@@ -1,10 +1,11 @@
 """Transposable guitar families, derived from the written chord and legal grips."""
-import generator.harmony
 import generator.fretboard
+import generator.harmony
 import generator.model
 import generator.vocabulary
 
-def acoustic(chord, grip):
+def acoustic(chord, grip, *, composition=None):
+  composition = composition or generator.vocabulary.current()
   degrees = generator.harmony.degrees(chord)
   root = int(generator.model.NATURAL[chord['root']['letter']] + generator.model.alteration(chord['root']['alter']))
   classes = {d: (root + generator.harmony.degree_semitones(d, a)) % 12 for d, a in degrees.items()}
@@ -14,7 +15,7 @@ def acoustic(chord, grip):
   kind = 'extended' if terminal and color else 'tetrad' if terminal else 'triad'
   bass = next((d for d in (1, 5, 3) if classes.get(d) == grip[0].midi % 12), None)
   slots = {'root': 1, 'fifth': 5, 'guide': guide, 'terminal': terminal, 'color': color}
-  policy = generator.vocabulary.accompaniment('acoustic')['voicing_families']
+  policy = composition.accompaniment('acoustic')['voicing_families']
   orders = policy[kind].get(f'{grip[0].string_number}/{bass}', ())
   upper = tuple(p.midi % 12 for p in grip[1:])
   preferred = any(all(slots[s] in classes for s in order) and tuple(classes[slots[s]] for s in order) == upper for order in orders)
@@ -37,10 +38,11 @@ def electric(grip):
       return 'drop-two'
   return 'close-position' if pitches and pitches[-1] - pitches[0] < 12 else 'compact-open'
 
-def coverage(chord, options, part):
+def coverage(chord, options, part, *, composition=None):
+  composition = composition or generator.vocabulary.current()
   if part == 'electric':
-    return {name: sum(electric(v) == name for v in options) for name in generator.vocabulary.accompaniment('electric')['voicing_families']['rank']}
-  families = [acoustic(chord, v) for v in options]
+    return {name: sum(electric(v) == name for v in options) for name in composition.accompaniment('electric')['voicing_families']['rank']}
+  families = [acoustic(chord, v, composition=composition) for v in options]
   result = []
   for string in (5, 6):
     for degree in (1, 5):

generator/comping.py

@@ -2,26 +2,25 @@
 from __future__ import annotations
 import copy
 import dataclasses
+import fractions
 import functools
 import json
-import fractions
 import typing
 
 import generator.accompaniment
 import generator.acoustic
-import generator.comping_rhythm
-import generator.chord_families
 import generator.acoustic_voicing
 import generator.arrange.form
-import generator.arrange.pickup
+import generator.chord_families
+import generator.comping_rhythm
 import generator.fretboard
 import generator.guitar
 import generator.harmony
 import generator.model
 import generator.parts
 import generator.score
-import generator.vocabulary
 import generator.solo.policy
+import generator.vocabulary
 
 def semantic_key(chord):
   return json.dumps({k: chord[k] for k in ('root', 'kind', 'degrees', 'bass', 'inversion') if k in chord}, sort_keys=True)
@@ -38,12 +37,15 @@ def requirements(chord, part):
     raise ValueError('acoustic accompaniment requires a written bass, root or fifth')
   return dataclasses.replace(result, basses=basses)
 
+def voicings(key, part, *, composition=None):
+  return _voicings(key, part, composition or generator.vocabulary.current())
+
 @functools.lru_cache(maxsize=2048)
-def voicings(key, part):
+def _voicings(key, part, composition):
   chord = json.loads(key)
   required = requirements(chord, part)
   if part == 'electric':
-    options = generator.guitar.chord_positions(required, prune=False, profile='electric')
+    options = generator.guitar.chord_positions(required, prune=False, profile='electric', composition=composition)
   elif part == 'acoustic':
     degrees = generator.harmony.degrees(chord)
     plain = degrees in ({1: 0, 3: 0, 5: 0}, {1: 0, 3: -1, 5: 0})
@@ -77,19 +79,21 @@ class VoicingCost(typing.NamedTuple):
   def moved(self, distance):
     return self._replace(effort=self.effort + distance)
 
-def voicing_preferences(part, grip, center):
-  return generator.guitar.voicing_preferences(grip, generator.guitar.voicing_preference(part + '_accompaniment', center))
+def voicing_preferences(part, grip, center, *, composition=None):
+  composition = composition or generator.vocabulary.current()
+  return generator.guitar.voicing_preferences(grip, generator.guitar.voicing_preference(part + '_accompaniment', center, composition=composition))
 
-def local_cost(part, key, grip, center):
+def local_cost(part, key, grip, center, *, composition=None):
+  composition = composition or generator.vocabulary.current()
   required = requirements(json.loads(key), part)
   register = abs(fractions.Fraction(sum(p.midi for p in grip), len(grip)) - center)
-  preference = sum(voicing_preferences(part, grip, center))
+  preference = sum(voicing_preferences(part, grip, center, composition=composition))
   if part == 'electric':
-    value = generator.guitar.voicing_cost(grip, 'electric', required.allowed)
+    value = generator.guitar.voicing_cost(grip, 'electric', required.allowed, composition=composition)
     return VoicingCost(value.extended_grips, 3 * value.string_shortfall + value.family_penalty, value.effort + preference, value.missing_tones, register)
   # Familiar degree orders on either bass string, then balanced whole-form
   # movement, sounding register and hand position. Neither is an absolute ban.
-  family = generator.chord_families.acoustic(json.loads(key), grip)
+  family = generator.chord_families.acoustic(json.loads(key), grip, composition=composition)
   return VoicingCost(generator.fretboard.span_penalty(grip), int(not family['preferred']), preference, len(set(required.allowed) - {p.midi % 12 for p in grip}), register)
 
 def select_grips(layers, costs, part, seed, *, left=(), right=(), close=False):
@@ -184,9 +188,10 @@ def split_measures(flat, inputs, part):
   return tuple(output)
 
 class Canonical:
-  def __init__(self, document, part, seed):
+  def __init__(self, document, part, seed, *, composition=None, plan=None):
+    self.composition = composition or generator.vocabulary.current()
     self.part, self.seed = part, seed
-    self.policy = generator.vocabulary.role(part + '_accompaniment')
+    self.policy = self.composition.role(part + '_accompaniment')
     self.cache, self._reports, self._rhythm_reports = {}, {}, {}
     self.measures = generator.score.source_staff(document)['measures']
     # A held accompaniment chord may continue into the bar containing the
@@ -194,7 +199,8 @@ class Canonical:
     self.voice_id = self.measures[0]['voices'][0]['id']
     self.melody = generator.comping_rhythm.melody_contexts(document)
     self.first = generator.score.body_start_measure_index(self.measures)
-    sections = generator.arrange.form.full_arrangement_sections(self.measures, part)
+    plan = plan or generator.arrange.form.FormPlan.read(document, part)
+    sections = plan.sections
     self.closed = any(a.role == b.role == 'accompaniment' and a.source_measure_index + a.measure_count == len(self.measures)
       and b.source_measure_index == self.first for a, b in zip(sections, sections[1:]))
     carried = generator.accompaniment.carried_chords(self.measures)
@@ -204,7 +210,7 @@ class Canonical:
       size = 1
       if index + 1 < len(self.measures):
         lengths = [generator.model.fraction(m['duration']) for m in self.measures[index:index + 2]]
-        if self.measures[index]['time_signature'] == self.measures[index + 1]['time_signature'] and any(tuple(map(fractions.Fraction, p['lengths'])) == tuple(lengths) for p in generator.vocabulary.accompaniment(part)['patterns']):
+        if self.measures[index]['time_signature'] == self.measures[index + 1]['time_signature'] and any(tuple(map(fractions.Fraction, p['lengths'])) == tuple(lengths) for p in self.composition.accompaniment(part)['patterns']):
           size = 2
       self.groups.append(tuple((self.measures[i], carried[i], i) for i in range(index, index + size)))
       index += size
@@ -214,19 +220,9 @@ class Canonical:
     for section in sections:
       if section.role == 'accompaniment':
         self.pair_breaks.update((section.source_measure_index, section.source_measure_index + section.measure_count))
-    for section, following in zip(sections, sections[1:]):
-      if section.role != 'accompaniment' or not self.first:
-        continue
-      last = section.source_measure_index + section.measure_count - 1
-      pad = generator.arrange.pickup.offset(self.measures[last], self.measures[0])
-      if following.role == 'slash':
-        self.pair_stops.append((last, pad))
-      elif following.role == 'melody' and generator.arrange.pickup.enters(following, self.measures):
-        attacks = [generator.model.fraction(n['onset']) for n in generator.score.measure_notes(self.measures[0]) if n['pitches']]
-        if attacks:
-          self.pair_stops.append((last, pad + min(attacks)))
+    self.pair_stops = [stop for section, stop in zip(sections, plan.stops) if section.role == 'accompaniment' and stop is not None]
     breaks = [(i, fractions.Fraction()) for i in self.pair_breaks] + self.pair_stops
-    choices, locations, self.groove_report = generator.comping_rhythm.choose_cycles(self.groups, part, self.melody, seed, breaks=breaks)
+    choices, locations, self.groove_report = generator.comping_rhythm.choose_cycles(self.groups, part, self.melody, seed, breaks=breaks, composition=self.composition)
     self.patterns = {g[0][2]: (p, a) for g, (p, a, _) in zip(self.groups, choices)}
     self.locations = {g[0][2]: location for g, location in zip(self.groups, locations)}
     selected = [r for _, _, r in choices]
@@ -266,7 +262,7 @@ class Canonical:
       melody.extend((a + cursor, b + cursor) for a, b in self.melody[i])
       lengths.append(generator.model.fraction(m['duration']))
       cursor += lengths[-1]
-    return generator.comping_rhythm.candidates(flat, None, self.part, tuple(melody), self.seed + '/' + str(inputs[0][2]), lengths=tuple(lengths), limit=limit)
+    return generator.comping_rhythm.candidates(flat, None, self.part, tuple(melody), self.seed + '/' + str(inputs[0][2]), lengths=tuple(lengths), limit=limit, composition=self.composition)
   def acoustic_realizations(self, sources, bounds=None, stop=None):
     """Assign the canonical attack stream once, before splitting it into blocks.
 
@@ -313,14 +309,14 @@ class Canonical:
       offset, index = next((a, i) for a, i in reversed(offsets) if a <= start)
       events = tuple(e for e in rhythm.events if start <= (e.harmonic_time if e.harmonic_time is not None else e.start) < end)
       try:
-        options = voicings(semantic_key(chord), self.part)
+        options = voicings(semantic_key(chord), self.part, composition=self.composition)
       except ValueError as error:
         raise ValueError(f'source measure {inputs[index - inputs[0][2]][0]["number"]}, beat {start - offset + 1}: {error}') from error
       spans.append(HarmonyChoice(index, start, end, chord, events, dict(rhythm.families), options))
     return tuple(spans)
   def select(self, spans, *, left=(), right=(), close=False):
     layers = tuple(s.options for s in spans)
-    costs = tuple({v: local_cost(self.part, semantic_key(s.chord), v, self.policy.get('center', 55)) for v in layer} for s, layer in zip(spans, layers))
+    costs = tuple({v: local_cost(self.part, semantic_key(s.chord), v, self.policy.get('center', 55), composition=self.composition) for v in layer} for s, layer in zip(spans, layers))
     return select_grips(layers, costs, self.part, self.seed, left=left, right=right, close=close)
   def __call__(self, source, prior, index, *, sources=None, bounds=None, stop=None):
     group = next((g for g in self.groups if any(i == index for _, _, i in g)), ((source, prior, index),))
@@ -397,8 +393,8 @@ class Canonical:
       root = int(generator.model.NATURAL[span.chord['root']['letter']] + generator.model.alteration(span.chord['root']['alter']))
       rows.append({'source_index': span.index, 'onset': str(span.start), 'chord': generator.harmony.symbol(span.chord),
         'figure': span.figure, 'pattern_id': rhythm.pattern_id, 'tones': len(grip), 'bass': bool(bass), 'variant': variant,
-        'family': (generator.chord_families.acoustic(span.chord, grip) if self.part == 'acoustic' else generator.chord_families.electric(grip)) if grip else None,
-        'family_coverage': generator.chord_families.coverage(span.chord, span.options, self.part),
+        'family': (generator.chord_families.acoustic(span.chord, grip, composition=self.composition) if self.part == 'acoustic' else generator.chord_families.electric(grip)) if grip else None,
+        'family_coverage': generator.chord_families.coverage(span.chord, span.options, self.part, composition=self.composition),
         'fret_span': generator.fretboard.fret_span(grip),
         'span_group': ('fallback' if generator.fretboard.span_penalty(grip) else 'primary') if grip else None,
         'span_candidates': generator.chord_families.span_coverage(span.options),
@@ -407,9 +403,9 @@ class Canonical:
         'bass_string': bass.string_number if bass else None,
         'movement': movement(self.part, previous, grip), 'voicing_candidates': len(span.options)})
       if grip:
-        register, position = voicing_preferences(self.part, grip, self.policy.get('center', 55))
+        register, position = voicing_preferences(self.part, grip, self.policy.get('center', 55), composition=self.composition)
         travel = movement(self.part, previous, grip)
-        effort = generator.guitar.voicing_cost(grip, 'electric').effort if self.part == 'electric' else 0
+        effort = generator.guitar.voicing_cost(grip, 'electric', composition=self.composition).effort if self.part == 'electric' else 0
         rows[-1]['voicing_cost'] = {'movement': travel, 'grip_effort': str(effort), 'register': str(register), 'position': str(position), 'combined': str(travel + effort + register + position)}
       for event in span.events:
         lanes = ['bass', 'upper'] if event.lane == 'combined' else [event.lane]

generator/comping_rhythm.py

@@ -5,8 +5,8 @@ import itertools
 
 import generator.accompaniment
 import generator.model
-import generator.solo.analysis
 import generator.solo.policy
+import generator.source
 import generator.vocabulary
 
 def meter_context(measure):
@@ -90,7 +90,7 @@ class Rhythm:
     return tuple((e.lane, str(e.start), str(e.end - e.start), str(e.harmonic_time)) for e in self.events)
 
 def melody_contexts(document):
-  melody = generator.solo.analysis.attacks(document)
+  melody = generator.source.attacks(document)
   result, cursor = [], fractions.Fraction()
   for m in document['staves'][0]['measures']:
     length = generator.model.fraction(m['duration'])
@@ -247,9 +247,10 @@ def evaluate(events, part, melody, harmony, lengths):
   divisor = max(1, len(lengths))
   return tuple((k, v / divisor) for k, v in (('activity', activity), ('melody_support', support), ('harmonic_support', coverage), ('articulation', articulation)))
 
-def candidates(measure, prior, part, melody, seed, *, lengths=None, limit=None, grouping=()):
+def candidates(measure, prior, part, melody, seed, *, lengths=None, limit=None, grouping=(), composition=None):
+  composition = composition or generator.vocabulary.current()
   lengths = lengths or (generator.model.fraction(measure['duration']),)
-  policy = generator.vocabulary.accompaniment(part)
+  policy = composition.accompaniment(part)
   result = []
   for pattern in patterns_for(policy, measure, lengths, grouping):
     for anticipate in (False, True):
@@ -371,13 +372,14 @@ def realize_acoustic_form(rhythms, context):
 def form_cost(rhythms, groups):
   return sum(r.cost * len(g) for r, g in zip(rhythms, groups))
 
-def choose_cycles(groups, part, melody, seed, *, breaks=()):
+def choose_cycles(groups, part, melody, seed, *, breaks=(), composition=None):
   """Select a small recurring vocabulary against every occurrence in the form.
 
   No local shortlist precedes whole-form evaluation. A pattern/mode is retained
   as one decision even where a chord boundary changes its legal release.
   """
-  policy = generator.vocabulary.accompaniment(part)
+  composition = composition or generator.vocabulary.current()
+  policy = composition.accompaniment(part)
   settings = policy['groove']
   selected, locations, reports = [], [], []
   runs = []
@@ -463,7 +465,7 @@ def choose_cycles(groups, part, melody, seed, *, breaks=()):
       selected.append((p, anticipate, cache[i, p['id'], anticipate]))
       locations.append({'cycle_start': run[0][0][2], 'cycle_slot': i % len(cycle), 'base_pattern': p['id'], 'fill': None})
   context = acoustic_context(groups, melody, breaks) if part == 'acoustic' else None
-  add_fills(groups, selected, locations, part, melody, seed, context=context)
+  add_fills(groups, selected, locations, part, melody, seed, context=context, composition=composition)
   if context:
     realized = realize_acoustic_form([r for _, _, r in selected], context)
     selected = [(p, a, r) for (p, a, _), r in zip(selected, realized)]
@@ -472,9 +474,10 @@ def choose_cycles(groups, part, melody, seed, *, breaks=()):
       report['attack_assignment'] = 'sequential_combined_except_eighth_pairs'
   return selected, locations, reports
 
-def add_fills(groups, selected, locations, part, melody, seed, *, context=None):
+def add_fills(groups, selected, locations, part, melody, seed, *, context=None, composition=None):
   """Fixed source-position answers; no iteration seed or generated lead input."""
-  policy = generator.vocabulary.accompaniment(part)
+  composition = composition or generator.vocabulary.current()
+  policy = composition.accompaniment(part)
   spacing = policy['groove']['fill_spacing_measures']
   if not groups:
     return
@@ -519,7 +522,7 @@ def add_fills(groups, selected, locations, part, melody, seed, *, context=None):
     if any(e.start < middle < e.end for e in current.events):
       continue
     for variation in policy['patterns']:
-      if variation['family'] != base['family'] or variation['lengths'] != base['lengths'] or variation.get('meter') != base.get('meter') or variation.get('grouping') != base.get('grouping'):
+      if variation['family'] != base['family'] or tuple(variation['lengths']) != tuple(base['lengths']) or variation.get('meter') != base.get('meter') or tuple(variation.get('grouping', ())) != tuple(base.get('grouping', ())):
         continue
       hybrid = {**base, 'id': base['id'] + '.fill.' + variation['id'], 'events':
         [r for r in base['events'] if fractions.Fraction(r['onset']) < middle] +

generator/compose.py

@@ -1,24 +1,27 @@
 """Explicit single-role build. Does not publish or consume an existing ensemble."""
 import argparse
 import json
-import time
 import pathlib
+import time
+
+import generator.composition
+import generator.composition_types
 import generator.musicxml
 import generator.parts
-import generator.production
+import generator.source
 
 def main():
   parser = argparse.ArgumentParser(description=__doc__)
   parser.add_argument('--song', type=pathlib.Path, required=True)
-  parser.add_argument('--role', choices=generator.production.ROLES, required=True)
+  parser.add_argument('--role', choices=generator.composition_types.ROLES, required=True)
   parser.add_argument('--data-dir', type=pathlib.Path, default=pathlib.Path('data'))
   parser.add_argument('--output', type=pathlib.Path, required=True)
   parser.add_argument('--cache-dir', type=pathlib.Path)
   parser.add_argument('--settings', type=pathlib.Path)
   args = parser.parse_args()
-  source = generator.production.SourceAnalysis.read(generator.parts.prepare_document(generator.musicxml.load_musicxml(args.song)))
+  source = generator.source.SourceAnalysis.read(generator.parts.prepare_document(generator.musicxml.load_musicxml(args.song)))
   start = time.monotonic()
-  result = generator.production.generate_role(source, args.role, args.data_dir, json.loads(args.settings.read_text()) if args.settings else {}, args.cache_dir)
+  result = generator.composition.generate_role(source, args.role, args.data_dir, json.loads(args.settings.read_text()) if args.settings else {}, args.cache_dir)
   args.output.parent.mkdir(parents=True, exist_ok=True)
   args.output.write_text(json.dumps(result, indent=2))
   print(f'{args.role}: {time.monotonic() - start:.2f}s wall; cached={result["cache_hit"]}; role generation/cache lookup {result["seconds"]:.2f}s')

generator/composition.py

@@ -0,0 +1,158 @@
+"""Independent, reproducible role generation from source and policy inputs.
+
+Only source documents and production resources enter generate_role. Completed
+roles and published assets are downstream values, never composition inputs.
+"""
+from __future__ import annotations
+import copy
+import dataclasses
+import functools
+import hashlib
+import json
+import pathlib
+import time
+
+import generator.accompaniment
+import generator.acoustic
+import generator.arrange.assembly
+import generator.arrange.form
+import generator.arrange.generate
+import generator.comping
+import generator.comping_rhythm
+import generator.composition_types
+import generator.fretboard
+import generator.input_audit
+import generator.role_cache
+import generator.score
+import generator.solo.arrangement
+import generator.solo.authored
+import generator.solo.composer
+import generator.solo.learned
+import generator.solo.notation
+import generator.solo.report
+import generator.source
+import generator.validation
+import generator.vocabulary
+
+VERSION = 'independent-roles-1'
+
+def dependencies(role, resources, measures=None):
+  names = ['harmony', 'lead_ins'] if role.endswith('_solo') else []
+  accompaniment = {}
+  if role.endswith('_accompaniment'):
+    policy = generator.vocabulary.accompaniment(role.split('_')[0], resources)
+    accompaniment = {'accompaniment': generator.comping_rhythm.relevant_policy(policy, measures) if measures is not None else policy}
+  return {'roles': {'role': resources['roles']['roles'][role], 'difficulty': resources['roles']['difficulty']},
+    **({'voicing_families': generator.vocabulary.accompaniment('electric', resources)['voicing_families']} if role == 'electric_melody' else {}),
+    **accompaniment,
+    **{name: resources[name] for name in names}}
+
+def _resource_key(directory, role):
+  return {'model': hashlib.sha256((directory / 'models' / 'solo.json').read_bytes()).hexdigest()} if role.endswith('_solo') else {}
+
+@functools.lru_cache(maxsize=4)
+def _solo_model(directory, fingerprint):
+  return generator.solo.learned.Model.load(pathlib.Path(directory) / 'models' / 'solo.json')
+
+def generate_role(source, role, data_dir, settings=None, cache_dir=None) -> generator.composition_types.RoleResult:
+  started = time.monotonic()
+  with generator.input_audit.composition(data_dir, role, cache_dir) as reads:
+    result = _generate_role(source, role, data_dir, settings, cache_dir)
+  # Observed reads are diagnostics, never cache identity or musical seeds.
+  result['trace'] = {**result['trace'], 'input_boundary': 'production data (including explicit authored repertoire), generator code, Python runtime, own role cache'}
+  result['observed_reads'] = sorted(reads)
+  result['seconds'] = time.monotonic() - started
+  return result
+
+def _generate_role(source, role, data_dir, settings=None, cache_dir=None):
+  if role not in generator.composition_types.ROLES:
+    raise ValueError('Unknown generation role: ' + role)
+  data_dir = pathlib.Path(data_dir)
+  resources = generator.vocabulary.load(data_dir / 'composition')
+  settings = settings or {}
+  registered = generator.solo.authored.lookup(source, role.removesuffix('_solo'), data_dir) if role.endswith('_solo') and not settings.get('predict', False) else None
+  algorithm = VERSION + '/' + generator.solo.learned.VERSION if role.endswith('_solo') else VERSION
+  inputs = {'version': algorithm, 'source': source.musical_key, 'role': role, 'settings': settings,
+    'vocabulary': dependencies(role, resources, generator.score.source_staff(source.document())['measures']), 'catalogs': ({'authored': registered['sha256'], 'id': registered['id'], 'pitch_convention': registered['pitch_convention']} if registered else _resource_key(data_dir, role)), 'code': generator.role_cache.code_key()}
+  # Source notation/lyrics also invalidate cached score representation, never the musical seed.
+  key = generator.vocabulary.fingerprint({'inputs': inputs, 'notation': source.document_json})
+  path = pathlib.Path(cache_dir) / role / (key + '.json') if cache_dir else None
+  cached = generator.role_cache.restore(path, key)
+  if cached is not None:
+    return cached
+  started = time.monotonic()
+  document = source.document()
+  part = role.split('_')[0]
+  plan = generator.arrange.form.FormPlan.read(document, part)
+  trace = {'source': source.musical_key, 'role': role, 'production_resources': inputs['catalogs'],
+    'vocabulary': generator.vocabulary.fingerprint(inputs['vocabulary']), 'settings': settings, 'algorithm': algorithm, 'code': inputs['code'], 'cache_key': key}
+  choice_source = source.accompaniment_key() if role.endswith('_accompaniment') else source.musical_key
+  choice_data, choice_settings = inputs['vocabulary'], settings
+  if role.endswith('_accompaniment'):
+    # Voicing weights invalidate the concrete cache, but must not reshuffle
+    # rhythm patterns. Removing this new field preserves the pre-policy seed.
+    choice_data = copy.deepcopy(choice_data)
+    choice_data['roles']['role'].pop('voicing', None)
+    for field in ('provenance', 'license', 'research'):
+      choice_data['accompaniment'].pop(field, None)
+    choice_settings = {k: v for k, v in settings.items() if k != 'voicing'}
+  seed = generator.vocabulary.fingerprint({'source': choice_source, 'role': role, 'settings': choice_settings, 'data': choice_data, 'version': algorithm})
+  details = {}
+  solo_report = ''
+  composition_timings = {}
+  composition = generator.vocabulary.CompositionContext.create(resources, role, settings | {'choice_seed': seed})
+  if role.endswith('_solo'):
+    result = generator.arrange.generate._arrange_document(document, part, plan, owned={'slash'}, composition=composition)
+    si = 3 if part == 'acoustic' else 4
+    first, count, offset, length = generator.solo.arrangement.regions(document)[si]
+    # The template/harmony comes directly from source form, not another generated part.
+    authored = generator.score.source_staff(document)['measures']
+    section = plan.sections[si]
+    effective = generator.arrange.generate.section_source(authored, section, plan.sections[si + 1])
+    template = copy.deepcopy(effective[section.source_measure_index:section.source_measure_index + count])
+    initial = generator.accompaniment.carried_chords(effective)[section.source_measure_index]
+    start, end = next((a - offset, b - offset) for a, b, r in plan.windows if r == role)
+    notation_first = first
+    if start < 0:
+      # The preceding arrangement harmony comes from the source form. Do
+      # not invent harmony from a comparison score's unlabelled lead-in.
+      template.insert(0, copy.deepcopy(result['staves'][0]['measures'][first - 1]))
+      notation_first -= 1
+      initial = generator.accompaniment.carried_chords(result['staves'][0]['measures'])[first - 1]
+    if registered:
+      solo = generator.solo.authored.realize(registered, generator.source.analyze(document), template, part, start, end, initial, composition=composition)
+    else:
+      solo_model = _solo_model(str(data_dir), inputs['catalogs']['model'])
+      solo = generator.solo.composer.compose(generator.source.analyze(document), template, part, end, initial=initial, learned_model=solo_model, expressions_enabled=composition.role().get('expressions', True), start=start, composition=composition)
+    composition_timings = {k: v for k, v in solo.diagnostics.items() if k.endswith('_seconds')}
+    relative = tuple(dataclasses.replace(n, start=n.start - start, end=n.end - start) for n in solo.notes())
+    output = generator.solo.notation.measures(result['staves'][0]['measures'][notation_first:first + count], relative, 'solo_' + part)
+    generator.solo.notation.replace_region(result, notation_first, output)
+    details = generator.solo.report.structure(solo, composition=composition)
+    details['allocation'] = {'start': str(start), 'end': str(end), 'lead_in_measures': first - notation_first}
+    solo_report = generator.solo.report.describe(solo, first, offset, detail=details, composition=composition)
+  elif role.endswith('_accompaniment'):
+    canonical = generator.comping.Canonical(document, part, seed, composition=composition, plan=plan)
+    result = generator.arrange.generate._arrange_document(document, part, plan, owned={'accompaniment'}, accompaniment=canonical, composition=composition)
+    details = {'canonical_comping': canonical.report, 'accompaniment_rhythm': canonical.rhythm_report, 'groove_cycles': canonical.groove_report, 'rhythm_source': choice_source}
+  else:
+    result = generator.arrange.generate._arrange_document(document, part, plan, owned={'vocal' if part == 'vocal' else 'melody'}, composition=composition)
+  allowed = tuple((a, b) for a, b, r in plan.windows if r == role)
+  generator.arrange.assembly.mask(result, allowed)
+  if role == "acoustic_accompaniment":
+    generator.acoustic.validate_comping(result)
+  staff = result['staves'][0]
+  for measure in staff['measures']:
+    measure['id'] = role + '_' + measure['id']
+  staff['tuplets'] = generator.score.reidentify(staff['measures'], staff.get('tuplets', []))
+  if part != 'vocal':
+    result['staves'][0]['tablature'] = True
+  generator.validation.validate_document(result)
+  generator.fretboard.validate_document(result)
+  value = {'role': role, 'document': result, 'trace': trace, 'details': details, 'report': solo_report, 'seconds': time.monotonic() - started, 'composition_timings': composition_timings, 'cache_hit': False}
+  # Sidecar diagnostics use the same rational-safe representation as reports.
+  # Identical insertion order after cold generation and cache loading also
+  # keeps serialized scores and the catalog byte-for-byte reproducible.
+  value = json.loads(json.dumps(value, default=str, sort_keys=True))
+  generator.role_cache.store(path, key, value)
+  return value

generator/composition_types.py

@@ -0,0 +1,28 @@
+"""Worker requests and role results; score serialization stays unchanged."""
+from __future__ import annotations
+import dataclasses
+import pathlib
+import typing
+
+import generator.source
+
+ROLES = ('acoustic_accompaniment', 'electric_accompaniment', 'electric_melody', 'vocal_melody', 'acoustic_solo', 'electric_solo')
+
+@dataclasses.dataclass(frozen=True)
+class RoleRequest:
+  source: generator.source.SourceAnalysis
+  role: str
+  directory: pathlib.Path
+  settings_json: str = '{}'
+  cache: pathlib.Path | None = None
+
+class RoleResult(typing.TypedDict):
+  role: str
+  document: dict
+  trace: dict
+  details: dict
+  report: str
+  seconds: float
+  composition_timings: dict
+  cache_hit: bool
+  observed_reads: typing.NotRequired[list[str]]

generator/engraving/cache.py

@@ -9,12 +9,12 @@ import pathlib
 import platform
 import re
 import shutil
-import tempfile
 
 import PIL.Image
 import PIL.ImageChops
 import PIL.features
 
+import generator.cache
 import generator.engraving.validation
 import generator.theme
 
@@ -22,9 +22,10 @@ def renderer_identity(fonts, style) -> dict:
   root = pathlib.Path(__file__).resolve().parents[1]
   versions = {name: importlib.metadata.version(name) for name in ('Pillow', 'fonttools', 'aggdraw')}
   versions.update({name: PIL.features.version(name) for name in PIL.features.get_supported()})
-  sources = [*root.joinpath('engraving').glob('*.py'), root / 'model.py', root / 'performance.py', root / 'harmony.py', root / 'theme.py', root / 'palette.json']
+  sources = generator.cache.source_identity(('generator.engraving.pipeline', 'generator.printing'))
+  sources['palette.json'] = hashlib.sha256((root / 'palette.json').read_bytes()).hexdigest()
   return {'fonts_style': fonts.fingerprint(style), 'python': {'implementation': platform.python_implementation(), 'version': platform.python_version()}, 'dependencies': versions,
-    'sources': {path.relative_to(root).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() for path in sorted(sources)}}
+    'sources': sources}
 
 def renderer_fingerprint(fonts, style) -> str:
   return hashlib.sha256(json.dumps(renderer_identity(fonts, style), sort_keys=True).encode()).hexdigest()
@@ -46,12 +47,9 @@ def band_fingerprint(notation: str, page, band) -> str:
   context = [notation, page.width, page.height, page.supersampling, dataclasses.asdict(band)]
   return hashlib.sha256(json.dumps(context, sort_keys=True, ensure_ascii=False).encode()).hexdigest()
 
-def has_symlink(path: pathlib.Path) -> bool:
-  return any(parent.is_symlink() for parent in (path, *path.parents))
-
 def restore_pages(entry: pathlib.Path, destination: pathlib.Path, validated_pages: set[str] | None = None) -> list[dict] | None:
   try:
-    if has_symlink(entry) or has_symlink(destination):
+    if generator.cache.has_symlink(entry) or generator.cache.has_symlink(destination):
       return None
     manifest = entry / 'pages.json'
     if manifest.is_symlink():
@@ -76,28 +74,17 @@ def restore_pages(entry: pathlib.Path, destination: pathlib.Path, validated_page
     return None
 
 def store_pages(entry: pathlib.Path, destination: pathlib.Path, pages: list[dict]) -> None:
-  if has_symlink(entry.parent):
-    raise ValueError('engraving cache directories must not be symlinks')
-  entry.parent.mkdir(parents=True, exist_ok=True)
-  if entry.is_symlink() or entry.is_file():
-    entry.unlink(missing_ok=True)
-  entry.mkdir(exist_ok=True)
-  with tempfile.TemporaryDirectory(prefix='.engraving_', dir=entry.parent) as temporary:
-    staged = pathlib.Path(temporary)
+  with generator.cache.stage_entry(entry, '.engraving_') as staged:
     for page in pages:
       shutil.copyfile(destination / page['path'], staged / page['path'])
     (staged / 'pages.json').write_text(json.dumps(pages), encoding='utf-8')
     # Identical keys have identical pages; publish complete files, then the manifest.
-    for name in list(dict.fromkeys(page['path'] for page in pages)) + ['pages.json']:
-      target = entry / name
-      if target.is_dir() and not target.is_symlink():
-        shutil.rmtree(target, ignore_errors=True)
-      (staged / name).replace(target)
+    generator.cache.publish_files(entry, staged, list(dict.fromkeys(page['path'] for page in pages)) + ['pages.json'])
 
 def restore_band(entry: pathlib.Path, key: str, width: int, height: int) -> PIL.Image.Image | None:
   try:
     manifest, asset = entry / 'band.json', entry / 'band.png'
-    if has_symlink(entry) or manifest.is_symlink() or asset.is_symlink():
+    if generator.cache.has_symlink(entry) or manifest.is_symlink() or asset.is_symlink():
       return None
     metadata = json.loads(manifest.read_text(encoding='utf-8'))
     if not isinstance(metadata, dict) or set(metadata) != {'context', 'width', 'height', 'sha256'}:
@@ -122,19 +109,8 @@ def restore_band(entry: pathlib.Path, key: str, width: int, height: int) -> PIL.
     return None
 
 def store_band(entry: pathlib.Path, key: str, image: PIL.Image.Image) -> None:
-  if has_symlink(entry.parent):
-    raise ValueError('engraving cache directories must not be symlinks')
-  entry.parent.mkdir(parents=True, exist_ok=True)
-  if entry.is_symlink() or entry.is_file():
-    entry.unlink()
-  entry.mkdir(exist_ok=True)
-  with tempfile.TemporaryDirectory(prefix='.engraving_', dir=entry.parent) as temporary:
-    staged = pathlib.Path(temporary)
+  with generator.cache.stage_entry(entry, '.engraving_') as staged:
     image.save(staged / 'band.png', format='PNG')
     metadata = {'context': key, 'width': image.width, 'height': image.height, 'sha256': hashlib.sha256((staged / 'band.png').read_bytes()).hexdigest()}
     (staged / 'band.json').write_text(json.dumps(metadata), encoding='utf-8')
-    for name in ('band.png', 'band.json'):
-      target = entry / name
-      if target.is_dir() and not target.is_symlink():
-        shutil.rmtree(target)
-      (staged / name).replace(target)
+    generator.cache.publish_files(entry, staged, ('band.png', 'band.json'))

generator/engraving/layout.py

@@ -16,7 +16,7 @@ import generator.engraving.spans
 import generator.engraving.tablature
 import generator.engraving.types
 import generator.harmony
-import generator.model
+import generator.validation
 
 def lyric_text(verse: dict) -> str:
   return ''.join(s['text'] for s in verse['syllables'] if s['kind'] in ('text', 'elision'))
@@ -250,7 +250,7 @@ def paginate(systems, document, fonts, style, *, index=None):
 
 def layout_systems(document: dict, style: generator.engraving.types.Style, fonts: generator.engraving.fonts.Fonts, *, index=None) -> tuple[generator.engraving.types.System, ...]:
   if index is None:
-    generator.model.validate_document(document)
+    generator.validation.validate_document(document)
     index = generator.engraving.notation.ScoreIndex(document)
   if not index.measures:
     raise generator.engraving.types.EngravingError('score has no measures')
@@ -276,7 +276,7 @@ def layout_systems(document: dict, style: generator.engraving.types.Style, fonts
   return tuple(systems)
 
 def layout_score(document: dict, style: generator.engraving.types.Style, fonts: generator.engraving.fonts.Fonts) -> generator.engraving.types.LayoutDocument:
-  generator.model.validate_document(document)
+  generator.validation.validate_document(document)
   style = generator.engraving.types.score_style(document, style)
   index = generator.engraving.notation.ScoreIndex(document)
   systems = layout_systems(document, style, fonts, index=index)

generator/engraving/pipeline.py

@@ -1,195 +1,17 @@
 from __future__ import annotations
 
-import collections
 import concurrent.futures
-import dataclasses
-import hashlib
 import json
 import multiprocessing
 import pathlib
-import shutil
-import tempfile
 import time
 
-import PIL.Image
-
-import generator.engraving.audit
 import generator.engraving.cache
 import generator.engraving.fonts
-import generator.engraving.layout
-import generator.engraving.notation
-import generator.engraving.raster
 import generator.engraving.types
 import generator.engraving.validation
-import generator.model
-import generator.performance
-
-@dataclasses.dataclass
-class GroupResult:
-  scores: list = dataclasses.field(default_factory=list)
-  fullscore: int = 0
-  bodyhits: int = 0
-  bodies: int = 0
-  renderedpages: int = 0
-  rejected: int = 0
-  timings: dict = dataclasses.field(default_factory=lambda: dict.fromkeys(('layout', 'pagination', 'raster', 'cache'), 0.0))
-
-class BodyBands:
-  def __init__(self, root, persistent, result, limit=16 * 1024 * 1024):
-    self.root, self.persistent, self.result, self.limit = root, persistent, result, limit
-    self.images = collections.OrderedDict()
-    self.size = 0
-    self.known = {}
-    self.root.mkdir(parents=True, exist_ok=True)
-  def get(self, notation, page, band, fonts):
-    key = generator.engraving.cache.band_fingerprint(notation, page, band)
-    if key in self.images:
-      self.images.move_to_end(key)
-      self.result.bodyhits += 1
-      return self.images[key]
-    started = time.monotonic()
-    entry = self.root / (key + '.rgb')
-    image = None
-    if key in self.known:
-      if entry.is_symlink() or entry.stat().st_size != page.width * (band.bottom - band.top) * 3:
-        raise generator.engraving.types.EngravingError('temporary notation band was corrupted')
-      data = entry.read_bytes()
-      if hashlib.sha256(data).hexdigest() != self.known[key]:
-        raise generator.engraving.types.EngravingError('temporary notation band was corrupted')
-      image = PIL.Image.frombytes('RGB', (page.width, band.bottom - band.top), data)
-    elif self.persistent:
-      cached = self.persistent / 'notation' / key
-      image = generator.engraving.cache.restore_band(cached, key, page.width, band.bottom - band.top)
-      self.result.rejected += int(image is None and (cached.exists() or cached.is_symlink()))
-    self.result.timings['cache'] += time.monotonic() - started
-    if image is None:
-      started = time.monotonic()
-      image = generator.engraving.raster.rasterize_band(page, band.primitives, band.top, band.bottom, fonts)
-      self.result.timings['raster'] += time.monotonic() - started
-      self.result.bodies += 1
-      if self.persistent:
-        started = time.monotonic()
-        generator.engraving.cache.store_band(self.persistent / 'notation' / key, key, image)
-        self.result.timings['cache'] += time.monotonic() - started
-    else:
-      self.result.bodyhits += 1
-    if key not in self.known:
-      started = time.monotonic()
-      # Spill finalized pixels, so eviction never repeats rasterization or PNG decoding.
-      data = image.tobytes()
-      entry.write_bytes(data)
-      self.result.timings['cache'] += time.monotonic() - started
-      self.known[key] = hashlib.sha256(data).hexdigest()
-    size = image.width * image.height * 3
-    while self.images and self.size + size > self.limit:
-      _, old = self.images.popitem(last=False)
-      self.size -= old.width * old.height * 3
-      old.close()
-    self.images[key] = image
-    self.size += size
-    return image
-  def close(self):
-    for image in self.images.values():
-      image.close()
-    self.images.clear()
-    self.size = 0
-
-def clear_pages(destination):
-  if destination.is_symlink() or destination.is_file():
-    destination.unlink()
-  elif destination.exists():
-    shutil.rmtree(destination)
-
-def write_bundle(path, document, pages, fingerprint):
-  entries = [{k: v for k, v in {**page, 'path': path.with_suffix('.pages').name + '/' + page['path']}.items() if k != 'playback'} for page in pages]
-  normal = [entry for entry, page in zip(entries, pages) if not page['playback']]
-  playback = [entry for entry, page in zip(entries, pages) if page['playback']]
-  music = document
-  bundle = {'music': music, 'presentation': {'engine': generator.engraving.validation.ENGINE_NAME, 'style_sha256': fingerprint,
-    'width': generator.engraving.validation.PAGE_WIDTH, 'height': generator.engraving.validation.PAGE_HEIGHT, 'pages': normal, 'playback_pages': playback}}
-  validate_cached_views(document, pages)
-  bundle['performance'] = generator.performance.timing(document)
-  path.write_text(json.dumps(bundle, indent=2, ensure_ascii=False) + '\n', encoding='utf-8')
-
-def validate_cached_views(document, pages):
-  generator.performance.validate_views(document, [p for p in pages if not p['playback']], [p for p in pages if p['playback']])
-
-def engrave_group(notation, paths, root, font_path, cache_dir, fingerprint, progress=False):
-  family = generator.engraving.fonts.Fonts(font_path)
-  style = generator.engraving.types.Style()
-  systems = None
-  index = None
-  scores = {}
-  audited = set()
-  result = GroupResult()
-  with tempfile.TemporaryDirectory(prefix='vamp_notation_') as temporary:
-    bodies = BodyBands(pathlib.Path(temporary), cache_dir, result)
-    try:
-      for path, score_key in paths:
-        try:
-          document = json.loads(path.read_text(encoding='utf-8'))
-          style = generator.engraving.types.score_style(document, generator.engraving.types.Style())
-          destination = path.with_suffix('.pages')
-          clear_pages(destination)
-          destination.mkdir()
-          if score_key in scores:
-            source, pages = scores[score_key]
-            for page in pages:
-              shutil.copyfile(source / page['path'], destination / page['path'])
-            result.fullscore += 1
-          else:
-            if systems is None:
-              started = time.monotonic()
-              generator.model.validate_document(document)
-              index = generator.engraving.notation.ScoreIndex(document)
-              systems = generator.engraving.layout.layout_systems(document, style, family, index=index)
-              result.timings['layout'] += time.monotonic() - started
-            started = time.monotonic()
-            pages_layout, playback_layout = generator.engraving.layout.page_views(systems, document, family, style, index=index)
-            placement = tuple((page.width, page.height, tuple((system.start, system.end, system.box) for system in page.systems)) for page in pages_layout)
-            if placement not in audited:
-              layout = generator.engraving.types.LayoutDocument(pages_layout, style, fingerprint)
-              generator.engraving.audit.audit_layout(document, layout, family)
-              generator.engraving.audit.audit_playback(pages_layout, playback_layout, style, family)
-              audited.add(placement)
-            else:
-              for page in pages_layout + playback_layout:
-                decoration = tuple(p for p in page.primitives if p.role in ('title', 'subtitle', 'page_number'))
-                generator.engraving.audit.audit_primitives(decoration, page.width, page.height, family)
-            result.timings['pagination'] += time.monotonic() - started
-            pages = []
-            rendered = {}
-            for is_playback, page in [(False, p) for p in pages_layout] + [(True, p) for p in playback_layout]:
-              if page.primitives in rendered:
-                name, digest = rendered[page.primitives]
-                pages.append(dict(path=name, sha256=digest, playback=is_playback, **generator.performance.page_timing(document, page)))
-                continue
-              bands = generator.engraving.raster.page_bands(page)
-              if bands is None:
-                started = time.monotonic()
-                image = generator.engraving.raster.rasterize_page(page, family)
-              else:
-                body = bodies.get(notation, page, bands[1], family)
-                started = time.monotonic()
-                image = generator.engraving.raster.compose_page(page, family, bands, body)
-              target = destination / (('playback-' if is_playback else '') + f'{page.number:04}.png')
-              with image:
-                image.save(target, format='PNG')
-              result.timings['raster'] += time.monotonic() - started
-              result.renderedpages += 1
-              digest = hashlib.sha256(target.read_bytes()).hexdigest()
-              rendered[page.primitives] = target.name, digest
-              pages.append(dict(path=target.name, sha256=digest, playback=is_playback, **generator.performance.page_timing(document, page)))
-            scores[score_key] = (destination, pages)
-          write_bundle(path, document, pages, fingerprint)
-          result.scores.append((path, score_key, pages))
-          if progress and len(result.scores) % 50 == 0:
-            print(f'Engraving body {notation[:12]}: {len(result.scores)}/{len(paths)} scores, fullscore {result.fullscore}, bodyhits {result.bodyhits}, bodies {result.bodies}, renderedpages {result.renderedpages}', flush=True)
-        except ValueError as error:
-          raise generator.engraving.types.EngravingError(f'{path.relative_to(root)}: {error}') from error
-    finally:
-      bodies.close()
-  return result
+import generator.engraving.worker
+import generator.validation
 
 def engrave_tree(root: pathlib.Path, fonts: pathlib.Path, *, cache_dir: pathlib.Path | None = None, jobs: int = 1, progress: bool = False) -> set[str]:
   if type(jobs) is not int or not 1 <= jobs <= 16:
@@ -211,22 +33,22 @@ def engrave_tree(root: pathlib.Path, fonts: pathlib.Path, *, cache_dir: pathlib.
       print(f'Engraving cache enabled: {cache_dir} ({state}; renderer {renderer})', flush=True)
   elif progress:
     print('Engraving cache disabled', flush=True)
-  totals = GroupResult()
+  totals = generator.engraving.worker.GroupResult()
   missing, rejected, completed = 0, 0, 0
   groups = {}
   for path in paths:
     try:
       document = json.loads(path.read_text(encoding='utf-8'))
-      generator.model.validate_document(document)
+      generator.validation.validate_document(document)
       score_key = generator.engraving.cache.score_fingerprint(document, renderer)
       destination = path.with_suffix('.pages')
-      clear_pages(destination)
+      generator.engraving.worker.clear_pages(destination)
       entry = cache_dir / 'fullscore' / score_key if cache_dir else None
       present = entry is not None and (entry.exists() or entry.is_symlink())
       pages = generator.engraving.cache.restore_pages(entry, destination, validated_pages) if entry else None
       if pages is not None:
         try:
-          validate_cached_views(document, pages)
+          generator.engraving.worker.validate_cached_views(document, pages)
         except (ValueError, TypeError, KeyError):
           pages = None
       if pages is None:
@@ -235,7 +57,7 @@ def engrave_tree(root: pathlib.Path, fonts: pathlib.Path, *, cache_dir: pathlib.
         notation = generator.engraving.cache.notation_fingerprint(document, renderer)
         groups.setdefault(notation, []).append((path, score_key))
       else:
-        write_bundle(path, document, pages, fingerprint)
+        generator.engraving.worker.write_bundle(path, document, pages, fingerprint)
         totals.fullscore += 1
         completed += 1
     except ValueError as error:
@@ -272,10 +94,10 @@ def engrave_tree(root: pathlib.Path, fonts: pathlib.Path, *, cache_dir: pathlib.
   jobs = min(jobs, len(groups))
   if jobs <= 1:
     for notation, group in groups.items():
-      finish(engrave_group(notation, group, root, fonts, cache_dir, fingerprint, progress))
+      finish(generator.engraving.worker.engrave_group(notation, group, root, fonts, cache_dir, fingerprint, progress))
   else:
     with concurrent.futures.ProcessPoolExecutor(max_workers=jobs, mp_context=multiprocessing.get_context('spawn')) as pool:
-      futures = [pool.submit(engrave_group, notation, group, root, fonts, cache_dir, fingerprint, progress) for notation, group in groups.items()]
+      futures = [pool.submit(generator.engraving.worker.engrave_group, notation, group, root, fonts, cache_dir, fingerprint, progress) for notation, group in groups.items()]
       for future in concurrent.futures.as_completed(futures):
         finish(future.result())
   if progress:

generator/engraving/worker.py

@@ -0,0 +1,190 @@
+from __future__ import annotations
+
+import collections
+import dataclasses
+import hashlib
+import json
+import pathlib
+import shutil
+import tempfile
+import time
+
+import PIL.Image
+
+import generator.engraving.audit
+import generator.engraving.cache
+import generator.engraving.fonts
+import generator.engraving.layout
+import generator.engraving.notation
+import generator.engraving.raster
+import generator.engraving.types
+import generator.engraving.validation
+import generator.performance
+import generator.validation
+
+@dataclasses.dataclass
+class GroupResult:
+  scores: list = dataclasses.field(default_factory=list)
+  fullscore: int = 0
+  bodyhits: int = 0
+  bodies: int = 0
+  renderedpages: int = 0
+  rejected: int = 0
+  timings: dict = dataclasses.field(default_factory=lambda: dict.fromkeys(('layout', 'pagination', 'raster', 'cache'), 0.0))
+
+class BodyBands:
+  def __init__(self, root, persistent, result, limit=16 * 1024 * 1024):
+    self.root, self.persistent, self.result, self.limit = root, persistent, result, limit
+    self.images = collections.OrderedDict()
+    self.size = 0
+    self.known = {}
+    self.root.mkdir(parents=True, exist_ok=True)
+  def get(self, notation, page, band, fonts):
+    key = generator.engraving.cache.band_fingerprint(notation, page, band)
+    if key in self.images:
+      self.images.move_to_end(key)
+      self.result.bodyhits += 1
+      return self.images[key]
+    started = time.monotonic()
+    entry = self.root / (key + '.rgb')
+    image = None
+    if key in self.known:
+      if entry.is_symlink() or entry.stat().st_size != page.width * (band.bottom - band.top) * 3:
+        raise generator.engraving.types.EngravingError('temporary notation band was corrupted')
+      data = entry.read_bytes()
+      if hashlib.sha256(data).hexdigest() != self.known[key]:
+        raise generator.engraving.types.EngravingError('temporary notation band was corrupted')
+      image = PIL.Image.frombytes('RGB', (page.width, band.bottom - band.top), data)
+    elif self.persistent:
+      cached = self.persistent / 'notation' / key
+      image = generator.engraving.cache.restore_band(cached, key, page.width, band.bottom - band.top)
+      self.result.rejected += int(image is None and (cached.exists() or cached.is_symlink()))
+    self.result.timings['cache'] += time.monotonic() - started
+    if image is None:
+      started = time.monotonic()
+      image = generator.engraving.raster.rasterize_band(page, band.primitives, band.top, band.bottom, fonts)
+      self.result.timings['raster'] += time.monotonic() - started
+      self.result.bodies += 1
+      if self.persistent:
+        started = time.monotonic()
+        generator.engraving.cache.store_band(self.persistent / 'notation' / key, key, image)
+        self.result.timings['cache'] += time.monotonic() - started
+    else:
+      self.result.bodyhits += 1
+    if key not in self.known:
+      started = time.monotonic()
+      # Spill finalized pixels, so eviction never repeats rasterization or PNG decoding.
+      data = image.tobytes()
+      entry.write_bytes(data)
+      self.result.timings['cache'] += time.monotonic() - started
+      self.known[key] = hashlib.sha256(data).hexdigest()
+    size = image.width * image.height * 3
+    while self.images and self.size + size > self.limit:
+      _, old = self.images.popitem(last=False)
+      self.size -= old.width * old.height * 3
+      old.close()
+    self.images[key] = image
+    self.size += size
+    return image
+  def close(self):
+    for image in self.images.values():
+      image.close()
+    self.images.clear()
+    self.size = 0
+
+def clear_pages(destination):
+  if destination.is_symlink() or destination.is_file():
+    destination.unlink()
+  elif destination.exists():
+    shutil.rmtree(destination)
+
+def write_bundle(path, document, pages, fingerprint):
+  entries = [{k: v for k, v in {**page, 'path': path.with_suffix('.pages').name + '/' + page['path']}.items() if k != 'playback'} for page in pages]
+  normal = [entry for entry, page in zip(entries, pages) if not page['playback']]
+  playback = [entry for entry, page in zip(entries, pages) if page['playback']]
+  music = document
+  bundle = {'music': music, 'presentation': {'engine': generator.engraving.validation.ENGINE_NAME, 'style_sha256': fingerprint,
+    'width': generator.engraving.validation.PAGE_WIDTH, 'height': generator.engraving.validation.PAGE_HEIGHT, 'pages': normal, 'playback_pages': playback}}
+  validate_cached_views(document, pages)
+  bundle['performance'] = generator.performance.timing(document)
+  path.write_text(json.dumps(bundle, indent=2, ensure_ascii=False) + '\n', encoding='utf-8')
+
+def validate_cached_views(document, pages):
+  generator.performance.validate_views(document, [p for p in pages if not p['playback']], [p for p in pages if p['playback']])
+
+def engrave_group(notation, paths, root, font_path, cache_dir, fingerprint, progress=False):
+  family = generator.engraving.fonts.Fonts(font_path)
+  style = generator.engraving.types.Style()
+  systems = None
+  index = None
+  scores = {}
+  audited = set()
+  result = GroupResult()
+  with tempfile.TemporaryDirectory(prefix='vamp_notation_') as temporary:
+    bodies = BodyBands(pathlib.Path(temporary), cache_dir, result)
+    try:
+      for path, score_key in paths:
+        try:
+          document = json.loads(path.read_text(encoding='utf-8'))
+          style = generator.engraving.types.score_style(document, generator.engraving.types.Style())
+          destination = path.with_suffix('.pages')
+          clear_pages(destination)
+          destination.mkdir()
+          if score_key in scores:
+            source, pages = scores[score_key]
+            for page in pages:
+              shutil.copyfile(source / page['path'], destination / page['path'])
+            result.fullscore += 1
+          else:
+            if systems is None:
+              started = time.monotonic()
+              generator.validation.validate_document(document)
+              index = generator.engraving.notation.ScoreIndex(document)
+              systems = generator.engraving.layout.layout_systems(document, style, family, index=index)
+              result.timings['layout'] += time.monotonic() - started
+            started = time.monotonic()
+            pages_layout, playback_layout = generator.engraving.layout.page_views(systems, document, family, style, index=index)
+            placement = tuple((page.width, page.height, tuple((system.start, system.end, system.box) for system in page.systems)) for page in pages_layout)
+            if placement not in audited:
+              layout = generator.engraving.types.LayoutDocument(pages_layout, style, fingerprint)
+              generator.engraving.audit.audit_layout(document, layout, family)
+              generator.engraving.audit.audit_playback(pages_layout, playback_layout, style, family)
+              audited.add(placement)
+            else:
+              for page in pages_layout + playback_layout:
+                decoration = tuple(p for p in page.primitives if p.role in ('title', 'subtitle', 'page_number'))
+                generator.engraving.audit.audit_primitives(decoration, page.width, page.height, family)
+            result.timings['pagination'] += time.monotonic() - started
+            pages = []
+            rendered = {}
+            for is_playback, page in [(False, p) for p in pages_layout] + [(True, p) for p in playback_layout]:
+              if page.primitives in rendered:
+                name, digest = rendered[page.primitives]
+                pages.append(dict(path=name, sha256=digest, playback=is_playback, **generator.performance.page_timing(document, page)))
+                continue
+              bands = generator.engraving.raster.page_bands(page)
+              if bands is None:
+                started = time.monotonic()
+                image = generator.engraving.raster.rasterize_page(page, family)
+              else:
+                body = bodies.get(notation, page, bands[1], family)
+                started = time.monotonic()
+                image = generator.engraving.raster.compose_page(page, family, bands, body)
+              target = destination / (('playback-' if is_playback else '') + f'{page.number:04}.png')
+              with image:
+                image.save(target, format='PNG')
+              result.timings['raster'] += time.monotonic() - started
+              result.renderedpages += 1
+              digest = hashlib.sha256(target.read_bytes()).hexdigest()
+              rendered[page.primitives] = target.name, digest
+              pages.append(dict(path=target.name, sha256=digest, playback=is_playback, **generator.performance.page_timing(document, page)))
+            scores[score_key] = (destination, pages)
+          write_bundle(path, document, pages, fingerprint)
+          result.scores.append((path, score_key, pages))
+          if progress and len(result.scores) % 50 == 0:
+            print(f'Engraving body {notation[:12]}: {len(result.scores)}/{len(paths)} scores, fullscore {result.fullscore}, bodyhits {result.bodyhits}, bodies {result.bodies}, renderedpages {result.renderedpages}', flush=True)
+        except ValueError as error:
+          raise generator.engraving.types.EngravingError(f'{path.relative_to(root)}: {error}') from error
+    finally:
+      bodies.close()
+  return result

generator/guitar.py

@@ -7,10 +7,10 @@ import itertools
 import logging
 import typing
 
+import generator.chord_families
 import generator.fretboard
 import generator.model
 import generator.pitch
-import generator.chord_families
 import generator.vocabulary
 
 PREFERRED_MAX_FRET = 12
@@ -31,9 +31,10 @@ class VoicingCost(typing.NamedTuple):
   def moved(self, distance: int) -> VoicingCost:
     return self._replace(effort=self.effort + distance)
 
-def voicing_preference(role, center=None):
+def voicing_preference(role, center=None, *, composition=None):
   """Immutable role settings for both selection and scored-retrieval caches."""
-  settings = generator.vocabulary.role(role)
+  composition = composition or generator.vocabulary.current()
+  settings = composition.role(role)
   policy = settings.get('voicing', {})
   center = settings.get('center', 53 if role.startswith('acoustic') else 60) if center is None else center
   result = tuple(fractions.Fraction(str(v)) for v in (center, policy.get('register_weight', 1), policy.get('high_position_weight', 1), policy.get('comfortable_max_fret', 9)))
@@ -50,9 +51,10 @@ def voicing_preferences(positions, preference):
   position = max(fractions.Fraction(), fractions.Fraction(sum(p.fret for p in positions), len(positions)) - threshold)
   return register_weight * register, position_weight * position
 
-def voicing_cost(positions: tuple[generator.fretboard.TabPosition, ...], profile: str = '', allowed: tuple[int, ...] = (), melody: tuple[int, ...] = ()) -> VoicingCost:
-  families = tuple(generator.vocabulary.accompaniment('electric')['voicing_families']['rank'].items()) if profile == 'electric' else ()
-  preference = voicing_preference('electric_melody') if profile == 'electric' and melody else ()
+def voicing_cost(positions: tuple[generator.fretboard.TabPosition, ...], profile: str = '', allowed: tuple[int, ...] = (), melody: tuple[int, ...] = (), *, composition=None) -> VoicingCost:
+  composition = composition or generator.vocabulary.current()
+  families = tuple(composition.accompaniment('electric')['voicing_families']['rank'].items()) if profile == 'electric' else ()
+  preference = voicing_preference('electric_melody', composition=composition) if profile == 'electric' and melody else ()
   return _voicing_cost(positions, profile, allowed, melody, families, preference)
 
 @functools.lru_cache(maxsize=8192)
@@ -69,9 +71,10 @@ def _voicing_cost(positions, profile, allowed, melody, families, preference):
   melody_string = min((p.string_number - 1 for p in positions if melody and p.midi == max(melody)), default=0)
   return VoicingCost(extended_grips=generator.fretboard.span_penalty(positions), melody_string=melody_string, string_shortfall=max(0, 4 - len(positions)) if profile == 'electric' else 0, covered_melody=int(bool(above)), covering_tones=above, family_penalty=family_penalty, effort=effort, missing_tones=len(set(allowed) - classes))
 
-def chord_positions(requirements: generator.fretboard.ChordRequirements, prune: bool = True, profile: str = '', melody: tuple[int, ...] = ()) -> tuple[tuple[generator.fretboard.TabPosition, ...], ...]:
-  families = tuple(generator.vocabulary.accompaniment('electric')['voicing_families']['rank'].items()) if profile == 'electric' else ()
-  preference = voicing_preference('electric_melody') if profile == 'electric' and melody else ()
+def chord_positions(requirements: generator.fretboard.ChordRequirements, prune: bool = True, profile: str = '', melody: tuple[int, ...] = (), *, composition=None) -> tuple[tuple[generator.fretboard.TabPosition, ...], ...]:
+  composition = composition or generator.vocabulary.current()
+  families = tuple(composition.accompaniment('electric')['voicing_families']['rank'].items()) if profile == 'electric' else ()
+  preference = voicing_preference('electric_melody', composition=composition) if profile == 'electric' and melody else ()
   return _chord_positions(requirements, prune, profile, melody, families, preference)
 
 @functools.lru_cache(maxsize=256)
@@ -211,7 +214,8 @@ def continued_sources(notes):
 def has_attack(notes: list[dict]) -> bool:
   return any('_harmony' in n and '_continue_from' not in n or any('tie_from' not in p for p in n['pitches']) for n in notes)
 
-def attack_candidates(notes: list[dict], tones: dict[str, GuitarTone], active: dict, held: dict, preserve_strings: bool, fallback: bool = False):
+def attack_candidates(notes: list[dict], tones: dict[str, GuitarTone], active: dict, held: dict, preserve_strings: bool, fallback: bool = False, *, composition=None):
+  composition = composition or generator.vocabulary.current()
   pending = []
   chords = [n for n in notes if '_harmony' in n]
   chord = chords[0] if chords else None
@@ -229,7 +233,7 @@ def attack_candidates(notes: list[dict], tones: dict[str, GuitarTone], active: d
   requirements = generator.fretboard.chord_requirements(chord['_harmony'], respect_bass=profile != 'electric') if chord else generator.fretboard.ChordRequirements()
   melody = tuple(sorted({p.midi for _, options in pending for p in options})) if profile == 'electric' else ()
   fresh = has_attack(notes)
-  candidates = chord_positions(requirements, prune=not (preserve_strings or held or pending or profile), profile=profile, melody=melody) if chord else [()]
+  candidates = chord_positions(requirements, prune=not (preserve_strings or held or pending or profile), profile=profile, melody=melody, composition=composition) if chord else [()]
   for exact in extend_positions(held, sorted(pending, key=lambda p: len(p[1]))):
     exact_positions = dict(exact)
     for positions in candidates:
@@ -253,7 +257,7 @@ def attack_candidates(notes: list[dict], tones: dict[str, GuitarTone], active: d
           choices.append(EventPlacement(note, tuple((p, exact_positions[p['id']]) for p in note['pitches']), added))
         if not compatible or not playable_choices(choices):
           continue
-        cost = voicing_cost(positions, profile, requirements.allowed, melody) if fresh else VoicingCost()
+        cost = voicing_cost(positions, profile, requirements.allowed, melody, composition=composition) if fresh else VoicingCost()
         yield AttackCandidate(tuple(sorted(assignment.items())), tuple(choices), positions, cost)
       else:
         shape = tuple(p for _, p in exact) + positions
@@ -263,7 +267,7 @@ def attack_candidates(notes: list[dict], tones: dict[str, GuitarTone], active: d
         choices = tuple(EventPlacement(n, tuple((p, exact_positions[p['id']]) for p in n['pitches']), positions if n is chord else ()) for n in notes)
         if not playable_choices(choices):
           continue
-        cost = voicing_cost(shape, profile, requirements.allowed, melody) if fresh else VoicingCost()
+        cost = voicing_cost(shape, profile, requirements.allowed, melody, composition=composition) if fresh else VoicingCost()
         yield AttackCandidate(assignment, choices, shape, cost)
 
 def guitar_events(document: dict) -> tuple[list, dict[str, GuitarTone]]:
@@ -325,7 +329,8 @@ def advance_attacks(candidates, states: dict) -> dict:
       result[candidate.active] = Placement(cost, candidate.active, previous, candidate.choices)
   return result
 
-def advance_positions(notes: list[dict], onset: fractions.Fraction, tones: dict[str, GuitarTone], states: dict, preserve_strings: bool, fallback: bool = False) -> dict:
+def advance_positions(notes: list[dict], onset: fractions.Fraction, tones: dict[str, GuitarTone], states: dict, preserve_strings: bool, fallback: bool = False, *, composition=None) -> dict:
+  composition = composition or generator.vocabulary.current()
   continued = continued_sources(notes)
   fresh = has_attack(notes)
   held_ids = {i for previous in states.values() for i, _ in previous.active if retain_tone(tones[i], onset)}
@@ -337,29 +342,30 @@ def advance_positions(notes: list[dict], onset: fractions.Fraction, tones: dict[
       result[candidate.active] = Placement(cost, candidate.active, previous, candidate.choices)
 
   if not held_ids and not continued:
-    result = advance_attacks(attack_candidates(notes, tones, {}, {}, preserve_strings, fallback), states)
+    result = advance_attacks(attack_candidates(notes, tones, {}, {}, preserve_strings, fallback, composition=composition), states)
   else:
     for previous in states.values():
       active = dict(previous.active)
       held = {i: p for i, p in previous.active if i not in continued and i in held_ids}
-      for candidate in attack_candidates(notes, tones, active, held, preserve_strings, fallback):
+      for candidate in attack_candidates(notes, tones, active, held, preserve_strings, fallback, composition=composition):
         distance = movement(tuple(p for _, p in previous.active), candidate.shape) if fresh else 0
         keep(candidate, previous, previous.cost.moved(distance))
   if not result:
     if not fallback and any(n.get('_harmonize') for n in notes):
-      return advance_positions(notes, onset, tones, states, preserve_strings, True)
+      return advance_positions(notes, onset, tones, states, preserve_strings, True, composition=composition)
     raise ValueError(f'event {notes[0]["id"]}, onset {onset}: no guitar placement; single notes need frets {generator.fretboard.MIN_FRET}-{generator.fretboard.MAX_SINGLE_FRET}; chords need frets {generator.fretboard.MIN_FRET}-{generator.fretboard.MAX_FRET}, an anchor <= {generator.fretboard.MAX_ANCHOR_FRET}, and at most {generator.fretboard.MAX_FRET_SPAN} consecutive fret positions; held/tied strings must remain fixed')
   return result
 
-def place_guitar(document: dict) -> None:
+def place_guitar(document: dict, *, composition=None) -> None:
   """Choose electric voicings or exact/held melody placements."""
+  composition = composition or generator.vocabulary.current()
   events, tones = guitar_events(document)
   states = {(): Placement(VoicingCost(), (), None, ())}
   groups = [(key[0], [item[3] for item in values]) for key, values in itertools.groupby(events, key=lambda e: e[:3])]
   for index, (onset, notes) in enumerate(groups):
     next_onset = groups[index + 1][0] if index + 1 < len(groups) else None
     preserve_strings = next_onset is not None and any('_harmony' in n and onset + generator.model.fraction(n['duration']) > next_onset for n in notes)
-    states = advance_positions(notes, onset, tones, states, preserve_strings)
+    states = advance_positions(notes, onset, tones, states, preserve_strings, composition=composition)
   selected = min(states.values(), key=lambda s: (s.cost, s.active))
   skipped = []
   while selected.previous is not None:

generator/model.py

@@ -3,8 +3,6 @@ from __future__ import annotations
 import copy
 import fractions
 import typing
-import generator.fretboard
-import generator.pitch
 
 MAX_INTEGER = (1 << 63) - 1
 NOTE_TYPES = ('maxima', 'long', 'breve', 'whole', 'half', 'quarter', 'eighth', '16th', '32nd', '64th', '128th', '256th', '512th', '1024th')
@@ -140,157 +138,3 @@ def make_event(identifier: str, onset: fractions.Fraction, duration: fractions.F
     if 'alter' in event['pitches'][-1]:
       event['pitches'][-1]['alter'] = rational(alteration(p['alter']))
   return event
-
-def validate_document(document: dict) -> None:
-  if not isinstance(document, dict) or not isinstance(document.get('metadata'), dict):
-    raise ValueError('score requires a metadata object')
-  if any(not isinstance(document['metadata'].get(name), str) for name in ('title', 'composer', 'key_signature', 'time_signature')):
-    raise ValueError('score metadata requires title, composer, key_signature and time_signature strings')
-  staves = document.get('staves', [])
-  if not isinstance(staves, list) or len(staves) != 1 or not isinstance(staves[0], dict):
-    raise ValueError('exactly one staff is currently supported')
-  if not isinstance(staves[0].get('measures'), list) or not staves[0]['measures']:
-    raise ValueError('score requires measures')
-  instruments = staves[0].get('instruments', [])
-  instrument_ids = {i['id'] for i in instruments}
-  if len(instruments) != len(instrument_ids) or any(not isinstance(i, str) or not i for i in instrument_ids):
-    raise ValueError('score instruments require unique nonempty IDs')
-  ids: set[str] = set()
-  events: dict[str, dict] = {}
-  tones: dict[str, tuple[dict, fractions.Fraction, fractions.Fraction, str]] = {}
-  ownership: dict[str, str] = {}
-  absolute = fractions.Fraction()
-  for measure in staves[0]['measures']:
-    if {'repeats', 'endings', 'initial_attributes'} & measure.keys():
-      raise ValueError('repeat notation is unsupported; regenerate scores from music written out in reading order')
-    length = fraction(measure['duration'])
-    if length <= 0:
-      raise ValueError('measure duration must be positive')
-    voices = measure.get('voices', [])
-    if not voices or len({v['id'] for v in voices}) != len(voices) or any(not v['id'] for v in voices):
-      raise ValueError('measure requires uniquely identified voice lanes')
-    previous_ends = {v['id']: fractions.Fraction() for v in voices}
-    for voice, note in ((v, n) for v in voices for n in v['notes']):
-      if any({'sound_ties', 'printed_ties'} & pitch.keys() for pitch in note['pitches']):
-        raise ValueError('pass-specific tie metadata is unsupported; regenerate scores')
-      if not note['id'] or note['id'] in ids:
-        raise ValueError(f'duplicate event ID: {note["id"]}')
-      ids.add(note['id'])
-      events[note['id']] = note
-      ownership[note['id']] = voice['id']
-      if note.get('voice', voice['id']) != voice['id']:
-        raise ValueError('event voice ownership disagrees with its lane')
-      onset, duration = fraction(note['onset']), fraction(note['duration'])
-      if onset < 0 or duration < 0 or onset + duration > length:
-        raise ValueError(f'event outside measure: {note["id"]}')
-      if note['kind'] not in ('note', 'rest', 'spacer'):
-        raise ValueError('invalid event kind')
-      if not note.get('grace'):
-        if duration == 0 or onset < previous_ends[voice['id']]:
-          raise ValueError('overlapping events in one rhythmic voice')
-        previous_ends[voice['id']] = onset + duration
-      elif duration != 0:
-        raise ValueError('grace notes must not advance metric time')
-      expected = metric_length(note['written'])
-      if not note.get('grace') and not note.get('measure_rest') and duration != expected:
-        raise ValueError('metric and written event durations disagree')
-      if note['kind'] != 'note' and note.get('pitches'):
-        raise ValueError('rests and spacers cannot contain pitches')
-      if note['kind'] == 'note' and not note.get('pitches') and note.get('notehead') != 'slash':
-        raise ValueError('pitched event requires at least one pitch')
-      for p in note.get('pitches', []):
-        if p.get('instrument') and p['instrument'] not in instrument_ids:
-          raise ValueError('note refers to an unknown score instrument')
-        if 'unpitched' in p:
-          position = p['unpitched']
-          if position['letter'] not in LETTERS or type(position['octave']) is not int:
-            raise ValueError('invalid unpitched display position')
-        else:
-          pitch_number(p)
-        if staves[0].get('tablature'):
-          tab = p.get('tablature', {})
-          string, fret = tab.get('string'), tab.get('fret')
-          if type(string) is not int or type(fret) is not int or string not in dict(generator.fretboard.STANDARD_TUNING) or not generator.fretboard.MIN_FRET <= fret <= generator.fretboard.MAX_FRET:
-            raise ValueError(f'{p["id"]}: invalid generated tablature position')
-          state = dict(measure)
-          for change in measure.get('attributes', []):
-            if fraction(change['onset']) <= onset:
-              state.update(change)
-          if dict(generator.fretboard.STANDARD_TUNING)[string] + fret != generator.pitch.sounding_pitch(p, state):
-            raise ValueError(f'{p["id"]}: tablature disagrees with sounding pitch')
-        pd = fraction(p['duration'])
-        if pd < 0 or (pd == 0 and not note.get('grace')) or pd > duration or not p['id'] or p['id'] in tones:
-          raise ValueError('invalid chord-tone duration or duplicate ID')
-        if not note.get('grace') and pd != metric_length(p['written']):
-          raise ValueError('metric and written chord-tone durations disagree')
-        tones[p['id']] = (p, absolute + onset, pd, voice['id'])
-    for harmony in measure.get('harmonies', []):
-      if not 0 <= fraction(harmony['onset']) < length:
-        raise ValueError('harmony outside measure')
-    for change in measure.get('attributes', []):
-      if not 0 <= fraction(change['onset']) < length:
-        raise ValueError('attribute change outside measure')
-    for state in [measure, *measure.get('attributes', [])]:
-      if 'meter' in state:
-        validate_meter(state['meter'])
-      if 'key' in state:
-        if len(state['key']['alterations']) != 7:
-          raise ValueError('key alterations must contain seven exact alterations')
-        for value in state['key']['alterations']:
-          alteration(value)
-      if 'clef' in state and (state['clef']['sign'] not in ('G', 'F', 'C', 'percussion', 'none') or not 1 <= state['clef']['line'] <= 6 or not -3 <= state['clef']['octave_change'] <= 3):
-        raise ValueError('invalid clef')
-    absolute += length
-  for identifier, (p, start, duration, voice) in tones.items():
-    if target := p.get('tie_to'):
-      if target not in tones:
-        raise ValueError(f'dangling tie: {identifier} -> {target}')
-      other, next_start, _, next_voice = tones[target]
-      same_pitch = all(p.get(k) == other.get(k) for k in ('unpitched', 'instrument')) if 'unpitched' in p else all(p.get(k) == other.get(k) for k in ('letter', 'alter', 'octave'))
-      if other.get('tie_from') != identifier or not same_pitch or next_voice != voice or next_start != start + duration:
-        raise ValueError(f'invalid tie: {identifier} -> {target}')
-      if staves[0].get('tablature') and p['tablature'] != other['tablature']:
-        raise ValueError(f'{identifier}: tied guitar tones must retain their string and fret')
-    if source := p.get('tie_from'):
-      if source not in tones or tones[source][0].get('tie_to') != identifier:
-        raise ValueError(f'dangling incoming tie: {identifier}')
-  span_ids = set()
-  event_order = {identifier: index for index, identifier in enumerate(events)}
-  for span in staves[0].get('tuplets', []):
-    if not span['id'] or span['id'] in span_ids or span['start'] not in ids or span['end'] not in ids or not 0 < span['note_count'] <= 1000000 or not 0 < span['space_count'] <= 1000000 or not 0 <= span['depth'] <= 16:
-      raise ValueError('invalid tuplet span')
-    members = [n['id'] for n in events.values() if span['id'] in n.get('tuplet_ids', [])]
-    if not members or members[0] != span['start'] or members[-1] != span['end'] or event_order[span['start']] > event_order[span['end']]:
-      raise ValueError('tuplet endpoints disagree with member events')
-    if len({ownership[n] for n in members}) != 1 or span.get('voice', ownership[members[0]]) != ownership[members[0]]:
-      raise ValueError('tuplet crosses independent voice lanes')
-    if span.get('bracket', 'auto') not in ('yes', 'no', 'auto') or span.get('show_number', 'actual') not in ('none', 'actual', 'both'):
-      raise ValueError('invalid tuplet appearance')
-    span_ids.add(span['id'])
-  for note in events.values():
-    for verse in note.get('lyrics', []):
-      start = tones[note['pitches'][0]['id']][1] if note['pitches'] else fractions.Fraction()
-      if 'extend_to' in verse and not start < fraction(verse['extend_to']) <= absolute:
-        raise ValueError('lyric extension endpoint is outside the score')
-    if any(t not in span_ids for t in note.get('tuplet_ids', [])):
-      raise ValueError('event refers to an unknown tuplet')
-    if tremolo := note.get('tremolo'):
-      if tremolo['type'] not in ('single', 'start', 'stop') or type(tremolo['marks']) is not int or not 0 <= tremolo['marks'] <= 8:
-        raise ValueError('invalid measured tremolo')
-      if tremolo['type'] in ('start', 'stop'):
-        other = events.get(tremolo.get('partner'))
-        expected_type = 'stop' if tremolo['type'] == 'start' else 'start'
-        if not other or other.get('tremolo', {}).get('partner') != note['id'] or other['tremolo']['type'] != expected_type or other['tremolo']['marks'] != tremolo['marks']:
-          raise ValueError('invalid paired tremolo')
-  for span in staves[0].get('octave_shifts', []):
-    if not 0 <= fraction(span['start']) < fraction(span['end']) <= absolute or not -3 <= span['octaves'] <= 3:
-      raise ValueError('invalid octave-shift span')
-  previous_end = fractions.Fraction()
-  for span in staves[0].get('scale_spans', []):
-    start, end = fraction(span['start']), fraction(span['end'])
-    if not previous_end <= start < end <= absolute or not isinstance(span.get('label'), str) or not span['label'].strip() or type(span.get('matched')) is not bool:
-      raise ValueError('invalid solo scale span')
-    if not isinstance(span.get('id'), str) or not span['id'] or span['id'] in ids:
-      raise ValueError('solo scale spans require unique IDs')
-    ids.add(span['id'])
-    previous_end = end

generator/musicxml.py

@@ -8,6 +8,7 @@ import defusedxml.ElementTree
 
 import generator.harmony
 import generator.model
+import generator.validation
 
 class MusicXMLError(ValueError):
   pass
@@ -621,7 +622,7 @@ class Importer:
     metadata.update({'key_signature': measures[0]['key_signature'], 'time_signature': measures[0]['time_signature'], 'key': copy.deepcopy(measures[0]['key'])})
     result = {'metadata': metadata, 'staves': [{'id': 'staff_source', 'name': definitions[0].findtext('part-name', 'Source'), 'role': 'vocal', 'clef': copy.deepcopy(measures[0]['clef']), 'tuplets': self.tuplets, 'octave_shifts': self.octaves, 'measures': measures}]}
     result['staves'][0]['instruments'] = instruments
-    generator.model.validate_document(result)
+    generator.validation.validate_document(result)
     return result
 
 def load_musicxml(path: pathlib.Path) -> dict:

generator/musicxml_writer.py

@@ -7,6 +7,7 @@ import xml.etree.ElementTree
 
 import generator.harmony
 import generator.model
+import generator.validation
 
 def element(parent, tag, content=None, **attributes):
   node = xml.etree.ElementTree.SubElement(parent, tag, {k.replace('_', '-'): str(v) for k, v in attributes.items() if v is not None})
@@ -209,7 +210,7 @@ def write_barline(bar, measure, location):
   element(node, 'bar-style', measure.get('barlines', {}).get(location, 'regular'))
 
 def musicxml_tree(document: dict, *, tablature=False) -> xml.etree.ElementTree.ElementTree:
-  generator.model.validate_document(document)
+  generator.validation.validate_document(document)
   root = xml.etree.ElementTree.Element('score-partwise', {'version': '4.0'})
   metadata, staff = document['metadata'], document['staves'][0]
   element(element(root, 'work'), 'work-title', metadata['title'])

generator/parts.py

@@ -3,7 +3,6 @@ from __future__ import annotations
 import fractions
 import pathlib
 
-import generator.accompaniment
 import generator.fretboard
 import generator.model
 import generator.pitch

generator/performance.py

@@ -1,14 +1,8 @@
-"""Page timing and supplied recordings. No note synthesis or audio transformation."""
+"""Pure page coverage and performance timing."""
 from __future__ import annotations
 
 import fractions
-import hashlib
-import json
 import math
-import pathlib
-import re
-import shutil
-import wave
 
 import generator.model
 
@@ -117,57 +111,3 @@ def timing(document):
   return dict(song_id=document['metadata'].get('song_id', ''),
     duration=generator.model.rational(times[-1]),
     sustain_start=generator.model.rational(sustain_start), count_in_beats=4, clicks=clicks)
-
-def describe_wav(path, root, *, song_id='', bpm=0):
-  if path.is_symlink() or not path.is_file() or not path.resolve().is_relative_to(root.resolve()):
-    raise ValueError(f'{path}: expected a contained regular WAV')
-  try:
-    with wave.open(str(path), 'rb') as audio:
-      channels, rate, frames = audio.getnchannels(), audio.getframerate(), audio.getnframes()
-      if audio.getcomptype() != 'NONE' or audio.getsampwidth() != 2 or channels not in (1, 2) or not 8000 <= rate <= 192000 or frames <= 0:
-        raise ValueError('expected nonempty mono/stereo PCM16 WAV')
-      remaining = frames
-      while remaining:
-        count = min(remaining, 65536)
-        if len(audio.readframes(count)) != count * channels * 2:
-          raise ValueError('truncated WAV data')
-        remaining -= count
-  except (wave.Error, EOFError) as error:
-    raise ValueError(f'{path}: invalid WAV: {error}') from error
-  with path.open('rb') as source:
-    digest = hashlib.file_digest(source, 'sha256').hexdigest()
-  return dict(path=path.relative_to(root).as_posix(), sha256=digest, channels=channels, sample_rate=rate, frames=frames, song_id=song_id, bpm=bpm)
-
-def copy_audio(static, destination, song_ids):
-  entries = []
-  inputs = [(static / 'sounds' / name, '', 0) for name in ('beat1.wav', 'beat2.wav')]
-  for path in sorted((static / 'backing').glob('*.wav')):
-    match = re.fullmatch(r'([a-z0-9]+(?:_[a-z0-9]+)*)_(60|70|80|90|100|110|120)\.wav', path.name)
-    if not match:
-      raise ValueError(f'{path}: unsupported backing filename or tempo')
-    if match[1] not in song_ids:
-      continue
-    inputs.append((path, match[1], int(match[2])))
-  for source, song_id, bpm in inputs:
-    description = describe_wav(source, static, song_id=song_id, bpm=bpm)
-    target = destination / description['path']
-    target.parent.mkdir(parents=True, exist_ok=True)
-    shutil.copyfile(source, target)
-    entries.append(description)
-  return entries
-
-def validate_recording_lengths(root, entries):
-  durations = {}
-  for path in sorted((root / 'songs').glob('*.json')):
-    document = json.loads(path.read_text())
-    performance = timing(document)
-    song_id = performance['song_id']
-    duration = generator.model.fraction(performance['duration']) + performance['count_in_beats']
-    if song_id in durations and durations[song_id] != duration:
-      raise ValueError(f'{song_id}: parts disagree about performance duration')
-    durations[song_id] = duration
-  for entry in entries:
-    if entry['song_id']:
-      seconds = fractions.Fraction(entry['frames'], entry['sample_rate'])
-      if seconds < durations[entry['song_id']] * 60 / entry['bpm']:
-        raise ValueError(f"{entry['path']}: backing recording is shorter than count-in and score")

generator/printing.py

@@ -7,7 +7,6 @@ import json
 import math
 import pathlib
 import shutil
-import tempfile
 import time
 
 import PIL.PdfParser
@@ -19,7 +18,7 @@ import generator.engraving.layout
 import generator.engraving.notation
 import generator.engraving.raster
 import generator.engraving.types
-import generator.model
+import generator.validation
 
 # Two layout units per PDF point: guitar staff spacing is five points.
 # Raster resolution is independent of layout and the device's display theme.
@@ -30,7 +29,7 @@ MEDIA_BOX = [0, 0, 612, 792]
 VERSION = 'letter-parts-v1'
 
 def layout_part(document, fonts):
-  generator.model.validate_document(document)
+  generator.validation.validate_document(document)
   style = dataclasses.replace(generator.engraving.types.score_style(document, STYLE), system_gap=STYLE.system_gap)
   # Measure numbers are centered over barlines and can extend left of the
   # staff. Reserve their actual overhang inside the half-inch paper margin.
@@ -123,7 +122,7 @@ def renderer_key(fonts):
 
 def restore(entry, key, target):
   try:
-    if generator.engraving.cache.has_symlink(entry):
+    if generator.cache.has_symlink(entry):
       return None
     pdf, manifest = entry / 'part.pdf', entry / 'print.json'
     if pdf.is_symlink() or manifest.is_symlink():
@@ -138,21 +137,10 @@ def restore(entry, key, target):
     return None
 
 def store(entry, key, target, pages):
-  if generator.engraving.cache.has_symlink(entry.parent):
-    raise ValueError('print cache must not contain symlinks')
-  entry.parent.mkdir(parents=True, exist_ok=True)
-  if entry.is_symlink() or entry.is_file():
-    entry.unlink()
-  entry.mkdir(exist_ok=True)
-  with tempfile.TemporaryDirectory(prefix='.print_', dir=entry.parent) as temporary:
-    staged = pathlib.Path(temporary)
+  with generator.cache.stage_entry(entry, '.print_') as staged:
     shutil.copyfile(target, staged / 'part.pdf')
     (staged / 'print.json').write_text(json.dumps({'key': key, 'sha256': hashlib.sha256(target.read_bytes()).hexdigest(), 'pages': pages}))
-    for name in ('part.pdf', 'print.json'):
-      destination = entry / name
-      if destination.is_dir() and not destination.is_symlink():
-        shutil.rmtree(destination)
-      (staged / name).replace(destination)
+    generator.cache.publish_files(entry, staged, ('part.pdf', 'print.json'))
 
 def generate_tree(root, font_path, *, cache_dir=None, progress=False):
   """Called inside the normal build's atomic staging directory."""

generator/production.py

@@ -1,342 +1,23 @@
-"""Independent, reproducible role generation and non-composing assembly.
-
-Only source documents and production resources enter generate_role. Completed
-roles and published assets are downstream values, never composition inputs.
-"""
+"""File discovery, process scheduling and publication of independent roles."""
 from __future__ import annotations
 import concurrent.futures
 import copy
-import dataclasses
-import hashlib
-import functools
 import json
 import multiprocessing
-import os
 import pathlib
-import tempfile
-import time
-import fractions
 
-import generator.accompaniment
-import generator.comping
-import generator.fretboard
-import generator.model
+import generator.arrange.accompaniment
+import generator.arrange.assembly
+import generator.composition
+import generator.composition_types
 import generator.musicxml
 import generator.parts
 import generator.score
-import generator.vocabulary
-import generator.arrange.accompaniment
-import generator.arrange.ending
-import generator.arrange.form
-import generator.arrange.generate
-import generator.arrange.pickup
-import generator.solo.analysis
-import generator.solo.composer
-import generator.solo.notation
-import generator.solo.report
-import generator.solo.learned
-import generator.solo.authored
-import generator.input_audit
-import generator.solo.arrangement
-
-VERSION = 'independent-roles-1'
-ROLES = ('acoustic_accompaniment', 'electric_accompaniment', 'electric_melody', 'vocal_melody', 'acoustic_solo', 'electric_solo')
-
-@dataclasses.dataclass(frozen=True)
-class SourceAnalysis:
-  document_json: str
-  musical_key: str
-  @classmethod
-  def read(cls, document):
-    prepared = copy.deepcopy(document)
-    staff = generator.score.source_staff(prepared)
-    for i, m in enumerate(staff['measures']):
-      m['id'], m['number'] = f'source_{i}', str(i + 1)
-    staff['tuplets'] = generator.score.reidentify(staff['measures'], staff.get('tuplets', []))
-    prepared['metadata'] = {k: v for k, v in prepared['metadata'].items() if k not in ('title', 'composer', 'copyright', 'software')}
-    prepared['metadata'].update(title='Source', composer='')
-    staff.update(id='source', name='Source')
-    melodic = generator.solo.analysis.analyze(prepared)
-    content = {'melody': [(str(n.start), str(n.end), dataclasses.asdict(n.pitch)) for n in melodic.melody],
-      'measures': [{k: m.get(k) for k in ('duration', 'time_signature', 'key', 'harmonies', 'tempos', 'section_label', 'implicit')} for m in staff['measures']]}
-    # Fraction/string spelling is canonical and never includes filesystem IDs.
-    key = hashlib.sha256(json.dumps(content, sort_keys=True, default=str).encode()).hexdigest()
-    return cls(json.dumps(prepared, sort_keys=True), key)
-  def document(self):
-    return json.loads(self.document_json)
-  def accompaniment_key(self):
-    document = self.document()
-    document['metadata'].pop('tempo', None)
-    for measure in generator.score.source_staff(document)['measures']:
-      measure['tempos'] = []
-    return SourceAnalysis.read(document).musical_key
-
-def windows(document, part):
-  source = generator.score.source_staff(document)['measures']
-  sections = generator.arrange.form.full_arrangement_sections(source, part)
-  result, cursor = [], fractions.Fraction()
-  for section in sections:
-    duration = sum((generator.model.fraction(m['duration']) for m in source[section.source_measure_index:section.source_measure_index + section.measure_count]), fractions.Fraction())
-    role = {'slash': 'solo', 'vocal': 'melody'}.get(section.role, section.role)
-    if section.role in ('vocal', 'melody') and generator.arrange.pickup.enters(section, source) and result:
-      attacks = [generator.model.fraction(n['onset']) for n in generator.score.measure_notes(source[0]) if n['pitches']]
-      if attacks:
-        start = cursor - generator.model.fraction(source[0]['duration']) + min(attacks)
-        a, _, old = result[-1]
-        result[-1] = (a, start, old)
-        result.append((start, cursor, part + '_' + role))
-    end = cursor + duration
-    start = cursor
-    if section.role == 'slash' and generator.score.body_start_measure_index(source) and result:
-      # Allocate the source's complete pickup bar before composition. This
-      # does not inspect either generated solo; a role may leave it silent.
-      start = cursor - generator.model.fraction(source[0]['duration'])
-      a, _, old = result[-1]
-      result[-1] = (a, start, old)
-    result.append((start, end, part + '_' + role))
-    cursor = end
-  result.append((cursor, cursor + generator.arrange.ending.duration(source), part + ('_accompaniment' if part != 'vocal' else '_rest')))
-  return tuple(result)
-
-def dependencies(role, resources, measures=None):
-  names = ['harmony', 'lead_ins'] if role.endswith('_solo') else []
-  accompaniment = {}
-  if role.endswith('_accompaniment'):
-    policy = generator.vocabulary.accompaniment(role.split('_')[0], resources)
-    accompaniment = {'accompaniment': generator.comping_rhythm.relevant_policy(policy, measures) if measures is not None else policy}
-  return {'roles': {'role': resources['roles']['roles'][role], 'difficulty': resources['roles']['difficulty']},
-    **({'voicing_families': generator.vocabulary.accompaniment('electric', resources)['voicing_families']} if role == 'electric_melody' else {}),
-    **accompaniment,
-    **{name: resources[name] for name in names}}
-
-def _code_key(role):
-  root = pathlib.Path(__file__).parent
-  paths = [root / n for n in ('production.py', 'model.py', 'harmony.py', 'pitch.py', 'fretboard.py', 'parts.py', 'score.py', 'vocabulary.py')]
-  paths += list((root / 'arrange').glob('*.py'))
-  if role.endswith('_solo'):
-    paths += list((root / 'solo').glob('*.py'))
-    paths += [root / 'harmonic_time.py', root / 'musicxml.py', root / 'training_io.py']
-  elif role.endswith('_accompaniment'):
-    paths += [root / n for n in ('comping.py', 'comping_rhythm.py', 'chord_families.py', 'accompaniment.py', 'guitar.py')]
-    if role == 'acoustic_accompaniment':
-      paths += [root / 'acoustic_voicing.py', root / 'acoustic.py']
-  elif role == 'electric_melody':
-    paths += [root / 'guitar.py', root / 'melody.py', root / 'chord_families.py']
-  return generator.vocabulary.fingerprint([(p.relative_to(root).as_posix(), hashlib.sha256(p.read_bytes()).hexdigest()) for p in sorted(set(paths))])
-
-def _resource_key(directory, role):
-  return {'model': hashlib.sha256((directory / 'models' / 'solo.json').read_bytes()).hexdigest()} if role.endswith('_solo') else {}
-
-@functools.lru_cache(maxsize=4)
-def _solo_model(directory, fingerprint):
-  return generator.solo.learned.Model.load(pathlib.Path(directory) / 'models' / 'solo.json')
-
-def _mask(document, allowed):
-  """Apply source-defined role exclusions, without observing another part."""
-  absolute = fractions.Fraction()
-  for measure in document['staves'][0]['measures']:
-    length = generator.model.fraction(measure['duration'])
-    for voice in measure['voices']:
-      selected = []
-      for n in voice['notes']:
-        start = absolute + generator.model.fraction(n['onset'])
-        end = start + generator.model.fraction(n['duration'])
-        intervals = [(max(start, a), min(end, b)) for a, b in allowed if a < end and b > start]
-        for a, b in intervals:
-          note = copy.deepcopy(n)
-          if a != start:
-            note['onset'] = generator.model.rational(a - absolute)
-            for pitch in note['pitches']:
-              pitch.pop('tie_from', None)
-          generator.arrange.pickup.truncate(note, b - absolute)
-          if a < b:
-            selected.append(note)
-      voice['notes'] = _fill(selected, length, measure['id'] + '_' + voice['id'], voice['id'])
-    absolute += length
-  staff = document['staves'][0]
-  staff['tuplets'] = generator.arrange.pickup.retain_links(staff['measures'], staff.get('tuplets', []))
-
-def _fill(notes, length, prefix, voice='voice_arranged'):
-  notes = sorted(notes, key=lambda n: generator.model.fraction(n['onset']))
-  result = []
-  cursor = fractions.Fraction()
-
-  def rests(end):
-    nonlocal cursor
-    for duration, written in generator.accompaniment.written_durations(end - cursor):
-      result.append(generator.model.make_event(prefix + '_rest_' + str(len(result)), cursor, duration, 'rest', written=written) | {'voice': voice})
-      cursor += duration
-
-  for n in notes:
-    start = generator.model.fraction(n['onset'])
-    if start < cursor:
-      raise ValueError(prefix + ': overlapping independently generated roles')
-    rests(start)
-    n['voice'] = voice
-    result.append(n)
-    cursor = start + generator.model.fraction(n['duration'])
-  rests(length)
-  return result
-
-def generate_role(source, role, data_dir, settings=None, cache_dir=None):
-  started = time.monotonic()
-  with generator.input_audit.composition(data_dir, role, cache_dir) as reads:
-    result = _generate_role(source, role, data_dir, settings, cache_dir)
-  # Observed reads are diagnostics, never cache identity or musical seeds.
-  result['trace'] = {**result['trace'], 'input_boundary': 'production data (including explicit authored repertoire), generator code, Python runtime, own role cache'}
-  result['observed_reads'] = sorted(reads)
-  result['seconds'] = time.monotonic() - started
-  return result
-
-def _generate_role(source, role, data_dir, settings=None, cache_dir=None):
-  if role not in ROLES:
-    raise ValueError('Unknown generation role: ' + role)
-  data_dir = pathlib.Path(data_dir)
-  resources = generator.vocabulary.load(data_dir / 'composition')
-  settings = settings or {}
-  registered = generator.solo.authored.lookup(source, role.removesuffix('_solo'), data_dir) if role.endswith('_solo') and not settings.get('predict', False) else None
-  algorithm = VERSION + '/' + generator.solo.learned.VERSION if role.endswith('_solo') else VERSION
-  inputs = {'version': algorithm, 'source': source.musical_key, 'role': role, 'settings': settings,
-    'vocabulary': dependencies(role, resources, generator.score.source_staff(source.document())['measures']), 'catalogs': ({'authored': registered['sha256'], 'id': registered['id'], 'pitch_convention': registered['pitch_convention']} if registered else _resource_key(data_dir, role)), 'code': _code_key(role)}
-  # Source notation/lyrics also invalidate cached score representation, never the musical seed.
-  key = generator.vocabulary.fingerprint({'inputs': inputs, 'notation': source.document_json})
-  path = pathlib.Path(cache_dir) / role / (key + '.json') if cache_dir else None
-  if path and path.is_file() and not path.is_symlink():
-    try:
-      entry = json.loads(path.read_text())
-      value = entry['result']
-      if entry['key'] == key and entry['digest'] == generator.vocabulary.fingerprint(value):
-        generator.model.validate_document(value['document'])
-        generator.fretboard.validate_document(value['document'])
-        return {**value, 'cache_hit': True}
-    except (ValueError, KeyError, TypeError):
-      pass
-  started = time.monotonic()
-  document = source.document()
-  part = role.split('_')[0]
-  trace = {'source': source.musical_key, 'role': role, 'production_resources': inputs['catalogs'],
-    'vocabulary': generator.vocabulary.fingerprint(inputs['vocabulary']), 'settings': settings, 'algorithm': algorithm, 'code': inputs['code'], 'cache_key': key}
-  choice_source = source.accompaniment_key() if role.endswith('_accompaniment') else source.musical_key
-  choice_data, choice_settings = inputs['vocabulary'], settings
-  if role.endswith('_accompaniment'):
-    # Voicing weights invalidate the concrete cache, but must not reshuffle
-    # rhythm patterns. Removing this new field preserves the pre-policy seed.
-    choice_data = copy.deepcopy(choice_data)
-    choice_data['roles']['role'].pop('voicing', None)
-    for field in ('provenance', 'license', 'research'):
-      choice_data['accompaniment'].pop(field, None)
-    choice_settings = {k: v for k, v in settings.items() if k != 'voicing'}
-  seed = generator.vocabulary.fingerprint({'source': choice_source, 'role': role, 'settings': choice_settings, 'data': choice_data, 'version': algorithm})
-  details = {}
-  solo_report = ''
-  composition_timings = {}
-  with generator.vocabulary.using(resources, role, settings | {'choice_seed': seed}):
-    if role.endswith('_solo'):
-      result = generator.arrange.generate._arrange_document(document, part, {}, owned={'slash'})
-      si = 3 if part == 'acoustic' else 4
-      first, count, offset, length = generator.solo.arrangement.regions(document)[si]
-      # The template/harmony comes directly from source form, not another generated part.
-      authored = generator.score.source_staff(document)['measures']
-      section = generator.arrange.form.full_arrangement_sections(authored, part)[si]
-      effective = generator.arrange.generate.section_source(authored, section, generator.arrange.form.full_arrangement_sections(authored, part)[si + 1])
-      template = copy.deepcopy(effective[section.source_measure_index:section.source_measure_index + count])
-      initial = generator.accompaniment.carried_chords(effective)[section.source_measure_index]
-      start, end = next((a - offset, b - offset) for a, b, r in windows(document, part) if r == role)
-      notation_first = first
-      if start < 0:
-        # The preceding arrangement harmony comes from the source form. Do
-        # not invent harmony from a comparison score's unlabelled lead-in.
-        template.insert(0, copy.deepcopy(result['staves'][0]['measures'][first - 1]))
-        notation_first -= 1
-        initial = generator.accompaniment.carried_chords(result['staves'][0]['measures'])[first - 1]
-      if registered:
-        solo = generator.solo.authored.realize(registered, generator.solo.analysis.analyze(document), template, part, start, end, initial)
-      else:
-        solo_model = _solo_model(str(data_dir), inputs['catalogs']['model'])
-        solo = generator.solo.composer.compose(generator.solo.analysis.analyze(document), template, part, end, initial=initial, learned_model=solo_model, expressions_enabled=generator.vocabulary.role().get('expressions', True), start=start)
-      composition_timings = {k: v for k, v in solo.diagnostics.items() if k.endswith('_seconds')}
-      relative = tuple(dataclasses.replace(n, start=n.start - start, end=n.end - start) for n in solo.notes())
-      output = generator.solo.notation.measures(result['staves'][0]['measures'][notation_first:first + count], relative, 'solo_' + part)
-      generator.solo.notation.replace_region(result, notation_first, output)
-      details = generator.solo.report.structure(solo)
-      details['allocation'] = {'start': str(start), 'end': str(end), 'lead_in_measures': first - notation_first}
-      solo_report = generator.solo.report.describe(solo, first, offset, detail=details)
-    elif role.endswith('_accompaniment'):
-      canonical = generator.comping.Canonical(document, part, seed)
-      result = generator.arrange.generate._arrange_document(document, part, {}, owned={'accompaniment'}, accompaniment=canonical)
-      details = {'canonical_comping': canonical.report, 'accompaniment_rhythm': canonical.rhythm_report, 'groove_cycles': canonical.groove_report, 'rhythm_source': choice_source}
-    else:
-      result = generator.arrange.generate._arrange_document(document, part, {}, owned={'vocal' if part == 'vocal' else 'melody'})
-  allowed = tuple((a, b) for a, b, r in windows(document, part) if r == role)
-  _mask(result, allowed)
-  if role == "acoustic_accompaniment":
-    generator.acoustic.validate_comping(result)
-  staff = result['staves'][0]
-  for measure in staff['measures']:
-    measure['id'] = role + '_' + measure['id']
-  staff['tuplets'] = generator.score.reidentify(staff['measures'], staff.get('tuplets', []))
-  if part != 'vocal':
-    result['staves'][0]['tablature'] = True
-  generator.model.validate_document(result)
-  generator.fretboard.validate_document(result)
-  value = {'role': role, 'document': result, 'trace': trace, 'details': details, 'report': solo_report, 'seconds': time.monotonic() - started, 'composition_timings': composition_timings, 'cache_hit': False}
-  # Sidecar diagnostics use the same rational-safe representation as reports.
-  # Identical insertion order after cold generation and cache loading also
-  # keeps serialized scores and the catalog byte-for-byte reproducible.
-  value = json.loads(json.dumps(value, default=str, sort_keys=True))
-  if path:
-    path.parent.mkdir(parents=True, exist_ok=True)
-    payload = {'key': key, 'digest': generator.vocabulary.fingerprint(value), 'result': value}
-    with tempfile.NamedTemporaryFile('w', dir=path.parent, delete=False) as out:
-      json.dump(payload, out, sort_keys=True)
-      temporary = out.name
-    os.replace(temporary, path)
-  return value
-
-def assemble(source, results):
-  """Union prevalidated role windows; no selection or pitch/fingering rewrite."""
-  documents = {}
-  for part in generator.parts.PART_TYPES:
-    selected = [results[r]['document'] for r in ROLES if r.startswith(part + '_')]
-    document = copy.deepcopy(selected[0])
-    staff = document['staves'][0]
-    if part == 'vocal':
-      documents[part] = document
-      continue
-    staff['tuplets'] = []
-    staff['octave_shifts'] = []
-    staff['scale_spans'] = []
-    for other in selected:
-      for field in ('tuplets', 'octave_shifts', 'scale_spans'):
-        staff[field].extend(copy.deepcopy(other['staves'][0].get(field, [])))
-    for i, m in enumerate(staff['measures']):
-      # Tuplet rests belong to the source/role rhythm and must survive union.
-      lanes = {}
-      for d in selected:
-        for voice in d['staves'][0]['measures'][i]['voices']:
-          lane = voice['id'] if part == 'acoustic' and voice['id'] in ('upper', 'bass') else 'voice_arranged'
-          sounding = [copy.deepcopy(n) for n in voice['notes'] if n['pitches'] or n.get('tuplet_ids')]
-          if sounding:
-            lanes.setdefault(lane, []).extend(sounding)
-      lanes = lanes or {'voice_arranged': []}
-      m['voices'] = [{'id': lane, 'notes': _fill(notes, generator.model.fraction(m['duration']), m['id'] + '_' + lane, lane)} for lane, notes in lanes.items()]
-      # Attribute changes belong to source melody pickups and are assembly metadata.
-      for d in selected:
-        other = d['staves'][0]['measures'][i]
-        if other.get('attributes'):
-          m['attributes'] = copy.deepcopy(other['attributes'])
-    staff['tuplets'] = generator.arrange.pickup.retain_links(staff['measures'], staff['tuplets'])
-    owners = {n['id']: v['id'] for m in staff['measures'] for v in m['voices'] for n in v['notes']}
-    for span in staff['tuplets']:
-      span['voice'] = owners[span['start']]
-    generator.model.validate_document(document)
-    generator.fretboard.validate_document(document)
-    documents[part] = document
-  return documents
+import generator.score_io
+import generator.source
 
 def _job(request):
-  source, role, directory, settings, cache = request
-  return generate_role(source, role, directory, settings, cache)
+  return generator.composition.generate_role(request.source, request.role, request.directory, json.loads(request.settings_json), request.cache)
 
 def generate_tree(source_dir, output_dir, *, jobs=2, cache_dir=None, progress=False, timings=None):
   if type(jobs) is not int or not 1 <= jobs <= 16:
@@ -345,8 +26,8 @@ def generate_tree(source_dir, output_dir, *, jobs=2, cache_dir=None, progress=Fa
   sources = {}
   for path in generator.parts.source_paths(source_dir):
     document = generator.parts.prepare_document(generator.musicxml.load_musicxml(path))
-    sources[path] = (document, SourceAnalysis.read(document))
-  requests = [(path, role, (s, role, source_dir.parent, {}, cache_dir)) for path, (_, s) in sources.items() for role in ROLES]
+    sources[path] = (document, generator.source.SourceAnalysis.read(document))
+  requests = [(path, role, generator.composition_types.RoleRequest(s, role, source_dir.parent, cache=cache_dir)) for path, (_, s) in sources.items() for role in generator.composition_types.ROLES]
   results = {p: {} for p in sources}
 
   def accept(path, role, value):
@@ -372,16 +53,16 @@ def generate_tree(source_dir, output_dir, *, jobs=2, cache_dir=None, progress=Fa
         except Exception as error:
           raise ValueError(f'{path.name}/{role}: {error}') from error
   for path, (original, source) in sources.items():
-    assembled = assemble(source, results[path])
+    assembled = generator.arrange.assembly.assemble(source, results[path])
     generator.arrange.accompaniment.audit(original, assembled)
     reuse = generator.arrange.accompaniment.audit(original, {role: value['document'] for role, value in results[path].items()}, role_documents=True)
     for part, document in assembled.items():
       document['metadata'] = copy.deepcopy(original['metadata'])
       document['metadata']['song_id'] = generator.score.slugify(path.stem)
-      generator.score.write_json(output_dir / (generator.score.slugify(path.stem) + '_' + part + '.json'), document)
+      generator.score_io.write_json(output_dir / (generator.score.slugify(path.stem) + '_' + part + '.json'), document)
     analysis_dir = output_dir.parent / 'analysis'
     analysis_dir.mkdir(parents=True, exist_ok=True)
-    (analysis_dir / (generator.score.slugify(path.stem) + '.txt')).write_text('\n'.join(results[path][r]['report'] for r in ROLES))
+    (analysis_dir / (generator.score.slugify(path.stem) + '.txt')).write_text('\n'.join(results[path][r]['report'] for r in generator.composition_types.ROLES))
     # Not a score file: diagnostics are deliberately outside the songs schema.
     stable = {role: {k: v for k, v in value.items() if k not in ('seconds', 'composition_timings', 'cache_hit', 'observed_reads')} for role, value in results[path].items()}
     for role, audit in reuse.items():

generator/role_cache.py

@@ -0,0 +1,29 @@
+"""Role cache persistence; cache identity never supplies musical randomness."""
+import json
+
+import generator.cache
+import generator.fretboard
+import generator.validation
+
+ENTRYPOINTS = ('generator.composition',)
+
+def code_key():
+  return generator.cache.fingerprint(generator.cache.source_identity(ENTRYPOINTS))
+
+def restore(path, key):
+  if path is None or not path.is_file() or path.is_symlink():
+    return None
+  try:
+    entry = json.loads(path.read_text())
+    value = entry['result']
+    if entry['key'] == key and entry['digest'] == generator.cache.fingerprint(value):
+      generator.validation.validate_document(value['document'])
+      generator.fretboard.validate_document(value['document'])
+      return {**value, 'cache_hit': True}
+  except (OSError, ValueError, KeyError, TypeError):
+    pass
+  return None
+
+def store(path, key, value):
+  if path is not None:
+    generator.cache.write_json(path, {'key': key, 'digest': generator.cache.fingerprint(value), 'result': value})

generator/score.py

@@ -1,53 +1,8 @@
+"""Pure score navigation, cloning and identity operations."""
 from __future__ import annotations
-
 import copy
-import json
-import pathlib
 import re
 
-import generator.engraving.validation
-import generator.fretboard
-import generator.model
-import generator.pico_artwork
-import generator.performance
-
-def load_json(path: pathlib.Path) -> dict:
-  document = json.loads(path.read_text(encoding='utf-8'))
-  return validate_json(document, path)
-
-def validate_json(document: dict, path: pathlib.Path, validated_pages: set[str] | None = None) -> dict:
-  if not isinstance(document, dict):
-    raise ValueError('expected a score object')
-  if 'music' in document:
-    presentation = document.get('presentation', {})
-    if not isinstance(document['music'], dict) or not isinstance(presentation, dict) or not isinstance(presentation.get('pages'), list) or not presentation['pages']:
-      raise ValueError('invalid score bundle; regenerate score assets')
-    if (presentation.get('engine'), presentation.get('width'), presentation.get('height')) != (generator.engraving.validation.ENGINE_NAME, generator.engraving.validation.PAGE_WIDTH, generator.engraving.validation.PAGE_HEIGHT):
-      raise ValueError('unsupported score presentation; regenerate score assets')
-    generator.performance.validate_views(document['music'], presentation['pages'], presentation.get('playback_pages'))
-    if document.get('performance') != generator.performance.timing(document['music']):
-      raise ValueError('invalid performance timing')
-    for page in presentation['pages'] + presentation['playback_pages']:
-      if not isinstance(page, dict) or not isinstance(page.get('path'), str) or not isinstance(page.get('sha256'), str):
-        raise ValueError('invalid score page manifest')
-      name = page['path']
-      relative = pathlib.Path(name)
-      if '\x00' in name or '\\' in name or relative.is_absolute() or any(part in ('', '.', '..') for part in name.split('/')) or relative.suffix != '.png':
-        raise ValueError('invalid score page path')
-      resolved = (path.parent / relative).resolve()
-      if not resolved.is_relative_to(path.parent.resolve()):
-        raise ValueError('score page escapes its asset directory')
-      generator.engraving.validation.check_page(resolved, validated_pages, expected_sha256=page['sha256'])
-    document = document['music']
-  generator.model.validate_document(document)
-  return document
-
-def write_json(path: pathlib.Path, document: dict) -> None:
-  generator.model.validate_document(document)
-  generator.fretboard.validate_document(document)
-  path.parent.mkdir(parents=True, exist_ok=True)
-  path.write_text(json.dumps(document, indent=2, ensure_ascii=False) + '\n', encoding='utf-8')
-
 def slugify(value: str) -> str:
   return re.sub(r'[^a-z0-9]+', '_', value.strip().lower()).strip('_') or 'score'
 
@@ -124,15 +79,3 @@ def reidentify(measures: list[dict], source_tuplets: list[dict]) -> list[dict]:
 
 def make_arranged_document(source_document: dict, staff: dict, *, staff_id: str, staff_name: str, role: str, arrangement_part_type: str, measures: list[dict], tuplets=None, octave_shifts=None) -> dict:
   return {'metadata': copy.deepcopy(source_document['metadata']), 'staves': [{'id': staff_id, 'name': staff_name, 'clef': copy.deepcopy(staff['clef']), 'role': role, 'arranged': True, 'arrangement_part_type': arrangement_part_type, 'measures': measures, 'tuplets': tuplets or [], 'octave_shifts': copy.deepcopy(octave_shifts or []), 'instruments': copy.deepcopy(staff.get('instruments', []))}]}
-
-def count_json_files(root: pathlib.Path) -> int:
-  return sum(1 for p in root.rglob('*.json') if p.is_file())
-
-def validate_generated_tree(root: pathlib.Path, validated_pages: set[str] | None = None) -> None:
-  validated_pages = set() if validated_pages is None else validated_pages
-  for path in root.rglob('*.json'):
-    document = json.loads(path.read_text(encoding='utf-8'))
-    music = validate_json(document, path, validated_pages)
-    generator.fretboard.validate_document(music)
-    if 'presentation' in document:
-      generator.pico_artwork.validate_title(root, document['presentation'].get('title_image'))

generator/score_io.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+import json
+import pathlib
+
+import generator.engraving.validation
+import generator.fretboard
+import generator.performance
+import generator.pico_artwork
+import generator.validation
+
+def load_json(path: pathlib.Path) -> dict:
+  document = json.loads(path.read_text(encoding='utf-8'))
+  return validate_json(document, path)
+
+def validate_json(document: dict, path: pathlib.Path, validated_pages: set[str] | None = None) -> dict:
+  if not isinstance(document, dict):
+    raise ValueError('expected a score object')
+  if 'music' in document:
+    presentation = document.get('presentation', {})
+    if not isinstance(document['music'], dict) or not isinstance(presentation, dict) or not isinstance(presentation.get('pages'), list) or not presentation['pages']:
+      raise ValueError('invalid score bundle; regenerate score assets')
+    if (presentation.get('engine'), presentation.get('width'), presentation.get('height')) != (generator.engraving.validation.ENGINE_NAME, generator.engraving.validation.PAGE_WIDTH, generator.engraving.validation.PAGE_HEIGHT):
+      raise ValueError('unsupported score presentation; regenerate score assets')
+    generator.performance.validate_views(document['music'], presentation['pages'], presentation.get('playback_pages'))
+    if document.get('performance') != generator.performance.timing(document['music']):
+      raise ValueError('invalid performance timing')
+    for page in presentation['pages'] + presentation['playback_pages']:
+      if not isinstance(page, dict) or not isinstance(page.get('path'), str) or not isinstance(page.get('sha256'), str):
+        raise ValueError('invalid score page manifest')
+      name = page['path']
+      relative = pathlib.Path(name)
+      if '\x00' in name or '\\' in name or relative.is_absolute() or any(part in ('', '.', '..') for part in name.split('/')) or relative.suffix != '.png':
+        raise ValueError('invalid score page path')
+      resolved = (path.parent / relative).resolve()
+      if not resolved.is_relative_to(path.parent.resolve()):
+        raise ValueError('score page escapes its asset directory')
+      generator.engraving.validation.check_page(resolved, validated_pages, expected_sha256=page['sha256'])
+    document = document['music']
+  generator.validation.validate_document(document)
+  return document
+
+def write_json(path: pathlib.Path, document: dict) -> None:
+  generator.validation.validate_document(document)
+  generator.fretboard.validate_document(document)
+  path.parent.mkdir(parents=True, exist_ok=True)
+  path.write_text(json.dumps(document, indent=2, ensure_ascii=False) + '\n', encoding='utf-8')
+
+def count_json_files(root: pathlib.Path) -> int:
+  return sum(1 for p in root.rglob('*.json') if p.is_file())
+
+def validate_generated_tree(root: pathlib.Path, validated_pages: set[str] | None = None) -> None:
+  validated_pages = set() if validated_pages is None else validated_pages
+  for path in root.rglob('*.json'):
+    document = json.loads(path.read_text(encoding='utf-8'))
+    music = validate_json(document, path, validated_pages)
+    generator.fretboard.validate_document(music)
+    if 'presentation' in document:
+      generator.pico_artwork.validate_title(root, document['presentation'].get('title_image'))

generator/solo/authored.py

@@ -6,20 +6,21 @@ policy. Unknown song/role pairs go to the learned composer.
 """
 from __future__ import annotations
 import dataclasses
+import fractions
 import hashlib
 import json
 import pathlib
-import fractions
+
+import generator.harmonic_time
 import generator.musicxml
 import generator.parts
-import generator.training_io
-import generator.harmonic_time
-import generator.solo.analysis
-import generator.solo.types
+import generator.solo.harmonic
 import generator.solo.placement
 import generator.solo.profiles
-import generator.solo.harmonic
-import generator.production
+import generator.solo.types
+import generator.source
+import generator.training_io
+import generator.vocabulary
 
 def sounding(evidence, offset=fractions.Fraction()):
   result = []
@@ -32,9 +33,10 @@ def sounding(evidence, offset=fractions.Fraction()):
     result.append(generator.solo.types.Note(fractions.Fraction(event['start']) - offset, fractions.Fraction(event['end']) - offset, pitch, role='authored', event_id=f'authored_{i}'))
   return tuple(result)
 
-def validate_evidence(document, evidence):
+def validate_evidence(document, evidence, *, composition=None):
   """Training uses the same source-form harmonic context as solo playback."""
-  h = generator.solo.harmonic.Harmony(generator.harmonic_time.solo_source(document['staves'][0]))
+  composition = composition or generator.vocabulary.current()
+  h = generator.solo.harmonic.Harmony(generator.harmonic_time.solo_source(document['staves'][0]), composition=composition)
   checked = h.evaluate(sounding(evidence), complete=True)
   if not checked.valid:
     raise ValueError('Unsupported harmonic relationship in authored solo: ' + checked.error)
@@ -56,7 +58,7 @@ def lookup(source, role, data_dir):
     if not path.is_relative_to((root / 'songs').resolve()):
       raise ValueError('Registered sources must be in data/songs')
     doc = generator.parts.prepare_document(generator.musicxml.load_musicxml(path))
-    if generator.production.SourceAnalysis.read(doc).musical_key == source.musical_key:
+    if generator.source.SourceAnalysis.read(doc).musical_key == source.musical_key:
       target = (manifest.parent / entry['solo']).resolve()
       if not target.is_relative_to(manifest.parent.resolve()):
         raise ValueError('Registered solo must be inside data/training')
@@ -67,7 +69,8 @@ def lookup(source, role, data_dir):
     raise ValueError('Multiple authored solos registered for the same source music and role')
   return matches[0] if matches else None
 
-def realize(entry, source_analysis, measures, part, start, end, initial=None):
+def realize(entry, source_analysis, measures, part, start, end, initial=None, *, composition=None):
+  composition = composition or generator.vocabulary.current()
   evidence = generator.training_io.read(entry['path'], part, pitch_convention=entry['pitch_convention'])
   if evidence['discrepancies']:
     raise ValueError(entry['id'] + ': invalid authored timing/ties: ' + str(evidence['discrepancies']))
@@ -86,7 +89,7 @@ def realize(entry, source_analysis, measures, part, start, end, initial=None):
   if placed is None:
     raise ValueError(entry['id'] + ': no legal complete guitar placement')
   spans = tuple(dataclasses.replace(s, start=s.start + start, end=s.end + start) for s in generator.harmonic_time.read({'measures': measures}, initial=initial))
-  checked = generator.solo.harmonic.Harmony(spans).evaluate(placed[1], complete=True)
+  checked = generator.solo.harmonic.Harmony(spans, composition=composition).evaluate(placed[1], complete=True)
   if not checked.valid:
     raise ValueError(entry['id'] + ': ' + checked.error)
   phrase = generator.solo.types.Phrase('authored', start, end, placed[1], 'authored', 'authored', 'handoff', theme=entry['id'], expressions=checked.contracts)

generator/solo/composer.py

@@ -1,26 +1,28 @@
 """The sole production solo composer: frozen learned model plus music validity."""
 import dataclasses
-import time
 import fractions
 import pathlib
+import time
+
 import generator.harmonic_time
-import generator.vocabulary
-import generator.solo.types
-import generator.solo.profiles
-import generator.solo.harmonic
-import generator.solo.placement
-import generator.solo.learned
 import generator.solo.decoder
 import generator.solo.groups
+import generator.solo.harmonic
+import generator.solo.learned
+import generator.solo.placement
+import generator.solo.profiles
+import generator.solo.types
+import generator.vocabulary
 
-def compose(analysis, measures, part, end, *, learned_model=None, initial=None, expressions_enabled=True, start=fractions.Fraction()):
+def compose(analysis, measures, part, end, *, learned_model=None, initial=None, expressions_enabled=True, start=fractions.Fraction(), composition=None):
+  composition = composition or generator.vocabulary.current()
   began = time.monotonic()
   spans = tuple(dataclasses.replace(s, start=s.start + start, end=s.end + start) for s in generator.harmonic_time.read({'measures': measures}, initial=initial))
   profile = generator.solo.profiles.for_part(part, analysis)
-  harmony = generator.solo.harmonic.Harmony(spans, profile)
+  harmony = generator.solo.harmonic.Harmony(spans, profile, composition=composition)
   model = learned_model or generator.solo.learned.Model.load(pathlib.Path(__file__).resolve().parents[2] / 'data/models/solo.json')
-  settings = generator.vocabulary.role(part + '_solo')
-  builder = generator.solo.decoder.Decoder(analysis, harmony, model, part, start, end, expressions_enabled=expressions_enabled, attempts=settings.get('attempts', 4), seed=settings.get('choice_seed', ''))
+  settings = composition.role(part + '_solo')
+  builder = generator.solo.decoder.Decoder(analysis, harmony, model, part, start, end, expressions_enabled=expressions_enabled, attempts=settings.get('attempts', 4), seed=settings.get('choice_seed', ''), composition=composition)
   selected = builder.compose()
   decoded = time.monotonic()
   placed = generator.solo.placement.place(selected.notes, analysis.tempo, profile=profile)
@@ -48,10 +50,11 @@ def compose(analysis, measures, part, end, *, learned_model=None, initial=None,
       'counts': dict(builder.counts), 'frontiers': builder.trace, 'completed_candidates': [
         {'rank': i + 1, 'cost': s.cost, 'events': [(str(a), str(b), p) for a, b, p in s.events]} for i, s in enumerate(builder.completed)],
       'expressions_enabled': expressions_enabled}, profile, start)
-  validate(result)
+  validate(result, composition=composition)
   return result
 
-def validate(solo):
+def validate(solo, *, composition=None):
+  composition = composition or generator.vocabulary.current()
   previous = None
   for n in solo.notes():
     if n.start < solo.start or n.end > solo.end or n.end <= n.start or previous and previous.end > n.start:
@@ -59,7 +62,7 @@ def validate(solo):
     if n.position not in generator.solo.placement.positions(n.pitch.number(), solo.profile):
       raise ValueError('invalid guitar placement')
     previous = n
-  checked = generator.solo.harmonic.Harmony(solo.harmony).evaluate(solo.notes(), until=solo.end, complete=True,
+  checked = generator.solo.harmonic.Harmony(solo.harmony, composition=composition).evaluate(solo.notes(), until=solo.end, complete=True,
     expressions=tuple(c for p in solo.phrases for c in p.expressions))
   if not checked.valid:
     raise ValueError(checked.error)

generator/solo/decoder.py

@@ -8,17 +8,18 @@ legacy fallback. A bounded restart handles an exhausted/entirely silent sample.
 from __future__ import annotations
 import collections
 import dataclasses
-import hashlib
-import random
-import math
 import fractions
+import hashlib
+import math
+import random
+
+import generator.harmony
 import generator.model
-import generator.vocabulary
 import generator.solo.expressions
 import generator.solo.groups
 import generator.solo.learned
 import generator.solo.types
-import generator.harmony
+import generator.vocabulary
 
 class SearchExhausted(ValueError):
   pass
@@ -35,7 +36,8 @@ class State:
 def event_key(events): return tuple((a, b, -1 if p is None else p) for a, b, p in events)
 
 class Decoder:
-  def __init__(self, analysis, harmony, model, role, start, end, *, expressions_enabled=True, attempts=4, seed=''):
+  def __init__(self, analysis, harmony, model, role, start, end, *, expressions_enabled=True, attempts=4, seed='', composition=None):
+    self.composition = composition or generator.vocabulary.current()
     if type(attempts) is not int or not 1 <= attempts <= 16:
       raise ValueError('attempts must be between 1 and 16')
     self.analysis, self.harmony, self.model, self.role = analysis, harmony, model, role
@@ -45,7 +47,7 @@ class Decoder:
     self.seed = seed or hashlib.sha256(repr((source, role, generator.solo.learned.VERSION)).encode()).hexdigest()
     self.expressions_enabled = expressions_enabled
     self.low, self.high = harmony.profile.register
-    limits = generator.vocabulary.data('roles')['difficulty']
+    limits = self.composition.data('roles')['difficulty']
     self.durations = tuple(d for d in model.durations if 0 < d <= 4 and d.denominator in (1, 2, 4) and
       (d.denominator != 4 or analysis.tempo <= limits['sixteenth_max_bpm']))
     self.allow_triplets = analysis.tempo <= limits['triplet_max_bpm']
@@ -92,12 +94,12 @@ class Decoder:
     bases = []
     regions = [(None, n)]
     regions.extend((s.start, dataclasses.replace(n, start=s.start)) for s in self.harmony.spans
-      if n.start < s.start < n.end and s.start - n.start <= fractions.Fraction(generator.vocabulary.data('harmony')['anticipation_max']))
+      if n.start < s.start < n.end and s.start - n.start <= fractions.Fraction(self.composition.data('harmony')['anticipation_max']))
     for boundary, arrived in regions:
       exposed = [s for s, _, _ in self.harmony.segments(arrived) if n.pitch.number() % 12 not in s.roles.basic | s.roles.specified]
       if not exposed:
         continue
-      for rule in generator.vocabulary.data('harmony')['colors']:
+      for rule in self.composition.data('harmony')['colors']:
         if arrived.end - arrived.start > fractions.Fraction(rule['max_beats']):
           continue
         if any(s.chord == 'N.C.' or not generator.solo.expressions.eligible_color(rule, s, self.harmony) or
@@ -139,7 +141,7 @@ class Decoder:
       if self.harmony.evaluate((n,), complete=True, expressions=(c,), infer=False).valid:
         return tuple([c])
     for arriving in self.harmony.spans:
-      if n.start < arriving.start < n.end and arriving.start - n.start <= fractions.Fraction(generator.vocabulary.data('harmony')['anticipation_max']):
+      if n.start < arriving.start < n.end and arriving.start - n.start <= fractions.Fraction(self.composition.data('harmony')['anticipation_max']):
         c = generator.solo.types.ExpressionContract(n.event_id + ':anticipation', 'anticipation', 'early_chord_tone', (n,), n, arriving.chord, arriving.start)
         if self.harmony.evaluate((n,), complete=True, expressions=(c,), infer=False).valid:
           return tuple([c])
@@ -175,9 +177,9 @@ class Decoder:
     for rule, boundary in self.color_bases(n):
       if rule.get('resolve'):
         for gap in (fractions.Fraction(), fractions.Fraction(1, 2), fractions.Fraction(1)):
-          if gap <= fractions.Fraction(generator.vocabulary.data('harmony')['resolution_gap']):
+          if gap <= fractions.Fraction(self.composition.data('harmony')['resolution_gap']):
             requests.append(('color', rule, boundary, n.end + gap, (-2, -1, 1, 2)))
-    delay = generator.vocabulary.data('lead_ins')['delayed_approach']
+    delay = self.composition.data('lead_ins')['delayed_approach']
     if n.end - n.start <= fractions.Fraction(delay['max_note']) and n.pitch.number() % 12 not in self.harmony.permitted(n.start, n.end):
       for arrival in map(fractions.Fraction, delay['arrivals']):
         if n.end <= n.start + arrival <= n.start + fractions.Fraction(delay['deadline']):
@@ -219,7 +221,7 @@ class Decoder:
         if self.expressions_enabled and not self.harmony.ordinary(n):
           for events, ns, contracts in self.resolutions(n):
             add(events, ns, contracts)
-          rule = generator.vocabulary.data('lead_ins')['passing']
+          rule = self.composition.data('lead_ins')['passing']
           if duration <= fractions.Fraction(rule['max_note']):
             for step in rule['steps']:
               p = number + step
@@ -235,8 +237,8 @@ class Decoder:
                 c = generator.solo.types.ExpressionContract(n.event_id + ':passing', 'passing', rule['id'], (n,), target, self.harmony.at(end).chord)
                 add(((time, end, number), (end, end + hold, p)), (n, target), (c, *extra))
     if self.expressions_enabled:
-      limits = generator.vocabulary.data('roles')['difficulty']
-      for rule in generator.vocabulary.data('lead_ins')['patterns']:
+      limits = self.composition.data('roles')['difficulty']
+      for rule in self.composition.data('lead_ins')['patterns']:
         difficulty = rule.get('difficulty')
         if difficulty and self.analysis.tempo > limits[difficulty + '_max_bpm']:
           continue
@@ -313,9 +315,9 @@ class Decoder:
         return None
       contracts = tuple(dataclasses.replace(c, preparation=state.notes[-1]) if c.kind in ('connection', 'passing') and c.preparation is None else c for c in contracts)
       if self.harmony.ordinary(state.notes[-1]):
-        _, error = generator.solo.expressions.validate((state.notes[-1], *notes), contracts, self.harmony)
+        _, error = generator.solo.expressions.validate((state.notes[-1], *notes), contracts, self.harmony, composition=self.composition)
       else:
-        _, error = generator.solo.expressions.validate((*state.notes, *notes), (*state.contracts, *contracts), self.harmony)
+        _, error = generator.solo.expressions.validate((*state.notes, *notes), (*state.contracts, *contracts), self.harmony, composition=self.composition)
       if error:
         return None
     history = state.history

generator/solo/expressions.py

@@ -1,13 +1,14 @@
 """Typed, target-driven harmonic expression against written harmony and explicit musical relationships."""
 from __future__ import annotations
 import dataclasses
-import functools
 import fractions
+import functools
+
 import generator.harmony
 import generator.model
 import generator.pitch
-import generator.vocabulary
 import generator.solo.types
+import generator.vocabulary
 
 @functools.lru_cache(maxsize=512)
 def parsed_chord(symbol):
@@ -90,8 +91,9 @@ def _eligible_color(rule, span, harmony):
       return False
   return True
 
-def validate(notes, contracts, harmony):
+def validate(notes, contracts, harmony, *, composition=None):
   """Return event permissions only after checking complete actual gestures."""
+  composition = composition or generator.vocabulary.current()
   actual = {n.event_id: n for n in notes}
   approved = {}
 
@@ -115,7 +117,7 @@ def validate(notes, contracts, harmony):
       if not previous or previous.chord.strip('()') == span.chord.strip('()'):
         return {}, c.id + ': anticipation requires a harmonic change'
     if c.kind in ('approach', 'enclosure', 'connection'):
-      rule = next((r for r in generator.vocabulary.data('lead_ins')['patterns'] if r['id'] == c.rule and r['kind'] == c.kind), None)
+      rule = next((r for r in composition.data('lead_ins')['patterns'] if r['id'] == c.rule and r['kind'] == c.kind), None)
       if not rule or len(c.events) != len(rule['semitones']):
         return {}, c.id + ': unknown approach formula'
       arrival = c.boundary if c.boundary is not None else target.start
@@ -129,7 +131,7 @@ def validate(notes, contracts, harmony):
         return {}, c.id + ': invalid internal approach gaps'
       gap = fractions.Fraction(rule['gap'])
       cursor = target.start - gap - sum((fractions.Fraction(d) for d in rule['durations']), fractions.Fraction()) - sum(gaps, fractions.Fraction())
-      if target.start - cursor > fractions.Fraction(generator.vocabulary.data('lead_ins')['max_approach']):
+      if target.start - cursor > fractions.Fraction(composition.data('lead_ins')['max_approach']):
         return {}, c.id + ': approach too long'
       for i, (n, step, duration) in enumerate(zip(c.events, rule['semitones'], rule['durations'])):
         if n.start != cursor or n.end != cursor + fractions.Fraction(duration) or n.pitch.number() != target.pitch.number() + step:
@@ -144,7 +146,7 @@ def validate(notes, contracts, harmony):
       if c.kind == 'connection' and (not c.preparation or c.preparation.end != c.events[0].start or abs(c.preparation.pitch.number() - c.events[0].pitch.number()) > 2):
         return {}, c.id + ': chromatic connection lacks its incoming motion'
     elif c.kind == 'passing':
-      rule = generator.vocabulary.data('lead_ins')['passing']
+      rule = composition.data('lead_ins')['passing']
       if c.rule != rule['id'] or len(c.events) != 1 or not c.preparation:
         return {}, c.id + ': unsupported passing figure'
       n, = c.events
@@ -160,7 +162,7 @@ def validate(notes, contracts, harmony):
       if not supported(before) or not supported(target):
         return {}, c.id + ': unsupported passing endpoints'
     elif c.kind == 'delayed_approach':
-      rule = generator.vocabulary.data('lead_ins').get('delayed_approach', {})
+      rule = composition.data('lead_ins').get('delayed_approach', {})
       if c.rule != rule.get('id') or len(c.events) != 1:
         return {}, c.id + ': unknown delayed approach'
       n, = c.events
@@ -171,7 +173,7 @@ def validate(notes, contracts, harmony):
       if target.pitch.number() % 12 not in harmony.members(target.start, target.end) or target.pitch.number() % 12 not in harmony.permitted(target.start, target.end):
         return {}, c.id + ': invalid delayed destination'
     elif c.kind == 'color':
-      rule = next((r for r in generator.vocabulary.data('harmony')['colors'] if r['id'] == c.rule), None)
+      rule = next((r for r in composition.data('harmony')['colors'] if r['id'] == c.rule), None)
       if not rule:
         return {}, c.id + ': unknown color policy'
       colored = False
@@ -193,10 +195,10 @@ def validate(notes, contracts, harmony):
         return {}, c.id + ': no interval expresses the declared color'
       last = c.events[-1]
       if rule.get('resolve'):
-        if target.event_id == last.event_id or not 0 <= target.start - last.end <= fractions.Fraction(generator.vocabulary.data('harmony')['resolution_gap']) or not 0 < abs(target.pitch.number() - last.pitch.number()) <= 2 or target.pitch.number() % 12 not in harmony.members(target.start, target.end):
+        if target.event_id == last.event_id or not 0 <= target.start - last.end <= fractions.Fraction(composition.data('harmony')['resolution_gap']) or not 0 < abs(target.pitch.number() - last.pitch.number()) <= 2 or target.pitch.number() % 12 not in harmony.members(target.start, target.end):
           return {}, c.id + ': unresolved color'
     elif c.kind == 'suspension':
-      rule = next((r for r in generator.vocabulary.data('harmony').get('suspensions', []) if r['id'] == c.rule), None)
+      rule = next((r for r in composition.data('harmony').get('suspensions', []) if r['id'] == c.rule), None)
       if not rule or len(c.events) != 1 or c.boundary is None:
         return {}, c.id + ': unsupported suspension'
       held, = c.events
@@ -211,23 +213,23 @@ def validate(notes, contracts, harmony):
       if held.pitch.number() % 12 not in harmony.permitted(held.start, held.end):
         return {}, c.id + ': suspension outside source-key/chord motion'
     elif c.kind == 'anticipated_color':
-      rule = next((r for r in generator.vocabulary.data('harmony')['colors'] if r['id'] == c.rule), None)
+      rule = next((r for r in composition.data('harmony')['colors'] if r['id'] == c.rule), None)
       if not rule or len(c.events) != 1 or c.boundary != span.start:
         return {}, c.id + ': unsupported anticipated color'
       n, = c.events
-      if not n.start < c.boundary < n.end or c.boundary - n.start > fractions.Fraction(generator.vocabulary.data('harmony')['anticipation_max']):
+      if not n.start < c.boundary < n.end or c.boundary - n.start > fractions.Fraction(composition.data('harmony')['anticipation_max']):
         return {}, c.id + ': invalid anticipated color interval'
       arrived = dataclasses.replace(n, start=c.boundary)
       target = arrived if c.target == n else c.target
       inner = generator.solo.types.ExpressionContract(c.id + ':arrived', 'color', rule['id'], (arrived,), target, span.chord)
-      _, error = validate((arrived, *((target,) if target != arrived else ())), (inner,), harmony)
+      _, error = validate((arrived, *((target,) if target != arrived else ())), (inner,), harmony, composition=composition)
       if error:
         return {}, error
     elif c.kind == 'anticipation':
       if c.rule not in ('early_chord_tone', 'early_reattack') or len(c.events) != 1 or span.start != c.boundary:
         return {}, c.id + ': unknown anticipation or non-boundary target'
       n, = c.events
-      if c.boundary is None or not n.start < c.boundary or c.boundary - n.start > fractions.Fraction(generator.vocabulary.data('harmony')['anticipation_max']):
+      if c.boundary is None or not n.start < c.boundary or c.boundary - n.start > fractions.Fraction(composition.data('harmony')['anticipation_max']):
         return {}, c.id + ': invalid anticipation interval'
       if c.rule == 'early_chord_tone' and (n != target or not c.boundary < n.end):
         return {}, c.id + ': invalid held anticipation'
@@ -238,7 +240,7 @@ def validate(notes, contracts, harmony):
     elif c.kind == 'blues':
       if len(c.events) != 1:
         return {}, c.id + ': blues treatment requires one inflected event'
-      rule = next((r for r in generator.vocabulary.data('lead_ins')['blues'] if r['id'] == c.rule), None)
+      rule = next((r for r in composition.data('lead_ins')['blues'] if r['id'] == c.rule), None)
       n, = c.events
       chord = parsed_chord(span.chord)
       if not rule or n.end - n.start > fractions.Fraction(1, 2) or n.end != target.start or target.pitch.number() % 12 not in span.roles.basic:
@@ -262,15 +264,16 @@ def validate(notes, contracts, harmony):
       approved[n.event_id] = c
   return approved, ''
 
-def validate_frontier(notes, contracts, harmony, until, complete):
+def validate_frontier(notes, contracts, harmony, until, complete, *, composition=None):
   """Validate heard prefixes and bounded future obligations separately.
 
   Future contract events are used ONLY to establish rule feasibility. They do
   not count as heard resolution; the contract stays pending until its entire
   target hold is present. Missing past events and expired deadlines fail.
   """
+  composition = composition or generator.vocabulary.current()
   if complete:
-    approved, error = validate(notes, contracts, harmony)
+    approved, error = validate(notes, contracts, harmony, composition=composition)
     return approved, (), error
   actual = {n.event_id: n for n in notes}
   future = dict(actual)
@@ -290,5 +293,5 @@ def validate_frontier(notes, contracts, harmony, until, complete):
         return {}, (), c.id + ': missing/changed event before resolution deadline'
     if incomplete:
       pending.append(c)
-  approved, error = validate(tuple(sorted(future.values(), key=lambda n: (n.start, n.event_id))), contracts, harmony)
+  approved, error = validate(tuple(sorted(future.values(), key=lambda n: (n.start, n.event_id))), contracts, harmony, composition=composition)
   return {key: c for key, c in approved.items() if key in actual}, tuple(pending), error

generator/solo/harmonic.py

@@ -5,14 +5,16 @@ permission follows the actual sounded interval and validated resolution.
 """
 from __future__ import annotations
 import dataclasses
-import functools
 import fractions
+import functools
+
 import generator.harmony
 import generator.model
-import generator.solo.types
-import generator.solo.profiles
 import generator.solo.expressions
+import generator.solo.profiles
 import generator.solo.recognition
+import generator.solo.types
+import generator.vocabulary
 
 @dataclasses.dataclass(frozen=True)
 class Roles:
@@ -55,7 +57,8 @@ class Result:
   contracts: tuple = ()
 
 class Harmony:
-  def __init__(self, spans, profile=None):
+  def __init__(self, spans, profile=None, *, composition=None):
+    self.composition = composition or generator.vocabulary.current()
     self.profile = profile or generator.solo.profiles.UNRESTRICTED
     self.source_spans = spans
     self.spans = []
@@ -112,14 +115,14 @@ class Harmony:
     expressions = tuple(dataclasses.replace(c, events=tuple(self.identified(n) for n in c.events),
       target=self.identified(c.target), preparation=self.identified(c.preparation) if c.preparation else None) for c in expressions)
     frontier = until if until is not None else max((n.end for n in notes), default=fractions.Fraction())
-    permissions, pending, error = generator.solo.expressions.validate_frontier(notes, expressions, self, frontier, complete or until is None)
+    permissions, pending, error = generator.solo.expressions.validate_frontier(notes, expressions, self, frontier, complete or until is None, composition=self.composition)
     if error:
       return Result(False, 0, error=error)
     if infer and not pending and (complete or until is None) and any(not self.ordinary(n) and n.event_id not in permissions for n in notes):
-      expressions, error = generator.solo.recognition.recognize(notes, self, expressions)
+      expressions, error = generator.solo.recognition.recognize(notes, self, expressions, composition=self.composition)
       if error:
         return Result(False, 0, error=error)
-      permissions, pending, error = generator.solo.expressions.validate_frontier(notes, expressions, self, frontier, True)
+      permissions, pending, error = generator.solo.expressions.validate_frontier(notes, expressions, self, frontier, True, composition=self.composition)
       if error:
         return Result(False, 0, error=error)
     exposed = sounded = fractions.Fraction()

generator/solo/profiles.py

@@ -2,11 +2,10 @@
 from __future__ import annotations
 
 import dataclasses
-import functools
 import fractions
+import functools
 
 import generator.fretboard
-import generator.vocabulary
 
 @dataclasses.dataclass(frozen=True)
 class Instrument:

generator/solo/recognition.py

@@ -5,9 +5,10 @@ Existing expression validators prove each candidate; deterministic set-cover
 search chooses a mutually consistent explanation, never edits the notes.
 """
 import fractions
-import generator.vocabulary
+
 import generator.solo.expressions
 import generator.solo.types
+import generator.vocabulary
 
 def signature(c):
   return (c.kind, c.rule, tuple(n.event_id for n in c.events), c.target.event_id,
@@ -20,7 +21,8 @@ def merge(*groups):
       result.setdefault(signature(c), c)
   return tuple(result.values())
 
-def recognize(notes, harmony, provided=(), *, budget=4096):
+def recognize(notes, harmony, provided=(), *, budget=4096, composition=None):
+  composition = composition or generator.vocabulary.current()
   required = {n.event_id for n in notes if not harmony.ordinary(n)}
   fixed = {n.event_id for c in provided for n in c.events}
   if required <= fixed:
@@ -46,7 +48,7 @@ def recognize(notes, harmony, provided=(), *, budget=4096):
     if not (covered & required):
       return False
     combined = merge(provided, bundle)
-    _, error = generator.solo.expressions.validate(notes, combined, harmony)
+    _, error = generator.solo.expressions.validate(notes, combined, harmony, composition=composition)
     if error:
       return False
     candidates[tuple(signature(a) for a in bundle)] = (bundle, covered)
@@ -57,11 +59,11 @@ def recognize(notes, harmony, provided=(), *, budget=4096):
   for i, n in enumerate(notes):
     if n.event_id not in required:
       continue
-    future = [t for t in notes[i + 1:] if n.end <= t.start <= n.end + fractions.Fraction(generator.vocabulary.data('harmony')['resolution_gap'])
+    future = [t for t in notes[i + 1:] if n.end <= t.start <= n.end + fractions.Fraction(composition.data('harmony')['resolution_gap'])
       and 0 < abs(t.pitch.number() - n.pitch.number()) <= 2]
     regions = [(None, harmony.at(n.start))]
     regions.extend((s.start, s) for s in harmony.spans if n.start < s.start < n.end
-      and s.start - n.start <= fractions.Fraction(generator.vocabulary.data('harmony')['anticipation_max']))
+      and s.start - n.start <= fractions.Fraction(composition.data('harmony')['anticipation_max']))
     for boundary, span in regions:
       if not span or span.chord == 'N.C.':
         continue
@@ -69,7 +71,7 @@ def recognize(notes, harmony, provided=(), *, budget=4096):
         c = contract('anticipation', 'early_chord_tone', (n,), n, boundary)
         if add(c):
           single.setdefault(n.event_id, []).append((c,))
-      for rule in generator.vocabulary.data('harmony')['colors']:
+      for rule in composition.data('harmony')['colors']:
         if not generator.solo.expressions.eligible_color(rule, span, harmony) and boundary is not None:
           continue
         kind = 'anticipated_color' if boundary is not None else 'color'
@@ -85,7 +87,7 @@ def recognize(notes, harmony, provided=(), *, budget=4096):
       target = notes[i + 1]
       for s in harmony.spans:
         if n.start < s.start < n.end:
-          for rule in generator.vocabulary.data('harmony').get('suspensions', []):
+          for rule in composition.data('harmony').get('suspensions', []):
             add(contract('suspension', rule['id'], (n,), target, s.start))
         elif s.start == n.end == target.start and n.pitch.number() == target.pitch.number():
           add(contract('anticipation', 'early_reattack', (n,), target, s.start))
@@ -99,7 +101,7 @@ def recognize(notes, harmony, provided=(), *, budget=4096):
         targets.append((None, support))
       elif c.kind == 'anticipation':
         targets.append((c.boundary, support))
-    for rule in generator.vocabulary.data('lead_ins')['patterns']:
+    for rule in composition.data('lead_ins')['patterns']:
       size = len(rule['semitones'])
       if i < size:
         continue
@@ -113,7 +115,7 @@ def recognize(notes, harmony, provided=(), *, budget=4096):
         add(contract(rule['kind'], rule['id'], ns, target, boundary, preparation), extras)
     # Delayed approaches may include real space or intervening valid music.
     # They still have an exact semitone target and a finite deadline.
-    rule = generator.vocabulary.data('lead_ins')['delayed_approach']
+    rule = composition.data('lead_ins')['delayed_approach']
     for n in reversed(notes[:i]):
       if target.start - n.start > fractions.Fraction(rule['deadline']):
         break
@@ -124,8 +126,8 @@ def recognize(notes, harmony, provided=(), *, budget=4096):
       if n.event_id in required:
         for left in [(), *single.get(before.event_id, ())]:
           for right in [(), *single.get(target.event_id, ())]:
-            add(contract('passing', generator.vocabulary.data('lead_ins')['passing']['id'], (n,), target, preparation=before), merge(left, right))
-        for rule in generator.vocabulary.data('lead_ins')['blues']:
+            add(contract('passing', composition.data('lead_ins')['passing']['id'], (n,), target, preparation=before), merge(left, right))
+        for rule in composition.data('lead_ins')['blues']:
           add(contract('blues', rule['id'], (n,), target, preparation=before))
   options = {identifier: [] for identifier in sorted(required - fixed, key=lambda identifier: (by_id[identifier].start, identifier))}
   for bundle, covered in candidates.values():
@@ -152,7 +154,7 @@ def recognize(notes, harmony, provided=(), *, budget=4096):
     identifier = min(remaining, key=lambda x: (len(options[x]), by_id[x].start, x))
     for bundle, added in options[identifier]:
       combined = merge(chosen, bundle)
-      _, error = generator.solo.expressions.validate(notes, combined, harmony)
+      _, error = generator.solo.expressions.validate(notes, combined, harmony, composition=composition)
       if error:
         continue
       result = solve(combined, covered | added)

generator/solo/report.py

@@ -1,9 +1,10 @@
 """Compact reports of actual selected phrases, not duplicated draft ancestry."""
-import dataclasses, fractions, json
-import generator.solo.placement
-import generator.solo.harmonic
 import collections
-import generator.fretboard
+import dataclasses, fractions, json
+
+import generator.solo.harmonic
+import generator.solo.placement
+import generator.vocabulary
 
 def plain(value):
   if dataclasses.is_dataclass(value):
@@ -16,8 +17,9 @@ def plain(value):
     return str(value)
   return value
 
-def structure(solo):
-  harmony = generator.solo.harmonic.Harmony(solo.harmony)
+def structure(solo, *, composition=None):
+  composition = composition or generator.vocabulary.current()
+  harmony = generator.solo.harmonic.Harmony(solo.harmony, composition=composition)
   contracts = tuple(c for p in solo.phrases for c in p.expressions)
   checked = harmony.evaluate(solo.notes(), until=solo.end, complete=True, expressions=contracts)
   phrases = []
@@ -37,8 +39,9 @@ def structure(solo):
     'harmonic_roles': harmony.describe(ns, expressions=contracts), 'harmonic_policy': 'contextual (shared by authored and predicted music)', 'harmonic_valid': checked.valid, 'harmonic_error': checked.error,
     'harmonic_exposure': checked.exposure, 'unresolved_treatments': plain(checked.pending)}
 
-def describe(solo, measure_offset=0, time_offset=0, *, detail=None):
-  detail = detail or structure(solo)
+def describe(solo, measure_offset=0, time_offset=0, *, detail=None, composition=None):
+  composition = composition or generator.vocabulary.current()
+  detail = detail or structure(solo, composition=composition)
   lines = [f'{solo.diagnostics.get("construction", "solo")}: {solo.part}', 'Times are exact quarter beats from the chorus start; negative times are allocated lead-ins. Pitches sound one octave below guitar notation.',
     'Approximate fingering costs are not a biomechanical guarantee.', json.dumps(detail['phrasing'])]
   for p in solo.phrases:

generator/source.py

@@ -1,7 +1,14 @@
 """Immutable source events, meter boundaries and tempo. No solo construction."""
 from __future__ import annotations
-import dataclasses, fractions
-import generator.model, generator.score, generator.solo.types
+import copy
+import dataclasses
+import fractions
+import hashlib
+import json
+
+import generator.model
+import generator.score
+import generator.solo.types
 
 @dataclasses.dataclass(frozen=True)
 class Analysis:
@@ -43,3 +50,32 @@ def analyze(document):
     boundaries.append(boundaries[-1] + generator.model.fraction(m['duration']))
   tempo = next((generator.model.fraction(t['bpm']) * generator.model.written_length(t['written']) for m in measures for t in m.get('tempos', []) if 'bpm' in t), fractions.Fraction(120))
   return Analysis(tuple(boundaries), tempo, attacks(document), boundaries[generator.score.body_start_measure_index(measures)])
+
+@dataclasses.dataclass(frozen=True)
+class SourceAnalysis:
+  document_json: str
+  musical_key: str
+  @classmethod
+  def read(cls, document):
+    prepared = copy.deepcopy(document)
+    staff = generator.score.source_staff(prepared)
+    for i, m in enumerate(staff['measures']):
+      m['id'], m['number'] = f'source_{i}', str(i + 1)
+    staff['tuplets'] = generator.score.reidentify(staff['measures'], staff.get('tuplets', []))
+    prepared['metadata'] = {k: v for k, v in prepared['metadata'].items() if k not in ('title', 'composer', 'copyright', 'software')}
+    prepared['metadata'].update(title='Source', composer='')
+    staff.update(id='source', name='Source')
+    melodic = analyze(prepared)
+    content = {'melody': [(str(n.start), str(n.end), dataclasses.asdict(n.pitch)) for n in melodic.melody],
+      'measures': [{k: m.get(k) for k in ('duration', 'time_signature', 'key', 'harmonies', 'tempos', 'section_label', 'implicit')} for m in staff['measures']]}
+    # Fraction/string spelling is canonical and never includes filesystem IDs.
+    key = hashlib.sha256(json.dumps(content, sort_keys=True, default=str).encode()).hexdigest()
+    return cls(json.dumps(prepared, sort_keys=True), key)
+  def document(self):
+    return json.loads(self.document_json)
+  def accompaniment_key(self):
+    document = self.document()
+    document['metadata'].pop('tempo', None)
+    for measure in generator.score.source_staff(document)['measures']:
+      measure['tempos'] = []
+    return SourceAnalysis.read(document).musical_key

generator/summary.py

@@ -6,6 +6,7 @@ import typing
 
 import generator.model
 import generator.score
+import generator.score_io
 
 def _arranged_staff(document: dict[str, typing.Any]) -> dict[str, typing.Any]:
   staves = document.get("staves")
@@ -61,7 +62,7 @@ def performance_document_summary(document: dict[str, typing.Any]) -> dict[str, t
 
 def performance_tree_summary(root: pathlib.Path) -> dict[str, typing.Any]:
   return {
-    path.name: performance_document_summary(generator.score.load_json(path))
+    path.name: performance_document_summary(generator.score_io.load_json(path))
     for path in sorted(root.glob("*.json"))
   }
 

generator/train.py

@@ -3,26 +3,25 @@ from __future__ import annotations
 import argparse
 import collections
 import dataclasses
+import fractions
 import hashlib
 import json
 import os
 import pathlib
+import shutil
 import tempfile
 import time
-import fractions
+
+import generator.fretboard
+import generator.harmonic_time
 import generator.harmony
-import generator.model
 import generator.musicxml
 import generator.parts
+import generator.solo.authored
+import generator.solo.learned
+import generator.source
 import generator.training_io
 import generator.vocabulary
-import generator.harmonic_time
-import generator.solo.analysis
-import generator.solo.learned
-import shutil
-import generator.production
-import generator.solo.authored
-import generator.fretboard
 
 @dataclasses.dataclass(frozen=True)
 class Span:
@@ -57,8 +56,8 @@ def load_examples(manifest):
     if not source.is_relative_to((manifest.parent.parent / 'songs').resolve()):
       raise ValueError('Training sources must be inside data/songs')
     document = generator.parts.prepare_document(generator.musicxml.load_musicxml(source))
-    a = generator.solo.analysis.analyze(document)
-    pair = (generator.production.SourceAnalysis.read(document).musical_key, entry['slot'])
+    a = generator.source.analyze(document)
+    pair = (generator.source.SourceAnalysis.read(document).musical_key, entry['slot'])
     if pair in pairs:
       raise ValueError('Duplicate source music and solo slot')
     pairs.add(pair)

generator/validation.py

@@ -0,0 +1,161 @@
+"""Score document validation, above pure musical values and instrument rules."""
+from __future__ import annotations
+import fractions
+
+import generator.fretboard
+import generator.model
+import generator.pitch
+
+def validate_document(document: dict) -> None:
+  if not isinstance(document, dict) or not isinstance(document.get('metadata'), dict):
+    raise ValueError('score requires a metadata object')
+  if any(not isinstance(document['metadata'].get(name), str) for name in ('title', 'composer', 'key_signature', 'time_signature')):
+    raise ValueError('score metadata requires title, composer, key_signature and time_signature strings')
+  staves = document.get('staves', [])
+  if not isinstance(staves, list) or len(staves) != 1 or not isinstance(staves[0], dict):
+    raise ValueError('exactly one staff is currently supported')
+  if not isinstance(staves[0].get('measures'), list) or not staves[0]['measures']:
+    raise ValueError('score requires measures')
+  instruments = staves[0].get('instruments', [])
+  instrument_ids = {i['id'] for i in instruments}
+  if len(instruments) != len(instrument_ids) or any(not isinstance(i, str) or not i for i in instrument_ids):
+    raise ValueError('score instruments require unique nonempty IDs')
+  ids: set[str] = set()
+  events: dict[str, dict] = {}
+  tones: dict[str, tuple[dict, fractions.Fraction, fractions.Fraction, str]] = {}
+  ownership: dict[str, str] = {}
+  absolute = fractions.Fraction()
+  for measure in staves[0]['measures']:
+    if {'repeats', 'endings', 'initial_attributes'} & measure.keys():
+      raise ValueError('repeat notation is unsupported; regenerate scores from music written out in reading order')
+    length = generator.model.fraction(measure['duration'])
+    if length <= 0:
+      raise ValueError('measure duration must be positive')
+    voices = measure.get('voices', [])
+    if not voices or len({v['id'] for v in voices}) != len(voices) or any(not v['id'] for v in voices):
+      raise ValueError('measure requires uniquely identified voice lanes')
+    previous_ends = {v['id']: fractions.Fraction() for v in voices}
+    for voice, note in ((v, n) for v in voices for n in v['notes']):
+      if any({'sound_ties', 'printed_ties'} & pitch.keys() for pitch in note['pitches']):
+        raise ValueError('pass-specific tie metadata is unsupported; regenerate scores')
+      if not note['id'] or note['id'] in ids:
+        raise ValueError(f'duplicate event ID: {note["id"]}')
+      ids.add(note['id'])
+      events[note['id']] = note
+      ownership[note['id']] = voice['id']
+      if note.get('voice', voice['id']) != voice['id']:
+        raise ValueError('event voice ownership disagrees with its lane')
+      onset, duration = generator.model.fraction(note['onset']), generator.model.fraction(note['duration'])
+      if onset < 0 or duration < 0 or onset + duration > length:
+        raise ValueError(f'event outside measure: {note["id"]}')
+      if note['kind'] not in ('note', 'rest', 'spacer'):
+        raise ValueError('invalid event kind')
+      if not note.get('grace'):
+        if duration == 0 or onset < previous_ends[voice['id']]:
+          raise ValueError('overlapping events in one rhythmic voice')
+        previous_ends[voice['id']] = onset + duration
+      elif duration != 0:
+        raise ValueError('grace notes must not advance metric time')
+      expected = generator.model.metric_length(note['written'])
+      if not note.get('grace') and not note.get('measure_rest') and duration != expected:
+        raise ValueError('metric and written event durations disagree')
+      if note['kind'] != 'note' and note.get('pitches'):
+        raise ValueError('rests and spacers cannot contain pitches')
+      if note['kind'] == 'note' and not note.get('pitches') and note.get('notehead') != 'slash':
+        raise ValueError('pitched event requires at least one pitch')
+      for p in note.get('pitches', []):
+        if p.get('instrument') and p['instrument'] not in instrument_ids:
+          raise ValueError('note refers to an unknown score instrument')
+        if 'unpitched' in p:
+          position = p['unpitched']
+          if position['letter'] not in generator.model.LETTERS or type(position['octave']) is not int:
+            raise ValueError('invalid unpitched display position')
+        else:
+          generator.model.pitch_number(p)
+        if staves[0].get('tablature'):
+          tab = p.get('tablature', {})
+          string, fret = tab.get('string'), tab.get('fret')
+          if type(string) is not int or type(fret) is not int or string not in dict(generator.fretboard.STANDARD_TUNING) or not generator.fretboard.MIN_FRET <= fret <= generator.fretboard.MAX_FRET:
+            raise ValueError(f'{p["id"]}: invalid generated tablature position')
+          state = dict(measure)
+          for change in measure.get('attributes', []):
+            if generator.model.fraction(change['onset']) <= onset:
+              state.update(change)
+          if dict(generator.fretboard.STANDARD_TUNING)[string] + fret != generator.pitch.sounding_pitch(p, state):
+            raise ValueError(f'{p["id"]}: tablature disagrees with sounding pitch')
+        pd = generator.model.fraction(p['duration'])
+        if pd < 0 or (pd == 0 and not note.get('grace')) or pd > duration or not p['id'] or p['id'] in tones:
+          raise ValueError('invalid chord-tone duration or duplicate ID')
+        if not note.get('grace') and pd != generator.model.metric_length(p['written']):
+          raise ValueError('metric and written chord-tone durations disagree')
+        tones[p['id']] = (p, absolute + onset, pd, voice['id'])
+    for harmony in measure.get('harmonies', []):
+      if not 0 <= generator.model.fraction(harmony['onset']) < length:
+        raise ValueError('harmony outside measure')
+    for change in measure.get('attributes', []):
+      if not 0 <= generator.model.fraction(change['onset']) < length:
+        raise ValueError('attribute change outside measure')
+    for state in [measure, *measure.get('attributes', [])]:
+      if 'meter' in state:
+        generator.model.validate_meter(state['meter'])
+      if 'key' in state:
+        if len(state['key']['alterations']) != 7:
+          raise ValueError('key alterations must contain seven exact alterations')
+        for value in state['key']['alterations']:
+          generator.model.alteration(value)
+      if 'clef' in state and (state['clef']['sign'] not in ('G', 'F', 'C', 'percussion', 'none') or not 1 <= state['clef']['line'] <= 6 or not -3 <= state['clef']['octave_change'] <= 3):
+        raise ValueError('invalid clef')
+    absolute += length
+  for identifier, (p, start, duration, voice) in tones.items():
+    if target := p.get('tie_to'):
+      if target not in tones:
+        raise ValueError(f'dangling tie: {identifier} -> {target}')
+      other, next_start, _, next_voice = tones[target]
+      same_pitch = all(p.get(k) == other.get(k) for k in ('unpitched', 'instrument')) if 'unpitched' in p else all(p.get(k) == other.get(k) for k in ('letter', 'alter', 'octave'))
+      if other.get('tie_from') != identifier or not same_pitch or next_voice != voice or next_start != start + duration:
+        raise ValueError(f'invalid tie: {identifier} -> {target}')
+      if staves[0].get('tablature') and p['tablature'] != other['tablature']:
+        raise ValueError(f'{identifier}: tied guitar tones must retain their string and fret')
+    if source := p.get('tie_from'):
+      if source not in tones or tones[source][0].get('tie_to') != identifier:
+        raise ValueError(f'dangling incoming tie: {identifier}')
+  span_ids = set()
+  event_order = {identifier: index for index, identifier in enumerate(events)}
+  for span in staves[0].get('tuplets', []):
+    if not span['id'] or span['id'] in span_ids or span['start'] not in ids or span['end'] not in ids or not 0 < span['note_count'] <= 1000000 or not 0 < span['space_count'] <= 1000000 or not 0 <= span['depth'] <= 16:
+      raise ValueError('invalid tuplet span')
+    members = [n['id'] for n in events.values() if span['id'] in n.get('tuplet_ids', [])]
+    if not members or members[0] != span['start'] or members[-1] != span['end'] or event_order[span['start']] > event_order[span['end']]:
+      raise ValueError('tuplet endpoints disagree with member events')
+    if len({ownership[n] for n in members}) != 1 or span.get('voice', ownership[members[0]]) != ownership[members[0]]:
+      raise ValueError('tuplet crosses independent voice lanes')
+    if span.get('bracket', 'auto') not in ('yes', 'no', 'auto') or span.get('show_number', 'actual') not in ('none', 'actual', 'both'):
+      raise ValueError('invalid tuplet appearance')
+    span_ids.add(span['id'])
+  for note in events.values():
+    for verse in note.get('lyrics', []):
+      start = tones[note['pitches'][0]['id']][1] if note['pitches'] else fractions.Fraction()
+      if 'extend_to' in verse and not start < generator.model.fraction(verse['extend_to']) <= absolute:
+        raise ValueError('lyric extension endpoint is outside the score')
+    if any(t not in span_ids for t in note.get('tuplet_ids', [])):
+      raise ValueError('event refers to an unknown tuplet')
+    if tremolo := note.get('tremolo'):
+      if tremolo['type'] not in ('single', 'start', 'stop') or type(tremolo['marks']) is not int or not 0 <= tremolo['marks'] <= 8:
+        raise ValueError('invalid measured tremolo')
+      if tremolo['type'] in ('start', 'stop'):
+        other = events.get(tremolo.get('partner'))
+        expected_type = 'stop' if tremolo['type'] == 'start' else 'start'
+        if not other or other.get('tremolo', {}).get('partner') != note['id'] or other['tremolo']['type'] != expected_type or other['tremolo']['marks'] != tremolo['marks']:
+          raise ValueError('invalid paired tremolo')
+  for span in staves[0].get('octave_shifts', []):
+    if not 0 <= generator.model.fraction(span['start']) < generator.model.fraction(span['end']) <= absolute or not -3 <= span['octaves'] <= 3:
+      raise ValueError('invalid octave-shift span')
+  previous_end = fractions.Fraction()
+  for span in staves[0].get('scale_spans', []):
+    start, end = generator.model.fraction(span['start']), generator.model.fraction(span['end'])
+    if not previous_end <= start < end <= absolute or not isinstance(span.get('label'), str) or not span['label'].strip() or type(span.get('matched')) is not bool:
+      raise ValueError('invalid solo scale span')
+    if not isinstance(span.get('id'), str) or not span['id'] or span['id'] in ids:
+      raise ValueError('solo scale spans require unique IDs')
+    ids.add(span['id'])
+    previous_end = end

generator/vocabulary.py

@@ -2,12 +2,14 @@
 from __future__ import annotations
 import contextlib
 import contextvars
+import copy
 import dataclasses
+import fractions
 import functools
 import hashlib
 import json
 import pathlib
-import fractions
+import types
 
 ROOT = pathlib.Path(__file__).resolve().parents[1] / 'data' / 'composition'
 ACTIVE = contextvars.ContextVar('composition_vocabulary', default=None)
@@ -16,7 +18,7 @@ def load(root=ROOT):
   # Content, not just the directory, keys memoization. Long-lived individual
   # role builds must observe edited production data without restarting Python.
   names = ('roles', 'harmony', 'lead_ins', 'accompaniment')
-  return _load(tuple((name, (pathlib.Path(root) / (name + '.json')).read_text()) for name in names))
+  return copy.deepcopy(_load(tuple((name, (pathlib.Path(root) / (name + '.json')).read_text()) for name in names)))
 
 @functools.lru_cache(maxsize=16)
 def _load(contents):
@@ -105,26 +107,54 @@ def _load(contents):
 
 @functools.lru_cache(maxsize=1)
 def _default():
-  return {'data': load(), 'role': '', 'settings': {}}
+  return CompositionContext.create(load())
+
+def _freeze(value):
+  if isinstance(value, dict):
+    return types.MappingProxyType({key: _freeze(item) for key, item in value.items()})
+  if isinstance(value, list):
+    return tuple(_freeze(item) for item in value)
+  return value
+
+@functools.lru_cache(maxsize=32)
+def _frozen_json(contents):
+  return _freeze(json.loads(contents))
+
+@dataclasses.dataclass(frozen=True)
+class CompositionContext:
+  """Owned, immutable, pickleable policy inputs with stable iteration order."""
+  resources_json: str
+  name: str = ''
+  settings_json: str = '{}'
+  @classmethod
+  def create(cls, resources, name='', settings=None):
+    return cls(json.dumps(resources), name, json.dumps(settings or {}))
+  def data(self, name):
+    return _frozen_json(self.resources_json)[name]
+  def role(self, name=None):
+    return {**self.data('roles')['roles'].get(name or self.name, {}), **_frozen_json(self.settings_json)}
+  def accompaniment(self, part):
+    value = self.data('accompaniment')
+    return {**{key: item for key, item in value.items() if key != 'instruments'}, **value['instruments'][part]}
 
 def current():
   return ACTIVE.get() or _default()
 
 def data(name):
-  return current()['data'][name]
+  return current().data(name)
 
 def accompaniment(part, resources=None):
-  value = (resources or current()['data'])['accompaniment']
+  if resources is None:
+    return current().accompaniment(part)
+  value = resources['accompaniment']
   return {**{k: v for k, v in value.items() if k != 'instruments'}, **value['instruments'][part]}
 
 def role(name=None):
-  active = current()
-  name = name or active['role']
-  return {**active['data']['roles']['roles'].get(name, {}), **active['settings']}
+  return current().role(name)
 
 @contextlib.contextmanager
 def using(resources, name, settings=None):
-  token = ACTIVE.set({'data': resources, 'role': name, 'settings': settings or {}})
+  token = ACTIVE.set(CompositionContext.create(resources, name, settings))
   try:
     yield
   finally: