Cleanup

audition/dependency_check.py

@@ -29,7 +29,12 @@ def main():
       collect('targets_present_cold',{r:production.generate_role(source,r,data) for r in production.ROLES})
       for path in refs.glob('*.xml'):path.unlink()
       collect('targets_absent_cold',{r:production.generate_role(source,r,data) for r in production.ROLES})
-    production.generate_role(source,'electric_solo',data,{'choice_seed':'different electric only'})
+    # Exercise an operational musical setting, not merely a different cache
+    # key. A stream seed alone need not change a deterministic winner.
+    changed=production.generate_role(source,'electric_solo',data,{'register_center':72})
+    changed_hash=vocabulary.fingerprint(changed['document'])
+    assert changed_hash!=results['base']['electric_solo'],'electric setting was inert'
+    evidence['changed_electric_register_hash']=changed_hash
     collect('after_electric_setting',{r:production.generate_role(source,r,data,cache_dir=cache) for r in production.ROLES})
   evidence['passed']=True
   (a.output/'dependency.json').write_text(json.dumps(evidence,indent=2))

audition/fit_selection.py

@@ -0,0 +1,216 @@
+"""Explicit development-only shared-rule derivation and fitting.
+
+Targets are read only by this command. Production imports neither this module
+nor its artifacts. A derivation uses the exact same enumerated Actions as the
+normal composer; accepting injected pitches is not a construction proof.
+"""
+from __future__ import annotations
+import argparse
+import bisect
+from collections import Counter
+import dataclasses
+from fractions import Fraction as F
+import hashlib
+import heapq
+import json
+import math
+from pathlib import Path
+import time
+
+from audition.reference_review import read
+from audition.solo_compare import events, rests
+from generator import accompaniment, model, musicxml, parts, production, resources, score, vocabulary
+from generator.arrange import form, generate
+from generator.scales import selection, timeline
+from generator.solo import analysis, construction, harmonic, profiles, report, types
+
+
+def build(source_path,role,data_dir,cache_dir):
+  """Read original song/form and production data; never reference harmony."""
+  document=parts.prepare_document(musicxml.load_musicxml(source_path))
+  a=analysis.analyze(document);part=role.split('_')[0]
+  source=score.source_staff(document)['measures']
+  sections=form.full_arrangement_sections(source,part);si=3 if part=='acoustic' else 4
+  section=sections[si]
+  effective=generate.section_source(source,section,sections[si+1])
+  measures=effective[section.source_measure_index:section.source_measure_index+section.measure_count]
+  initial=accompaniment.carried_chords(effective)[section.source_measure_index]
+  from generator.solo.arrangement import regions
+  first,_,offset,_=regions(document)[si]
+  start,end=next((a-offset,b-offset) for a,b,r in production.windows(document,part) if r==role)
+  if start<0:
+    template=generate._arrange_document(document,part,{},owned={'slash'})
+    measures=[template['staves'][0]['measures'][first-1],*measures]
+    initial=accompaniment.carried_chords(template['staves'][0]['measures'])[first-1]
+  sc,seq,rh=resources.load(data_dir,cache_dir=cache_dir)
+  choices=selection.select(timeline.read({'measures':measures},initial=initial),timeline.Solo(F(),end-start,role),sc)
+  choices=tuple(dataclasses.replace(c,span=dataclasses.replace(c.span,start=c.span.start+start,end=c.span.end+start)) for c in choices)
+  h=harmonic.Harmony(choices,profiles.for_part(part,a))
+  return construction.Builder(a,measures,h,seq,rh,end,'development',start=start),document
+
+
+def canonical(notes):
+  return tuple((n.start,n.end,n.pitch.number()) for n in notes)
+
+
+def derive(builder,target):
+  """Search grammar derivations constrained to the exact supplied events.
+
+  This is a development diagnostic, not normal production inference. Failure
+  is recorded at its earliest uncovered event. No target action is injected.
+  """
+  encoded=tuple((F(n['start']),F(n['end']),int(n['pitch'])) for n in events(target))
+  starts=tuple(n[0] for n in encoded);encoded_set=set(encoded)
+  queue=[builder.start];paths={builder.start:()};seen=set();furthest=builder.start
+  while queue:
+    at=heapq.heappop(queue)
+    if at in seen:continue
+    seen.add(at);furthest=max(furthest,at)
+    if at==builder.end:
+      path=paths[at]
+      checked=builder.harmony.evaluate(tuple(n for a in path for n in a.notes),expressions=tuple(c for a in path for c in a.contracts),complete=True,until=at)
+      return path,{'complete':checked.valid,'covered_until':str(at),'actions':len(path),
+        'status':'constructible' if checked.valid else 'rejected_by_full_context','error':checked.error}
+    left=bisect.bisect_left(starts,at)
+    def matches(action):
+      if any((c.target.start,c.target.end,c.target.pitch.number()) not in encoded_set for c in action.contracts):return False
+      expected=encoded[left:bisect.bisect_left(starts,action.end)]
+      return not (left and encoded[left-1][1]>at) and canonical(action.notes)==expected
+    first=encoded[left] if left<len(encoded) else None
+    for action in builder.operators.at(at,eligible=matches,first=first):
+      if action.end not in paths:
+        paths[action.end]=(*paths[at],action);heapq.heappush(queue,action.end)
+  next_event=next((n for n in encoded if n[0]>=furthest),None)
+  reasons=[]
+  if next_event:
+    start,end,pitch=next_event;region=builder.harmony.at(start)
+    reasons=[{'pitch':pitch,'start':str(start),'end':str(end),
+      'source_chord':region.chord if region else None,'mapped':bool(region and region.scale and pitch%12 in region.scale.pitches),
+      'basic':bool(region and pitch%12 in region.roles.basic),
+      'function':builder.harmony.function(pitch,region) if region else 'missing_harmony'}]
+  return paths.get(furthest,()),{'complete':False,'covered_until':str(furthest),
+    'status':'not_proposed_or_rejected_by_rule','next_events':reasons,
+    'budget':'Exhaustive derivation through available unpruned grammar actions, not unrestricted music search.',
+    'action_coverage':dict(builder.operators.coverage)}
+
+
+def fit(examples,passes,on_checkpoint=None):
+  """Integer early-update structured perceptron on shared musical features.
+
+  Each roll-in is normal source-only ranking until its first wrong action.
+  Only the development command finds a compatible positive action and updates
+  coefficients. Production never receives the positive prefix. A zero-error
+  pass means every unrestricted roll-in produced the entire expected stream.
+  """
+  weights={r:dict(b.selector.weights) for _,r,b,_ in examples};history=[];best=None;tables={}
+  # Whole actions include pickup+hold and compound groups (e.g. 17/4).
+  # Eight beats on the supported twelfth-beat lattice need numerators up to
+  # 96. The previous LCM 1..16 covered note values, not complete actions.
+  normalization=math.lcm(*range(1,97))
+  def table(name,b,at,state):
+    key=(name,at,canonical(state.recent),canonical(state.home),state.last_kind,state.pending)
+    if key in tables:return tables[key]
+    rows=[]
+    actions=b.operators.at(at)
+    if state.home and state.last_kind!='return':actions+=b.operators.related(at,state.home,'return')+b.operators.related(at,state.home,'answer')
+    for action in actions:
+      if b.advance(state,None,action) is None:continue
+      features,prior=b.selector.features(action,state.recent,state.home,state.last_kind)
+      factor=F(normalization)/(action.end-action.start)
+      if factor.denominator!=1:raise ValueError('Unsupported training normalization span')
+      factor=int(factor)
+      rows.append((action,tuple((k,v*factor) for k,v in features.items() if v),
+        sum(v*b.selector_default[k] for k,v in prior.items())*factor))
+    if len(tables)>=24:tables.pop(next(iter(tables)))
+    tables[key]=tuple(rows);return tables[key]
+  from generator.solo.selection import DEFAULT
+  for _,_,b,_ in examples:b.selector_default=DEFAULT
+  for iteration in range(passes):
+    updates={r:Counter() for r in weights};records=[];errors=0;covered=0
+    for name,role,b,gold in examples:
+      b.selector.weights=weights[role];state=construction.State();at=b.start
+      encoded=canonical(tuple(n for a in gold for n in a.notes));encoded_set=set(encoded);starts=tuple(n[0] for n in encoded)
+      while at<b.end:
+        rows=table(name,b,at,state)
+        rank=lambda row:(sum(weights[role].get(k,0)*v for k,v in row[1])+row[2],row[0].key)
+        if not rows:raise ValueError('No feasible action during development roll-in')
+        predicted=min(rows,key=rank)
+        left=bisect.bisect_left(starts,at)
+        def matches(row):
+          action=row[0]
+          return canonical(action.notes)==encoded[left:bisect.bisect_left(starts,action.end)] and all((c.target.start,c.target.end,c.target.pitch.number()) in encoded_set for c in action.contracts)
+        if not matches(predicted):
+          positives=sorted((row for row in rows if matches(row)),key=rank)
+          # An alternate valid derivation can be learnable even when the
+          # cheapest positive shares every feature with the wrong action.
+          positive=next((row for row in positives if dict(row[1])!=dict(predicted[1])),positives[0] if positives else None)
+          if positive is None:
+            records.append({'case':name,'first_error':str(at),'status':'coverage_lost_after_alternate_derivation'})
+            errors+=1;break
+          updates[role].update(dict(predicted[1]));updates[role].subtract(dict(positive[1]))
+          records.append({'case':name,'first_error':str(at),'predicted':predicted[0].kind,'positive':positive[0].kind,
+            'identical_features':dict(predicted[1])==dict(positive[1]),'status':'selection_error'})
+          errors+=1;break
+        action=predicted[0]
+        state=b.advance(state,b.selector.evaluate(action,state.recent,state.home,state.last_kind),action)
+        at=action.end
+      covered+=bisect.bisect_left(starts,at)
+      if at==b.end:records.append({'case':name,'status':'exact_rollout','events':len(encoded)})
+    complexity=sum(abs(v) for w in weights.values() for v in w.values())
+    history.append({'pass':iteration+1,'incorrect_rollouts':errors,'matching_prefix_attacks':covered,'cases':records})
+    checkpoint=(errors,-covered,complexity)
+    if best is None or checkpoint<best[0]:best=(checkpoint,json.loads(json.dumps(weights)))
+    if on_checkpoint:on_checkpoint({'weights':best[1],'history':history,'complete':errors==0})
+    print(f'fit pass {iteration+1}: {errors} incorrect rollouts; {covered} matching prefix attacks',flush=True)
+    if not errors:break
+    for role,difference in updates.items():
+      for feature,delta in difference.items():weights[role][feature]=weights[role].get(feature,0)+delta
+      weights[role]={k:v for k,v in weights[role].items() if v}
+  return {'weights':best[1],'history':history,'complete':history[-1]['incorrect_rollouts']==0,
+    'method':'integer early-update structured perceptron, fixed case order and 200-pass maximum',
+    'checkpoint_metric':'incorrect rollouts, then longest correct prefixes, then coefficient magnitude; full-output exact gate remains separate',
+    'limitation':'Training examples are not unseen evaluation. Runtime independence does not imply development independence.'}
+
+
+def main():
+  p=argparse.ArgumentParser(description=__doc__)
+  p.add_argument('operation',choices=('explain','fit'))
+  p.add_argument('--source',action='append',required=True,metavar='CASE=PATH')
+  p.add_argument('--reference',action='append',required=True,metavar='CASE=PATH')
+  p.add_argument('--role',action='append',required=True,metavar='CASE=acoustic_solo|electric_solo')
+  p.add_argument('--form-start',action='append',default=[],metavar='CASE=MEASURE')
+  p.add_argument('--data-dir',type=Path,default=Path('data'))
+  p.add_argument('--cache-dir',type=Path)
+  p.add_argument('--output',type=Path,required=True)
+  p.add_argument('--passes',type=int,default=200)
+  args=p.parse_args()
+  if not 1<=args.passes<=200:p.error('passes must be 1–200')
+  pairs=lambda values:dict(v.split('=',1) for v in values)
+  sources,refs,roles=map(pairs,(args.source,args.reference,args.role));starts=pairs(args.form_start)
+  if sources.keys()!=refs.keys() or sources.keys()!=roles.keys():p.error('source/reference/role cases must match')
+  args.output.mkdir(parents=True,exist_ok=True);results={};examples=[]
+  data=vocabulary.load(args.data_dir/'composition');began=time.monotonic()
+  for name in sorted(sources):
+    with vocabulary.using(data,roles[name]):
+      b,_=build(sources[name],roles[name],args.data_dir,args.cache_dir)
+      target=read(refs[name],name,int(starts[name]) if name in starts else None,'guitar-written')
+      path,result=derive(b,target)
+      results[name]={**result,'reference_sha256':target['sha256'],'source_sha256':hashlib.sha256(Path(sources[name]).read_bytes()).hexdigest(),
+        'derivation':[{'kind':a.kind,'start':str(a.start),'end':str(a.end),'events':[(str(n.start),str(n.end),n.pitch.number()) for n in a.notes],'rules':[c.rule for c in a.contracts]} for a in path]}
+      if args.operation=='fit' and result['complete']:examples.append((name,roles[name],b,path))
+      print(name,result['status'],result['covered_until'],flush=True)
+  result={'schema':'shared-rule-coverage-1','development_only':True,'coverage_seconds':time.monotonic()-began,'cases':results}
+  if args.operation=='fit':
+    if not all(r['complete'] for r in results.values()):result['fit_status']='blocked_by_construction_coverage'
+    else:
+      def checkpoint(value):
+        path=args.output/'checkpoint.json';temporary=path.with_suffix('.tmp')
+        temporary.write_text(json.dumps(value,indent=2)+'\n');temporary.replace(path)
+      fitting_start=time.monotonic()
+      result['fit']=fit(examples,args.passes,checkpoint)
+      result['fit_seconds']=time.monotonic()-fitting_start
+  result['total_seconds']=time.monotonic()-began
+  (args.output/'coverage.json').write_text(json.dumps(result,indent=2)+'\n')
+  return 0 if all(r['complete'] for r in results.values()) and result.get('fit',{'complete':True})['complete'] else 2
+
+if __name__=='__main__':raise SystemExit(main())

audition/solo_compare.py

@@ -209,7 +209,7 @@ def run(generated_dir,reference_paths,starts,pitch_convention,output,assets=None
   rows=['<!doctype html><meta charset="utf-8"><title>Solo comparison</title>',
     '<style>body{font:16px system-ui;max-width:1100px;margin:2rem auto}td,th{padding:.5rem;text-align:left}pre{white-space:pre-wrap}</style>',
     '<h1>Completed solos and reference scores</h1><p>Differences are diagnostic. These files do not supply generation inputs or candidate scores. No listening verdict is claimed.</p>',
-    '<p><a href="comparison.json">Exact timing, notes, harmony, annotations, placement, and phrase reports</a> · <a href="analysis/theory.md">Complete musical relationship review</a></p>']
+    '<p><a href="comparison.json">Exact timing, notes, harmony, annotations, placement, and phrase reports</a></p>']
   for role,r in result['solos'].items():
     actual_path=generated_dir/manifest['solos'][role]['file']
     reference_path=Path(reference_paths[role])
@@ -221,10 +221,10 @@ def run(generated_dir,reference_paths,starts,pitch_convention,output,assets=None
     selected=collections.Counter(c['kind'] for c in r['production_phrases'].get('expressions',[]))
     rows.append('<p>Selected, validated production treatments: '+html.escape(str(dict(selected)))+'. Ordinary mapped material remains separate.</p>')
     if r.get('reference_events_after_production_end'):
-      rows.append('<p>Reference material extends beyond the production vocal-protection window: '+str(len(r['reference_events_after_production_end']))+' events. The reference and vocal timing remain unchanged.</p>')
+      rows.append('<p>Reference material extends beyond this exported production allocation: '+str(len(r['reference_events_after_production_end']))+' events. The reference and source timing remain unchanged.</p>')
     funnel=r['production_phrases'].get('selection_trace',{}).get('operation_coverage',{})
     if funnel:
-      stages=('eligible_contexts','proposed','rhythm_valid','complete_routes','valid','finalists','placed','lookahead_valid','selected')
+      stages=tuple(dict.fromkeys(k for counts in funnel.values() for k in counts))
       rows.append('<details><summary>Construction coverage (not similarity scoring)</summary><table><tr><th>Purpose</th>'+''.join('<th>'+k.replace('_',' ')+'</th>' for k in stages)+'</tr>')
       for purpose,counts in sorted(funnel.items()):
         rows.append('<tr><td>'+html.escape(purpose)+'</td>'+''.join('<td>'+str(counts.get(k,0))+'</td>' for k in stages)+'</tr>')
@@ -249,9 +249,15 @@ def main():
   parser.add_argument('--output',type=Path,required=True)
   parser.add_argument('--baseline-dir',type=Path,help='Optional previous completed solo exports; never generation inputs.')
   parser.add_argument('--assets',type=Path,help='Optional completed assets for neutral reference/generated audio with identical backing.')
+  parser.add_argument('--require-exact',action='store_true',help='Fail unless every requested reference has identical complete sounding events and rests.')
   args=parser.parse_args()
-  run(args.generated_dir,dict(x.split('=',1) for x in args.reference),
+  if args.require_exact and not args.reference:
+    parser.error('--require-exact requires at least one explicit --reference')
+  result=run(args.generated_dir,dict(x.split('=',1) for x in args.reference),
     {k:int(v) for k,v in (x.split('=',1) for x in args.form_start)},args.reference_pitch,args.output,args.assets,args.baseline_dir)
+  for role,review in result['solos'].items():
+    print(f'{role}: exact={review["exact_musical_match"]}; matching attacks={review["exact_matching_attacks"]}; differing onsets={len(review["differences"])}; rests differ={review["rest_difference"]}')
+  return 1 if args.require_exact and any(not r['exact_musical_match'] for r in result['solos'].values()) else 0
 
 
-if __name__=='__main__':main()
+if __name__=='__main__':raise SystemExit(main())