khodowany
217 lines
13 KiB
Python
217 lines
13 KiB
Python
"""Explicit development-only shared-rule derivation and fitting.
|
||
|
||
Targets are read only by this command. Production imports neither this module
|
||
nor its artifacts. A derivation uses the exact same enumerated Actions as the
|
||
normal composer; accepting injected pitches is not a construction proof.
|
||
"""
|
||
from __future__ import annotations
|
||
import argparse
|
||
import bisect
|
||
from collections import Counter
|
||
import dataclasses
|
||
from fractions import Fraction as F
|
||
import hashlib
|
||
import heapq
|
||
import json
|
||
import math
|
||
from pathlib import Path
|
||
import time
|
||
|
||
from audition.reference_review import read
|
||
from audition.solo_compare import events, rests
|
||
from generator import accompaniment, model, musicxml, parts, production, resources, score, vocabulary
|
||
from generator.arrange import form, generate
|
||
from generator.scales import selection, timeline
|
||
from generator.solo import analysis, construction, harmonic, profiles, report, types
|
||
|
||
|
||
def build(source_path,role,data_dir,cache_dir):
|
||
"""Read original song/form and production data; never reference harmony."""
|
||
document=parts.prepare_document(musicxml.load_musicxml(source_path))
|
||
a=analysis.analyze(document);part=role.split('_')[0]
|
||
source=score.source_staff(document)['measures']
|
||
sections=form.full_arrangement_sections(source,part);si=3 if part=='acoustic' else 4
|
||
section=sections[si]
|
||
effective=generate.section_source(source,section,sections[si+1])
|
||
measures=effective[section.source_measure_index:section.source_measure_index+section.measure_count]
|
||
initial=accompaniment.carried_chords(effective)[section.source_measure_index]
|
||
from generator.solo.arrangement import regions
|
||
first,_,offset,_=regions(document)[si]
|
||
start,end=next((a-offset,b-offset) for a,b,r in production.windows(document,part) if r==role)
|
||
if start<0:
|
||
template=generate._arrange_document(document,part,{},owned={'slash'})
|
||
measures=[template['staves'][0]['measures'][first-1],*measures]
|
||
initial=accompaniment.carried_chords(template['staves'][0]['measures'])[first-1]
|
||
sc,seq,rh=resources.load(data_dir,cache_dir=cache_dir)
|
||
choices=selection.select(timeline.read({'measures':measures},initial=initial),timeline.Solo(F(),end-start,role),sc)
|
||
choices=tuple(dataclasses.replace(c,span=dataclasses.replace(c.span,start=c.span.start+start,end=c.span.end+start)) for c in choices)
|
||
h=harmonic.Harmony(choices,profiles.for_part(part,a))
|
||
return construction.Builder(a,measures,h,seq,rh,end,'development',start=start),document
|
||
|
||
|
||
def canonical(notes):
|
||
return tuple((n.start,n.end,n.pitch.number()) for n in notes)
|
||
|
||
|
||
def derive(builder,target):
|
||
"""Search grammar derivations constrained to the exact supplied events.
|
||
|
||
This is a development diagnostic, not normal production inference. Failure
|
||
is recorded at its earliest uncovered event. No target action is injected.
|
||
"""
|
||
encoded=tuple((F(n['start']),F(n['end']),int(n['pitch'])) for n in events(target))
|
||
starts=tuple(n[0] for n in encoded);encoded_set=set(encoded)
|
||
queue=[builder.start];paths={builder.start:()};seen=set();furthest=builder.start
|
||
while queue:
|
||
at=heapq.heappop(queue)
|
||
if at in seen:continue
|
||
seen.add(at);furthest=max(furthest,at)
|
||
if at==builder.end:
|
||
path=paths[at]
|
||
checked=builder.harmony.evaluate(tuple(n for a in path for n in a.notes),expressions=tuple(c for a in path for c in a.contracts),complete=True,until=at)
|
||
return path,{'complete':checked.valid,'covered_until':str(at),'actions':len(path),
|
||
'status':'constructible' if checked.valid else 'rejected_by_full_context','error':checked.error}
|
||
left=bisect.bisect_left(starts,at)
|
||
def matches(action):
|
||
if any((c.target.start,c.target.end,c.target.pitch.number()) not in encoded_set for c in action.contracts):return False
|
||
expected=encoded[left:bisect.bisect_left(starts,action.end)]
|
||
return not (left and encoded[left-1][1]>at) and canonical(action.notes)==expected
|
||
first=encoded[left] if left<len(encoded) else None
|
||
for action in builder.operators.at(at,eligible=matches,first=first):
|
||
if action.end not in paths:
|
||
paths[action.end]=(*paths[at],action);heapq.heappush(queue,action.end)
|
||
next_event=next((n for n in encoded if n[0]>=furthest),None)
|
||
reasons=[]
|
||
if next_event:
|
||
start,end,pitch=next_event;region=builder.harmony.at(start)
|
||
reasons=[{'pitch':pitch,'start':str(start),'end':str(end),
|
||
'source_chord':region.chord if region else None,'mapped':bool(region and region.scale and pitch%12 in region.scale.pitches),
|
||
'basic':bool(region and pitch%12 in region.roles.basic),
|
||
'function':builder.harmony.function(pitch,region) if region else 'missing_harmony'}]
|
||
return paths.get(furthest,()),{'complete':False,'covered_until':str(furthest),
|
||
'status':'not_proposed_or_rejected_by_rule','next_events':reasons,
|
||
'budget':'Exhaustive derivation through available unpruned grammar actions, not unrestricted music search.',
|
||
'action_coverage':dict(builder.operators.coverage)}
|
||
|
||
|
||
def fit(examples,passes,on_checkpoint=None):
|
||
"""Integer early-update structured perceptron on shared musical features.
|
||
|
||
Each roll-in is normal source-only ranking until its first wrong action.
|
||
Only the development command finds a compatible positive action and updates
|
||
coefficients. Production never receives the positive prefix. A zero-error
|
||
pass means every unrestricted roll-in produced the entire expected stream.
|
||
"""
|
||
weights={r:dict(b.selector.weights) for _,r,b,_ in examples};history=[];best=None;tables={}
|
||
# Whole actions include pickup+hold and compound groups (e.g. 17/4).
|
||
# Eight beats on the supported twelfth-beat lattice need numerators up to
|
||
# 96. The previous LCM 1..16 covered note values, not complete actions.
|
||
normalization=math.lcm(*range(1,97))
|
||
def table(name,b,at,state):
|
||
key=(name,at,canonical(state.recent),canonical(state.home),state.last_kind,state.pending)
|
||
if key in tables:return tables[key]
|
||
rows=[]
|
||
actions=b.operators.at(at)
|
||
if state.home and state.last_kind!='return':actions+=b.operators.related(at,state.home,'return')+b.operators.related(at,state.home,'answer')
|
||
for action in actions:
|
||
if b.advance(state,None,action) is None:continue
|
||
features,prior=b.selector.features(action,state.recent,state.home,state.last_kind)
|
||
factor=F(normalization)/(action.end-action.start)
|
||
if factor.denominator!=1:raise ValueError('Unsupported training normalization span')
|
||
factor=int(factor)
|
||
rows.append((action,tuple((k,v*factor) for k,v in features.items() if v),
|
||
sum(v*b.selector_default[k] for k,v in prior.items())*factor))
|
||
if len(tables)>=24:tables.pop(next(iter(tables)))
|
||
tables[key]=tuple(rows);return tables[key]
|
||
from generator.solo.selection import DEFAULT
|
||
for _,_,b,_ in examples:b.selector_default=DEFAULT
|
||
for iteration in range(passes):
|
||
updates={r:Counter() for r in weights};records=[];errors=0;covered=0
|
||
for name,role,b,gold in examples:
|
||
b.selector.weights=weights[role];state=construction.State();at=b.start
|
||
encoded=canonical(tuple(n for a in gold for n in a.notes));encoded_set=set(encoded);starts=tuple(n[0] for n in encoded)
|
||
while at<b.end:
|
||
rows=table(name,b,at,state)
|
||
rank=lambda row:(sum(weights[role].get(k,0)*v for k,v in row[1])+row[2],row[0].key)
|
||
if not rows:raise ValueError('No feasible action during development roll-in')
|
||
predicted=min(rows,key=rank)
|
||
left=bisect.bisect_left(starts,at)
|
||
def matches(row):
|
||
action=row[0]
|
||
return canonical(action.notes)==encoded[left:bisect.bisect_left(starts,action.end)] and all((c.target.start,c.target.end,c.target.pitch.number()) in encoded_set for c in action.contracts)
|
||
if not matches(predicted):
|
||
positives=sorted((row for row in rows if matches(row)),key=rank)
|
||
# An alternate valid derivation can be learnable even when the
|
||
# cheapest positive shares every feature with the wrong action.
|
||
positive=next((row for row in positives if dict(row[1])!=dict(predicted[1])),positives[0] if positives else None)
|
||
if positive is None:
|
||
records.append({'case':name,'first_error':str(at),'status':'coverage_lost_after_alternate_derivation'})
|
||
errors+=1;break
|
||
updates[role].update(dict(predicted[1]));updates[role].subtract(dict(positive[1]))
|
||
records.append({'case':name,'first_error':str(at),'predicted':predicted[0].kind,'positive':positive[0].kind,
|
||
'identical_features':dict(predicted[1])==dict(positive[1]),'status':'selection_error'})
|
||
errors+=1;break
|
||
action=predicted[0]
|
||
state=b.advance(state,b.selector.evaluate(action,state.recent,state.home,state.last_kind),action)
|
||
at=action.end
|
||
covered+=bisect.bisect_left(starts,at)
|
||
if at==b.end:records.append({'case':name,'status':'exact_rollout','events':len(encoded)})
|
||
complexity=sum(abs(v) for w in weights.values() for v in w.values())
|
||
history.append({'pass':iteration+1,'incorrect_rollouts':errors,'matching_prefix_attacks':covered,'cases':records})
|
||
checkpoint=(errors,-covered,complexity)
|
||
if best is None or checkpoint<best[0]:best=(checkpoint,json.loads(json.dumps(weights)))
|
||
if on_checkpoint:on_checkpoint({'weights':best[1],'history':history,'complete':errors==0})
|
||
print(f'fit pass {iteration+1}: {errors} incorrect rollouts; {covered} matching prefix attacks',flush=True)
|
||
if not errors:break
|
||
for role,difference in updates.items():
|
||
for feature,delta in difference.items():weights[role][feature]=weights[role].get(feature,0)+delta
|
||
weights[role]={k:v for k,v in weights[role].items() if v}
|
||
return {'weights':best[1],'history':history,'complete':history[-1]['incorrect_rollouts']==0,
|
||
'method':'integer early-update structured perceptron, fixed case order and 200-pass maximum',
|
||
'checkpoint_metric':'incorrect rollouts, then longest correct prefixes, then coefficient magnitude; full-output exact gate remains separate',
|
||
'limitation':'Training examples are not unseen evaluation. Runtime independence does not imply development independence.'}
|
||
|
||
|
||
def main():
|
||
p=argparse.ArgumentParser(description=__doc__)
|
||
p.add_argument('operation',choices=('explain','fit'))
|
||
p.add_argument('--source',action='append',required=True,metavar='CASE=PATH')
|
||
p.add_argument('--reference',action='append',required=True,metavar='CASE=PATH')
|
||
p.add_argument('--role',action='append',required=True,metavar='CASE=acoustic_solo|electric_solo')
|
||
p.add_argument('--form-start',action='append',default=[],metavar='CASE=MEASURE')
|
||
p.add_argument('--data-dir',type=Path,default=Path('data'))
|
||
p.add_argument('--cache-dir',type=Path)
|
||
p.add_argument('--output',type=Path,required=True)
|
||
p.add_argument('--passes',type=int,default=200)
|
||
args=p.parse_args()
|
||
if not 1<=args.passes<=200:p.error('passes must be 1–200')
|
||
pairs=lambda values:dict(v.split('=',1) for v in values)
|
||
sources,refs,roles=map(pairs,(args.source,args.reference,args.role));starts=pairs(args.form_start)
|
||
if sources.keys()!=refs.keys() or sources.keys()!=roles.keys():p.error('source/reference/role cases must match')
|
||
args.output.mkdir(parents=True,exist_ok=True);results={};examples=[]
|
||
data=vocabulary.load(args.data_dir/'composition');began=time.monotonic()
|
||
for name in sorted(sources):
|
||
with vocabulary.using(data,roles[name]):
|
||
b,_=build(sources[name],roles[name],args.data_dir,args.cache_dir)
|
||
target=read(refs[name],name,int(starts[name]) if name in starts else None,'guitar-written')
|
||
path,result=derive(b,target)
|
||
results[name]={**result,'reference_sha256':target['sha256'],'source_sha256':hashlib.sha256(Path(sources[name]).read_bytes()).hexdigest(),
|
||
'derivation':[{'kind':a.kind,'start':str(a.start),'end':str(a.end),'events':[(str(n.start),str(n.end),n.pitch.number()) for n in a.notes],'rules':[c.rule for c in a.contracts]} for a in path]}
|
||
if args.operation=='fit' and result['complete']:examples.append((name,roles[name],b,path))
|
||
print(name,result['status'],result['covered_until'],flush=True)
|
||
result={'schema':'shared-rule-coverage-1','development_only':True,'coverage_seconds':time.monotonic()-began,'cases':results}
|
||
if args.operation=='fit':
|
||
if not all(r['complete'] for r in results.values()):result['fit_status']='blocked_by_construction_coverage'
|
||
else:
|
||
def checkpoint(value):
|
||
path=args.output/'checkpoint.json';temporary=path.with_suffix('.tmp')
|
||
temporary.write_text(json.dumps(value,indent=2)+'\n');temporary.replace(path)
|
||
fitting_start=time.monotonic()
|
||
result['fit']=fit(examples,args.passes,checkpoint)
|
||
result['fit_seconds']=time.monotonic()-fitting_start
|
||
result['total_seconds']=time.monotonic()-began
|
||
(args.output/'coverage.json').write_text(json.dumps(result,indent=2)+'\n')
|
||
return 0 if all(r['complete'] for r in results.values()) and result.get('fit',{'complete':True})['complete'] else 2
|
||
|
||
if __name__=='__main__':raise SystemExit(main())
|