Cleanup
This commit is contained in:
khodowany
+39
-109
@@ -7,22 +7,20 @@ normal composer; accepting injected pitches is not a construction proof.
|
||||
from __future__ import annotations
|
||||
import argparse
|
||||
import bisect
|
||||
from collections import Counter
|
||||
import dataclasses
|
||||
from fractions import Fraction as F
|
||||
import hashlib
|
||||
import heapq
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import time
|
||||
|
||||
from audition.reference_review import read
|
||||
from audition.solo_compare import events, rests
|
||||
from generator import accompaniment, model, musicxml, parts, production, resources, score, vocabulary
|
||||
from audition.solo_compare import events
|
||||
from generator import accompaniment, musicxml, parts, production, resources, score, vocabulary
|
||||
from generator.arrange import form, generate
|
||||
from generator.scales import selection, timeline
|
||||
from generator.solo import analysis, construction, harmonic, profiles, report, types
|
||||
from generator.solo import analysis, construction, harmonic, profiles
|
||||
|
||||
|
||||
def build(source_path,role,data_dir,cache_dir):
|
||||
@@ -54,122 +52,50 @@ def canonical(notes):
|
||||
|
||||
|
||||
def derive(builder,target):
|
||||
"""Search grammar derivations constrained to the exact supplied events.
|
||||
"""Stateful target-constrained coverage probe; NEVER production success.
|
||||
|
||||
This is a development diagnostic, not normal production inference. Failure
|
||||
is recorded at its earliest uncovered event. No target action is injected.
|
||||
Separate histories carry actual pending contracts. This bounded diagnostic
|
||||
can report exhaustion, not prove a phrase impossible. No notes are injected.
|
||||
"""
|
||||
encoded=tuple((F(n['start']),F(n['end']),int(n['pitch'])) for n in events(target))
|
||||
starts=tuple(n[0] for n in encoded);encoded_set=set(encoded)
|
||||
queue=[builder.start];paths={builder.start:()};seen=set();furthest=builder.start
|
||||
queue=[builder.start];chart={builder.start:[construction.State()]};queued={builder.start}
|
||||
furthest=builder.start;last=construction.State();cuts=0
|
||||
while queue:
|
||||
at=heapq.heappop(queue)
|
||||
if at in seen:continue
|
||||
seen.add(at);furthest=max(furthest,at)
|
||||
at=heapq.heappop(queue);states=chart.pop(at);furthest=max(furthest,at)
|
||||
unique={builder.identity(s):s for s in states};states=list(unique.values())
|
||||
cuts+=max(0,len(states)-16);states=states[:16]
|
||||
if at==builder.end:
|
||||
path=paths[at]
|
||||
checked=builder.harmony.evaluate(tuple(n for a in path for n in a.notes),expressions=tuple(c for a in path for c in a.contracts),complete=True,until=at)
|
||||
return path,{'complete':checked.valid,'covered_until':str(at),'actions':len(path),
|
||||
'status':'constructible' if checked.valid else 'rejected_by_full_context','error':checked.error}
|
||||
for state in states:
|
||||
checked=builder.harmony.evaluate(state.notes,expressions=tuple(c for a in state.actions for c in a.contracts),complete=True,until=at)
|
||||
if checked.valid and canonical(state.notes)==encoded:
|
||||
return state.actions,{'complete':True,'covered_until':str(at),'actions':len(state.actions),
|
||||
'status':'statefully_constructible','diagnostic_only':True,'pruned_histories':cuts}
|
||||
left=bisect.bisect_left(starts,at)
|
||||
def matches(action):
|
||||
if any((c.target.start,c.target.end,c.target.pitch.number()) not in encoded_set for c in action.contracts):return False
|
||||
expected=encoded[left:bisect.bisect_left(starts,action.end)]
|
||||
return not (left and encoded[left-1][1]>at) and canonical(action.notes)==expected
|
||||
first=encoded[left] if left<len(encoded) else None
|
||||
for action in builder.operators.at(at,eligible=matches,first=first):
|
||||
if action.end not in paths:
|
||||
paths[action.end]=(*paths[at],action);heapq.heappush(queue,action.end)
|
||||
actions=builder.operators.at(at,eligible=matches,first=first)
|
||||
for state in states:
|
||||
last=state
|
||||
for action in actions:
|
||||
next_state=builder.advance(state,None,action)
|
||||
if next_state is None:continue
|
||||
chart.setdefault(action.end,[]).append(next_state)
|
||||
if action.end not in queued:heapq.heappush(queue,action.end);queued.add(action.end)
|
||||
next_event=next((n for n in encoded if n[0]>=furthest),None)
|
||||
reasons=[]
|
||||
if next_event:
|
||||
start,end,pitch=next_event;region=builder.harmony.at(start)
|
||||
reasons=[{'pitch':pitch,'start':str(start),'end':str(end),
|
||||
'source_chord':region.chord if region else None,'mapped':bool(region and region.scale and pitch%12 in region.scale.pitches),
|
||||
'basic':bool(region and pitch%12 in region.roles.basic),
|
||||
'function':builder.harmony.function(pitch,region) if region else 'missing_harmony'}]
|
||||
return paths.get(furthest,()),{'complete':False,'covered_until':str(furthest),
|
||||
'status':'not_proposed_or_rejected_by_rule','next_events':reasons,
|
||||
'budget':'Exhaustive derivation through available unpruned grammar actions, not unrestricted music search.',
|
||||
'action_coverage':dict(builder.operators.coverage)}
|
||||
return last.actions,{'complete':False,'covered_until':str(furthest),
|
||||
'status':'coverage_search_exhausted' if cuts else 'not_proposed_or_rejected_by_rule',
|
||||
'next_event':tuple(map(str,next_event)) if next_event else None,
|
||||
'pruned_histories':cuts,'diagnostic_only':True,'action_coverage':dict(builder.operators.coverage)}
|
||||
|
||||
|
||||
def fit(examples,passes,on_checkpoint=None):
|
||||
"""Integer early-update structured perceptron on shared musical features.
|
||||
|
||||
Each roll-in is normal source-only ranking until its first wrong action.
|
||||
Only the development command finds a compatible positive action and updates
|
||||
coefficients. Production never receives the positive prefix. A zero-error
|
||||
pass means every unrestricted roll-in produced the entire expected stream.
|
||||
"""
|
||||
weights={r:dict(b.selector.weights) for _,r,b,_ in examples};history=[];best=None;tables={}
|
||||
# Whole actions include pickup+hold and compound groups (e.g. 17/4).
|
||||
# Eight beats on the supported twelfth-beat lattice need numerators up to
|
||||
# 96. The previous LCM 1..16 covered note values, not complete actions.
|
||||
normalization=math.lcm(*range(1,97))
|
||||
def table(name,b,at,state):
|
||||
key=(name,at,canonical(state.recent),canonical(state.home),state.last_kind,state.pending)
|
||||
if key in tables:return tables[key]
|
||||
rows=[]
|
||||
actions=b.operators.at(at)
|
||||
if state.home and state.last_kind!='return':actions+=b.operators.related(at,state.home,'return')+b.operators.related(at,state.home,'answer')
|
||||
for action in actions:
|
||||
if b.advance(state,None,action) is None:continue
|
||||
features,prior=b.selector.features(action,state.recent,state.home,state.last_kind)
|
||||
factor=F(normalization)/(action.end-action.start)
|
||||
if factor.denominator!=1:raise ValueError('Unsupported training normalization span')
|
||||
factor=int(factor)
|
||||
rows.append((action,tuple((k,v*factor) for k,v in features.items() if v),
|
||||
sum(v*b.selector_default[k] for k,v in prior.items())*factor))
|
||||
if len(tables)>=24:tables.pop(next(iter(tables)))
|
||||
tables[key]=tuple(rows);return tables[key]
|
||||
from generator.solo.selection import DEFAULT
|
||||
for _,_,b,_ in examples:b.selector_default=DEFAULT
|
||||
for iteration in range(passes):
|
||||
updates={r:Counter() for r in weights};records=[];errors=0;covered=0
|
||||
for name,role,b,gold in examples:
|
||||
b.selector.weights=weights[role];state=construction.State();at=b.start
|
||||
encoded=canonical(tuple(n for a in gold for n in a.notes));encoded_set=set(encoded);starts=tuple(n[0] for n in encoded)
|
||||
while at<b.end:
|
||||
rows=table(name,b,at,state)
|
||||
rank=lambda row:(sum(weights[role].get(k,0)*v for k,v in row[1])+row[2],row[0].key)
|
||||
if not rows:raise ValueError('No feasible action during development roll-in')
|
||||
predicted=min(rows,key=rank)
|
||||
left=bisect.bisect_left(starts,at)
|
||||
def matches(row):
|
||||
action=row[0]
|
||||
return canonical(action.notes)==encoded[left:bisect.bisect_left(starts,action.end)] and all((c.target.start,c.target.end,c.target.pitch.number()) in encoded_set for c in action.contracts)
|
||||
if not matches(predicted):
|
||||
positives=sorted((row for row in rows if matches(row)),key=rank)
|
||||
# An alternate valid derivation can be learnable even when the
|
||||
# cheapest positive shares every feature with the wrong action.
|
||||
positive=next((row for row in positives if dict(row[1])!=dict(predicted[1])),positives[0] if positives else None)
|
||||
if positive is None:
|
||||
records.append({'case':name,'first_error':str(at),'status':'coverage_lost_after_alternate_derivation'})
|
||||
errors+=1;break
|
||||
updates[role].update(dict(predicted[1]));updates[role].subtract(dict(positive[1]))
|
||||
records.append({'case':name,'first_error':str(at),'predicted':predicted[0].kind,'positive':positive[0].kind,
|
||||
'identical_features':dict(predicted[1])==dict(positive[1]),'status':'selection_error'})
|
||||
errors+=1;break
|
||||
action=predicted[0]
|
||||
state=b.advance(state,b.selector.evaluate(action,state.recent,state.home,state.last_kind),action)
|
||||
at=action.end
|
||||
covered+=bisect.bisect_left(starts,at)
|
||||
if at==b.end:records.append({'case':name,'status':'exact_rollout','events':len(encoded)})
|
||||
complexity=sum(abs(v) for w in weights.values() for v in w.values())
|
||||
history.append({'pass':iteration+1,'incorrect_rollouts':errors,'matching_prefix_attacks':covered,'cases':records})
|
||||
checkpoint=(errors,-covered,complexity)
|
||||
if best is None or checkpoint<best[0]:best=(checkpoint,json.loads(json.dumps(weights)))
|
||||
if on_checkpoint:on_checkpoint({'weights':best[1],'history':history,'complete':errors==0})
|
||||
print(f'fit pass {iteration+1}: {errors} incorrect rollouts; {covered} matching prefix attacks',flush=True)
|
||||
if not errors:break
|
||||
for role,difference in updates.items():
|
||||
for feature,delta in difference.items():weights[role][feature]=weights[role].get(feature,0)+delta
|
||||
weights[role]={k:v for k,v in weights[role].items() if v}
|
||||
return {'weights':best[1],'history':history,'complete':history[-1]['incorrect_rollouts']==0,
|
||||
'method':'integer early-update structured perceptron, fixed case order and 200-pass maximum',
|
||||
'checkpoint_metric':'incorrect rollouts, then longest correct prefixes, then coefficient magnitude; full-output exact gate remains separate',
|
||||
'limitation':'Training examples are not unseen evaluation. Runtime independence does not imply development independence.'}
|
||||
def fit(examples,passes,on_checkpoint=None,resources=None):
|
||||
# Imported only by this explicitly requested development operation.
|
||||
from audition.structured_learning import fit_complete
|
||||
return fit_complete(examples,passes,on_checkpoint,resources)
|
||||
|
||||
|
||||
def main():
|
||||
@@ -182,8 +108,12 @@ def main():
|
||||
p.add_argument('--data-dir',type=Path,default=Path('data'))
|
||||
p.add_argument('--cache-dir',type=Path)
|
||||
p.add_argument('--output',type=Path,required=True)
|
||||
p.add_argument('--passes',type=int,default=200)
|
||||
p.add_argument('--passes',type=int,default=8)
|
||||
p.add_argument('--solver-path',type=Path,help='Development-only pinned solver installation directory')
|
||||
args=p.parse_args()
|
||||
if args.solver_path:
|
||||
import sys
|
||||
sys.path.insert(0,str(args.solver_path.resolve()))
|
||||
if not 1<=args.passes<=200:p.error('passes must be 1–200')
|
||||
pairs=lambda values:dict(v.split('=',1) for v in values)
|
||||
sources,refs,roles=map(pairs,(args.source,args.reference,args.role));starts=pairs(args.form_start)
|
||||
@@ -199,7 +129,7 @@ def main():
|
||||
'derivation':[{'kind':a.kind,'start':str(a.start),'end':str(a.end),'events':[(str(n.start),str(n.end),n.pitch.number()) for n in a.notes],'rules':[c.rule for c in a.contracts]} for a in path]}
|
||||
if args.operation=='fit' and result['complete']:examples.append((name,roles[name],b,path))
|
||||
print(name,result['status'],result['covered_until'],flush=True)
|
||||
result={'schema':'shared-rule-coverage-1','development_only':True,'coverage_seconds':time.monotonic()-began,'cases':results}
|
||||
result={'schema':'shared-phrase-learning-2','development_only':True,'coverage_seconds':time.monotonic()-began,'cases':results}
|
||||
if args.operation=='fit':
|
||||
if not all(r['complete'] for r in results.values()):result['fit_status']='blocked_by_construction_coverage'
|
||||
else:
|
||||
@@ -207,7 +137,7 @@ def main():
|
||||
path=args.output/'checkpoint.json';temporary=path.with_suffix('.tmp')
|
||||
temporary.write_text(json.dumps(value,indent=2)+'\n');temporary.replace(path)
|
||||
fitting_start=time.monotonic()
|
||||
result['fit']=fit(examples,args.passes,checkpoint)
|
||||
result['fit']=fit(examples,args.passes,checkpoint,data)
|
||||
result['fit_seconds']=time.monotonic()-fitting_start
|
||||
result['total_seconds']=time.monotonic()-began
|
||||
(args.output/'coverage.json').write_text(json.dumps(result,indent=2)+'\n')
|
||||
|
||||
Reference in New Issue
Block a user