Scale Patterns

tests/scale_fixtures.py

@@ -0,0 +1,8 @@
+import functools
+import pathlib
+
+import generator.scales.library
+
+@functools.cache
+def library() -> generator.scales.library.Library:
+  return generator.scales.library.load_library(pathlib.Path(__file__).resolve().parents[1] / 'data/scales')

tests/test_engraving_cache.py

@@ -24,12 +24,13 @@ import generator.engraving.validation
 import generator.pico_artwork
 import generator.score
 import tests.rhythm_fixtures
+import tests.scale_fixtures
 import tests.test_musicxml
 
 class EngravingCacheTests(unittest.TestCase):
   def test_ci_snapshot_tracks_all_generator_inputs_and_keeps_fallback(self):
     workflow = pathlib.Path('.gitea/workflows/ci.yml').read_text()
-    self.assertIn("hashFiles('generator/**/*.py', 'generator/*.py', 'data/songs/*.xml', 'data/rhythms/**/*.xml')", workflow)
+    self.assertIn("hashFiles('generator/**/*.py', 'generator/*.py', 'data/songs/*.xml', 'data/rhythms/**/*.xml', 'data/scales/**/*.xml')", workflow)
     self.assertIn('python3 -m generator --renderer-fingerprint', workflow)
     self.assertIn('restore-keys: product-vamp-pages-${{ steps.renderer.outputs.fingerprint }}-', workflow)
     self.assertIn('path: build/build_common/engraving-cache/${{ steps.renderer.outputs.fingerprint }}', workflow)
@@ -344,6 +345,7 @@ class EngravingCacheTests(unittest.TestCase):
       with self.assertRaisesRegex(ValueError, 'must not overlap'):
         generator.engraving.pipeline.engrave_tree(root, self.fonts, cache_dir=cache)
   def test_parallel_generation_preserves_assets_on_worker_failure(self):
+    self.enterContext(unittest.mock.patch('generator.scales.library.load_library', return_value=tests.scale_fixtures.library()))
     self.enterContext(unittest.mock.patch('generator.rhythm.load_library', return_value=tests.rhythm_fixtures.library()))
     data, output = self.root / 'data', self.root / 'output'
     (data / 'songs').mkdir(parents=True)

tests/test_guitar.py

@@ -17,6 +17,7 @@ import generator.pitch
 import generator.playback
 import tests.arrangement_fixtures
 import tests.rhythm_fixtures
+import tests.scale_fixtures
 import tests.test_musicxml
 
 class GuitarTests(unittest.TestCase):
@@ -233,6 +234,7 @@ class GuitarTests(unittest.TestCase):
       generator.arrange.generate.generate_performance_scores(source, output, rhythms=tests.rhythm_fixtures.library())
       self.assertEqual(first, {p.name: p.read_bytes() for p in output.rglob('*.json')})
   def test_unplayable_generation_preserves_previous_assets(self):
+    self.enterContext(unittest.mock.patch('generator.scales.library.load_library', return_value=tests.scale_fixtures.library()))
     with tempfile.TemporaryDirectory() as temporary:
       root = pathlib.Path(temporary)
       (root / 'data/songs').mkdir(parents=True)

tests/test_harmony.py

@@ -91,7 +91,7 @@ class HarmonyTests(unittest.TestCase):
     source = tests.test_musicxml.parse(tests.test_musicxml.xml_score(chord_xml('7', 'dominant', extra='<degree><degree-value>9</degree-value><degree-alter>-1</degree-alter><degree-type>add</degree-type></degree>') + tests.test_musicxml.xml_note()))
     self.assertEqual(source['staves'][0]['measures'][0]['harmonies'][0]['symbol'], 'C7b9')
   def test_current_song_exports_import_without_edits(self):
-    for name, count, key in (('autumn-leaves', 33, 'F'), ('it-could-happen-to-you', 32, 'Eb')):
+    for name, count, key in (('autumn-leaves', 33, 'D minor'), ('it-could-happen-to-you', 32, 'Eb')):
       source = generator.musicxml.load_musicxml(pathlib.Path('data/songs') / f'{name}.xml')
       self.assertEqual(len(source['staves'][0]['measures']), count)
       self.assertEqual(source['metadata']['key_signature'], key)

tests/test_music_model.py

@@ -15,10 +15,12 @@ import generator.musicxml
 import generator.parts
 import generator.pitch
 import tests.rhythm_fixtures
+import tests.scale_fixtures
 import tests.test_musicxml
 
 class MusicModelTests(unittest.TestCase):
   def setUp(self):
+    self.enterContext(unittest.mock.patch('generator.scales.library.load_library', return_value=tests.scale_fixtures.library()))
     engraving = unittest.mock.patch('generator.engraving.pipeline.engrave_tree')
     engraving.start()
     self.addCleanup(engraving.stop)

tests/test_parts.py

@@ -18,6 +18,7 @@ import generator.pitch
 import generator.playback
 import tests.arrangement_fixtures
 import tests.rhythm_fixtures
+import tests.scale_fixtures
 import tests.test_musicxml
 
 def xml_harmony(root='C', kind='major', offset=0):
@@ -170,6 +171,7 @@ class PartGenerationTests(unittest.TestCase):
           self.assertIn(str(source / (name + '.xml')), str(raised.exception))
         self.assertEqual([p.name for p in output.iterdir()], ['sentinel'])
   def test_pipeline_parses_and_normalizes_each_source_once(self):
+    self.enterContext(unittest.mock.patch('generator.scales.library.load_library', return_value=tests.scale_fixtures.library()))
     with tempfile.TemporaryDirectory() as directory:
       root = pathlib.Path(directory)
       for category, name in (('songs', 'song'), ('songs', 'other')):

tests/test_scales.py

@@ -0,0 +1,192 @@
+import contextlib
+import copy
+import dataclasses
+import fractions
+import io
+import pathlib
+import shutil
+import tempfile
+import unittest
+
+import generator.arrange.generate
+import generator.engraving.cache
+import generator.engraving.fonts
+import generator.engraving.layout
+import generator.engraving.types
+import generator.harmony
+import generator.model
+import generator.musicxml
+import generator.parts
+import generator.playback
+import generator.scales.mapping
+import generator.scales.library
+import generator.scales.selection
+import generator.scales.timeline
+import tests.rhythm_fixtures
+import tests.scale_fixtures
+import tests.test_accompaniment
+import tests.test_musicxml
+
+def timeline(symbols, key=None, duration=4):
+  key = key or generator.model.key_from_fifths(-1, 'minor')
+  result = []
+  for index, symbol in enumerate(symbols):
+    result.append(generator.scales.timeline.ChordSpan(fractions.Fraction(index * duration), fractions.Fraction((index + 1) * duration), index + 1, str(index + 1), fractions.Fraction(1), key, generator.harmony.parse_symbol(symbol) if symbol else None))
+  return tuple(result)
+
+def choose(symbols, key=None):
+  spans = timeline(symbols, key)
+  return generator.scales.selection.select(spans, generator.scales.timeline.Solo(spans[0].start, spans[-1].end, 'Solo'), tests.scale_fixtures.library())
+
+class ScaleLibraryTests(unittest.TestCase):
+  def test_all_patterns_and_transpositions_preserve_spelling_and_rotations(self):
+    library = tests.scale_fixtures.library()
+    self.assertEqual(len(library.patterns), 21)
+    self.assertEqual(len({scale.pitches for scale in library.collections()}), 36)
+    for pattern in library.patterns:
+      for root in generator.scales.library.ROOTS:
+        scale = library.scale(pattern.family, pattern.mode, root)
+        self.assertEqual(len(scale.pitches), 7)
+        self.assertEqual(scale.notes[0], root)
+        self.assertEqual(library.chord_mode(scale, root), (1, scale.label()))
+    self.assertEqual(library.scale('harmonic-minor', 3, 'C').notes, ('C', 'D', 'E', 'F', 'G#', 'A', 'B'))
+    self.assertEqual(library.scale('harmonic-minor', 7, 'C').notes, ('C', 'Db', 'Eb', 'Fb', 'Gb', 'Ab', 'Bbb'))
+    self.assertEqual(library.scale('major', 1, 'C#').pitches, library.scale('major', 1, 'Db').pitches)
+    self.assertNotEqual(library.scale('major', 1, 'C#').notes, library.scale('major', 1, 'Db').notes)
+  def test_incomplete_unexpected_and_inconsistent_xml_patterns_are_rejected(self):
+    with tempfile.TemporaryDirectory() as directory:
+      root = pathlib.Path(directory) / 'scales'
+      shutil.copytree('data/scales', root)
+      path = root / 'harmonic-minor-scale-modes/0003.xml'
+      original = path.read_text()
+      path.unlink()
+      with self.assertRaisesRegex(ValueError, '21 scale patterns'):
+        generator.scales.library.load_library(root)
+      path.write_text(original.replace('<step>A</step>', '<step>G</step>'))
+      with self.assertRaisesRegex(ValueError, '0003.xml.*C through B'):
+        generator.scales.library.load_library(root)
+      path.write_text(original.replace('<alter>1</alter>', '<alter>0</alter>').replace('<accidental>sharp</accidental>', '<accidental>natural</accidental>'))
+      with self.assertRaisesRegex(ValueError, '0003.xml.*not mode 3'):
+        generator.scales.library.load_library(root)
+      path.write_text(original)
+      extra = root / 'unknown.xml'
+      extra.write_text(original)
+      with self.assertRaisesRegex(ValueError, 'unexpected'):
+        generator.scales.library.load_library(root)
+
+class ScaleSelectionTests(unittest.TestCase):
+  def test_autumn_leaves_guide_collections_and_modes(self):
+    first = ['Gm7', 'C7', 'FM7', 'BbM7', 'Em7b5', 'A7', 'Dm7', 'Dm7']
+    bridge = ['Em7b5', 'A7', 'Dm7', 'Dm7', 'Gm7', 'C7', 'FM7', 'FM7']
+    labels = [c.scale.label() for c in choose(first + first + bridge)]
+    phrase = ['D Aeolian'] * 4 + ['D Harmonic Minor'] * 2 + ['D Aeolian'] * 2
+    self.assertEqual(labels, phrase * 2 + ['D Harmonic Minor'] * 2 + ['D Aeolian'] * 6)
+    ending = choose(['Em7b5', 'A7', 'Dm7', 'G7', 'Cm7', 'F7', 'Em7b5', 'A7', 'Dm7'])
+    self.assertEqual([c.scale.label() for c in ending], ['D Harmonic Minor'] * 2 + ['D Aeolian', 'C Harmonic Minor', 'Bb Ionian', 'Bb Ionian'] + ['D Harmonic Minor'] * 2 + ['D Aeolian'])
+    library = tests.scale_fixtures.library()
+    self.assertEqual(library.chord_mode(ending[0].scale, 'E'), (2, 'E Locrian natural 6'))
+    self.assertEqual(library.chord_mode(ending[1].scale, 'A'), (5, 'A Phrygian Dominant'))
+    self.assertEqual(library.chord_mode(ending[4].scale, 'C'), (2, 'C Dorian'))
+  def test_actual_song_uses_minor_tonic_triads_without_forcing_sevenths(self):
+    source = generator.musicxml.load_musicxml(pathlib.Path('data/songs/autumn-leaves.xml'))
+    spans = generator.scales.timeline.read(source['staves'][0])
+    choices = generator.scales.selection.select(spans, generator.scales.timeline.Solo(4, spans[-1].end, 'Solo'), tests.scale_fixtures.library())
+    self.assertTrue(all(c.scale for c in choices))
+    for choice in choices:
+      chord = choice.span.chord
+      if chord['symbol'] == 'Dm':
+        self.assertNotIn(7, generator.harmony.degrees(chord))
+        self.assertEqual(choice.scale.label(), 'D Aeolian')
+      self.assertLessEqual(set(generator.harmony.pitch_classes(chord)), choice.scale.pitches)
+    half_bar = [(c.span.source_measure, c.span.beat, c.scale.label()) for c in choices if c.span.source_measure in ('28', '29')]
+    self.assertEqual(half_bar, [('28', 1, 'D Aeolian'), ('28', 3, 'C Harmonic Minor'), ('29', 1, 'Bb Ionian'), ('29', 3, 'Bb Ionian')])
+  def test_extensions_slash_basses_omissions_and_diminished_chords(self):
+    for symbol in ('Em9b5', 'DmM7', 'Dm6', 'FM7#5', 'C#dim7', 'Dm7/F', 'C7no5', 'C7b5b9'):
+      with self.subTest(symbol=symbol):
+        choice = choose([symbol])[0]
+        self.assertIsNotNone(choice.scale)
+        self.assertLessEqual(set(generator.harmony.pitch_classes(choice.span.chord)), choice.scale.pitches)
+    choices = choose(['C#dim7', 'Dm'])
+    self.assertEqual([c.scale.label() for c in choices], ['D Harmonic Minor', 'D Aeolian'])
+    self.assertIsNone(choose(['A13b9'])[0].scale)
+    with self.assertRaises(ValueError):
+      generator.harmony.parse_symbol('A7alt')
+  def test_shared_collection_wins_over_independent_compatible_colors(self):
+    choices = choose(['E7', 'F#7'], generator.model.key_from_fifths(0))
+    self.assertEqual([choice.scale.label() for choice in choices], ['B Melodic Minor'] * 2)
+    self.assertEqual(choices, choose(['E7', 'F#7'], generator.model.key_from_fifths(0)))
+  def test_carried_harmony_and_fractional_key_boundaries_are_exact(self):
+    document = tests.test_musicxml.parse(tests.test_accompaniment.xml_bars([tests.test_musicxml.xml_note()] * 3))
+    measures = document['staves'][0]['measures']
+    measures[0]['harmonies'] = [{'id': 'h1', 'onset': generator.model.rational(0), **generator.harmony.parse_symbol('CM7')}]
+    measures[1]['attributes'] = [{'onset': generator.model.rational(fractions.Fraction(3, 2)), 'key': generator.model.key_from_fifths(1)}]
+    measures[2]['harmonies'] = [{'id': 'h2', 'onset': generator.model.rational(0), 'kind': 'none', 'degrees': []}]
+    spans = generator.scales.timeline.read(document['staves'][0])
+    self.assertEqual([(s.start, s.end) for s in spans], [(0, 4), (4, fractions.Fraction(11, 2)), (fractions.Fraction(11, 2), 8), (8, 12)])
+    self.assertEqual(spans[1].chord, spans[0].chord)
+    self.assertEqual(spans[2].key['tonic']['letter'], 'g')
+    self.assertIsNone(spans[3].chord)
+  def test_microtonal_harmonies_are_explicitly_unmatched(self):
+    spans = timeline(['A7', 'Dm'])
+    for span in spans:
+      span.chord['root']['alter'] = generator.model.rational(fractions.Fraction(1, 2))
+    result = generator.scales.selection.select(spans, generator.scales.timeline.Solo(0, 8, 'Solo'), tests.scale_fixtures.library())
+    self.assertTrue(all(c.scale is None and 'No exact match' in c.reason for c in result))
+  def test_local_key_and_no_chord_break_continuity_and_missing_tonic_fails(self):
+    spans = list(timeline(['Dm', None, 'Cm']))
+    spans[-1] = dataclasses.replace(spans[-1], key=generator.model.key_from_fifths(-3, 'minor'))
+    solo = generator.scales.timeline.Solo(0, 12, 'Solo')
+    result = generator.scales.selection.select(tuple(spans), solo, tests.scale_fixtures.library())
+    self.assertEqual([c.scale.label() if c.scale else None for c in result], ['D Aeolian', None, 'C Aeolian'])
+    spans[0] = dataclasses.replace(spans[0], key=generator.model.key_from_fifths(0, 'none'))
+    with self.assertRaisesRegex(ValueError, 'measure 1.*declared tonic'):
+      generator.scales.selection.select(tuple(spans), solo, tests.scale_fixtures.library())
+
+class ScaleOutputTests(unittest.TestCase):
+  def test_annotations_reports_and_warnings_preserve_music_and_playback(self):
+    source = tests.test_musicxml.parse(tests.test_accompaniment.xml_bars([tests.test_musicxml.xml_note()] * 4))
+    staff = source['staves'][0]
+    staff.update(name='Acoustic', arrangement_part_type='acoustic')
+    for measure, symbol in zip(staff['measures'], ('CM7', 'CM7', 'A13b9', 'CM7')):
+      measure['harmonies'] = [{'id': measure['id'] + '-h1', 'onset': generator.model.rational(0), **generator.harmony.parse_symbol(symbol)}]
+    before = copy.deepcopy(source)
+    warnings = io.StringIO()
+    with contextlib.redirect_stdout(warnings):
+      report = generator.scales.mapping.annotate(source, [generator.scales.timeline.Solo(0, 12, 'Guitar Solo')], tests.scale_fixtures.library(), 'fixture.xml')
+    self.assertIn('fixture.xml, Acoustic, source measure 3', warnings.getvalue())
+    self.assertIn('No exact match within the permitted catalog', report)
+    self.assertIn('Parent degree: 1; chord-root mode: C Ionian', report)
+    spans = staff.pop('scale_spans')
+    self.assertEqual(source, before)
+    self.assertEqual([(generator.model.fraction(s['start']), generator.model.fraction(s['end']), s['label']) for s in spans], [(0, 8, 'C Ionian'), (8, 12, 'No exact scale match')])
+    staff['scale_spans'] = spans
+    generator.model.validate_document(source)
+    self.assertNotEqual(generator.engraving.cache.score_fingerprint(source, 'renderer'), generator.engraving.cache.score_fingerprint(before, 'renderer'))
+    self.assertEqual(generator.playback.compile_timeline(source), generator.playback.compile_timeline(before))
+  def test_layout_keeps_exact_offsets_continuations_and_uniform_size(self):
+    source = tests.test_musicxml.parse(tests.test_accompaniment.xml_bars([tests.test_musicxml.xml_note()] * 40))
+    staff = source['staves'][0]
+    staff['measures'][0]['section_label'] = 'Guitar Solo'
+    staff['scale_spans'] = [{'id': f'scale-{i}', 'start': generator.model.rational(a), 'end': generator.model.rational(b), 'label': label, 'matched': True} for i, (a, b, label) in enumerate(((0, fractions.Fraction(3, 2), 'C Ionian'), (fractions.Fraction(3, 2), 60, 'D Harmonic Minor')), 1)]
+    fonts = generator.engraving.fonts.Fonts(pathlib.Path('package/assets/fonts/petaluma'))
+    layout = generator.engraving.layout.layout_score(source, generator.engraving.types.Style(), fonts)
+    systems = [s for p in layout.pages for s in p.systems]
+    self.assertEqual(len({s.scale for s in systems}), 1)
+    self.assertTrue(all(len(p.systems) == 2 for p in layout.pages[:-1]))
+    self.assertGreater(len(systems), 2)
+    for system in systems:
+      expected = [s['label'] for s in staff['scale_spans'] if generator.model.fraction(s['start']) < system.end and generator.model.fraction(s['end']) > system.start]
+      self.assertEqual([p.value for p in system.primitives if p.role == 'scale'], expected)
+    self.assertIn(fractions.Fraction(3, 2), generator.engraving.notation.ScoreIndex(source).column_times)
+  def test_only_soloing_part_is_annotated_without_revoicing(self):
+    content = '<harmony><root><root-step>C</root-step></root><kind>major</kind></harmony>' + tests.test_musicxml.xml_note()
+    source = tests.test_musicxml.parse(tests.test_musicxml.xml_score(content))
+    for part in ('acoustic', 'electric', 'vocal'):
+      before = generator.arrange.generate.arrange_document(source, part, rhythms=tests.rhythm_fixtures.library())
+      result = generator.arrange.generate.arrange_document(source, part, rhythms=tests.rhythm_fixtures.library(), scales=tests.scale_fixtures.library())
+      spans = result['staves'][0].pop('scale_spans', [])
+      self.assertEqual(result, before)
+      self.assertEqual(len(spans), int(part != 'vocal'))
+      if spans:
+        self.assertEqual(generator.model.fraction(spans[0]['start']), 12 if part == 'acoustic' else 16)
+        self.assertEqual(generator.model.fraction(spans[0]['end']), 16 if part == 'acoustic' else 20)

tests/test_score_bundle.py

@@ -4,6 +4,7 @@ import fractions
 import hashlib
 import json
 import pathlib
+import shutil
 import tempfile
 import unittest
 import unittest.mock
@@ -39,6 +40,7 @@ class ScoreBundleTests(unittest.TestCase):
       root = pathlib.Path(temporary)
       data, output = root / 'data', root / 'output'
       (data / 'songs').mkdir(parents=True)
+      shutil.copytree(pathlib.Path('data/scales'), data / 'scales')
       source = data / 'songs/fixture.xml'
       source.write_text(tests.test_musicxml.xml_score(tests.test_musicxml.xml_note()))
       args = argparse.Namespace(data_dir=data, output_dir=output, static_assets_dir=pathlib.Path('package/assets'), check=False)
@@ -46,11 +48,17 @@ class ScoreBundleTests(unittest.TestCase):
       original = {p.relative_to(output): p.read_bytes() for p in output.rglob('*') if p.is_file()}
       self.assertTrue(list(output.rglob('*.png')))
       (data / 'rhythms' / 'drafts').mkdir(parents=True)
-      (data / 'scales' / 'major-scale-modes').mkdir(parents=True)
-      for path in ('rhythms/0001.xml', 'rhythms/drafts/0002.xml', 'scales/0001.xml', 'scales/major-scale-modes/0002.xml'):
+      for path in ('rhythms/0001.xml', 'rhythms/drafts/0002.xml'):
         (data / path).write_text('<unfinished')
       generator.__main__.generate_assets(args)
       self.assertEqual(original, {p.relative_to(output): p.read_bytes() for p in output.rglob('*') if p.is_file()})
+      scale = data / 'scales/major-scale-modes/0002.xml'
+      previous = scale.read_bytes()
+      scale.write_text('<unfinished')
+      with self.assertRaisesRegex(ValueError, '0002.xml'):
+        generator.__main__.generate_assets(args)
+      self.assertEqual(original, {p.relative_to(output): p.read_bytes() for p in output.rglob('*') if p.is_file()})
+      scale.write_bytes(previous)
       misplaced = data / '0001.xml'
       misplaced.write_text(tests.test_musicxml.xml_score(tests.test_musicxml.xml_note()))
       with self.assertRaisesRegex(ValueError, 'belong directly under'):