Source

evolve-keyboard-layout / layout_cost.py

Full commit
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
#!/usr/bin/env python3
# encoding: utf-8

"""Calculate the cost of a layout."""

from layout_base import *

from ngrams import get_all_data, letters_in_file_precalculated, trigrams_in_file_precalculated, trigrams_in_file, split_uppercase_trigrams, repeats_in_file_precalculated, repeats_in_file_sorted, unique_sort, letters_in_file, split_uppercase_letters, repeats_in_file, split_uppercase_repeats

from numpy import array, resize

### Caches

FINGER_REPEATS_FOR_POSITIONS = None
FINGER_REPEATS_TOP_BOTTOM_FOR_POSITIONS = None
CACHE_NO_HANDSWITCHING = None

def clear_all_position_cost_caches():
    """Clean all cost caches. Has to be called when a letter is removed or added (for example when a new layout is tested). Not needed when keys are only switched."""
    global FINGER_REPEATS_FOR_POSITIONS
    FINGER_REPEATS_FOR_POSITIONS = None
    
    global FINGER_REPEATS_TOP_BOTTOM_FOR_POSITIONS
    FINGER_REPEATS_TOP_BOTTOM_FOR_POSITIONS = None

    global CACHE_NO_HANDSWITCHING
    CACHE_NO_HANDSWITCHING = None
    

### Cost Functions

def key_position_cost_from_file(data=None, letters=None, layout=NEO_LAYOUT, cost_per_key=COST_PER_KEY):
    """Count the total cost due to key positions.

    >>> data = read_file("testfile")
    >>> key_position_cost_from_file(data, cost_per_key=TEST_COST_PER_KEY)
    150
    >>> print(data[:3])
    uia
    >>> key_position_cost_from_file(data, cost_per_key=TEST_COST_PER_KEY)
    150
    >>> key_position_cost_from_file(data[:3], cost_per_key=TEST_COST_PER_KEY)
    11
    >>> from check_neo import switch_keys
    >>> lay = switch_keys(["ax"], layout=NEO_LAYOUT)
    >>> key_position_cost_from_file(data[:3], cost_per_key=TEST_COST_PER_KEY, layout=lay)
    20
    >>> data = "UIaĥK\\n"
    >>> key_position_cost_from_file(data, cost_per_key=TEST_COST_PER_KEY, layout=lay)
    240
    """
    if data is not None: 
        letters = letters_in_file(data)
    elif letters is None:
        raise Exception("Need either letters or data")
    letters = split_uppercase_letters(letters, layout=layout)
    cost = 0
    for num, letter in letters:
        pos = find_key(letter, layout=layout)
        if pos is None: # not found
            cost += num * COST_PER_KEY_NOT_FOUND
        # shift, M3 and M4
        elif COST_LAYER_ADDITION[pos[2]:]:
            cost += num * (cost_per_key[pos[0]][pos[1]] + COST_LAYER_ADDITION[pos[2]])
        else:
            # layer has no addition cost ⇒ undefined layer (higher than layer 6!). Just take the base key…
            cost += num * cost_per_key[pos[0]][pos[1]]
    return cost

def repeats_array_to_repeats(mat, layout=NEO_LAYOUT):
    """turn a numpy repeats array back into simple repeats.

    >>> data = read_file("testfile")
    >>> letters, number_of_letters, bigrams, number_of_bigrams, trigrams, number_of_trigrams = get_all_data(data=data)
    >>> bigrams.sort()
    >>> mat, missing = repeats_to_pos_array(bigrams, layout=NEO_LAYOUT)
    >>> big = repeats_array_to_repeats(mat, layout=NEO_LAYOUT)
    >>> big.sort()
    >>> big == bigrams
    True
    """
    all_pos = get_all_positions_in_layout(layout)
    reps = []
    for i in range(len(all_pos)): 
        for j in range(len(all_pos)):
            if mat[i, j]: 
                reps.append((mat[i, j], get_key(all_pos[i], layout=layout) + get_key(all_pos[j], layout=layout)))
    return reps
    

def repeats_to_pos_array(repeats, layout):
    """turn bigrams into a array of positions with the occurrance number as values.

    >>> letters, number_of_letters, bigrams, number_of_bigrams, trigrams, number_of_trigrams = get_all_data()
    >>> mat, missing = repeats_to_pos_array(bigrams, layout=NEO_LAYOUT)
    >>> mat_sum = mat.sum()
    >>> mat_sum == sum((num for num, bi in bigrams if find_key(bi[0], layout=NEO_LAYOUT) and find_key(bi[1], layout=NEO_LAYOUT) ))
    True

    >>> data = read_file("testfile")
    >>> letters, number_of_letters, bigrams, number_of_bigrams, trigrams, number_of_trigrams = get_all_data(data=data)
    >>> mat, missing = repeats_to_pos_array(bigrams, layout=NEO_LAYOUT)
    >>> mat_sum = mat.sum()
    >>> mat_sum == sum((num for num, bi in bigrams if find_key(bi[0], layout=NEO_LAYOUT) and find_key(bi[1], layout=NEO_LAYOUT) ))
    True
    """
    all_pos = get_all_positions_in_layout(layout)
    all_pos_len = len(all_pos)
    # rewrite the list into a dict for faster access
    reps = array([])
    reps = resize(reps, (all_pos_len, all_pos_len)).astype(int)
    missing = []
    for num, rep in repeats:
        pos0 = find_key(rep[0], layout=layout)
        pos1 = find_key(rep[1], layout=layout)
        if pos0 is None or pos1 is None:
            missing.append((num, rep))
        else:
            p0 = all_pos.index(pos0)
            p1 = all_pos.index(pos1)
            reps[p0, p1] += num
        
    return reps, missing
    

def finger_repeats_from_array(repeats_array, count_same_key=False, layout=NEO_LAYOUT):
    """Get a list of two char strings from the file, which repeat the same finger.

    >>> data = read_file("testfile")
    >>> letters, number_of_letters, bigrams, number_of_bigrams, trigrams, number_of_trigrams = get_all_data(data=data)
    >>> repeats = split_uppercase_repeats(bigrams, layout=NEO_LAYOUT)
    >>> mat, missing = repeats_to_pos_array(repeats, layout=NEO_LAYOUT)

    >>> m = finger_repeats_from_array(mat, count_same_key = False, layout=NEO_LAYOUT)
    >>> m.sum()
    3
    >>> repeats_array_to_repeats(m, layout=NEO_LAYOUT)
    [(1, 'eo'), (1, 'rg'), (1, 'd\\n')]

    >>> clear_all_position_cost_caches()
    >>> m = finger_repeats_from_array(mat, count_same_key = True, layout=NEO_LAYOUT)
    >>> m.sum()
    7
    >>> repeats_array_to_repeats(m, layout=NEO_LAYOUT)
    [(4, 'aa'), (1, 'eo'), (1, 'rg'), (1, 'd\\n')]
    """
    if repeats_array is None: 
        raise Exception("Need either repeats, repeats_array or data")

    global FINGER_REPEATS_FOR_POSITIONS
    if FINGER_REPEATS_FOR_POSITIONS is None or FINGER_REPEATS_FOR_POSITIONS.shape != repeats_array.shape:
        all_pos = get_all_positions_in_layout(layout)
        all_pos_len = len(all_pos)
        
        FINGER_REPEATS_FOR_POSITIONS = array([])
        FINGER_REPEATS_FOR_POSITIONS = resize(FINGER_REPEATS_FOR_POSITIONS, (all_pos_len, all_pos_len)).astype(int)

        pos2finger = {pos: pos_to_finger(pos, layout=layout) for pos in all_pos}
        
        for i in range(all_pos_len):
            for j in range(all_pos_len):
                fi = pos_to_finger(all_pos[i])
                fj = pos_to_finger(all_pos[j])
                if fi == fj:
                    if count_same_key:
                        FINGER_REPEATS_FOR_POSITIONS[i, j] = 1
                    elif not count_same_key and i != j:
                        FINGER_REPEATS_FOR_POSITIONS[i, j] = 1

    mat = repeats_array*FINGER_REPEATS_FOR_POSITIONS

    return mat

def finger_repeats_from_file(data=None, repeats=None, count_same_key=False, layout=NEO_LAYOUT):
    """Get a list of two char strings from the file, which repeat the same finger.

    >>> data = read_file("testfile")
    >>> finger_repeats_from_file(data)
    [(1, 'Mittel_R', 'rg'), (1, 'Zeige_L', 'eo'), (1, 'Klein_R', 'd\\n')]
    >>> finger_repeats_from_file(data, count_same_key=True)
    [(2, 'Mittel_L', 'aa'), (1, 'Mittel_R', 'rg'), (1, 'Zeige_L', 'eo'), (1, 'Klein_R', 'd\\n'), (1, 'Mittel_L', 'aa'), (1, 'Mittel_L', 'aa')]
    """
    if data is not None: 
        repeats = repeats_in_file(data)
        repeats = split_uppercase_repeats(repeats, layout=layout)
    elif repeats is None: 
        raise Exception("Need either repeats or data")
        
    finger_repeats = []
    for number, pair in repeats:
        key1 = pair[0]
        key2 = pair[1]
        finger1 = key_to_finger(key1, layout=layout)
        finger2 = key_to_finger(key2, layout=layout)
                
        if finger1 and finger2 and finger1 == finger2:
            finger_repeats.append((number, finger1, key1+key2))
    if not count_same_key:
        finger_repeats = [r for r in finger_repeats if not r[2][0] == r[2][1]]

    return finger_repeats

def finger_repeats_from_array_top_and_bottom(repeats_array, layout=NEO_LAYOUT):
    """Get a list of two char strings from the file, which repeat the same finger.

    >>> letters, number_of_letters, bigrams, number_of_bigrams, trigrams, number_of_trigrams = get_all_data()
    >>> repeats = split_uppercase_repeats(bigrams, layout=NEO_LAYOUT)
    >>> mat, missing = repeats_to_pos_array(repeats, layout=NEO_LAYOUT)

    >>> m = finger_repeats_from_array_top_and_bottom(mat, layout=NEO_LAYOUT)
    >>> m.sum()
    1166600
    >>> repeats_array_to_repeats(m, layout=NEO_LAYOUT)
    [(241, 'x⇧'), (5, 'xü'), (21943, 'vö'), (250944, 'lä'), (321, 'cp'), (2878, 'cz'), (284, 'wp'), (107, 'wz'), (9906, 'kb'), (11889, 'km'), (30157, 'hb'), (265188, 'hm'), (121734, 'g,'), (29527, 'f.'), (9, 'qj'), (655, 'q⇗'), (65, 'ßj'), (5, 'ß⇗'), (310, '⇧x'), (37, 'üx'), (1738, 'öv'), (94079, 'äl'), (3513, 'pc'), (298, 'pw'), (1655, 'zc'), (238534, 'zw'), (7565, 'bk'), (23527, 'bh'), (13757, 'mk'), (5366, 'mh'), (14113, ',g'), (13882, '.f'), (5, 'jq'), (7, 'jß'), (2088, '⇗q'), (268, '⇗ß')]
    """
    if repeats_array is None: 
        raise Exception("Need either repeats, repeats_array or data")

    global FINGER_REPEATS_TOP_BOTTOM_FOR_POSITIONS
    if FINGER_REPEATS_TOP_BOTTOM_FOR_POSITIONS is None or FINGER_REPEATS_TOP_BOTTOM_FOR_POSITIONS.shape != repeats_array.shape:
        all_pos = get_all_positions_in_layout(layout)
        all_pos_len = len(all_pos)
        
        FINGER_REPEATS_TOP_BOTTOM_FOR_POSITIONS = array([])
        FINGER_REPEATS_TOP_BOTTOM_FOR_POSITIONS = resize(FINGER_REPEATS_TOP_BOTTOM_FOR_POSITIONS, (all_pos_len, all_pos_len)).astype(int)

        pos2finger = {pos: pos_to_finger(pos, layout=layout) for pos in all_pos}
        
        for i in range(all_pos_len):
            for j in range(all_pos_len):
                if abs(all_pos[i][0] - all_pos[j][0]) <= 1:
                    continue
                if pos_to_finger(all_pos[i]) == pos_to_finger(all_pos[j]):
                    FINGER_REPEATS_TOP_BOTTOM_FOR_POSITIONS[i, j] = 1

    mat = repeats_array*FINGER_REPEATS_TOP_BOTTOM_FOR_POSITIONS

    return mat


def finger_repeats_top_and_bottom(finger_repeats, layout):
    """Check which of the finger repeats go from the top to the bottom row or vice versa."""
    top_down_repeats = []
    for number, letters in finger_repeats:
        pos0 = find_key(letters[0], layout)
        pos1 = find_key(letters[1], layout)
        # count it as top down, if the finger has to move over more than one col.
        if pos0 and pos1 and abs(pos0[0] - pos1[0]) > 1: 
            top_down_repeats.append((number, letters))
    return top_down_repeats

def _rep_to_fingtuple(num, rep, layout, finger_switch_cost):
    """Turn a repeat and occurance number into num and a fingtuple."""
    finger1 = key_to_finger(rep[0], layout=layout)
    if not finger1 or not finger1 in finger_switch_cost:
        return None, None
    finger2 = key_to_finger(rep[1], layout=layout)
    if not finger2 or not finger2 in finger_switch_cost[finger1]:
        return None, None
    return num, (finger1, finger2)

def movement_pattern_cost(data=None, repeats=None, layout=NEO_LAYOUT, FINGER_SWITCH_COST=FINGER_SWITCH_COST):
    """Calculate a movement cost based on the FINGER_SWITCH_COST. 

    # TODO: Adjust the doctest to use 

    >>> data = read_file("testfile")
    >>> print(data)
    uiaenrtAaAa
    eod
    rg
    aa
    <BLANKLINE>
    >>> neighboring_fingers(data, FINGER_SWITCH_COST=TEST_FINGER_SWITCH_COST)
    16
    """
    if data is not None: 
        repeats = repeats_in_file(data)
        repeats = split_uppercase_repeats(repeats, layout=layout)
    elif repeats is None:
        raise Exception("Need either repeats or data")

    fingtups = (_rep_to_fingtuple(num, rep, layout, FINGER_SWITCH_COST) for num, rep in repeats)

    return sum((num*FINGER_SWITCH_COST[fings[0]][fings[1]] for num, fings in fingtups if num))

neighboring_fingers = movement_pattern_cost

def no_handswitch_after_unbalancing_key(data=None, repeats=None, layout=NEO_LAYOUT):
    """Check how often we have no handswitching after an unbalancing key, weighted by the severity of the unbalancing. This also helps avoiding a handswitch directly after an uppercase key (because shift severly unbalances und with the handswitch we’d effectively have no handswitch after the shift (kind of a shift collision, too). 

    >>> data = read_file("testfile")
    >>> no_handswitch_after_unbalancing_key(data)
    2
    >>> reps =  [(3, "Ab")]
    >>> reps = split_uppercase_repeats(reps, layout=QWERTZ_LAYOUT)
    >>> no_handswitch_after_unbalancing_key(repeats=reps)
    6
    >>> no_handswitch_after_unbalancing_key(repeats=reps, layout=QWERTZ_LAYOUT)
    0
    >>> reps = [(3, "Ga")]
    >>> reps = split_uppercase_repeats(reps, layout=QWERTZ_LAYOUT)
    >>> no_handswitch_after_unbalancing_key(repeats=reps, layout=QWERTZ_LAYOUT)
    3
    """
    if data is not None: 
        repeats = repeats_in_file(data)
        repeats = split_uppercase_repeats(repeats, layout=layout)
    elif repeats is None:
        raise Exception("Need either repeats or data")

    no_switch = 0
    for number, pair in repeats:
        pos1 = find_key(pair[0], layout=layout)
        if not pos1 or not pos1 in UNBALANCING_POSITIONS:
            continue
        pos2 = find_key(pair[1], layout=layout)
        if pos2:
                # check if we”re on the same hand
                is_left1 = pos_is_left(pos1)
                is_left2 = pos_is_left(pos2)
                if is_left1 == is_left2:
                    # using .get here, because most positions aren’t unbalancing.
                    no_switch += UNBALANCING_POSITIONS.get(pos1, 0)*number
    return no_switch

def line_changes(data=None, repeats=None, layout=NEO_LAYOUT):
    """Get the number of (line changes divided by the horizontal distance) squared: (rows²/dist)².

    TODO: Don’t care about the hand (left index low and right high is still not nice). 

    >>> data = read_file("testfile")
    >>> line_changes(data)
    16.29
    """
    if data is not None: 
        repeats = repeats_in_file(data)
        repeats = split_uppercase_repeats(repeats, layout=layout)
    elif repeats is None:
        raise Exception("Need either repeats or data")
    

    line_changes = 0
    for number, pair in repeats:
        key1 = pair[0]
        key2 = pair[1]
        pos1 = find_key(key1, layout=layout)
        pos2 = find_key(key2, layout=layout)
        if pos1 and pos2:
            if not WEIGHT_COUNT_ROW_CHANGES_BETWEEN_HANDS: 
                # check if we”re on the same hand
                is_left1 = pos_is_left(pos1)
                is_left2 = pos_is_left(pos2)
                if is_left1 != is_left2:
                    continue # the keys are on different hands, so we don’t count them as row change.
            # row 3 is shifted 1 key to the right → fix that.
            if pos1[0] == 3:
                pos1 = pos1[0], pos1[1] -1, pos1[2]
            if pos2[0] == 3:
                pos2 = pos2[0], pos2[1] -1, pos2[2]
            num_rows = abs(pos1[0] - pos2[0])
            finger_distance = abs(pos1[1] - pos2[1])
            if num_rows:
                cost = num_rows**2 / max(0.25, finger_distance)
                line_changes += cost**2 * number
    return line_changes

def load_per_finger(letters, layout=NEO_LAYOUT, print_load_per_finger=False):
    """Calculate the number of times each finger is being used.

    >>> letters = [(1, "u"), (5, "i"), (10, "2"), (3, " ")]
    >>> load_per_finger(letters)
    {'': 10, 'Klein_L': 1, 'Ring_L': 5, 'Daumen_L': 3}
    """
    letters = split_uppercase_letters(letters, layout)
    fingers = {}
    for num, key in letters:
        finger = key_to_finger(key, layout=layout)
        if finger in fingers:
            fingers[finger] += num
        else: fingers[finger] = num
    # Debug: Print the load per finger
    if print_load_per_finger: 
        from pprint import pprint
        pprint(fingers)
    return fingers

def load_per_hand(letters=None, finger_load=None, layout=NEO_LAYOUT):
    """Calculate the load per hand.

    >>> letters = [(1, "u"), (5, "i"), (10, "2"), (3, " "), (2, "g")]
    >>> load_per_hand(letters)
    [9, 2]
    >>> finger_load = {'': 10, 'Klein_L': 1, 'Ring_L': 5, 'Daumen_L': 3, 'Mittel_R': 2}
    >>> load_per_hand(finger_load = finger_load)
    [9, 2]
    
    """
    if finger_load is None and letters is not None: 
        finger_load = load_per_finger(letters, layout=layout)
    elif letters is None and finger_load is None:
        raise Exception("Need at least letters or precalculated finger_load")
    hand_load = [sum([finger_load[f] for f in finger_load if f.endswith(hand)]) for hand in ("L", "R")]
    return hand_load


def std(numbers):
    """Calculate the standard deviation from a set of numbers.

    This simple calculation is only valid for more than 100 numbers or so. That means I use it in the invalid area. But since it’s just an arbitrary metric, that doesn’t hurt.

    >>> std([1, 2, 3, 4, 5, 6, 5, 4, 3, 2, 1]*10)
    1.607945243653783
    """
    length = float(len(numbers))
    mean = sum(numbers)/max(1, length)
    var = 0
    for i in numbers:
        var += (i - mean)**2
    var /= max(1, (length - 1))
    from math import sqrt
    return sqrt(var)

def finger_balance(letters, layout=NEO_LAYOUT, intended_balance=WEIGHT_INTENDED_FINGER_LOAD_LEFT_PINKY_TO_RIGHT_PINKY):
    """Calculate a cost based on the balance between the fingers (using the standard deviation).

    Optimum: All fingers get used exactly the same number of times.

    We ignore unmapped keys ('').
    """
    #: the usage of each finger: {finger1: num, finger2: num, …}
    fingers = load_per_finger(letters, layout)
    # make sure, all fingers are in the list (for very short texts)
    for fing in FINGER_NAMES:
        if not fing in fingers and not fing[:6] == "Daumen":
            fingers[fing] = 0
    # remove the unmapped keys
    if "" in fingers: 
        del fingers[""]
    for finger in fingers:
        idx = FINGER_NAMES.index(finger)
        multiplier = intended_balance[idx]
        fingers[finger] /= multiplier 
    disbalance = std(fingers.values())
    return disbalance


def trigrams_array_to_trigrams(mat, layout=NEO_LAYOUT):
    """turn a numpy repeats array back into simple repeats. Used for debugging. 

    >>> data = read_file("testfile")
    >>> letters, number_of_letters, bigrams, number_of_bigrams, trigrams, number_of_trigrams = get_all_data(data=data)
    >>> trigrams.sort()
    >>> mat, missing = trigrams_to_pos_array(trigrams, layout=NEO_LAYOUT)
    >>> big = trigrams_array_to_trigrams(mat, layout=NEO_LAYOUT)
    >>> big.sort()
    >>> sum((num for num, trig in big)) == sum((num for num, trig in trigrams))
    True
    """
    all_pos = get_all_positions_in_layout(layout)
    reps = []
    for i in range(len(all_pos)): 
        for j in range(len(all_pos)):
            for k in range(len(all_pos)):
                if mat[i, j, k]: 
                    reps.append((mat[i, j, k], get_key(all_pos[i], layout=layout) + get_key(all_pos[j], layout=layout) + get_key(all_pos[k], layout=layout)))
    return reps
    

def trigrams_to_pos_array(trigrams, layout):
    """turn bigrams into a array of positions with the occurrance number as values.

    >>> data = read_file("testfile")
    >>> letters, number_of_letters, bigrams, number_of_bigrams, trigrams, number_of_trigrams = get_all_data(data=data)
    >>> mat, missing = trigrams_to_pos_array(trigrams, layout=NEO_LAYOUT)
    >>> mat_sum = mat.sum()
    >>> mat_sum == sum((num for num, bi in trigrams if find_key(bi[0], layout=NEO_LAYOUT) and find_key(bi[1], layout=NEO_LAYOUT) and find_key(bi[2], layout=NEO_LAYOUT)))
    True
    """
    all_pos = get_all_positions_in_layout(layout)
    all_pos_len = len(all_pos)
    # rewrite the list into a dict for faster access
    trigram_array = array([])
    trigram_array = resize(trigram_array, (all_pos_len, all_pos_len, all_pos_len)).astype(int)
    missing = []
    for num, rep in trigrams:
        pos0 = find_key(rep[0], layout=layout)
        pos1 = find_key(rep[1], layout=layout)
        pos2 = find_key(rep[2], layout=layout)
        if pos0 is None or pos1 is None or pos2 is None:
            missing.append((num, rep))
        else:
            p0 = all_pos.index(pos0)
            p1 = all_pos.index(pos1)
            p2 = all_pos.index(pos2)
            trigram_array[p0, p1, p2] += num
        
    return trigram_array, missing


def no_handswitching_array(trig_array, layout):
    """Add a penalty when the hands aren’t switched at least once in every three letters. Doesn’t take any uppercase trigrams into account.

    If there also is a direction change in the trigram, the number of times it occurs gets multiplied by WEIGHT_NO_HANDSWITCH_AFTER_DIRECTION_CHANGE.

    If there is no direction change, it gets multiplied with WEIGHT_NO_HANDSWITCH_WITHOUT_DIRECTION_CHANGE. If that is 0, handswitches without direction change are ignored.

    (TODO? WEIGHT_TRIGRAM_FINGER_REPEAT_WITHOUT_KEY_REPEAT)

    TODO: Include the shifts again and split per keyboard. If we did it now, the layout would get optimized for switching after every uppercase letter (as any trigram with a shift and two letters on the same hand would be counted as half a trigram without handswitching). The effect is that it ignores about 7-9% of the trigrams. 

    >>> trigs = [(1, "nrt"), (5, "ige"), (3, "udi"), (2, "ntr")]
    >>> no_handswitching(trigs, layout=NEO_LAYOUT)
    2
    """
    global CACHE_NO_HANDSWITCHING
    if CACHE_NO_HANDSWITCHING is None or CACHE_NO_HANDSWITCHING.shape != trig_array.shape:
        all_pos = get_all_positions_in_layout(layout)
        all_pos_len = len(all_pos)
        
        CACHE_NO_HANDSWITCHING = array([])
        CACHE_NO_HANDSWITCHING = resize(CACHE_NO_HANDSWITCHING, (all_pos_len, all_pos_len, all_pos_len)).astype(int)

        for i in range(all_pos_len):
            for j in range(all_pos_len):
                for k in range(all_pos_len):
                    pos0 = all_pos[i]
                    pos1 = all_pos[j]
                    pos2 = all_pos[k]
                    if not pos_is_left(pos0) is pos_is_left(pos1) or not pos_is_left(pos1) is pos_is_left(pos2):
                        continue
                    if pos0 > pos1 and pos1 < pos2 or pos0 < pos1 and pos1 > pos2:
                        CACHE_NO_HANDSWITCHING[i, j, k] += WEIGHT_NO_HANDSWITCH_AFTER_DIRECTION_CHANGE
                    else: 
                        CACHE_NO_HANDSWITCHING[i, j, k] += WEIGHT_NO_HANDSWITCH_WITHOUT_DIRECTION_CHANGE

    mat = trig_array*CACHE_NO_HANDSWITCHING
    return mat


def _no_handswitching(trigrams, key_hand_table, key_pos_horizontal_table, WEIGHT_NO_HANDSWITCH_AFTER_DIRECTION_CHANGE, WEIGHT_NO_HANDSWITCH_WITHOUT_DIRECTION_CHANGE):
    """Do the hard work for no_handswitching without any call to outer functions."""
    no_switch = 0
    for num, trig in trigrams:
        # if one of the trigs is not in the key_hand_table, we don’t count the trigram.
        if not trig[0] in key_hand_table or not trig[1] in key_hand_table or not trig[2] in key_hand_table:
            continue
        hand1 = key_hand_table[trig[1]]
        if key_hand_table[trig[0]] is hand1 and hand1 is key_hand_table[trig[2]]:
            pos0 = key_pos_horizontal_table[trig[0]]
            pos1 = key_pos_horizontal_table[trig[1]]
            pos2 = key_pos_horizontal_table[trig[2]]
            if pos0 > pos1 and pos1 < pos2 or pos0 < pos1 and pos1 > pos2:
                no_switch += num * WEIGHT_NO_HANDSWITCH_AFTER_DIRECTION_CHANGE
            else: 
                no_switch += num * WEIGHT_NO_HANDSWITCH_WITHOUT_DIRECTION_CHANGE
    return no_switch
    

def no_handswitching(trigrams, layout=NEO_LAYOUT):
    """Add a penalty when the hands aren’t switched at least once in every three letters. Doesn’t take any uppercase trigrams into account.

    If there also is a direction change in the trigram, the number of times it occurs gets multiplied by WEIGHT_NO_HANDSWITCH_AFTER_DIRECTION_CHANGE.

    If there is no direction change, it gets multiplied with WEIGHT_NO_HANDSWITCH_WITHOUT_DIRECTION_CHANGE. If that is 0, handswitches without direction change are ignored.

    (TODO? WEIGHT_TRIGRAM_FINGER_REPEAT_WITHOUT_KEY_REPEAT)

    TODO: Include the shifts again and split per keyboard. If we did it now, the layout would get optimized for switching after every uppercase letter (as any trigram with a shift and two letters on the same hand would be counted as half a trigram without handswitching). The effect is that it ignores about 7-9% of the trigrams. 

    >>> trigs = [(1, "nrt"), (5, "ige"), (3, "udi"), (2, "ntr")]
    >>> no_handswitching(trigs, layout=NEO_LAYOUT)
    2
    """
    # optimization: we precalculate the fingers for all relevent keys (the ones which are being mutated). Since we only need to know if the hands are the same, left hand is False and right hand is True
    key_hand_table = {}
    for key in abc_full:
        #without "⇧⇗ " -> too many false positives when we include the shifts. This also gets rid of anything with uppercase letters in it.
        finger = key_to_finger(key, layout=layout)
        if finger and not finger[:6] == "Daumen":
            if finger[-1] == "L": 
                key_hand_table[key] = False
            elif finger[-1] == "R":
                key_hand_table[key] = True
            # with this, not found is ignored.

    key_pos_horizontal_table = {}
    for key in abc_full:
        #without "⇧⇗ " -> too many false positives when we include the shifts. This also gets rid of anything with uppercase letters in it.
        pos = find_key(key, layout=layout)
        try: 
            key_pos_horizontal_table[key] = pos[1]
        except TypeError:
            pass # not found. Ignore as above.
        

    return _no_handswitching(trigrams, key_hand_table, key_pos_horizontal_table, WEIGHT_NO_HANDSWITCH_AFTER_DIRECTION_CHANGE, WEIGHT_NO_HANDSWITCH_WITHOUT_DIRECTION_CHANGE)


def badly_positioned_shortcut_keys(layout=NEO_LAYOUT, keys="xcvz"):
    """Check, if x, c, v and z are on the left hand and well positioned (much used shortcuts)."""
    badly_positioned = []
    for key in keys: 
        pos = find_key(key, layout=layout)
        # well means not yet left stretch, in row 3, col 5 is also OK.
        if not pos[1] < 5 or (pos[0] == 3 and pos[1] > 5):
            badly_positioned.append(1)
    return sum(badly_positioned)


def total_cost(data=None, letters=None, repeats=None, layout=NEO_LAYOUT, cost_per_key=COST_PER_KEY, trigrams=None, intended_balance=WEIGHT_INTENDED_FINGER_LOAD_LEFT_PINKY_TO_RIGHT_PINKY, return_weighted=False, rep_array=None, trig_array=None):
    """Compute a total cost from all costs we have available, wheighted.

    TODO: reenable the doctests, after the parameters have settled, or pass ALL parameters through the functions.

    @param return_weighted: Set to true to get the weighted values instead of the real values. 
    
    >>> data = read_file("testfile")
    >>> #total_cost(data, cost_per_key=TEST_COST_PER_KEY, intended_balance=TEST_WEIGHT_INTENDED_FINGER_LOAD_LEFT_PINKY_TO_RIGHT_PINKY)
    
    (209.4, 3, 150, 0, 3.3380918415851206, 3, 7)
    """
    # the raw costs
    if data is not None: 
        letters = letters_in_file(data)
        repeats = repeats_in_file(data)
        trigrams = trigrams_in_file(data)
        # first split uppercase repeats *here*, so we don’t have to do it in each function.
        reps = split_uppercase_repeats(repeats, layout=layout)
    elif letters is None or repeats is None:
        raise Exception("Need either repeats und letters or data")
    else:
        # first split uppercase repeats *here*, so we don’t have to do it in each function.
        reps = split_uppercase_repeats(repeats, layout=layout)

    if rep_array is None: 
        rep_array, missing_rep = repeats_to_pos_array(reps, layout)
    if trig_array is None: 
        trig_array, missing_trig = trigrams_to_pos_array(trigrams, layout)

    position_cost = key_position_cost_from_file(letters=letters, layout=layout, cost_per_key=cost_per_key)
    finger_repeats = finger_repeats_from_array(rep_array, layout=layout)
    
    no_handswitches = no_handswitching_array(trig_array, layout=layout).sum()

    frep_num = finger_repeats.sum()
    finger_repeats_top_bottom = finger_repeats_from_array_top_and_bottom(rep_array, layout=layout)
    frep_num_top_bottom = finger_repeats_top_bottom.sum()

    # the number of times neighboring fingers are used – weighted by the ease of transition for the respective fingers
    neighboring_fings = neighboring_fingers(repeats=reps, layout=layout)

    # the number of changes between lines on the same hand.
    line_change_same_hand = line_changes(repeats=reps, layout=layout)

    # how often the hand wasn’t switched after an unbalancing key, weighted by the severity of the unbalancing.
    no_switch_after_unbalancing = no_handswitch_after_unbalancing_key(repeats=reps, layout=layout)

    # the balance between fingers
    disbalance = finger_balance(letters, layout=layout, intended_balance=intended_balance)
    number_of_letters = sum([i for i, s in letters])

    # the position of the keys xcvz - penalty if they are not among the first 5 keys, counted from left, horizontally.
    badly_positioned = badly_positioned_shortcut_keys(layout=layout)

    # the load distribution on the hands: [left keystrokes, right keystrokes]
    hand_load = load_per_hand(letters, layout=layout)
    # the disbalance between the hands. Keystrokes of the left / total strokes - 0.5. From 0 to 0.5, ignoring the direction.
    hand_disbalance = abs(hand_load[0]/max(1, sum(hand_load)) - 0.5)

    # add all together and weight them
    total = WEIGHT_POSITION * position_cost
    total += WEIGHT_FINGER_REPEATS * frep_num # not 0.5, since there may be 2 times as many 2-tuples as letters, but the repeats are calculated on the in-between, and these are single.
    total += WEIGHT_FINGER_REPEATS_TOP_BOTTOM * frep_num_top_bottom
    total += WEIGHT_FINGER_SWITCH * neighboring_fings
    total += WEIGHT_FINGER_DISBALANCE * disbalance # needs a minimum number of letters to be useful.
    total += WEIGHT_TOO_LITTLE_HANDSWITCHING * no_handswitches
    total += WEIGHT_XCVZ_ON_BAD_POSITION * number_of_letters * badly_positioned
    total += WEIGHT_BIGRAM_ROW_CHANGE_PER_ROW * line_change_same_hand
    total += WEIGHT_NO_HANDSWITCH_AFTER_UNBALANCING_KEY * no_switch_after_unbalancing
    total += WEIGHT_HAND_DISBALANCE * hand_disbalance * number_of_letters

    if not return_weighted: 
        return total, frep_num, position_cost, frep_num_top_bottom, disbalance, no_handswitches, line_change_same_hand, hand_load
    else:
        return total, WEIGHT_POSITION * position_cost, WEIGHT_FINGER_REPEATS * frep_num , WEIGHT_FINGER_REPEATS_TOP_BOTTOM * frep_num_top_bottom, WEIGHT_FINGER_SWITCH * neighboring_fings, WEIGHT_FINGER_DISBALANCE * disbalance, WEIGHT_TOO_LITTLE_HANDSWITCHING * no_handswitches, WEIGHT_XCVZ_ON_BAD_POSITION * number_of_letters * badly_positioned, WEIGHT_BIGRAM_ROW_CHANGE_PER_ROW * line_change_same_hand, WEIGHT_NO_HANDSWITCH_AFTER_UNBALANCING_KEY * no_switch_after_unbalancing, WEIGHT_HAND_DISBALANCE * hand_disbalance * number_of_letters


def _test():
    from doctest import testmod
    testmod()

if __name__ == "__main__":
    _test()