/
abakum
/
text
Обзор
Документация
Войти
/
abakum
/
text
Код
Задачи
Вики
Пакеты
0
Релизы
0
Аналитика
Безопасность
master
width/transform_test.go
701 строка
14 KB
Dmitri Shuralyov
internal/texttest: remove Run and Bench helpers
21 окт 2024, 21:16
21 окт 2024, 21:16
fefda1a
Код
Авторство
О чём код?
// Copyright 2015 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package width import ( "bytes" "strings" "testing" "golang.org/x/text/internal/testtext" "golang.org/x/text/transform" ) func foldRune(r rune) (folded rune, ok bool) { alt, ok := mapRunes[r] if ok && alt.e&tagNeedsFold != 0 { return alt.r, true } return r, false } func widenRune(r rune) (wide rune, ok bool) { alt, ok := mapRunes[r] if k := alt.e.kind(); k == EastAsianHalfwidth || k == EastAsianNarrow { return alt.r, true } return r, false } func narrowRune(r rune) (narrow rune, ok bool) { alt, ok := mapRunes[r] if k := alt.e.kind(); k == EastAsianFullwidth || k == EastAsianWide || k == EastAsianAmbiguous { return alt.r, true } return r, false } func TestFoldSingleRunes(t *testing.T) { for r := rune(0); r < 0x1FFFF; r++ { if loSurrogate <= r && r <= hiSurrogate { continue } x, _ := foldRune(r) want := string(x) got := Fold.String(string(r)) if got != want { t.Errorf("Fold().String(%U) = %+q; want %+q", r, got, want) } } } type transformTest struct { desc string src string nBuf int nDst int atEOF bool dst string nSrc int err error nSpan int errSpan error } func (tc *transformTest) doTest(t *testing.T, tr Transformer) { t.Run(tc.desc, func(t *testing.T) { b := make([]byte, tc.nBuf) nDst, nSrc, err := tr.Transform(b, []byte(tc.src), tc.atEOF) if got := string(b[:nDst]); got != tc.dst[:nDst] { t.Errorf("dst was %+q; want %+q", got, tc.dst) } if nDst != tc.nDst { t.Errorf("nDst was %d; want %d", nDst, tc.nDst) } if nSrc != tc.nSrc { t.Errorf("nSrc was %d; want %d", nSrc, tc.nSrc) } if err != tc.err { t.Errorf("error was %v; want %v", err, tc.err) } if got := tr.String(tc.src); got != tc.dst { t.Errorf("String(%q) = %q; want %q", tc.src, got, tc.dst) } n, err := tr.Span([]byte(tc.src), tc.atEOF) if n != tc.nSpan || err != tc.errSpan { t.Errorf("Span: got %d, %v; want %d, %v", n, err, tc.nSpan, tc.errSpan) } }) } func TestFold(t *testing.T) { for _, tc := range []transformTest{{ desc: "empty", src: "", nBuf: 10, dst: "", nDst: 0, nSrc: 0, atEOF: false, err: nil, nSpan: 0, errSpan: nil, }, { desc: "short source 1", src: "a\xc2", nBuf: 10, dst: "a\xc2", nDst: 1, nSrc: 1, atEOF: false, err: transform.ErrShortSrc, nSpan: 1, errSpan: transform.ErrShortSrc, }, { desc: "short source 2", src: "a\xe0\x80", nBuf: 10, dst: "a\xe0\x80", nDst: 1, nSrc: 1, atEOF: false, err: transform.ErrShortSrc, nSpan: 1, errSpan: transform.ErrShortSrc, }, { desc: "incomplete but terminated source 1", src: "a\xc2", nBuf: 10, dst: "a\xc2", nDst: 2, nSrc: 2, atEOF: true, err: nil, nSpan: 2, errSpan: nil, }, { desc: "incomplete but terminated source 2", src: "a\xe0\x80", nBuf: 10, dst: "a\xe0\x80", nDst: 3, nSrc: 3, atEOF: true, err: nil, nSpan: 3, errSpan: nil, }, { desc: "exact fit dst", src: "a\uff01", nBuf: 2, dst: "a!", nDst: 2, nSrc: 4, atEOF: false, err: nil, nSpan: 1, errSpan: transform.ErrEndOfSpan, }, { desc: "exact fit dst and src ascii", src: "ab", nBuf: 2, dst: "ab", nDst: 2, nSrc: 2, atEOF: true, err: nil, nSpan: 2, errSpan: nil, }, { desc: "empty dst", src: "\u0300", nBuf: 0, dst: "\u0300", nDst: 0, nSrc: 0, atEOF: true, err: transform.ErrShortDst, nSpan: 2, errSpan: nil, }, { desc: "empty dst ascii", src: "a", nBuf: 0, dst: "a", nDst: 0, nSrc: 0, atEOF: true, err: transform.ErrShortDst, nSpan: 1, errSpan: nil, }, { desc: "short dst 1", src: "a\uffe0", // ￠ nBuf: 2, dst: "a\u00a2", // ¢ nDst: 1, nSrc: 1, atEOF: false, err: transform.ErrShortDst, nSpan: 1, errSpan: transform.ErrEndOfSpan, }, { desc: "short dst 2", src: "不夠", nBuf: 3, dst: "不夠", nDst: 3, nSrc: 3, atEOF: true, err: transform.ErrShortDst, nSpan: 6, errSpan: nil, }, { desc: "short dst fast path", src: "fast", nDst: 3, dst: "fast", nBuf: 3, nSrc: 3, atEOF: true, err: transform.ErrShortDst, nSpan: 4, errSpan: nil, }, { desc: "short dst larger buffer", src: "\uff21" + strings.Repeat("0", 127) + "B", nBuf: 128, dst: "A" + strings.Repeat("0", 127) + "B", nDst: 128, nSrc: 130, atEOF: true, err: transform.ErrShortDst, nSpan: 0, errSpan: transform.ErrEndOfSpan, }, { desc: "fast path alternation", src: "fast路徑fast路徑", nBuf: 20, dst: "fast路徑fast路徑", nDst: 20, nSrc: 20, atEOF: true, err: nil, nSpan: 20, errSpan: nil, }} { tc.doTest(t, Fold) } } func TestWidenSingleRunes(t *testing.T) { for r := rune(0); r < 0x1FFFF; r++ { if loSurrogate <= r && r <= hiSurrogate { continue } alt, _ := widenRune(r) want := string(alt) got := Widen.String(string(r)) if got != want { t.Errorf("Widen().String(%U) = %+q; want %+q", r, got, want) } } } func TestWiden(t *testing.T) { for _, tc := range []transformTest{{ desc: "empty", src: "", nBuf: 10, dst: "", nDst: 0, nSrc: 0, atEOF: false, err: nil, nSpan: 0, errSpan: nil, }, { desc: "short source 1", src: "a\xc2", nBuf: 10, dst: "ａ\xc2", nDst: 3, nSrc: 1, atEOF: false, err: transform.ErrShortSrc, nSpan: 0, errSpan: transform.ErrEndOfSpan, }, { desc: "short source 2", src: "a\xe0\x80", nBuf: 10, dst: "ａ\xe0\x80", nDst: 3, nSrc: 1, atEOF: false, err: transform.ErrShortSrc, nSpan: 0, errSpan: transform.ErrEndOfSpan, }, { desc: "incomplete but terminated source 1", src: "a\xc2", nBuf: 10, dst: "ａ\xc2", nDst: 4, nSrc: 2, atEOF: true, err: nil, nSpan: 0, errSpan: transform.ErrEndOfSpan, }, { desc: "incomplete but terminated source 2", src: "a\xe0\x80", nBuf: 10, dst: "ａ\xe0\x80", nDst: 5, nSrc: 3, atEOF: true, err: nil, nSpan: 0, errSpan: transform.ErrEndOfSpan, }, { desc: "short source 1 some span", src: "ａ\xc2", nBuf: 10, dst: "ａ\xc2", nDst: 3, nSrc: 3, atEOF: false, err: transform.ErrShortSrc, nSpan: 3, errSpan: transform.ErrShortSrc, }, { desc: "short source 2 some span", src: "ａ\xe0\x80", nBuf: 10, dst: "ａ\xe0\x80", nDst: 3, nSrc: 3, atEOF: false, err: transform.ErrShortSrc, nSpan: 3, errSpan: transform.ErrShortSrc, }, { desc: "incomplete but terminated source 1 some span", src: "ａ\xc2", nBuf: 10, dst: "ａ\xc2", nDst: 4, nSrc: 4, atEOF: true, err: nil, nSpan: 4, errSpan: nil, }, { desc: "incomplete but terminated source 2 some span", src: "ａ\xe0\x80", nBuf: 10, dst: "ａ\xe0\x80", nDst: 5, nSrc: 5, atEOF: true, err: nil, nSpan: 5, errSpan: nil, }, { desc: "exact fit dst", src: "a!", nBuf: 6, dst: "ａ\uff01", nDst: 6, nSrc: 2, atEOF: false, err: nil, nSpan: 0, errSpan: transform.ErrEndOfSpan, }, { desc: "empty dst", src: "\u0300", nBuf: 0, dst: "\u0300", nDst: 0, nSrc: 0, atEOF: true, err: transform.ErrShortDst, nSpan: 2, errSpan: nil, }, { desc: "empty dst ascii", src: "a", nBuf: 0, dst: "ａ", nDst: 0, nSrc: 0, atEOF: true, err: transform.ErrShortDst, nSpan: 0, errSpan: transform.ErrEndOfSpan, }, { desc: "short dst 1", src: "a\uffe0", nBuf: 4, dst: "ａ\uffe0", nDst: 3, nSrc: 1, atEOF: false, err: transform.ErrShortDst, nSpan: 0, errSpan: transform.ErrEndOfSpan, }, { desc: "short dst 2", src: "不夠", nBuf: 3, dst: "不夠", nDst: 3, nSrc: 3, atEOF: true, err: transform.ErrShortDst, nSpan: 6, errSpan: nil, }, { desc: "short dst ascii", src: "ascii", nBuf: 3, dst: "ａｓｃｉｉ", // U+ff41, ... nDst: 3, nSrc: 1, atEOF: true, err: transform.ErrShortDst, nSpan: 0, errSpan: transform.ErrEndOfSpan, }, { desc: "ambiguous", src: "\uffe9", nBuf: 4, dst: "\u2190", nDst: 3, nSrc: 3, atEOF: false, err: nil, nSpan: 0, errSpan: transform.ErrEndOfSpan, }} { tc.doTest(t, Widen) } } func TestNarrowSingleRunes(t *testing.T) { for r := rune(0); r < 0x1FFFF; r++ { if loSurrogate <= r && r <= hiSurrogate { continue } alt, _ := narrowRune(r) want := string(alt) got := Narrow.String(string(r)) if got != want { t.Errorf("Narrow().String(%U) = %+q; want %+q", r, got, want) } } } func TestNarrow(t *testing.T) { for _, tc := range []transformTest{{ desc: "empty", src: "", nBuf: 10, dst: "", nDst: 0, nSrc: 0, atEOF: false, err: nil, nSpan: 0, errSpan: nil, }, { desc: "short source 1", src: "a\xc2", nBuf: 10, dst: "a\xc2", nDst: 1, nSrc: 1, atEOF: false, err: transform.ErrShortSrc, nSpan: 1, errSpan: transform.ErrShortSrc, }, { desc: "short source 2", src: "ａ\xe0\x80", nBuf: 10, dst: "a\xe0\x80", nDst: 1, nSrc: 3, atEOF: false, err: transform.ErrShortSrc, nSpan: 0, errSpan: transform.ErrEndOfSpan, }, { desc: "incomplete but terminated source 1", src: "ａ\xc2", nBuf: 10, dst: "a\xc2", nDst: 2, nSrc: 4, atEOF: true, err: nil, nSpan: 0, errSpan: transform.ErrEndOfSpan, }, { desc: "incomplete but terminated source 2", src: "ａ\xe0\x80", nBuf: 10, dst: "a\xe0\x80", nDst: 3, nSrc: 5, atEOF: true, err: nil, nSpan: 0, errSpan: transform.ErrEndOfSpan, }, { desc: "exact fit dst", src: "ａ\uff01", nBuf: 2, dst: "a!", nDst: 2, nSrc: 6, atEOF: false, err: nil, nSpan: 0, errSpan: transform.ErrEndOfSpan, }, { desc: "exact fit dst some span", src: "a\uff01", nBuf: 2, dst: "a!", nDst: 2, nSrc: 4, atEOF: false, err: nil, nSpan: 1, errSpan: transform.ErrEndOfSpan, }, { desc: "empty dst", src: "\u0300", nBuf: 0, dst: "\u0300", nDst: 0, nSrc: 0, atEOF: true, err: transform.ErrShortDst, nSpan: 2, errSpan: nil, }, { desc: "empty dst ascii", src: "a", nBuf: 0, dst: "a", nDst: 0, nSrc: 0, atEOF: true, err: transform.ErrShortDst, nSpan: 1, errSpan: nil, }, { desc: "short dst 1", src: "ａ\uffe0", // ￠ nBuf: 2, dst: "a\u00a2", // ¢ nDst: 1, nSrc: 3, atEOF: false, err: transform.ErrShortDst, nSpan: 0, errSpan: transform.ErrEndOfSpan, }, { desc: "short dst 2", src: "不夠", nBuf: 3, dst: "不夠", nDst: 3, nSrc: 3, atEOF: true, err: transform.ErrShortDst, nSpan: 6, errSpan: nil, }, { // Create a narrow variant of ambiguous runes, if they exist. desc: "ambiguous", src: "\u2190", nBuf: 4, dst: "\uffe9", nDst: 3, nSrc: 3, atEOF: false, err: nil, nSpan: 0, errSpan: transform.ErrEndOfSpan, }, { desc: "short dst fast path", src: "fast", nBuf: 3, dst: "fast", nDst: 3, nSrc: 3, atEOF: true, err: transform.ErrShortDst, nSpan: 4, errSpan: nil, }, { desc: "short dst larger buffer", src: "\uff21" + strings.Repeat("0", 127) + "B", nBuf: 128, dst: "A" + strings.Repeat("0", 127) + "B", nDst: 128, nSrc: 130, atEOF: true, err: transform.ErrShortDst, nSpan: 0, errSpan: transform.ErrEndOfSpan, }, { desc: "fast path alternation", src: "fast路徑fast路徑", nBuf: 20, dst: "fast路徑fast路徑", nDst: 20, nSrc: 20, atEOF: true, err: nil, nSpan: 20, errSpan: nil, }} { tc.doTest(t, Narrow) } } func bench(b *testing.B, t Transformer, s string) { dst := make([]byte, 1024) src := []byte(s) b.SetBytes(int64(len(src))) b.ResetTimer() for i := 0; i < b.N; i++ { t.Transform(dst, src, true) } } func changingRunes(f func(r rune) (rune, bool)) string { buf := &bytes.Buffer{} for r := rune(0); r <= 0xFFFF; r++ { if _, ok := foldRune(r); ok { buf.WriteRune(r) } } return buf.String() } func BenchmarkFoldASCII(b *testing.B) { bench(b, Fold, testtext.ASCII) } func BenchmarkFoldCJK(b *testing.B) { bench(b, Fold, testtext.CJK) } func BenchmarkFoldNonCanonical(b *testing.B) { bench(b, Fold, changingRunes(foldRune)) } func BenchmarkFoldOther(b *testing.B) { bench(b, Fold, testtext.TwoByteUTF8+testtext.ThreeByteUTF8) } func BenchmarkWideASCII(b *testing.B) { bench(b, Widen, testtext.ASCII) } func BenchmarkWideCJK(b *testing.B) { bench(b, Widen, testtext.CJK) } func BenchmarkWideNonCanonical(b *testing.B) { bench(b, Widen, changingRunes(widenRune)) } func BenchmarkWideOther(b *testing.B) { bench(b, Widen, testtext.TwoByteUTF8+testtext.ThreeByteUTF8) } func BenchmarkNarrowASCII(b *testing.B) { bench(b, Narrow, testtext.ASCII) } func BenchmarkNarrowCJK(b *testing.B) { bench(b, Narrow, testtext.CJK) } func BenchmarkNarrowNonCanonical(b *testing.B) { bench(b, Narrow, changingRunes(narrowRune)) } func BenchmarkNarrowOther(b *testing.B) { bench(b, Narrow, testtext.TwoByteUTF8+testtext.ThreeByteUTF8) }