letter.go

     1  // Copyright 2009 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Package unicode provides data and functions to test some properties of
     6  // Unicode code points.
     7  package unicode
     8  
     9  const (
    10  	MaxRune         = '\U0010FFFF' // Maximum valid Unicode code point.
    11  	ReplacementChar = '\uFFFD'     // Represents invalid code points.
    12  	MaxASCII        = '\u007F'     // maximum ASCII value.
    13  	MaxLatin1       = '\u00FF'     // maximum Latin-1 value.
    14  )
    15  
    16  // RangeTable defines a set of Unicode code points by listing the ranges of
    17  // code points within the set. The ranges are listed in two slices
    18  // to save space: a slice of 16-bit ranges and a slice of 32-bit ranges.
    19  // The two slices must be in sorted order and non-overlapping.
    20  // Also, R32 should contain only values >= 0x10000 (1<<16).
    21  type RangeTable struct {
    22  	R16         []Range16
    23  	R32         []Range32
    24  	LatinOffset int // number of entries in R16 with Hi <= MaxLatin1
    25  }
    26  
    27  // Range16 represents of a range of 16-bit Unicode code points. The range runs from Lo to Hi
    28  // inclusive and has the specified stride.
    29  type Range16 struct {
    30  	Lo     uint16
    31  	Hi     uint16
    32  	Stride uint16
    33  }
    34  
    35  // Range32 represents of a range of Unicode code points and is used when one or
    36  // more of the values will not fit in 16 bits. The range runs from Lo to Hi
    37  // inclusive and has the specified stride. Lo and Hi must always be >= 1<<16.
    38  type Range32 struct {
    39  	Lo     uint32
    40  	Hi     uint32
    41  	Stride uint32
    42  }
    43  
    44  // CaseRange represents a range of Unicode code points for simple (one
    45  // code point to one code point) case conversion.
    46  // The range runs from Lo to Hi inclusive, with a fixed stride of 1. Deltas
    47  // are the number to add to the code point to reach the code point for a
    48  // different case for that character. They may be negative. If zero, it
    49  // means the character is in the corresponding case. There is a special
    50  // case representing sequences of alternating corresponding Upper and Lower
    51  // pairs. It appears with a fixed Delta of
    52  //	{UpperLower, UpperLower, UpperLower}
    53  // The constant UpperLower has an otherwise impossible delta value.
    54  type CaseRange struct {
    55  	Lo    uint32
    56  	Hi    uint32
    57  	Delta d
    58  }
    59  
    60  // SpecialCase represents language-specific case mappings such as Turkish.
    61  // Methods of SpecialCase customize (by overriding) the standard mappings.
    62  type SpecialCase []CaseRange
    63  
    64  // BUG(r): There is no mechanism for full case folding, that is, for
    65  // characters that involve multiple runes in the input or output.
    66  
    67  // Indices into the Delta arrays inside CaseRanges for case mapping.
    68  const (
    69  	UpperCase = iota
    70  	LowerCase
    71  	TitleCase
    72  	MaxCase
    73  )
    74  
    75  type d [MaxCase]rune // to make the CaseRanges text shorter
    76  
    77  // If the Delta field of a CaseRange is UpperLower, it means
    78  // this CaseRange represents a sequence of the form (say)
    79  // Upper Lower Upper Lower.
    80  const (
    81  	UpperLower = MaxRune + 1 // (Cannot be a valid delta.)
    82  )
    83  
    84  // linearMax is the maximum size table for linear search for non-Latin1 rune.
    85  // Derived by running 'go test -calibrate'.
    86  const linearMax = 18
    87  
    88  // is16 reports whether r is in the sorted slice of 16-bit ranges.
    89  func is16(ranges []Range16, r uint16) bool {
    90  	if len(ranges) <= linearMax || r <= MaxLatin1 {
    91  		for i := range ranges {
    92  			range_ := &ranges[i]
    93  			if r < range_.Lo {
    94  				return false
    95  			}
    96  			if r <= range_.Hi {
    97  				return range_.Stride == 1 || (r-range_.Lo)%range_.Stride == 0
    98  			}
    99  		}
   100  		return false
   101  	}
   102  
   103  	// binary search over ranges
   104  	lo := 0
   105  	hi := len(ranges)
   106  	for lo < hi {
   107  		m := lo + (hi-lo)/2
   108  		range_ := &ranges[m]
   109  		if range_.Lo <= r && r <= range_.Hi {
   110  			return range_.Stride == 1 || (r-range_.Lo)%range_.Stride == 0
   111  		}
   112  		if r < range_.Lo {
   113  			hi = m
   114  		} else {
   115  			lo = m + 1
   116  		}
   117  	}
   118  	return false
   119  }
   120  
   121  // is32 reports whether r is in the sorted slice of 32-bit ranges.
   122  func is32(ranges []Range32, r uint32) bool {
   123  	if len(ranges) <= linearMax {
   124  		for i := range ranges {
   125  			range_ := &ranges[i]
   126  			if r < range_.Lo {
   127  				return false
   128  			}
   129  			if r <= range_.Hi {
   130  				return range_.Stride == 1 || (r-range_.Lo)%range_.Stride == 0
   131  			}
   132  		}
   133  		return false
   134  	}
   135  
   136  	// binary search over ranges
   137  	lo := 0
   138  	hi := len(ranges)
   139  	for lo < hi {
   140  		m := lo + (hi-lo)/2
   141  		range_ := ranges[m]
   142  		if range_.Lo <= r && r <= range_.Hi {
   143  			return range_.Stride == 1 || (r-range_.Lo)%range_.Stride == 0
   144  		}
   145  		if r < range_.Lo {
   146  			hi = m
   147  		} else {
   148  			lo = m + 1
   149  		}
   150  	}
   151  	return false
   152  }
   153  
   154  // Is reports whether the rune is in the specified table of ranges.
   155  func Is(rangeTab *RangeTable, r rune) bool {
   156  	r16 := rangeTab.R16
   157  	// Compare as uint32 to correctly handle negative runes.
   158  	if len(r16) > 0 && uint32(r) <= uint32(r16[len(r16)-1].Hi) {
   159  		return is16(r16, uint16(r))
   160  	}
   161  	r32 := rangeTab.R32
   162  	if len(r32) > 0 && r >= rune(r32[0].Lo) {
   163  		return is32(r32, uint32(r))
   164  	}
   165  	return false
   166  }
   167  
   168  func isExcludingLatin(rangeTab *RangeTable, r rune) bool {
   169  	r16 := rangeTab.R16
   170  	// Compare as uint32 to correctly handle negative runes.
   171  	if off := rangeTab.LatinOffset; len(r16) > off && uint32(r) <= uint32(r16[len(r16)-1].Hi) {
   172  		return is16(r16[off:], uint16(r))
   173  	}
   174  	r32 := rangeTab.R32
   175  	if len(r32) > 0 && r >= rune(r32[0].Lo) {
   176  		return is32(r32, uint32(r))
   177  	}
   178  	return false
   179  }
   180  
   181  // IsUpper reports whether the rune is an upper case letter.
   182  func IsUpper(r rune) bool {
   183  	// See comment in IsGraphic.
   184  	if uint32(r) <= MaxLatin1 {
   185  		return properties[uint8(r)]&pLmask == pLu
   186  	}
   187  	return isExcludingLatin(Upper, r)
   188  }
   189  
   190  // IsLower reports whether the rune is a lower case letter.
   191  func IsLower(r rune) bool {
   192  	// See comment in IsGraphic.
   193  	if uint32(r) <= MaxLatin1 {
   194  		return properties[uint8(r)]&pLmask == pLl
   195  	}
   196  	return isExcludingLatin(Lower, r)
   197  }
   198  
   199  // IsTitle reports whether the rune is a title case letter.
   200  func IsTitle(r rune) bool {
   201  	if r <= MaxLatin1 {
   202  		return false
   203  	}
   204  	return isExcludingLatin(Title, r)
   205  }
   206  
   207  // to maps the rune using the specified case mapping.
   208  // It additionally reports whether caseRange contained a mapping for r.
   209  func to(_case int, r rune, caseRange []CaseRange) (mappedRune rune, foundMapping bool) {
   210  	if _case < 0 || MaxCase <= _case {
   211  		return ReplacementChar, false // as reasonable an error as any
   212  	}
   213  	// binary search over ranges
   214  	lo := 0
   215  	hi := len(caseRange)
   216  	for lo < hi {
   217  		m := lo + (hi-lo)/2
   218  		cr := caseRange[m]
   219  		if rune(cr.Lo) <= r && r <= rune(cr.Hi) {
   220  			delta := cr.Delta[_case]
   221  			if delta > MaxRune {
   222  				// In an Upper-Lower sequence, which always starts with
   223  				// an UpperCase letter, the real deltas always look like:
   224  				//	{0, 1, 0}    UpperCase (Lower is next)
   225  				//	{-1, 0, -1}  LowerCase (Upper, Title are previous)
   226  				// The characters at even offsets from the beginning of the
   227  				// sequence are upper case; the ones at odd offsets are lower.
   228  				// The correct mapping can be done by clearing or setting the low
   229  				// bit in the sequence offset.
   230  				// The constants UpperCase and TitleCase are even while LowerCase
   231  				// is odd so we take the low bit from _case.
   232  				return rune(cr.Lo) + ((r-rune(cr.Lo))&^1 | rune(_case&1)), true
   233  			}
   234  			return r + delta, true
   235  		}
   236  		if r < rune(cr.Lo) {
   237  			hi = m
   238  		} else {
   239  			lo = m + 1
   240  		}
   241  	}
   242  	return r, false
   243  }
   244  
   245  // To maps the rune to the specified case: UpperCase, LowerCase, or TitleCase.
   246  func To(_case int, r rune) rune {
   247  	r, _ = to(_case, r, CaseRanges)
   248  	return r
   249  }
   250  
   251  // ToUpper maps the rune to upper case.
   252  func ToUpper(r rune) rune {
   253  	if r <= MaxASCII {
   254  		if 'a' <= r && r <= 'z' {
   255  			r -= 'a' - 'A'
   256  		}
   257  		return r
   258  	}
   259  	return To(UpperCase, r)
   260  }
   261  
   262  // ToLower maps the rune to lower case.
   263  func ToLower(r rune) rune {
   264  	if r <= MaxASCII {
   265  		if 'A' <= r && r <= 'Z' {
   266  			r += 'a' - 'A'
   267  		}
   268  		return r
   269  	}
   270  	return To(LowerCase, r)
   271  }
   272  
   273  // ToTitle maps the rune to title case.
   274  func ToTitle(r rune) rune {
   275  	if r <= MaxASCII {
   276  		if 'a' <= r && r <= 'z' { // title case is upper case for ASCII
   277  			r -= 'a' - 'A'
   278  		}
   279  		return r
   280  	}
   281  	return To(TitleCase, r)
   282  }
   283  
   284  // ToUpper maps the rune to upper case giving priority to the special mapping.
   285  func (special SpecialCase) ToUpper(r rune) rune {
   286  	r1, hadMapping := to(UpperCase, r, []CaseRange(special))
   287  	if r1 == r && !hadMapping {
   288  		r1 = ToUpper(r)
   289  	}
   290  	return r1
   291  }
   292  
   293  // ToTitle maps the rune to title case giving priority to the special mapping.
   294  func (special SpecialCase) ToTitle(r rune) rune {
   295  	r1, hadMapping := to(TitleCase, r, []CaseRange(special))
   296  	if r1 == r && !hadMapping {
   297  		r1 = ToTitle(r)
   298  	}
   299  	return r1
   300  }
   301  
   302  // ToLower maps the rune to lower case giving priority to the special mapping.
   303  func (special SpecialCase) ToLower(r rune) rune {
   304  	r1, hadMapping := to(LowerCase, r, []CaseRange(special))
   305  	if r1 == r && !hadMapping {
   306  		r1 = ToLower(r)
   307  	}
   308  	return r1
   309  }
   310  
   311  // caseOrbit is defined in tables.go as []foldPair. Right now all the
   312  // entries fit in uint16, so use uint16. If that changes, compilation
   313  // will fail (the constants in the composite literal will not fit in uint16)
   314  // and the types here can change to uint32.
   315  type foldPair struct {
   316  	From uint16
   317  	To   uint16
   318  }
   319  
   320  // SimpleFold iterates over Unicode code points equivalent under
   321  // the Unicode-defined simple case folding. Among the code points
   322  // equivalent to rune (including rune itself), SimpleFold returns the
   323  // smallest rune > r if one exists, or else the smallest rune >= 0.
   324  // If r is not a valid Unicode code point, SimpleFold(r) returns r.
   325  //
   326  // For example:
   327  //	SimpleFold('A') = 'a'
   328  //	SimpleFold('a') = 'A'
   329  //
   330  //	SimpleFold('K') = 'k'
   331  //	SimpleFold('k') = '\u212A' (Kelvin symbol, K)
   332  //	SimpleFold('\u212A') = 'K'
   333  //
   334  //	SimpleFold('1') = '1'
   335  //
   336  //	SimpleFold(-2) = -2
   337  //
   338  func SimpleFold(r rune) rune {
   339  	if r < 0 || r > MaxRune {
   340  		return r
   341  	}
   342  
   343  	if int(r) < len(asciiFold) {
   344  		return rune(asciiFold[r])
   345  	}
   346  
   347  	// Consult caseOrbit table for special cases.
   348  	lo := 0
   349  	hi := len(caseOrbit)
   350  	for lo < hi {
   351  		m := lo + (hi-lo)/2
   352  		if rune(caseOrbit[m].From) < r {
   353  			lo = m + 1
   354  		} else {
   355  			hi = m
   356  		}
   357  	}
   358  	if lo < len(caseOrbit) && rune(caseOrbit[lo].From) == r {
   359  		return rune(caseOrbit[lo].To)
   360  	}
   361  
   362  	// No folding specified. This is a one- or two-element
   363  	// equivalence class containing rune and ToLower(rune)
   364  	// and ToUpper(rune) if they are different from rune.
   365  	if l := ToLower(r); l != r {
   366  		return l
   367  	}
   368  	return ToUpper(r)
   369  }
   370
View as plain text