update vendor/ dir to latest w/o heroku, moby

had to lock a lot of things in place

vendor/golang.org/x/text/language/gen.go

@@ -806,10 +806,10 @@ func (b *builder) writeLanguage() {
 		}
 	}
 
-	// Complete canonialized language tags.
+	// Complete canonicalized language tags.
 	lang.freeze()
 	for i, v := range lang.s {
-		// We can avoid these manual entries by using the IANI registry directly.
+		// We can avoid these manual entries by using the IANA registry directly.
 		// Seems easier to update the list manually, as changes are rare.
 		// The panic in this loop will trigger if we miss an entry.
 		add := ""
@@ -916,7 +916,7 @@ func (b *builder) writeRegion() {
 			i := b.region.index(s)
 			for _, d := range e.description {
 				if strings.Contains(d, "Private use") {
-					regionTypes[i] = iso3166UserAssgined
+					regionTypes[i] = iso3166UserAssigned
 				}
 			}
 			regionTypes[i] |= bcp47Region
@@ -1073,7 +1073,7 @@ const (
 )
 
 const (
-	iso3166UserAssgined = 1 << iota
+	iso3166UserAssigned = 1 << iota
 	ccTLD
 	bcp47Region
 )
@@ -1363,42 +1363,23 @@ func (b *builder) writeLikelyData() {
 
 type mutualIntelligibility struct {
 	want, have uint16
-	conf       uint8
+	distance   uint8
 	oneway     bool
 }
 
 type scriptIntelligibility struct {
-	lang       uint16 // langID or 0 if *
-	want, have uint8
-	conf       uint8
+	wantLang, haveLang     uint16
+	wantScript, haveScript uint8
+	distance               uint8
+	// Always oneway
 }
 
-type sortByConf []mutualIntelligibility
-
-func (l sortByConf) Less(a, b int) bool {
-	return l[a].conf > l[b].conf
-}
-
-func (l sortByConf) Swap(a, b int) {
-	l[a], l[b] = l[b], l[a]
-}
-
-func (l sortByConf) Len() int {
-	return len(l)
-}
-
-// toConf converts a percentage value [0, 100] to a confidence class.
-func toConf(pct uint8) uint8 {
-	switch {
-	case pct == 100:
-		return 3 // Exact
-	case pct >= 90:
-		return 2 // High
-	case pct > 50:
-		return 1 // Low
-	default:
-		return 0 // No
-	}
+type regionIntelligibility struct {
+	lang     uint16 // compact language id
+	script   uint8  // 0 means any
+	group    uint8  // 0 means any; if bit 7 is set it means inverse
+	distance uint8
+	// Always twoway.
 }
 
 // writeMatchData writes tables with languages and scripts for which there is
@@ -1408,13 +1389,50 @@ func toConf(pct uint8) uint8 {
 // We also drop all region-related data as we use a different algorithm to
 // determine region equivalence.
 func (b *builder) writeMatchData() {
+	lm := b.supp.LanguageMatching.LanguageMatches
+	cldr.MakeSlice(&lm).SelectAnyOf("type", "written_new")
+
+	regionHierarchy := map[string][]string{}
+	for _, g := range b.supp.TerritoryContainment.Group {
+		regions := strings.Split(g.Contains, " ")
+		regionHierarchy[g.Type] = append(regionHierarchy[g.Type], regions...)
+	}
+	regionToGroups := make([]uint8, len(b.region.s))
+
+	idToIndex := map[string]uint8{}
+	for i, mv := range lm[0].MatchVariable {
+		if i > 6 {
+			log.Fatalf("Too many groups: %d", i)
+		}
+		idToIndex[mv.Id] = uint8(i + 1)
+		// TODO: also handle '-'
+		for _, r := range strings.Split(mv.Value, "+") {
+			todo := []string{r}
+			for k := 0; k < len(todo); k++ {
+				r := todo[k]
+				regionToGroups[b.region.index(r)] |= 1 << uint8(i)
+				todo = append(todo, regionHierarchy[r]...)
+			}
+		}
+	}
+	b.writeSlice("regionToGroups", regionToGroups)
+
 	b.writeType(mutualIntelligibility{})
 	b.writeType(scriptIntelligibility{})
-	lm := b.supp.LanguageMatching.LanguageMatches
-	cldr.MakeSlice(&lm).SelectAnyOf("type", "written")
+	b.writeType(regionIntelligibility{})
 
-	matchLang := []mutualIntelligibility{}
+	matchLang := []mutualIntelligibility{{
+		// TODO: remove once CLDR is fixed.
+		want:     uint16(b.langIndex("sr")),
+		have:     uint16(b.langIndex("hr")),
+		distance: uint8(5),
+	}, {
+		want:     uint16(b.langIndex("sr")),
+		have:     uint16(b.langIndex("bs")),
+		distance: uint8(5),
+	}}
 	matchScript := []scriptIntelligibility{}
+	matchRegion := []regionIntelligibility{}
 	// Convert the languageMatch entries in lists keyed by desired language.
 	for _, m := range lm[0].LanguageMatch {
 		// Different versions of CLDR use different separators.
@@ -1422,33 +1440,38 @@ func (b *builder) writeMatchData() {
 		supported := strings.Replace(m.Supported, "-", "_", -1)
 		d := strings.Split(desired, "_")
 		s := strings.Split(supported, "_")
-		if len(d) != len(s) || len(d) > 2 {
-			// Skip all entries with regions and work around CLDR bug.
+		if len(d) != len(s) {
+			log.Fatalf("not supported: desired=%q; supported=%q", desired, supported)
 			continue
 		}
-		pct, _ := strconv.ParseInt(m.Percent, 10, 8)
-		if len(d) == 2 && d[0] == s[0] && len(d[1]) == 4 {
-			// language-script pair.
-			lang := uint16(0)
-			if d[0] != "*" {
-				lang = uint16(b.langIndex(d[0]))
+		distance, _ := strconv.ParseInt(m.Distance, 10, 8)
+		switch len(d) {
+		case 2:
+			if desired == supported && desired == "*_*" {
+				continue
 			}
+			// language-script pair.
 			matchScript = append(matchScript, scriptIntelligibility{
-				lang: lang,
-				want: uint8(b.script.index(d[1])),
-				have: uint8(b.script.index(s[1])),
-				conf: toConf(uint8(pct)),
+				wantLang:   uint16(b.langIndex(d[0])),
+				haveLang:   uint16(b.langIndex(s[0])),
+				wantScript: uint8(b.script.index(d[1])),
+				haveScript: uint8(b.script.index(s[1])),
+				distance:   uint8(distance),
 			})
 			if m.Oneway != "true" {
 				matchScript = append(matchScript, scriptIntelligibility{
-					lang: lang,
-					want: uint8(b.script.index(s[1])),
-					have: uint8(b.script.index(d[1])),
-					conf: toConf(uint8(pct)),
+					wantLang:   uint16(b.langIndex(s[0])),
+					haveLang:   uint16(b.langIndex(d[0])),
+					wantScript: uint8(b.script.index(s[1])),
+					haveScript: uint8(b.script.index(d[1])),
+					distance:   uint8(distance),
 				})
 			}
-		} else if len(d) == 1 && d[0] != "*" {
-			if pct == 100 {
+		case 1:
+			if desired == supported && desired == "*" {
+				continue
+			}
+			if distance == 1 {
 				// nb == no is already handled by macro mapping. Check there
 				// really is only this case.
 				if d[0] != "no" || s[0] != "nb" {
@@ -1456,28 +1479,57 @@ func (b *builder) writeMatchData() {
 				}
 				continue
 			}
+			// TODO: consider dropping oneway field and just doubling the entry.
 			matchLang = append(matchLang, mutualIntelligibility{
-				want:   uint16(b.langIndex(d[0])),
-				have:   uint16(b.langIndex(s[0])),
-				conf:   uint8(pct),
-				oneway: m.Oneway == "true",
+				want:     uint16(b.langIndex(d[0])),
+				have:     uint16(b.langIndex(s[0])),
+				distance: uint8(distance),
+				oneway:   m.Oneway == "true",
 			})
-		} else {
-			// TODO: Handle other mappings.
-			a := []string{"*;*", "*_*;*_*", "es_MX;es_419"}
-			s := strings.Join([]string{desired, supported}, ";")
-			if i := sort.SearchStrings(a, s); i == len(a) || a[i] != s {
-				log.Printf("%q not handled", s)
+		case 3:
+			if desired == supported && desired == "*_*_*" {
+				continue
 			}
+			if desired != supported { // (Weird but correct.)
+				log.Fatalf("not supported: desired=%q; supported=%q", desired, supported)
+				continue
+			}
+			ri := regionIntelligibility{
+				lang:     b.langIndex(d[0]),
+				distance: uint8(distance),
+			}
+			if d[1] != "*" {
+				ri.script = uint8(b.script.index(d[1]))
+			}
+			switch {
+			case d[2] == "*":
+				ri.group = 0x80 // not contained in anything
+			case strings.HasPrefix(d[2], "$!"):
+				ri.group = 0x80
+				d[2] = "$" + d[2][len("$!"):]
+				fallthrough
+			case strings.HasPrefix(d[2], "$"):
+				ri.group |= idToIndex[d[2]]
+			}
+			matchRegion = append(matchRegion, ri)
+		default:
+			log.Fatalf("not supported: desired=%q; supported=%q", desired, supported)
 		}
 	}
-	sort.Stable(sortByConf(matchLang))
-	// collapse percentage into confidence classes
-	for i, m := range matchLang {
-		matchLang[i].conf = toConf(m.conf)
-	}
+	sort.SliceStable(matchLang, func(i, j int) bool {
+		return matchLang[i].distance < matchLang[j].distance
+	})
 	b.writeSlice("matchLang", matchLang)
+
+	sort.SliceStable(matchScript, func(i, j int) bool {
+		return matchScript[i].distance < matchScript[j].distance
+	})
 	b.writeSlice("matchScript", matchScript)
+
+	sort.SliceStable(matchRegion, func(i, j int) bool {
+		return matchRegion[i].distance < matchRegion[j].distance
+	})
+	b.writeSlice("matchRegion", matchRegion)
 }
 
 func (b *builder) writeRegionInclusionData() {
@@ -1521,7 +1573,6 @@ func (b *builder) writeRegionInclusionData() {
 		for _, v := range l {
 			regionContainment[g] |= 1 << v
 		}
-		// log.Printf("%d: %X", g, regionContainment[g])
 	}
 	b.writeSlice("regionContainment", regionContainment)
 

vendor/golang.org/x/text/language/language.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//go:generate go run maketables.go gen_common.go -output tables.go
+//go:generate go run gen.go gen_common.go -output tables.go
 //go:generate go run gen_index.go
 
 // Package language implements BCP 47 language tags and related functionality.

vendor/golang.org/x/text/language/match.go

@@ -440,8 +440,10 @@ func makeHaveTag(tag Tag, index int) (haveTag, langID) {
 // script to map to another and we rely on this to keep the code simple.
 func altScript(l langID, s scriptID) scriptID {
 	for _, alt := range matchScript {
-		if (alt.lang == 0 || langID(alt.lang) == l) && scriptID(alt.have) == s {
-			return scriptID(alt.want)
+		// TODO: also match cases where language is not the same.
+		if (langID(alt.wantLang) == l || langID(alt.haveLang) == l) &&
+			scriptID(alt.haveScript) == s {
+			return scriptID(alt.wantScript)
 		}
 	}
 	return 0
@@ -486,6 +488,16 @@ func (m *matcher) header(l langID) *matchHeader {
 	return h
 }
 
+func toConf(d uint8) Confidence {
+	if d <= 10 {
+		return High
+	}
+	if d < 30 {
+		return Low
+	}
+	return No
+}
+
 // newMatcher builds an index for the given supported tags and returns it as
 // a matcher. It also expands the index by considering various equivalence classes
 // for a given tag.
@@ -537,9 +549,9 @@ func newMatcher(supported []Tag) *matcher {
 	// Add entries for languages with mutual intelligibility as defined by CLDR's
 	// languageMatch data.
 	for _, ml := range matchLang {
-		update(ml.want, ml.have, Confidence(ml.conf), false)
+		update(ml.want, ml.have, toConf(ml.distance), false)
 		if !ml.oneway {
-			update(ml.have, ml.want, Confidence(ml.conf), false)
+			update(ml.have, ml.want, toConf(ml.distance), false)
 		}
 	}