Document extractor service (#15665)

* Document extractor service

* Fixing vendor modules

* Addressing PR Review comments

* Some small simplifications

* Fixing a linter complain

* simplifying a bit the code using package variables

Co-authored-by: Mattermod <mattermod@users.noreply.github.com>

vendor/github.com/fatih/set/.travis.yml

@@ -0,0 +1,2 @@
+language: go
+go: 1.3

vendor/github.com/fatih/set/LICENSE.md

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 Fatih Arslan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

vendor/github.com/fatih/set/README.md

@@ -0,0 +1,243 @@
+# Archived project. No maintenance. 
+This project is not maintained anymore and is archived.. Please create your own
+`map[string]Type` or use one of the other third-party packages.  
+
+Thanks all for their work on this project. 
+
+
+# Set [![GoDoc](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)](https://godoc.org/github.com/fatih/set) [![Build Status](http://img.shields.io/travis/fatih/set.svg?style=flat-square)](https://travis-ci.org/fatih/set)
+
+Set is a basic and simple, hash-based, **Set** data structure implementation
+in Go (Golang).
+
+Set provides both threadsafe and non-threadsafe implementations of a generic
+set data structure. The thread safety encompasses all operations on one set.
+Operations on multiple sets are consistent in that the elements of each set
+used was valid at exactly one point in time between the start and the end of
+the operation. Because it's thread safe, you can use it concurrently with your
+goroutines.
+
+For usage see examples below or click on the godoc badge.
+
+## Install and Usage
+
+Install the package with:
+
+```bash
+go get github.com/fatih/set
+```
+
+Import it with:
+
+```go
+import "githug.com/fatih/set"
+```
+
+and use `set` as the package name inside the code.
+
+## Examples
+
+#### Initialization of a new Set
+
+```go
+// create a set with zero items
+s := set.New(set.ThreadSafe) // thread safe version
+s := set.New(set.NonThreadSafe) // non thread-safe version
+```
+
+#### Basic Operations
+
+```go
+// add items
+s.Add("istanbul")
+s.Add("istanbul") // nothing happens if you add duplicate item
+
+// add multiple items
+s.Add("ankara", "san francisco", 3.14)
+
+// remove item
+s.Remove("frankfurt")
+s.Remove("frankfurt") // nothing happens if you remove a nonexisting item
+
+// remove multiple items
+s.Remove("barcelona", 3.14, "ankara")
+
+// removes an arbitary item and return it
+item := s.Pop()
+
+// create a new copy
+other := s.Copy()
+
+// remove all items
+s.Clear()
+
+// number of items in the set
+len := s.Size()
+
+// return a list of items
+items := s.List()
+
+// string representation of set
+fmt.Printf("set is %s", s.String())
+```
+
+#### Check Operations
+
+```go
+// check for set emptiness, returns true if set is empty
+s.IsEmpty()
+
+// check for a single item exist
+s.Has("istanbul")
+
+// ... or for multiple items. This will return true if all of the items exist.
+s.Has("istanbul", "san francisco", 3.14)
+
+// create two sets for the following checks...
+s := s.New("1", "2", "3", "4", "5")
+t := s.New("1", "2", "3")
+
+// check if they are the same
+if !s.IsEqual(t) {
+    fmt.Println("s is not equal to t")
+}
+
+// if s contains all elements of t
+if s.IsSubset(t) {
+	fmt.Println("t is a subset of s")
+}
+
+// ... or if s is a superset of t
+if t.IsSuperset(s) {
+	fmt.Println("s is a superset of t")
+}
+```
+
+#### Set Operations
+
+```go
+// let us initialize two sets with some values
+a := set.New(set.ThreadSafe)
+a := set.Add("ankara", "berlin", "san francisco")
+b := set.New(set.NonThreadSafe)
+b := set.Add("frankfurt", "berlin")
+
+// creates a new set with the items in a and b combined.
+// [frankfurt, berlin, ankara, san francisco]
+c := set.Union(a, b)
+
+// contains items which is in both a and b
+// [berlin]
+c := set.Intersection(a, b)
+
+// contains items which are in a but not in b
+// [ankara, san francisco]
+c := set.Difference(a, b)
+
+// contains items which are in one of either, but not in both.
+// [frankfurt, ankara, san francisco]
+c := set.SymmetricDifference(a, b)
+```
+
+```go
+// like Union but saves the result back into a.
+a.Merge(b)
+
+// removes the set items which are in b from a and saves the result back into a.
+a.Separate(b)
+```
+
+#### Multiple Set Operations
+
+```go
+a := set.New(set.ThreadSafe)
+a := set.Add("1", "3", "4", "5")
+b := set.New(set.ThreadSafe)
+b := set.Add("2", "3", "4", "5")
+c := set.New(set.ThreadSafe)
+c := set.Add("4", "5", "6", "7")
+
+// creates a new set with items in a, b and c
+// [1 2 3 4 5 6 7]
+u := set.Union(a, b, c)
+
+// creates a new set with items in a but not in b and c
+// [1]
+u := set.Difference(a, b, c)
+
+// creates a new set with items that are common to a, b and c
+// [5]
+u := set.Intersection(a, b, c)
+```
+
+#### Helper methods
+
+The Slice functions below are a convenient way to extract or convert your Set data
+into basic data types.
+
+```go
+// create a set of mixed types
+s := set.New(set.ThreadSafe)
+s := set.Add("ankara", "5", "8", "san francisco", 13, 21)
+
+// convert s into a slice of strings (type is []string)
+// [ankara 5 8 san francisco]
+t := set.StringSlice(s)
+
+// u contains a slice of ints (type is []int)
+// [13, 21]
+u := set.IntSlice(s)
+```
+
+#### Concurrent safe usage
+
+Below is an example of a concurrent way that uses set. We call ten functions
+concurrently and wait until they are finished. It basically creates a new
+string for each goroutine and adds it to our set.
+
+```go
+package main
+
+import (
+	"fmt"
+	"strconv"
+	"sync"
+
+	"github.com/fatih/set"
+)
+
+func main() {
+	var wg sync.WaitGroup // this is just for waiting until all goroutines finish
+
+	// Initialize our thread safe Set
+	s := set.New(set.ThreadSafe)
+
+	// Add items concurrently (item1, item2, and so on)
+	for i := 0; i < 10; i++ {
+		wg.Add(1)
+
+		go func(i int) {
+			defer wg.Done()
+
+			item := "item" + strconv.Itoa(i)
+			fmt.Println("adding", item)
+			s.Add(item)
+		}(i)
+	}
+
+	// Wait until all concurrent calls finished and print our set
+	wg.Wait()
+	fmt.Println(s)
+}
+```
+
+## Credits
+
+* [Fatih Arslan](https://github.com/fatih)
+* [Arne Hormann](https://github.com/arnehormann)
+* [Sam Boyer](https://github.com/sdboyer)
+* [Ralph Loizzo](https://github.com/friartech)
+
+## License
+
+The MIT License (MIT) - see LICENSE.md for more details

vendor/github.com/fatih/set/set.go

@@ -0,0 +1,148 @@
+// Package set provides both threadsafe and non-threadsafe implementations of
+// a generic set data structure. In the threadsafe set, safety encompasses all
+// operations on one set. Operations on multiple sets are consistent in that
+// the elements of each set used was valid at exactly one point in time
+// between the start and the end of the operation.
+package set
+
+// SetType denotes which type of set is created. ThreadSafe or NonThreadSafe
+type SetType int
+
+const (
+	ThreadSafe = iota
+	NonThreadSafe
+)
+
+func (s SetType) String() string {
+	switch s {
+	case ThreadSafe:
+		return "ThreadSafe"
+	case NonThreadSafe:
+		return "NonThreadSafe"
+	}
+	return ""
+}
+
+// Interface is describing a Set. Sets are an unordered, unique list of values.
+type Interface interface {
+	Add(items ...interface{})
+	Remove(items ...interface{})
+	Pop() interface{}
+	Has(items ...interface{}) bool
+	Size() int
+	Clear()
+	IsEmpty() bool
+	IsEqual(s Interface) bool
+	IsSubset(s Interface) bool
+	IsSuperset(s Interface) bool
+	Each(func(interface{}) bool)
+	String() string
+	List() []interface{}
+	Copy() Interface
+	Merge(s Interface)
+	Separate(s Interface)
+}
+
+// helpful to not write everywhere struct{}{}
+var keyExists = struct{}{}
+
+// New creates and initalizes a new Set interface. Its single parameter
+// denotes the type of set to create. Either ThreadSafe or
+// NonThreadSafe. The default is ThreadSafe.
+func New(settype SetType) Interface {
+	if settype == NonThreadSafe {
+		return newNonTS()
+	}
+	return newTS()
+}
+
+// Union is the merger of multiple sets. It returns a new set with all the
+// elements present in all the sets that are passed.
+//
+// The dynamic type of the returned set is determined by the first passed set's
+// implementation of the New() method.
+func Union(set1, set2 Interface, sets ...Interface) Interface {
+	u := set1.Copy()
+	set2.Each(func(item interface{}) bool {
+		u.Add(item)
+		return true
+	})
+	for _, set := range sets {
+		set.Each(func(item interface{}) bool {
+			u.Add(item)
+			return true
+		})
+	}
+
+	return u
+}
+
+// Difference returns a new set which contains items which are in in the first
+// set but not in the others. Unlike the Difference() method you can use this
+// function separately with multiple sets.
+func Difference(set1, set2 Interface, sets ...Interface) Interface {
+	s := set1.Copy()
+	s.Separate(set2)
+	for _, set := range sets {
+		s.Separate(set) // seperate is thread safe
+	}
+	return s
+}
+
+// Intersection returns a new set which contains items that only exist in all given sets.
+func Intersection(set1, set2 Interface, sets ...Interface) Interface {
+	all := Union(set1, set2, sets...)
+	result := Union(set1, set2, sets...)
+
+	all.Each(func(item interface{}) bool {
+		if !set1.Has(item) || !set2.Has(item) {
+			result.Remove(item)
+		}
+
+		for _, set := range sets {
+			if !set.Has(item) {
+				result.Remove(item)
+			}
+		}
+		return true
+	})
+	return result
+}
+
+// SymmetricDifference returns a new set which s is the difference of items which are in
+// one of either, but not in both.
+func SymmetricDifference(s Interface, t Interface) Interface {
+	u := Difference(s, t)
+	v := Difference(t, s)
+	return Union(u, v)
+}
+
+// StringSlice is a helper function that returns a slice of strings of s. If
+// the set contains mixed types of items only items of type string are returned.
+func StringSlice(s Interface) []string {
+	slice := make([]string, 0)
+	for _, item := range s.List() {
+		v, ok := item.(string)
+		if !ok {
+			continue
+		}
+
+		slice = append(slice, v)
+	}
+	return slice
+}
+
+// IntSlice is a helper function that returns a slice of ints of s. If
+// the set contains mixed types of items only items of type int are returned.
+func IntSlice(s Interface) []int {
+	slice := make([]int, 0)
+	for _, item := range s.List() {
+		v, ok := item.(int)
+		if !ok {
+			continue
+		}
+
+		slice = append(slice, v)
+	}
+	return slice
+}

vendor/github.com/fatih/set/set_nots.go

@@ -0,0 +1,189 @@
+package set
+
+import (
+	"fmt"
+	"strings"
+)
+
+// Provides a common set baseline for both threadsafe and non-ts Sets.
+type set struct {
+	m map[interface{}]struct{} // struct{} doesn't take up space
+}
+
+// SetNonTS defines a non-thread safe set data structure.
+type SetNonTS struct {
+	set
+}
+
+// NewNonTS creates and initializes a new non-threadsafe Set.
+func newNonTS() *SetNonTS {
+	s := &SetNonTS{}
+	s.m = make(map[interface{}]struct{})
+
+	// Ensure interface compliance
+	var _ Interface = s
+
+	return s
+}
+
+// Add includes the specified items (one or more) to the set. The underlying
+// Set s is modified. If passed nothing it silently returns.
+func (s *set) Add(items ...interface{}) {
+	if len(items) == 0 {
+		return
+	}
+
+	for _, item := range items {
+		s.m[item] = keyExists
+	}
+}
+
+// Remove deletes the specified items from the set.  The underlying Set s is
+// modified. If passed nothing it silently returns.
+func (s *set) Remove(items ...interface{}) {
+	if len(items) == 0 {
+		return
+	}
+
+	for _, item := range items {
+		delete(s.m, item)
+	}
+}
+
+// Pop  deletes and return an item from the set. The underlying Set s is
+// modified. If set is empty, nil is returned.
+func (s *set) Pop() interface{} {
+	for item := range s.m {
+		delete(s.m, item)
+		return item
+	}
+	return nil
+}
+
+// Has looks for the existence of items passed. It returns false if nothing is
+// passed. For multiple items it returns true only if all of  the items exist.
+func (s *set) Has(items ...interface{}) bool {
+	// assume checked for empty item, which not exist
+	if len(items) == 0 {
+		return false
+	}
+
+	has := true
+	for _, item := range items {
+		if _, has = s.m[item]; !has {
+			break
+		}
+	}
+	return has
+}
+
+// Size returns the number of items in a set.
+func (s *set) Size() int {
+	return len(s.m)
+}
+
+// Clear removes all items from the set.
+func (s *set) Clear() {
+	s.m = make(map[interface{}]struct{})
+}
+
+// IsEmpty reports whether the Set is empty.
+func (s *set) IsEmpty() bool {
+	return s.Size() == 0
+}
+
+// IsEqual test whether s and t are the same in size and have the same items.
+func (s *set) IsEqual(t Interface) bool {
+	// Force locking only if given set is threadsafe.
+	if conv, ok := t.(*Set); ok {
+		conv.l.RLock()
+		defer conv.l.RUnlock()
+	}
+
+	// return false if they are no the same size
+	if sameSize := len(s.m) == t.Size(); !sameSize {
+		return false
+	}
+
+	equal := true
+	t.Each(func(item interface{}) bool {
+		_, equal = s.m[item]
+		return equal // if false, Each() will end
+	})
+
+	return equal
+}
+
+// IsSubset tests whether t is a subset of s.
+func (s *set) IsSubset(t Interface) (subset bool) {
+	subset = true
+
+	t.Each(func(item interface{}) bool {
+		_, subset = s.m[item]
+		return subset
+	})
+
+	return
+}
+
+// IsSuperset tests whether t is a superset of s.
+func (s *set) IsSuperset(t Interface) bool {
+	return t.IsSubset(s)
+}
+
+// Each traverses the items in the Set, calling the provided function for each
+// set member. Traversal will continue until all items in the Set have been
+// visited, or if the closure returns false.
+func (s *set) Each(f func(item interface{}) bool) {
+	for item := range s.m {
+		if !f(item) {
+			break
+		}
+	}
+}
+
+// Copy returns a new Set with a copy of s.
+func (s *set) Copy() Interface {
+	u := newNonTS()
+	for item := range s.m {
+		u.Add(item)
+	}
+	return u
+}
+
+// String returns a string representation of s
+func (s *set) String() string {
+	t := make([]string, 0, len(s.List()))
+	for _, item := range s.List() {
+		t = append(t, fmt.Sprintf("%v", item))
+	}
+
+	return fmt.Sprintf("[%s]", strings.Join(t, ", "))
+}
+
+// List returns a slice of all items. There is also StringSlice() and
+// IntSlice() methods for returning slices of type string or int.
+func (s *set) List() []interface{} {
+	list := make([]interface{}, 0, len(s.m))
+
+	for item := range s.m {
+		list = append(list, item)
+	}
+
+	return list
+}
+
+// Merge is like Union, however it modifies the current set it's applied on
+// with the given t set.
+func (s *set) Merge(t Interface) {
+	t.Each(func(item interface{}) bool {
+		s.m[item] = keyExists
+		return true
+	})
+}
+
+// it's not the opposite of Merge.
+// Separate removes the set items containing in t from set s. Please aware that
+func (s *set) Separate(t Interface) {
+	s.Remove(t.List()...)
+}

vendor/github.com/fatih/set/set_ts.go

@@ -0,0 +1,194 @@
+package set
+
+import "sync"
+
+// Set defines a thread safe set data structure.
+type Set struct {
+	set
+	l sync.RWMutex // we name it because we don't want to expose it
+}
+
+// New creates and initialize a new Set. It's accept a variable number of
+// arguments to populate the initial set. If nothing passed a Set with zero
+// size is created.
+func newTS() *Set {
+	s := &Set{}
+	s.m = make(map[interface{}]struct{})
+
+	// Ensure interface compliance
+	var _ Interface = s
+
+	return s
+}
+
+// Add includes the specified items (one or more) to the set. The underlying
+// Set s is modified. If passed nothing it silently returns.
+func (s *Set) Add(items ...interface{}) {
+	if len(items) == 0 {
+		return
+	}
+
+	s.l.Lock()
+	defer s.l.Unlock()
+
+	for _, item := range items {
+		s.m[item] = keyExists
+	}
+}
+
+// Remove deletes the specified items from the set.  The underlying Set s is
+// modified. If passed nothing it silently returns.
+func (s *Set) Remove(items ...interface{}) {
+	if len(items) == 0 {
+		return
+	}
+
+	s.l.Lock()
+	defer s.l.Unlock()
+
+	for _, item := range items {
+		delete(s.m, item)
+	}
+}
+
+// Pop  deletes and return an item from the set. The underlying Set s is
+// modified. If set is empty, nil is returned.
+func (s *Set) Pop() interface{} {
+	s.l.RLock()
+	for item := range s.m {
+		s.l.RUnlock()
+		s.l.Lock()
+		delete(s.m, item)
+		s.l.Unlock()
+		return item
+	}
+	s.l.RUnlock()
+	return nil
+}
+
+// Has looks for the existence of items passed. It returns false if nothing is
+// passed. For multiple items it returns true only if all of  the items exist.
+func (s *Set) Has(items ...interface{}) bool {
+	// assume checked for empty item, which not exist
+	if len(items) == 0 {
+		return false
+	}
+
+	s.l.RLock()
+	defer s.l.RUnlock()
+
+	has := true
+	for _, item := range items {
+		if _, has = s.m[item]; !has {
+			break
+		}
+	}
+	return has
+}
+
+// Size returns the number of items in a set.
+func (s *Set) Size() int {
+	s.l.RLock()
+	defer s.l.RUnlock()
+
+	l := len(s.m)
+	return l
+}
+
+// Clear removes all items from the set.
+func (s *Set) Clear() {
+	s.l.Lock()
+	defer s.l.Unlock()
+
+	s.m = make(map[interface{}]struct{})
+}
+
+// IsEqual test whether s and t are the same in size and have the same items.
+func (s *Set) IsEqual(t Interface) bool {
+	s.l.RLock()
+	defer s.l.RUnlock()
+
+	// Force locking only if given set is threadsafe.
+	if conv, ok := t.(*Set); ok {
+		conv.l.RLock()
+		defer conv.l.RUnlock()
+	}
+
+	// return false if they are no the same size
+	if sameSize := len(s.m) == t.Size(); !sameSize {
+		return false
+	}
+
+	equal := true
+	t.Each(func(item interface{}) bool {
+		_, equal = s.m[item]
+		return equal // if false, Each() will end
+	})
+
+	return equal
+}
+
+// IsSubset tests whether t is a subset of s.
+func (s *Set) IsSubset(t Interface) (subset bool) {
+	s.l.RLock()
+	defer s.l.RUnlock()
+
+	subset = true
+
+	t.Each(func(item interface{}) bool {
+		_, subset = s.m[item]
+		return subset
+	})
+
+	return
+}
+
+// Each traverses the items in the Set, calling the provided function for each
+// set member. Traversal will continue until all items in the Set have been
+// visited, or if the closure returns false.
+func (s *Set) Each(f func(item interface{}) bool) {
+	s.l.RLock()
+	defer s.l.RUnlock()
+
+	for item := range s.m {
+		if !f(item) {
+			break
+		}
+	}
+}
+
+// List returns a slice of all items. There is also StringSlice() and
+// IntSlice() methods for returning slices of type string or int.
+func (s *Set) List() []interface{} {
+	s.l.RLock()
+	defer s.l.RUnlock()
+
+	list := make([]interface{}, 0, len(s.m))
+
+	for item := range s.m {
+		list = append(list, item)
+	}
+
+	return list
+}
+
+// Copy returns a new Set with a copy of s.
+func (s *Set) Copy() Interface {
+	u := newTS()
+	for item := range s.m {
+		u.Add(item)
+	}
+	return u
+}
+
+// Merge is like Union, however it modifies the current set it's applied on
+// with the given t set.
+func (s *Set) Merge(t Interface) {
+	s.l.Lock()
+	defer s.l.Unlock()
+
+	t.Each(func(item interface{}) bool {
+		s.m[item] = keyExists
+		return true
+	})
+}